diff --git a/extensions/client_cpp/hl2ss.cpp b/extensions/client_cpp/hl2ss.cpp index 2bd9263a..04dd2b5b 100644 --- a/extensions/client_cpp/hl2ss.cpp +++ b/extensions/client_cpp/hl2ss.cpp @@ -299,7 +299,7 @@ void create_configuration_for_h26x_encoding(std::vector& sc, std::vecto } } -void create_configuration_for_mrc_video(std::vector& sc, bool enable_mrc, bool hologram_composition, bool recording_indicator, bool video_stabilization, bool blank_protected, bool show_mesh, float global_opacity, float output_width, float output_height, uint32_t video_stabilization_length, uint32_t hologram_perspective) +void create_configuration_for_mrc_video(std::vector& sc, bool enable_mrc, bool hologram_composition, bool recording_indicator, bool video_stabilization, bool blank_protected, bool show_mesh, bool shared, float global_opacity, float output_width, float output_height, uint32_t video_stabilization_length, uint32_t hologram_perspective) { push_u8(sc, enable_mrc); push_u8(sc, hologram_composition); @@ -307,6 +307,7 @@ void create_configuration_for_mrc_video(std::vector& sc, bool enable_mr push_u8(sc, video_stabilization); push_u8(sc, blank_protected); push_u8(sc, show_mesh); + push_u8(sc, shared); push_float(sc, global_opacity); push_float(sc, output_width); push_float(sc, output_height); @@ -408,13 +409,13 @@ uint8_t const STOP = 0x08; uint8_t const MODE_3 = 0x03; }; -void start_subsystem_pv(char const* host, uint16_t port, bool enable_mrc, bool hologram_composition, bool recording_indicator, bool video_stabilization, bool blank_protected, bool show_mesh, float global_opacity, float output_width, float output_height, uint32_t video_stabilization_length, uint32_t hologram_perspective) +void start_subsystem_pv(char const* host, uint16_t port, bool enable_mrc, bool hologram_composition, bool recording_indicator, bool video_stabilization, bool blank_protected, bool show_mesh, bool shared, float global_opacity, float output_width, float output_height, uint32_t video_stabilization_length, uint32_t hologram_perspective) { std::vector sc; client c; create_configuration_for_pv_mode_2(sc, pv_control::START | pv_control::MODE_3, 1920, 1080, 30); - create_configuration_for_mrc_video(sc, enable_mrc, hologram_composition, recording_indicator, video_stabilization, blank_protected, show_mesh, global_opacity, output_width, output_height, video_stabilization_length, hologram_perspective); + create_configuration_for_mrc_video(sc, enable_mrc, hologram_composition, recording_indicator, video_stabilization, blank_protected, show_mesh, shared, global_opacity, output_width, output_height, video_stabilization_length, hologram_perspective); c.open(host, port); c.sendall(sc.data(), sc.size()); @@ -708,11 +709,11 @@ void collect_i420(uint8_t* dst, int width, int height, uint8_t* data[8], int lin void collect_nv12(uint8_t* dst, uint16_t width, uint16_t height, uint8_t const* src, uint16_t stride) { - trim_plane(dst, src, height, width, stride); - trim_plane(dst + (height * width), src + (height * stride), height, width / 2, stride / 2); + trim_plane(dst, src, height, width, stride); + trim_plane(dst + (height * width), src + (height * stride), height / 2, width, stride); } -uint16_t get_video_stride(uint16_t width) +constexpr uint16_t get_video_stride(uint16_t width) { return width + ((64 - (width & 63)) & 63); } @@ -862,6 +863,37 @@ uint8_t decoder_pv::decoded_bpp(uint8_t decoded_format) return cv_format[decoded_format][0]; } +void decoder_pv::resolution(uint32_t bytes, uint16_t& width, uint16_t& height, uint16_t& stride) +{ + switch (bytes) + { + case get_video_stride(1952)*1100: width = 1952; height = 1100; stride = get_video_stride(1952); break; + case get_video_stride(1504)* 846: width = 1504; height = 846; stride = get_video_stride(1504); break; + case get_video_stride(1920)*1080: width = 1920; height = 1080; stride = get_video_stride(1920); break; + case get_video_stride(1280)* 720: width = 1280; height = 720; stride = get_video_stride(1280); break; + case get_video_stride( 640)* 360: width = 640; height = 360; stride = get_video_stride( 640); break; + case get_video_stride( 760)* 428: width = 760; height = 428; stride = get_video_stride( 760); break; + case get_video_stride( 960)* 540: width = 960; height = 540; stride = get_video_stride( 960); break; + case get_video_stride(1128)* 636: width = 1128; height = 636; stride = get_video_stride(1128); break; + case get_video_stride( 424)* 240: width = 424; height = 240; stride = get_video_stride( 424); break; + case get_video_stride( 500)* 282: width = 500; height = 282; stride = get_video_stride( 500); break; + case 1952*1100: width = 1952; height = 1100; stride = 1952; break; + case 1504* 846: width = 1504; height = 846; stride = 1504; break; + case 760* 428: width = 760; height = 428; stride = 760; break; + case 1128* 636: width = 1128; height = 636; stride = 1128; break; + case 424* 240: width = 424; height = 240; stride = 424; break; + case 500* 282: width = 500; height = 282; stride = 500; break; + default: throw std::runtime_error("hl2ss::decoder_pv::resolution : unknown resolution"); + } +} + +void decoder_pv::resolution_decoded(uint32_t payload_size, uint8_t decoded_format, uint16_t& width, uint16_t& height, uint8_t& channels) +{ + uint16_t stride; + channels = decoded_bpp(decoded_format); + resolution((payload_size - K_SIZE) / channels, width, height, stride); +} + void decoder_pv::open(uint16_t width, uint16_t height, uint8_t profile) { m_width = width; @@ -871,38 +903,38 @@ void decoder_pv::open(uint16_t width, uint16_t height, uint8_t profile) if (m_profile != video_profile::RAW) { m_codec.open(get_video_codec_id(m_profile)); } } -uint32_t decoder_pv::decoded_size(uint8_t decoded_format) +std::unique_ptr decoder_pv::decode(uint8_t* data, uint32_t size, uint8_t decoded_format, uint32_t& decoded_size) { - return m_width * m_height * decoded_bpp(decoded_format) + K_SIZE; -} - -std::unique_ptr decoder_pv::decode(uint8_t* data, uint32_t size, uint8_t decoded_format) -{ - uint32_t full_size = decoded_size(decoded_format); uint32_t h26x_size = size - K_SIZE; - - std::unique_ptr out = std::make_unique(full_size); - - cv::Mat input = cv::Mat((m_height * 3) / 2, m_width, CV_8UC1); - cv::Mat output; - int code; + uint16_t width; + uint16_t height; + uint16_t stride; + cv::Mat input; + int code; if (m_profile != video_profile::RAW) { std::shared_ptr f = m_codec.decode(data, h26x_size); - collect_i420(input.data, m_width, m_height, f->av_frame->data, f->av_frame->linesize); + width = f->av_frame->width; + height = f->av_frame->height; + input = cv::Mat((height * 3) / 2, width, CV_8UC1); + collect_i420(input.data, width, height, f->av_frame->data, f->av_frame->linesize); code = cv_format[decoded_format][2]; } else { - collect_nv12(input.data, m_width, m_height, data, get_video_stride(m_width)); + resolution((h26x_size * 2) / 3, width, height, stride); + input = cv::Mat((height * 3) / 2, width, CV_8UC1); + collect_nv12(input.data, width, height, data, stride); code = cv_format[decoded_format][3]; } - - output = cv::Mat(m_height, m_width, cv_format[decoded_format][1], out.get()); + + decoded_size = width * height * decoded_bpp(decoded_format) + K_SIZE; + std::unique_ptr out = std::make_unique(decoded_size); + cv::Mat output = cv::Mat(height, width, cv_format[decoded_format][1], out.get()); cv::cvtColor(input, output, code); + memcpy(out.get() + decoded_size - K_SIZE, data + h26x_size, K_SIZE); - memcpy(out.get() + full_size - K_SIZE, data + h26x_size, K_SIZE); return out; } @@ -1027,7 +1059,9 @@ void rx_decoded_pv::open() std::shared_ptr rx_decoded_pv::get_next_packet() { std::shared_ptr p = rx_pv::get_next_packet(); - p->set_payload(m_decoder.decoded_size(decoded_format), m_decoder.decode(p->payload.get(), p->sz_payload, decoded_format)); + uint32_t size; + std::unique_ptr payload = m_decoder.decode(p->payload.get(), p->sz_payload, decoded_format, size); + p->set_payload(size, std::move(payload)); return p; } @@ -1187,6 +1221,7 @@ std::shared_ptr download_calibration_pv(char const* host, uint16 c.download(data->radial_distortion, sizeof(data->radial_distortion), chunk_size::SINGLE_TRANSFER); c.download(data->tangential_distortion, sizeof(data->tangential_distortion), chunk_size::SINGLE_TRANSFER); c.download(data->projection, sizeof(data->projection), chunk_size::SINGLE_TRANSFER); + c.download(data->extrinsics, sizeof(data->extrinsics), chunk_size::SINGLE_TRANSFER); c.close(); diff --git a/extensions/client_cpp/hl2ss.h b/extensions/client_cpp/hl2ss.h index a67ee701..fcda2508 100644 --- a/extensions/client_cpp/hl2ss.h +++ b/extensions/client_cpp/hl2ss.h @@ -386,7 +386,7 @@ struct pv_intrinsics float cy; }; -void start_subsystem_pv(char const* host, uint16_t port, bool enable_mrc, bool hologram_composition, bool recording_indicator, bool video_stabilization, bool blank_protected, bool show_mesh, float global_opacity, float output_width, float output_height, uint32_t video_stabilization_length, uint32_t hologram_perspective); +void start_subsystem_pv(char const* host, uint16_t port, bool enable_mrc, bool hologram_composition, bool recording_indicator, bool video_stabilization, bool blank_protected, bool show_mesh, bool shared, float global_opacity, float output_width, float output_height, uint32_t video_stabilization_length, uint32_t hologram_perspective); void stop_subsystem_pv(char const* host, uint16_t port); //------------------------------------------------------------------------------ @@ -629,10 +629,11 @@ class decoder_pv static uint32_t const K_SIZE = sizeof(pv_intrinsics); static uint8_t decoded_bpp(uint8_t decoded_format); + static void resolution(uint32_t bytes, uint16_t& width, uint16_t& height, uint16_t& stride); + static void resolution_decoded(uint32_t payload_size, uint8_t decoded_format, uint16_t& width, uint16_t& height, uint8_t& channels); void open(uint16_t width, uint16_t height, uint8_t profile); - uint32_t decoded_size(uint8_t decoded_format); - std::unique_ptr decode(uint8_t* data, uint32_t size, uint8_t decoded_format); + std::unique_ptr decode(uint8_t* data, uint32_t size, uint8_t decoded_format, uint32_t& decoded_size); void close(); }; @@ -777,6 +778,7 @@ struct calibration_pv float radial_distortion[3]; float tangential_distortion[2]; float projection[4][4]; + float extrinsics[4][4]; }; std::shared_ptr download_calibration_rm_vlc(char const* host, uint16_t port); diff --git a/extensions/client_cpp/hl2ss_lnm.cpp b/extensions/client_cpp/hl2ss_lnm.cpp index 58f091da..e0e65956 100644 --- a/extensions/client_cpp/hl2ss_lnm.cpp +++ b/extensions/client_cpp/hl2ss_lnm.cpp @@ -78,9 +78,9 @@ uint64_t get_sync_period(hl2ss::rx* rx) // Control //------------------------------------------------------------------------------ -void start_subsystem_pv(char const* host, uint16_t port, bool enable_mrc, bool hologram_composition, bool recording_indicator, bool video_stabilization, bool blank_protected, bool show_mesh, float global_opacity, float output_width, float output_height, uint32_t video_stabilization_length, uint32_t hologram_perspective) +void start_subsystem_pv(char const* host, uint16_t port, bool enable_mrc, bool hologram_composition, bool recording_indicator, bool video_stabilization, bool blank_protected, bool show_mesh, bool shared, float global_opacity, float output_width, float output_height, uint32_t video_stabilization_length, uint32_t hologram_perspective) { - hl2ss::start_subsystem_pv(host, port, enable_mrc, hologram_composition, recording_indicator, video_stabilization, blank_protected, show_mesh, global_opacity, output_width, output_height, video_stabilization_length, hologram_perspective); + hl2ss::start_subsystem_pv(host, port, enable_mrc, hologram_composition, recording_indicator, video_stabilization, blank_protected, show_mesh, shared, global_opacity, output_width, output_height, video_stabilization_length, hologram_perspective); } void stop_subsystem_pv(char const* host, uint16_t port) diff --git a/extensions/client_cpp/hl2ss_lnm.h b/extensions/client_cpp/hl2ss_lnm.h index d13ef3ac..835440ce 100644 --- a/extensions/client_cpp/hl2ss_lnm.h +++ b/extensions/client_cpp/hl2ss_lnm.h @@ -26,7 +26,7 @@ uint64_t get_sync_period(hl2ss::rx* rx); // Control //------------------------------------------------------------------------------ -void start_subsystem_pv(char const* host, uint16_t port, bool enable_mrc=false, bool hologram_composition=true, bool recording_indicator=false, bool video_stabilization=false, bool blank_protected=false, bool show_mesh=false, float global_opacity=0.9f, float output_width=0.0f, float output_height=0.0f, uint32_t video_stabilization_length=0, uint32_t hologram_perspective=hl2ss::hologram_perspective::PV); +void start_subsystem_pv(char const* host, uint16_t port, bool enable_mrc=false, bool hologram_composition=true, bool recording_indicator=false, bool video_stabilization=false, bool blank_protected=false, bool show_mesh=false, bool shared=false, float global_opacity=0.9f, float output_width=0.0f, float output_height=0.0f, uint32_t video_stabilization_length=0, uint32_t hologram_perspective=hl2ss::hologram_perspective::PV); void stop_subsystem_pv(char const* host, uint16_t port); //------------------------------------------------------------------------------ diff --git a/extensions/client_cpp/main.cpp b/extensions/client_cpp/main.cpp index 6ab52229..bcda74ad 100644 --- a/extensions/client_cpp/main.cpp +++ b/extensions/client_cpp/main.cpp @@ -8,11 +8,8 @@ // Common //----------------------------------------------------------------------------- -void print_packet_metadata(uint64_t timestamp, hl2ss::matrix_4x4* matrix) +void print_matrix(hl2ss::matrix_4x4* matrix) { - std::cout << "Pose at time " << timestamp << std::endl; - if (matrix) - { std::cout << "[" << std::endl; for (int row = 0; row < 4; ++row) { @@ -23,6 +20,14 @@ void print_packet_metadata(uint64_t timestamp, hl2ss::matrix_4x4* matrix) std::cout << std::endl; } std::cout << "]" << std::endl; +} + +void print_packet_metadata(uint64_t timestamp, hl2ss::matrix_4x4* matrix) +{ + std::cout << "Pose at time " << timestamp << std::endl; + if (matrix) + { + print_matrix(matrix); } else { diff --git a/extensions/client_matlab/+hl2ss/+mt/sink_pv.m b/extensions/client_matlab/+hl2ss/+mt/sink_pv.m index 1c700719..a93aa511 100644 --- a/extensions/client_matlab/+hl2ss/+mt/sink_pv.m +++ b/extensions/client_matlab/+hl2ss/+mt/sink_pv.m @@ -75,7 +75,7 @@ function close(self) response = self.module('download_calibration', self.host, self.port, self.width, self.height, self.framerate); end - function start_subsystem(self, enable_mrc, hologram_composition, recording_indicator, video_stabilization, blank_protected, show_mesh, global_opacity, output_width, output_height, video_stabilization_length, hologram_perspective) + function start_subsystem(self, enable_mrc, hologram_composition, recording_indicator, video_stabilization, blank_protected, show_mesh, shared, global_opacity, output_width, output_height, video_stabilization_length, hologram_perspective) arguments self enable_mrc = false @@ -84,6 +84,7 @@ function start_subsystem(self, enable_mrc, hologram_composition, recording_indic video_stabilization = false blank_protected = false show_mesh = false + shared = false global_opacity = 0.9 output_width = 0.0 output_height = 0.0 @@ -91,7 +92,7 @@ function start_subsystem(self, enable_mrc, hologram_composition, recording_indic hologram_perspective = 1 end - self.module('start_subsystem_pv', self.host, self.port, logical(enable_mrc), logical(hologram_composition), logical(recording_indicator), logical(video_stabilization), logical(blank_protected), logical(show_mesh), single(global_opacity), single(output_width), single(output_height), uint32(video_stabilization_length), uint32(hologram_perspective)); + self.module('start_subsystem_pv', self.host, self.port, logical(enable_mrc), logical(hologram_composition), logical(recording_indicator), logical(video_stabilization), logical(blank_protected), logical(show_mesh), logical(shared), single(global_opacity), single(output_width), single(output_height), uint32(video_stabilization_length), uint32(hologram_perspective)); end function stop_subsystem(self) diff --git a/extensions/client_matlab/hl2ss_matlab.cpp b/extensions/client_matlab/hl2ss_matlab.cpp index 0ada6550..4dc45571 100644 --- a/extensions/client_matlab/hl2ss_matlab.cpp +++ b/extensions/client_matlab/hl2ss_matlab.cpp @@ -455,7 +455,7 @@ class MexFunction : public matlab::mex::Function outputs[0] = std::move(o); } - void pack_pv(int64_t frame_index, int32_t status, hl2ss::packet* packet, uint16_t width, uint16_t height, matlab::mex::ArgumentList outputs) + void pack_pv(int64_t frame_index, int32_t status, hl2ss::packet* packet, uint16_t width, uint16_t height, uint8_t channels, matlab::mex::ArgumentList outputs) { matlab::data::StructArray o = m_factory.createStructArray({ 1 }, { "frame_index", "status", "timestamp", "image", "intrinsics", "pose" }); @@ -468,7 +468,7 @@ class MexFunction : public matlab::mex::Function if (packet->payload) { uint32_t image_size = packet->sz_payload - hl2ss::decoder_pv::K_SIZE; - o[0]["image"] = unpack_payload(packet->payload.get(), 0, image_size, { height, width, image_size / (sizeof(uint8_t) * height * width) }); + o[0]["image"] = unpack_payload(packet->payload.get(), 0, image_size, { height, width, channels }); o[0]["intrinsics"] = unpack_payload( packet->payload.get(), image_size, hl2ss::decoder_pv::K_SIZE, { 4 }); } if (packet->pose) @@ -676,9 +676,22 @@ class MexFunction : public matlab::mex::Function int64_t frame_index; int32_t status; std::shared_ptr packet = grab(source, frame_index, status, inputs); - hl2ss::rx_pv const* p_rx = source->get_rx(); - pack_pv(frame_index, status, packet.get(), p_rx->width, p_rx->height, outputs); + uint16_t width; + uint16_t height; + uint8_t channels; + + if (packet) + { + hl2ss::rx_decoded_pv const* p_rx = source->get_rx(); + hl2ss::decoder_pv::resolution_decoded(packet->sz_payload, p_rx->decoded_format, width, height, channels); + } + else + { + width = height = channels = 0; + } + + pack_pv(frame_index, status, packet.get(), width, height, channels, outputs); } void get_packet_microphone(uint16_t port, matlab::mex::ArgumentList outputs, matlab::mex::ArgumentList inputs) @@ -776,13 +789,14 @@ class MexFunction : public matlab::mex::Function bool video_stabilization = get_argument(inputs); bool blank_protected = get_argument(inputs); bool show_mesh = get_argument(inputs); + bool shared = get_argument(inputs); float global_opacity = get_argument(inputs); float output_width = get_argument(inputs); float output_height = get_argument(inputs); uint32_t video_stabilization_length = get_argument(inputs); uint32_t hologram_perspective = get_argument(inputs); - hl2ss::lnm::start_subsystem_pv(host.c_str(), port, enable_mrc, hologram_composition, recording_indicator, video_stabilization, blank_protected, show_mesh, global_opacity, output_width, output_height, video_stabilization_length, hologram_perspective); + hl2ss::lnm::start_subsystem_pv(host.c_str(), port, enable_mrc, hologram_composition, recording_indicator, video_stabilization, blank_protected, show_mesh, shared, global_opacity, output_width, output_height, video_stabilization_length, hologram_perspective); } void stop_subsystem_pv(matlab::mex::ArgumentList outputs, matlab::mex::ArgumentList inputs) @@ -881,13 +895,14 @@ class MexFunction : public matlab::mex::Function std::shared_ptr data = hl2ss::lnm::download_calibration_pv(host, port, width, height, framerate); - matlab::data::StructArray o = m_factory.createStructArray({ 1 }, { "focal_length", "principal_point", "radial_distortion", "tangential_distortion", "projection" }); + matlab::data::StructArray o = m_factory.createStructArray({ 1 }, { "focal_length", "principal_point", "radial_distortion", "tangential_distortion", "projection", "extrinsics" }); o[0]["focal_length"] = to_typed_array(data->focal_length, { sizeof(data->focal_length) / sizeof(float) }); o[0]["principal_point"] = to_typed_array(data->principal_point, { sizeof(data->principal_point) / sizeof(float) }); o[0]["radial_distortion"] = to_typed_array(data->radial_distortion, { sizeof(data->radial_distortion) / sizeof(float) }); o[0]["tangential_distortion"] = to_typed_array(data->tangential_distortion, { sizeof(data->tangential_distortion) / sizeof(float) }); o[0]["projection"] = unpack_payload(&data->projection, 0, sizeof(data->projection), { 4, 4 }); + o[0]["extrinsics"] = unpack_payload(&data->extrinsics, 0, sizeof(data->extrinsics), { 4, 4 }); outputs[0] = std::move(o); } diff --git a/hl2ss/hl2ss/custom_video_effect.h b/hl2ss/hl2ss/custom_video_effect.h index 430086b1..b8ff79cc 100644 --- a/hl2ss/hl2ss/custom_video_effect.h +++ b/hl2ss/hl2ss/custom_video_effect.h @@ -13,7 +13,8 @@ struct MRCVideoOptions bool video_stabilization; bool blank_protected; bool show_mesh; - uint8_t _reserved[2]; + bool shared; + uint8_t _reserved[1]; float global_opacity; float output_width; float output_height; diff --git a/hl2ss/hl2ss/ipc_sc.cpp b/hl2ss/hl2ss/ipc_sc.cpp index 7912ebaa..6e4d449f 100644 --- a/hl2ss/hl2ss/ipc_sc.cpp +++ b/hl2ss/hl2ss/ipc_sc.cpp @@ -187,6 +187,8 @@ bool ReceiveMRCVideoOptions(SOCKET clientsocket, MRCVideoOptions& options) if (!ok) { return false; } ok = recv_u8(clientsocket, *(uint8_t*)&options.show_mesh); if (!ok) { return false; } + ok = recv_u8(clientsocket, *(uint8_t*)&options.shared); + if (!ok) { return false; } ok = recv_u32(clientsocket, *(uint32_t*)&options.global_opacity); if (!ok) { return false; } ok = recv_u32(clientsocket, *(uint32_t*)&options.output_width); diff --git a/hl2ss/hl2ss/personal_video.cpp b/hl2ss/hl2ss/personal_video.cpp index 570a03c3..7ae26ae2 100644 --- a/hl2ss/hl2ss/personal_video.cpp +++ b/hl2ss/hl2ss/personal_video.cpp @@ -19,6 +19,7 @@ using namespace winrt::Windows::Media::Capture::Frames; static CRITICAL_SECTION g_lock; // DeleteCriticalSection static bool g_ready = false; +static bool g_shared = false; static MediaCapture g_mediaCapture = nullptr; static MediaFrameSource g_videoSource = nullptr; @@ -111,12 +112,20 @@ void PersonalVideo_Open(MRCVideoOptions const& options) PersonalVideo_FindMediaSourceGroup(width, height, framerate, sourceGroup, profile, description); + if (!options.shared) + { settings.VideoProfile(profile); settings.RecordMediaDescription(description); + settings.SharingMode(MediaCaptureSharingMode::ExclusiveControl); + } + else + { + settings.SharingMode(MediaCaptureSharingMode::SharedReadOnly); + } + settings.VideoDeviceId(sourceGroup.Id()); settings.StreamingCaptureMode(StreamingCaptureMode::Video); settings.MemoryPreference(MediaCaptureMemoryPreference::Cpu); - settings.SharingMode(MediaCaptureSharingMode::ExclusiveControl); settings.SourceGroup(sourceGroup); settings.MediaCategory(MediaCategory::Media); @@ -126,7 +135,8 @@ void PersonalVideo_Open(MRCVideoOptions const& options) PersonalVideo_FindVideoSource(g_mediaCapture, g_videoSource); - g_ready = true; + g_shared = options.shared; + g_ready = true; } // OK @@ -148,11 +158,19 @@ bool PersonalVideo_Status() } // OK -bool PersonalVideo_SetFormat(uint32_t width, uint32_t height, uint32_t framerate) +bool PersonalVideo_SetFormat(uint16_t& width, uint16_t& height, uint8_t& framerate) { MediaFrameFormat selectedFormat = nullptr; bool ok; + if (g_shared) + { + width = (uint16_t)g_videoSource.CurrentFormat().VideoFormat().Width(); + height = (uint16_t)g_videoSource.CurrentFormat().VideoFormat().Height(); + framerate = (uint8_t) g_videoSource.CurrentFormat().FrameRate().Numerator(); + return true; + } + ok = PersonalVideo_FindVideoFormat(g_videoSource, width, height, framerate, selectedFormat); if (!ok) { return false; } @@ -175,7 +193,7 @@ void PersonalVideo_SetFocus(uint32_t focusmode, uint32_t autofocusrange, uint32_ FocusSettings fs; CriticalSection cs(&g_lock); - if (!g_ready) { return; } + if (!g_ready || g_shared) { return; } switch (focusmode) { @@ -219,7 +237,7 @@ void PersonalVideo_SetVideoTemporalDenoising(uint32_t mode) VideoTemporalDenoisingMode vtdm; CriticalSection cs(&g_lock); - if (!g_ready) { return; } + if (!g_ready || g_shared) { return; } switch (mode) { @@ -237,7 +255,7 @@ void PersonalVideo_SetWhiteBalance_Preset(uint32_t preset) ColorTemperaturePreset ctp; CriticalSection cs(&g_lock); - if (!g_ready) { return; } + if (!g_ready || g_shared) { return; } switch (preset) { @@ -261,7 +279,7 @@ void PersonalVideo_SetWhiteBalance_Value(uint32_t value) uint32_t temperature; CriticalSection cs(&g_lock); - if (!g_ready) { return; } + if (!g_ready || g_shared) { return; } temperature = value * 25; if ((temperature >= 2300) && (temperature <= 7500)) { g_mediaCapture.VideoDeviceController().WhiteBalanceControl().SetValueAsync(temperature).get(); } @@ -274,7 +292,7 @@ void PersonalVideo_SetExposure(uint32_t setauto, uint32_t value) bool mode; CriticalSection cs(&g_lock); - if (!g_ready) { return; } + if (!g_ready || g_shared) { return; } mode = setauto != 0; g_mediaCapture.VideoDeviceController().ExposureControl().SetAutoAsync(mode).get(); @@ -287,7 +305,7 @@ void PersonalVideo_SetExposure(uint32_t setauto, uint32_t value) void PersonalVideo_SetExposurePriorityVideo(uint32_t enabled) { CriticalSection cs(&g_lock); - if (!g_ready) { return; } + if (!g_ready || g_shared) { return; } g_mediaCapture.VideoDeviceController().ExposurePriorityVideoControl().Enabled(enabled != 0); } @@ -297,7 +315,7 @@ void PersonalVideo_SetSceneMode(uint32_t mode) CaptureSceneMode value; CriticalSection cs(&g_lock); - if (!g_ready) { return; } + if (!g_ready || g_shared) { return; } switch (mode) { @@ -323,7 +341,7 @@ void PersonalVideo_SetSceneMode(uint32_t mode) void PersonalVideo_SetIsoSpeed(uint32_t setauto, uint32_t value) { CriticalSection cs(&g_lock); - if (!g_ready) { return; } + if (!g_ready || g_shared) { return; } if (setauto != 0) { g_mediaCapture.VideoDeviceController().IsoSpeedControl().SetAutoAsync().get(); } else if ((value >= 100) && (value <= 3200)) { g_mediaCapture.VideoDeviceController().IsoSpeedControl().SetValueAsync(value).get(); } } @@ -331,6 +349,6 @@ void PersonalVideo_SetIsoSpeed(uint32_t setauto, uint32_t value) void PersonalVideo_SetBacklightCompensation(bool enable) { CriticalSection cs(&g_lock); - if (!g_ready) { return; } + if (!g_ready || g_shared) { return; } g_mediaCapture.VideoDeviceController().BacklightCompensation().TrySetValue(enable ? 1.0 : 0.0); } diff --git a/hl2ss/hl2ss/personal_video.h b/hl2ss/hl2ss/personal_video.h index e2f46988..8b3fdd12 100644 --- a/hl2ss/hl2ss/personal_video.h +++ b/hl2ss/hl2ss/personal_video.h @@ -9,7 +9,7 @@ void PersonalVideo_Cleanup(); void PersonalVideo_Open(MRCVideoOptions const& options); void PersonalVideo_Close(); bool PersonalVideo_Status(); -bool PersonalVideo_SetFormat(uint32_t width, uint32_t height, uint32_t framerate); +bool PersonalVideo_SetFormat(uint16_t& width, uint16_t& height, uint8_t& framerate); winrt::Windows::Media::Capture::Frames::MediaFrameReader PersonalVideo_CreateFrameReader(); void PersonalVideo_SetFocus(uint32_t focusmode, uint32_t autofocusrange, uint32_t distance, uint32_t value, uint32_t disabledriverfallback); diff --git a/hl2ss/hl2ss/research_mode.cpp b/hl2ss/hl2ss/research_mode.cpp index 855f3948..9f81ecc7 100644 --- a/hl2ss/hl2ss/research_mode.cpp +++ b/hl2ss/hl2ss/research_mode.cpp @@ -1,6 +1,12 @@ #include "research_mode.h" +#include +#include + +using namespace winrt::Windows::Perception::Spatial; +using namespace winrt::Windows::Perception::Spatial::Preview; + extern "C" { HMODULE LoadLibraryA(LPCSTR lpLibFileName); } typedef HRESULT(__cdecl* PFN_CREATEPROVIDER)(IResearchModeSensorDevice**); @@ -30,7 +36,7 @@ static HANDLE g_imu_consent_event = NULL; // CloseHandle static ResearchModeSensorConsent g_camera_consent_value = ResearchModeSensorConsent::UserPromptRequired; static ResearchModeSensorConsent g_imu_consent_value = ResearchModeSensorConsent::UserPromptRequired; static std::vector g_sensors; // Release -static GUID g_rigNodeId; +static SpatialLocator g_locator = nullptr; //----------------------------------------------------------------------------- // Functions @@ -51,14 +57,14 @@ static void ResearchMode_IMUAccessCallback(ResearchModeSensorConsent consent) } // OK -bool ResearchMode_WaitForCameraConsent() +static bool ResearchMode_WaitForCameraConsent() { WaitForSingleObject(g_camera_consent_event, INFINITE); return g_camera_consent_value == ResearchModeSensorConsent::Allowed; } // OK -bool ResearchMode_WaitForIMUConsent() +static bool ResearchMode_WaitForIMUConsent() { WaitForSingleObject(g_imu_consent_event, INFINITE); return g_imu_consent_value == ResearchModeSensorConsent::Allowed; @@ -92,6 +98,7 @@ bool ResearchMode_Initialize() PFN_CREATEPROVIDER pfnCreate; size_t sensorcount; size_t sensorsloaded; + GUID rigNodeId; g_hrResearchMode = LoadLibraryA("ResearchModeAPI"); if (!g_hrResearchMode) { return false; } @@ -139,11 +146,13 @@ bool ResearchMode_Initialize() hr = g_pSensorDevice->QueryInterface(IID_PPV_ARGS(&pSensorDevicePerception)); if (FAILED(hr)) { return false; } - hr = pSensorDevicePerception->GetRigNodeId(&g_rigNodeId); + hr = pSensorDevicePerception->GetRigNodeId(&rigNodeId); if (FAILED(hr)) { return false; } pSensorDevicePerception->Release(); + g_locator = SpatialGraphInteropPreview::CreateLocatorForNode(rigNodeId); + return true; } @@ -168,7 +177,7 @@ void ResearchMode_Cleanup() g_camera_consent_value = ResearchModeSensorConsent::UserPromptRequired; g_imu_consent_value = ResearchModeSensorConsent::UserPromptRequired; g_sensors.clear(); - memset(&g_rigNodeId, 0, sizeof(g_rigNodeId)); + g_locator = nullptr; } // OK @@ -323,7 +332,7 @@ bool ResearchMode_GetExtrinsics(IResearchModeSensor* sensor, DirectX::XMFLOAT4X4 } // OK -GUID ResearchMode_GetRigNodeId() +SpatialLocator ResearchMode_GetLocator() { - return g_rigNodeId; + return g_locator; } diff --git a/hl2ss/hl2ss/research_mode.h b/hl2ss/hl2ss/research_mode.h index b2a73005..6959112b 100644 --- a/hl2ss/hl2ss/research_mode.h +++ b/hl2ss/hl2ss/research_mode.h @@ -4,6 +4,8 @@ #include #include +#include + int const RM_VLC_WIDTH = 640; int const RM_VLC_HEIGHT = 480; int const RM_VLC_FPS = 30; @@ -28,8 +30,6 @@ int const RM_ZLT_ABSIZE = RM_ZLT_PIXELS * sizeof(uint16_t); uint8_t const NV12_ZERO_CHROMA = 0x80; -bool ResearchMode_WaitForCameraConsent(); -bool ResearchMode_WaitForIMUConsent(); bool ResearchMode_WaitForConsent(IResearchModeSensor* sensor); bool ResearchMode_Initialize(); @@ -42,4 +42,4 @@ int ResearchMode_GetSensorTypeCount(); bool ResearchMode_GetIntrinsics(IResearchModeSensor* sensor, std::vector& uv2x, std::vector& uv2y, std::vector& mapx, std::vector& mapy, float K[4]); bool ResearchMode_GetExtrinsics(IResearchModeSensor* sensor, DirectX::XMFLOAT4X4& extrinsics); -GUID ResearchMode_GetRigNodeId(); +winrt::Windows::Perception::Spatial::SpatialLocator ResearchMode_GetLocator(); diff --git a/hl2ss/hl2ss/stream_pv.cpp b/hl2ss/hl2ss/stream_pv.cpp index 37fcc86a..08b419e4 100644 --- a/hl2ss/hl2ss/stream_pv.cpp +++ b/hl2ss/hl2ss/stream_pv.cpp @@ -8,6 +8,7 @@ #include "ports.h" #include "timestamps.h" #include "ipc_sc.h" +#include "research_mode.h" #include #include @@ -46,6 +47,7 @@ static uint32_t g_divisor = 1; // Mode: 2 static HANDLE g_event_intrinsic = NULL; // alias static float g_intrinsics[2 + 2 + 3 + 2 + 16]; +static float4x4 g_extrinsics; //----------------------------------------------------------------------------- // Functions @@ -135,6 +137,8 @@ static void PV_OnVideoFrameArrived_Intrinsics(MediaFrameReader const& sender, Me memcpy(&g_intrinsics[7], &t, sizeof(t)); memcpy(&g_intrinsics[9], &p, sizeof(p)); + g_extrinsics = Locator_Locate(QPCTimestampToPerceptionTimestamp(frame.SystemRelativeTime().Value().count()), ResearchMode_GetLocator(), frame.CoordinateSystem()); + SetEvent(g_event_intrinsic); } @@ -224,7 +228,7 @@ void PV_Stream(SOCKET clientsocket, HANDLE clientevent, MediaFrameReader const& // OK static void PV_Intrinsics(SOCKET clientsocket, HANDLE clientevent, MediaFrameReader const& reader) { - WSABUF wsaBuf[1]; + WSABUF wsaBuf[2]; g_event_intrinsic = clientevent; @@ -234,7 +238,8 @@ static void PV_Intrinsics(SOCKET clientsocket, HANDLE clientevent, MediaFrameRea WaitForSingleObject(g_event_intrinsic, INFINITE); reader.StopAsync().get(); - pack_buffer(wsaBuf, 0, g_intrinsics, sizeof(g_intrinsics)); + pack_buffer(wsaBuf, 0, g_intrinsics, sizeof(g_intrinsics)); + pack_buffer(wsaBuf, 1, &g_extrinsics, sizeof(g_extrinsics)); send_multiple(clientsocket, wsaBuf, sizeof(wsaBuf) / sizeof(WSABUF)); } diff --git a/hl2ss/hl2ss/stream_rm.cpp b/hl2ss/hl2ss/stream_rm.cpp index 180f4016..c5114e4d 100644 --- a/hl2ss/hl2ss/stream_rm.cpp +++ b/hl2ss/hl2ss/stream_rm.cpp @@ -69,7 +69,6 @@ static DWORD WINAPI RM_EntryPoint(void* param) SOCKET clientsocket; // closesocket IResearchModeSensor* sensor; ResearchModeSensorType type; - GUID nodeId; char const* port; bool ok; @@ -81,9 +80,7 @@ static DWORD WINAPI RM_EntryPoint(void* param) ok = ResearchMode_WaitForConsent(sensor); if (!ok) { return false; } - nodeId = ResearchMode_GetRigNodeId(); - locator = SpatialGraphInteropPreview::CreateLocatorForNode(nodeId); - + locator = ResearchMode_GetLocator(); port = g_research_sensor_port[type]; listensocket = CreateSocket(port); diff --git a/tools/pv_extrinsic_calibration.py b/tools/pv_extrinsic_calibration.py deleted file mode 100644 index 6a935fe4..00000000 --- a/tools/pv_extrinsic_calibration.py +++ /dev/null @@ -1,80 +0,0 @@ - -from scipy.spatial.transform import Rotation -from tqdm import tqdm - -import numpy as np -import argparse -import hl2ss -import hl2ss_lnm -import hl2ss_3dcv - -parser = argparse.ArgumentParser(description='HL2SS PV Extrinsic Calibration Tool. This tool requires that the HL2 is stationary. For best results, place the HL2 on a flat surface such as a table.') -parser.add_argument('--host', help='HL2 IP address (e.g. 192.168.1.0)', required=True) -parser.add_argument('--path', help='HL2 calibration folder (e.g. ../calibration/)', required=True) - -args = parser.parse_args() - -host = args.host -path = args.path - -samples = 32 - -client_rc = hl2ss.ipc_rc(host, hl2ss.IPCPort.REMOTE_CONFIGURATION) -client_pv = hl2ss_lnm.rx_pv(host, hl2ss.StreamPort.PERSONAL_VIDEO) -client_rn = hl2ss_lnm.rx_rm_vlc(host, hl2ss.StreamPort.RM_VLC_LEFTFRONT) - -list_pv = [] -list_rn = [] - -invalid = False - -print('Connecting to HL2 (' + host + ')...') - -hl2ss.start_subsystem_pv(host, hl2ss.StreamPort.PERSONAL_VIDEO) - -client_rc.open() -client_rc.wait_for_pv_subsystem(True) - -client_pv.open() -client_rn.open() - -for i in tqdm(range(0, samples)): - data_pv = client_pv.get_next_packet() - data_rn = client_rn.get_next_packet() - - invalid = (not hl2ss.is_valid_pose(data_pv.pose)) or (not hl2ss.is_valid_pose(data_rn.pose)) - if (invalid): - break - - list_pv.append(data_pv.pose) - list_rn.append(data_rn.pose) - -client_pv.close() -client_rn.close() - -hl2ss.stop_subsystem_pv(host, hl2ss.StreamPort.PERSONAL_VIDEO) -client_rc.wait_for_pv_subsystem(False) -client_rc.close() - -if (invalid): - print('Invalid pose detected') - print('Please make sure none of the HL2 sensors are covered and try again') - quit() - -# camera_to_world -# inv(extrinsics_pv) @ rn_pose = pv_pose -# inv(extrinsics_pv) = pv_pose @ inv(rn_pose) -# extrinsics_pv = rn_pose @ inv(pv_pose) - -list_extrinsics = [rn_pose @ np.linalg.inv(pv_pose) for rn_pose, pv_pose in zip(list_rn, list_pv)] - -extrinsics = np.zeros((4, 4), dtype=np.float32) -extrinsics[ 3, :3] = np.mean(np.array([list_extrinsics[i][3, 0:3] for i in range(0, samples)], dtype=np.float32), axis=0) -extrinsics[:3, :3] = Rotation.from_matrix(np.vstack([list_extrinsics[i][:3, :3].reshape((1, 3, 3)) for i in range(0, samples)])).mean().as_matrix() -extrinsics[ 3, 3] = 1 - -hl2ss_3dcv.save_extrinsics_pv(hl2ss.StreamPort.PERSONAL_VIDEO, extrinsics, path) - -print('PV extrinsics saved to ' + path) -print('PV extrinsics:') -print(extrinsics) diff --git a/tools/pv_intrinsics_downloader.py b/tools/pv_intrinsics_downloader.py index e507bf9e..529a0e55 100644 --- a/tools/pv_intrinsics_downloader.py +++ b/tools/pv_intrinsics_downloader.py @@ -33,7 +33,7 @@ print('Setting PV focus to {focus}'.format(focus=focus)) client_rc.set_pv_focus(hl2ss.PV_FocusMode.Manual, hl2ss.PV_AutoFocusRange.Normal, hl2ss.PV_ManualFocusDistance.Infinity, focus, hl2ss.PV_DriverFallback.Disable) print('Fetching PV calibration for {width}x{height}@{framerate}'.format(width=width, height=height, framerate=framerate)) -calibration = hl2ss_3dcv.get_calibration_pv(host, hl2ss.StreamPort.PERSONAL_VIDEO, path, focus, width, height, framerate, False) +calibration = hl2ss_3dcv.get_calibration_pv(host, hl2ss.StreamPort.PERSONAL_VIDEO, path, focus, width, height, framerate) print('PV calibration saved to ' + path) print('PV intrinsics:') print(calibration.intrinsics) diff --git a/unity/Assets/Plugins/WSA/hl2ss.dll b/unity/Assets/Plugins/WSA/hl2ss.dll index 4fdbfd3a..71980e24 100644 Binary files a/unity/Assets/Plugins/WSA/hl2ss.dll and b/unity/Assets/Plugins/WSA/hl2ss.dll differ diff --git a/viewer/client_stream_pv.py b/viewer/client_stream_pv.py index e048d9c2..55026d33 100644 --- a/viewer/client_stream_pv.py +++ b/viewer/client_stream_pv.py @@ -65,6 +65,8 @@ print(data.projection) print('Intrinsics') print(data.intrinsics) + print('RigNode Extrinsics') + print(data.extrinsics) else: enable = True diff --git a/viewer/hl2ss.py b/viewer/hl2ss.py index 6b1544d7..a6fa89e5 100644 --- a/viewer/hl2ss.py +++ b/viewer/hl2ss.py @@ -462,8 +462,8 @@ def _create_configuration_for_h26x_encoding(options): return bytes(configuration) -def _create_configuration_for_mrc_video(enable, hologram_composition, recording_indicator, video_stabilization, blank_protected, show_mesh, global_opacity, output_width, output_height, video_stabilization_length, hologram_perspective): - return struct.pack('