Create TextureStream.md

specs/APIReview_TextureStream.md

@@ -0,0 +1,353 @@
+
+# Background
+Many native apps use a native engine for real-time communication scenarios, which include video 
+capture, networking and video rendering. However, often, these apps still use WebView or 
+Electrion for UI rendering. The separation between real-time video rendering and UI rendering 
+prevents apps from rendering real-time video inside the web contents. This forces apps to
+render the real-time video on top of the web contents, which is limiting. Rendering video on 
+top constrains the user experience and it may also cause performance problems. 
+We can ask the native apps to use web renderer for video handling because web standard already 
+provides these features through WebRTC APIs. The end developers, however, prefer to use 
+their existing engine such as capturing and composition, meanwhile using WebRTC API for rendering.
+
+# Description
+The proposed APIs will allow the end developers to stream the captured or composed video frame to 
+the WebView renderer where Javascript is able to insert the frame to the page through W3C standard 
+API of Video, MediaStream element for displaying it.
+The API will use the shared GPU texture buffer so that it can minimize the overall cost with 
+regards to frame copy.
+
+# Examples
+Javascript
+// User click the video capture button.
+document.querySelector('#showVideo').addEventListener('click',
+ e => getStreamFromTheHost(e));
+async function getStreamFromTheHost(e) {
+ try {
+ // Request stream to the host with unique stream id.
+ const stream = await window.chrome.webview.getTextureStream('webview2-abcd1234'); 
+ // The MediaStream object is returned and it gets video MediaStreamTrack element from it.
+ const video_tracks = stream.getVideoTracks();
+ const videoTrack = video_tracks[0];
+ // Show the video via Video Element.
+ document.getElementById(video_id).srcObject = stream;
+ } catch (error) {
+ console.log(error);
+ }
+}
+Win32 C++
+UINT32 luid,
+// Get the LUID (Graphic adapter) that the WebView renderer uses.
+coreWebView->GetRenderAdapterLUID(&luid);
+// Create D3D device based on the WebView's LUID.
+ComPtr<D3D11Device> d3d_device = MyCreateD3DDevice(luid);
+// Register unique texture stream that the host can provide. 
+ComPtr<ICoreWebView2TextureStream> webviewTextureStream;
+g_webviewStaging3->CreateTextureStream(L"webview2-abcd1234", d3d_device.Get(), &webviewTextureStream);
+// Register the Origin URL that the target renderer could stream of the registered stream id. The request from not registered origin will fail to stream.
+webviewTextureStream->AddRequestedFilter(L"https://edge-webscratch");
+// Listen to Start request
+EventRegistrationToken start_token;
+webviewTextureStream->add_StartRequested(Callback<ICoreWebView2StagingTextureStreamStartRequestedEventHandler>(
+ [hWnd](ICoreWebView2StagingTextureStream* webview, IUnknown* eventArgs) -> HRESULT {
+ // Capture video stream by using native API, for example, Media Foundation on Windows.
+ StartMediaFoundationCapture(hWnd);
+ return S_OK;
+ }).Get(), &start_token);
+EventRegistrationToken stop_token;
+webviewTextureStream->add_StopRequested(Callback<ICoreWebView2StagingTextureStreamStopRequestedEventHandler>(
+ [hWnd](ICoreWebView2StagingTextureStream* webview, IUnknown* eventArgs) -> HRESULT {
+ StopMediaFoundationCapture();
+ return S_OK;
+ }).Get(), &stop_token);
+EventRegistrationToken texture_token;
+webviewTextureStream->add_TextureError(Callback<ICoreWebView2StagingTextureStreamTextureErrorEventHandler>(
+ [hWnd](ICoreWebView2StagingTextureStream* sender, ICoreWebView2StagingTextureStreamTextureErrorEventArgs* args) {
+ COREWEBVIEW2_TEXTURE_STREAM_ERROR_KIND kind;
+ HRESULT hr = args->get_Kind(&kind);
+ assert(SUCCEEDED(hr));
+ switch (kind)
+ {
+ case COREWEBVIEW2_TEXTURE_STREAM_ERROR_NO_VIDEO_TRACK_STARTED:
+ case COREWEBVIEW2_TEXTURE_STREAM_ERROR_BUFFER_NOT_FOUND:
+ case COREWEBVIEW2_TEXTURE_STREAM_ERROR_BUFFER_IN_USE:
+ // assert(false);
+ break;
+ default:
+ break;
+ }
+ return S_OK;
+ }).Get(), &texture_token);
+
+// TextureStream APIs are called in the UI thread on the WebView2 process meanwhile Video capture
+// and composition could happen in worker thread or out of process.
+LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam)
+{
+ static ICoreWebView2Staging3* webview2_17 = nullptr;
+ TCHAR greeting[] = _T("Hello, Windows desktop!");
+ ComPtr<ID3D11Device> d3d_device;
+ HANDLE slimCoreHandle;
+ HRESULT hr;
+ ComPtr<ICoreWebView2StagingTexture> texture_buffer;
+ int64_t bufferId = -1;
+ switch (message)
+ {
+ case IDC_TEST_SEND_TEXTURE:
+ if (webviewTextureStream) {
+ // Present API should be called on same thread where main WebView
+ // object is created.
+ bufferId = (int)wParam;
+ texture_buffer = texture_address_to_buffer_ids[(HANDLE)bufferId];
+ // assert(texture_buffer != nullptr);
+ if (texture_buffer) {
+ // Notify the renderer for updated texture on the shared buffer.
+ webviewTextureStream->SetBuffer(texture_buffer.Get(), texture_buffer_info_->timestamp);
+ webviewTextureStream->Present();
+ }
+ }
+ break;
+ case IDC_TEST_REQUEST_BUFFER:
+ // Retrieve available shared buffer.
+ hr = webviewTextureStream->GetAvailableBuffer(&texture_buffer);
+ if (SUCCEEDED(hr)) {
+ texture_buffer->get_Handle((HANDLE*)&bufferId);
+ }
+ SendBufferIdToOOFCaptureEngine(false, nullptr, bufferId);
+ break;
+ case IDC_TEST_CREATE_NEW_BUFFER:
+ if (webviewTextureStream) {
+ ComPtr<ICoreWebView2StagingTexture> texture_buffer;
+ UINT32 width = (UINT32)wParam;
+ UINT32 height = (UINT32)lParam;
+ // Create shared buffer.
+ webviewTextureStream->CreateSharedBuffer(width, height, &texture_buffer);
+ texture_buffer->get_Handle(&slimCoreHandle);
+ texture_address_to_buffer_ids[slimCoreHandle] = texture_buffer;
+ SendBufferIdToOOFCaptureEngine(true, slimCoreHandle, (int)slimCoreHandle);
+ }
+ break;
+ default:
+ return DefWindowProc(hWnd, message, wParam, lParam);
+ break;
+ }
+}
+# API Details
+Win32 C++
+[v1_enum]
+typedef enum COREWEBVIEW2_TEXTURE_STREAM_ERROR_KIND {
+ /// The host can't create a TextureStream instance more than once
+ /// for a specific stream id.
+ COREWEBVIEW2_TEXTURE_STREAM_ERROR_STREAM_ID_ALREADY_REGISTERED,
+ /// Occurs when the host calls CreateBuffer or Present
+ /// APIs without being called of Start event. Or, 10 seconds passed before
+ /// calling these APIs since the OnStart event.
+ COREWEBVIEW2_TEXTURE_STREAM_ERROR_NO_VIDEO_TRACK_STARTED,
+ /// The buffer has been removed using RemoveBuffer.
+ COREWEBVIEW2_TEXTURE_STREAM_ERROR_BUFFER_NOT_FOUND,
+ /// The texture to be presented is already in use for rendering.
+ /// Call GetAvailableBuffer to determine an available buffer to present.
+ /// The developer can technically call SetBuffer multiple times.
+ /// But once they call Present, the buffer becomes "in use" until
+ /// they call SetBuffer and Present on a different buffer and wait a bit
+ /// for the original buffer to stop being used.
+ COREWEBVIEW2_TEXTURE_STREAM_ERROR_BUFFER_IN_USE,
+} COREWEBVIEW2_TEXTURE_STREAM_ERROR_KIND;
+/// This is ICoreWebView2Staging3 that returns the texture stream interface.
+[uuid(96c27a45-f142-4873-80ad-9d0cd899b2b9), object, pointer_default(unique)]
+interface ICoreWebView2Staging3 : IUnknown {
+ /// Registers the stream id that the host can handle, providing a
+ /// texture stream when requested from the WebView2's JavaScript code.
+ /// The host can register multiple unique stream instances, each with
+ /// a unique stream ID, enabling the host to stream from different sources
+ /// concurrently.
+ /// The host should call this only once for unique streamId. The second
+ /// call of already created streamId without destroying
+ /// ICoreWebView2StagingTextureStream object will return an error.
+ /// 'd3dDevice' is used for creating shared IDXGI resource and NT shared
+ /// of it. The host should use Adapter of the LUID from the GetRenderAdapterLUID
+ /// for creating the D3D Device.
+ HRESULT CreateTextureStream(
+ [in] LPCWSTR streamId,
+ [in] IUnknown* d3dDevice,
+ [out, retval ] ICoreWebView2StagingTextureStream** value);
+ /// Get the graphics adapter LUID of the renderer. The host should use this
+ /// LUID adapter when creating D3D device to use with CreateTextureStream().
+HRESULT GetRenderAdapterLUID([out, retval] LUID* luid);
+ /// Listens for change of graphics adapter LUID of the browser.
+ /// The host can get the updated LUID by GetRenderAdapterLUID. It is expected
+ /// that the host updates texture's d3d Device with UpdateD3DDevice,
+ /// removes existing buffers and creates new buffer.
+ HRESULT add_RenderAdapterLUIDUpdated(
+ [in] ICoreWebView2StagingRenderAdapterLUIDUpdatedEventHandler* eventHandler,
+ [out] EventRegistrationToken* token);
+ /// Remove listener for start stream request.
+ HRESULT remove_RenderAdapterLUIDUpdated(
+ [in] EventRegistrationToken token);
+}
+/// This is the interface that handles texture streaming.
+/// The most of APIs have to be called on UI thread.
+[uuid(afca8431-633f-4528-abfe-7fc3bedd8962), object, pointer_default(unique)]
+interface ICoreWebView2StagingTextureStream : IUnknown {
+ /// Get the stream ID of the object that is used when calling CreateTextureStream.
+ /// The caller must free the returned string with CoTaskMemFree. See
+ /// [API Conventions](/microsoft-edge/webview2/concepts/win32-api-conventions#strings).
+ // MSOWNERS: TBD (wv2core@microsoft.com)
+ [propget] HRESULT StreamId([out, retval] LPWSTR* id);
+ /// Adds an allowed url origin for the given stream id. The stream requests
+ /// could be made from any frame, including iframes, but these origins
+ /// must be registered first in order for the request to succeed.
+ /// The added filter will be persistent until
+ /// ICoreWebView2StagingTextureStream is destroyed or
+ /// RemoveRequestedFilter is called.
+ /// The renderer does not support wildcard so it will compare
+ /// literal string input to the requesting frame origin. So, the input string
+ /// should have a scheme like https://.
+ /// For example, https://www.valid-host.com, http://www.valid-host.com are
+ /// valid origins but www.valid-host.com, or *.valid-host.com. are not
+ /// valid origins.
+ /// getTextureStream() will fail unless the requesting frame's origin URL is
+ /// added to the request filter.
+ HRESULT AddRequestedFilter([in] LPCWSTR origin);
+ /// Remove added origin, which was added by AddRequestedFilter.
+ HRESULT RemoveRequestedFilter([in] LPCWSTR origin);
+ /// Listens for stream requests from the Javascript's getTextureStream call
+ /// for the given stream id. It is called for the first request only, the
+ /// subsequent requests of same stream id will not be called.
+ /// It is expected that the host provides the stream within 10s after
+ /// being requested. The first call to Present() fulfills the stream request.
+ HRESULT add_StartRequested(
+ [in] ICoreWebView2StagingTextureStreamStartRequestedEventHandler* eventHandler,
+ [out] EventRegistrationToken* token);
+ /// Remove listener for start stream request.
+ HRESULT remove_StartRequested(
+ [in] EventRegistrationToken token);
+ /// Listen to stop stream request once the stream started.
+ /// It is called when user stop all streaming requests from
+ /// the renderers (Javascript) or the host calls the Stop API. The renderer
+ /// can stream again by calling the streaming request API.
+ /// The renderer cleared all registered buffers before sending
+ /// the stop request event so that the callback of the next start request
+ /// should register the textures again.
+ /// The event is triggered when all requests for given stream id closed
+ /// by the Javascript, or the host's Stop API call.
+ HRESULT add_StopRequested(
+ [in] ICoreWebView2StagingTextureStreamStopRequestedEventHandler* eventHandler,
+ [out] EventRegistrationToken* token);
+ /// Remove listener for stop stream request.
+ HRESULT remove_StopRequested(
+ [in] EventRegistrationToken token);
+ /// Creates shared buffer that will be referenced by the host and the browser.
+ /// By using the shared buffer mechanism, the host does not have to
+ /// send respective texture to the renderer, instead it notifies it
+ /// with internal buffer id, which is the identity of the shared buffer.
+ /// The shared buffer is 2D texture, IDXGIResource, format and will be
+ /// exposed through shared HANDLE or IUnknown type through ICoreWebView2StagingTexture.
+ /// Whenever the host has new texture to write, it should ask
+ /// reusable ICoreWebView2StagingTexture from the GetAvailableBuffer,
+ /// which returns ICoreWebView2StagingTexture.
+ /// If the GetAvailableBuffer returns an error, then the host calls the
+ /// CreateBuffer to allocate new shared buffer.
+ /// The API also registers created shared handle to the browser once it
+ /// created the resource.
+ HRESULT CreateBuffer(
+ [in] UINT32 width,
+ [in] UINT32 height,
+ [out, retval] ICoreWebView2StagingTexture** buffer);
+ /// GetAvailableBuffer can be called on any thread like SetBuffer.
+ HRESULT GetAvailableBuffer([out, retval] ICoreWebView2StagingTexture** buffer);
+ /// Remove texture buffer when the host removes the backed 2D texture.
+ /// The host can save the existing resources by deleting 2D textures
+ /// when it changes the frame sizes.
+ HRESULT RemoveBuffer([in] ICoreWebView2StagingTexture* buffer);
+ /// Indicates that the buffer is ready to present.
+ /// The buffer must be retrieved from the GetAvailableBuffer.
+ /// The host writes new texture to the local shared 2D texture of
+ /// the buffer id, which is created via CreateBuffer.
+ /// SetBuffer API can be called in any thread.
+ HRESULT SetBuffer([in] ICoreWebView2StagingTexture* buffer,
+ [in] ULONGLONG timestamp);
+ /// Render texture that is current set ICoreWebView2StagingTexture.
+ HRESULT Present();
+ /// Stop streaming of the current stream id.
+ /// API calls of Present, CreateBuffer will fail after this
+ /// with an error of COREWEBVIEW2_TEXTURE_STREAM_ERROR_NO_VIDEO_TRACK_STARTED.
+ /// The Javascript can restart the stream with getTextureStream.
+ HRESULT Stop();
+ /// Event handler for those that occur at the Renderer side, the example
+ /// are CreateBuffer, Present, or Stop.
+ HRESULT add_TextureError(
+ [in] ICoreWebView2StagingTextureStreamTextureErrorEventHandler* eventHandler,
+ [out] EventRegistrationToken* token);
+ /// Remove listener for texture error event.
+ HRESULT remove_TextureError([in] EventRegistrationToken token);
+ /// Updates d3d Device when it is updated by RenderAdapterLUIDUpdated
+ /// event.
+ HRESULT UpdateD3DDevice([in] IUnknown* d3dDevice);
+}
+/// Texture stream buffer that the host writes to so that the Renderer
+/// will render on it.
+[uuid(0836f09c-34bd-47bf-914a-99fb56ae2d07), object, pointer_default(unique)]
+interface ICoreWebView2StagingTexture : IUnknown {
+ /// Returns shared Windows NT handle. The caller expected to open it with
+ /// ID3D11Device1::OpenSharedResource1 and writes the incoming texture to it.
+ [propget] HRESULT Handle([out, retval] HANDLE* handle);
+ /// Returns IUnknown type that could be query interface to IDXGIResource.
+ /// The caller can write incoming texture to it.
+ [propget] HRESULT Resource([out, retval] IUnknown** resource);
+}
+/// This is the callback for new texture stream request.
+[uuid(62d09330-00a9-41bf-a9ae-55aaef8b3c44), object, pointer_default(unique)]
+interface ICoreWebView2StagingTextureStreamStartRequestedEventHandler : IUnknown {
+ //// Called to provide the implementer with the event args for the
+ //// corresponding event. There are no event args and the args
+ //// parameter will be null.
+ HRESULT Invoke(
+ [in] ICoreWebView2StagingTextureStream* sender,
+ [in] IUnknown* args);
+}
+/// This is the callback for stop request of texture stream.
+[uuid(4111102a-d19f-4438-af46-efc563b2b9cf), object, pointer_default(unique)]
+interface ICoreWebView2StagingTextureStreamStopRequestedEventHandler : IUnknown {
+ /// Called to provide the implementer with the event args for the
+ /// corresponding event. There are no event args and the args
+ /// parameter will be null.
+ HRESULT Invoke(
+ [in] ICoreWebView2StagingTextureStream* sender,
+ [in] IUnknown* args);
+}
+/// This is the callback for texture stream rendering error.
+[uuid(52cb8898-c711-401a-8f97-3646831ba72d), object, pointer_default(unique)]
+interface ICoreWebView2StagingTextureStreamTextureErrorEventHandler : IUnknown {
+ /// Called to provide the implementer with the event args for the
+ /// corresponding event.
+ HRESULT Invoke(
+ [in] ICoreWebView2StagingTextureStream* sender,
+ [in] ICoreWebView2StagingTextureStreamTextureErrorEventArgs* args);
+}
+/// This is the event args interface for texture stream error callback.
+[uuid(0e1730c1-03df-4ad2-b847-be4d63adf700), object, pointer_default(unique)]
+interface ICoreWebView2StagingTextureStreamTextureErrorEventArgs : IUnknown {
+ /// Error kind.
+ [propget] HRESULT Kind([out, retval]
+ COREWEBVIEW2_TEXTURE_STREAM_ERROR_KIND* value);
+ // Texture buffer that the error is associated with.
+ HRESULT GetBuffer([out, retval] ICoreWebView2StagingTexture** buffer);
+}
+[uuid(431721e0-0f18-4d7b-bd4d-e5b1522bb110), object, pointer_default(unique)]
+interface ICoreWebView2StagingRenderAdapterLUIDUpdatedEventHandler : IUnknown {
+ /// Called to provide the implementer with the event args for the
+ /// corresponding event.
+ HRESULT Invoke(
+ [in] ICoreWebView2StagingTextureStream* sender,
+ [in] IUnknown* args);
+}
+[uuid(431721e0-0f18-4d7b-bd4d-e5b1522bb110), object, pointer_default(unique)]
+interface ICoreWebView2StagingRenderAdapterLUIDUpdatedEventHandler : IUnknown {
+ /// Called to provide the implementer with the event args for the
+ /// corresponding event.
+ HRESULT Invoke(
+ [in] ICoreWebView2Staging3 * sender,
+ [in] IUnknown* args);
+}
+
+