Skip to content

Commit

Permalink
Added support for 0-copy decode and display using D3D11
Browse files Browse the repository at this point in the history 
FIXME: We need a way to do this that doesn't involve reaching into the D3D11 texture internals
  • Loading branch information
@slouken
slouken committed Oct 10, 2023
1 parent b4a3ae1 commit b32fe2c
Show file tree
Hide file tree
Showing 2 changed files with 251 additions and 18 deletions.
4 changes: 2 additions & 2 deletions test/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,21 +183,21 @@ if(HAVE_LIBUDEV_H)
endif()

include("${SDL3_SOURCE_DIR}/cmake/FindFFmpeg.cmake")
if(FFmpeg_FOUND AND FFmpeg_AVCODEC_VERSION VERSION_GREATER_EQUAL "60")
if(FFmpeg_FOUND AND (FFmpeg_AVCODEC_VERSION STREQUAL "" OR FFmpeg_AVCODEC_VERSION VERSION_GREATER_EQUAL "60"))
if(APPLE)
add_sdl_test_executable(testffmpeg NO_C90 SOURCES testffmpeg.c testffmpeg_videotoolbox.m ${icon_bmp_header})
target_link_options(testffmpeg PRIVATE "-Wl,-framework,CoreFoundation" "-Wl,-framework,CoreVideo" "-Wl,-framework,Metal")
else()
add_sdl_test_executable(testffmpeg NO_C90 SOURCES testffmpeg.c ${icon_bmp_header})
endif()
target_link_libraries(testffmpeg PRIVATE ${FFMPEG_LIBRARIES})
if(HAVE_OPENGLES_V2)
#message(STATUS "Enabling EGL support in testffmpeg")
target_compile_definitions(testffmpeg PRIVATE HAVE_EGL)
if(TARGET OpenGL::EGL)
target_link_libraries(testffmpeg PRIVATE OpenGL::EGL)
endif()
endif()
target_link_libraries(testffmpeg PRIVATE ${FFMPEG_LIBRARIES})
else()
message(STATUS "Can't find ffmpeg 6.0 or newer, skipping testffmpeg")
endif()
Expand Down
265 changes: 249 additions & 16 deletions test/testffmpeg.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,86 @@
#include "testffmpeg_videotoolbox.h"
#endif

#ifdef __WIN32__
#define COBJMACROS
#include <d3d11.h>

struct D3D11_TextureData
{
ID3D11Texture2D *mainTexture;
ID3D11ShaderResourceView *mainTextureResourceView;
ID3D11RenderTargetView *mainTextureRenderTargetView;
ID3D11Texture2D *stagingTexture;
int lockedTexturePositionX;
int lockedTexturePositionY;
D3D11_FILTER scaleMode;

/* YV12 texture support */
SDL_bool yuv;
ID3D11Texture2D *mainTextureU;
ID3D11ShaderResourceView *mainTextureResourceViewU;
ID3D11Texture2D *mainTextureV;
ID3D11ShaderResourceView *mainTextureResourceViewV;

/* NV12 texture support */
SDL_bool nv12;
ID3D11Texture2D *mainTextureNV;
ID3D11ShaderResourceView *mainTextureResourceViewNV;

Uint8 *pixels;
int pitch;
SDL_Rect locked_rect;
};

/* Rendering view state */
typedef struct SDL_RenderViewState
{
int pixel_w;
int pixel_h;
SDL_Rect viewport;
SDL_Rect clip_rect;
SDL_bool clipping_enabled;
SDL_FPoint scale;

} SDL_RenderViewState;

struct SDL_Texture
{
const void *magic;
Uint32 format; /**< The pixel format of the texture */
int access; /**< SDL_TextureAccess */
int w; /**< The width of the texture */
int h; /**< The height of the texture */
int modMode; /**< The texture modulation mode */
SDL_BlendMode blendMode; /**< The texture blend mode */
SDL_ScaleMode scaleMode; /**< The texture scale mode */
SDL_Color color; /**< Texture modulation values */
SDL_RenderViewState view; /**< Target texture view state */

SDL_Renderer *renderer;

/* Support for formats not supported directly by the renderer */
SDL_Texture *native;
void /*SDL_SW_YUVTexture*/ *yuv;
void *pixels;
int pitch;
SDL_Rect locked_rect;
SDL_Surface *locked_surface; /**< Locked region exposed as a SDL surface */

Uint32 last_command_generation; /* last command queue generation this texture was in. */

struct D3D11_TextureData /*void*/ *driverdata; /**< Driver specific texture representation */
void *userdata;

SDL_Texture *prev;
SDL_Texture *next;
};
#endif /* __WIN32__ */

#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/avutil.h>
#include <libavutil/hwcontext_d3d11va.h>
#include <libavutil/hwcontext_drm.h>
#include <libavutil/pixdesc.h>

Expand All @@ -51,6 +128,7 @@
#define WINDOW_WIDTH 640
#define WINDOW_HEIGHT 480


static SDL_Texture *sprite;
static SDL_FRect *positions;
static SDL_FRect *velocities;
Expand All @@ -71,21 +149,25 @@ static PFNGLEGLIMAGETARGETTEXTURE2DOESPROC glEGLImageTargetTexture2DOESFunc;
#ifdef __APPLE__
static SDL_bool has_videotoolbox_output;
#endif
#ifdef __WIN32__
static ID3D11Device *d3d11_device;
static ID3D11DeviceContext *d3d11_context;
#endif
static int done;

static SDL_bool CreateWindow(Uint32 window_flags, SDL_bool useEGL)
static SDL_bool CreateWindowAndRenderer(Uint32 window_flags, const char *driver)
{
SDL_RendererInfo info;
SDL_bool useEGL = (driver && SDL_strcmp(driver, "opengles2") == 0);

SDL_SetHint(SDL_HINT_RENDER_DRIVER, driver);
if (useEGL) {
SDL_SetHint( SDL_HINT_VIDEO_FORCE_EGL, "1" );
SDL_SetHint( SDL_HINT_RENDER_DRIVER, "opengles2" );
SDL_SetHint(SDL_HINT_VIDEO_FORCE_EGL, "1");
SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, SDL_GL_CONTEXT_PROFILE_ES);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 2);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 0);
} else {
SDL_SetHint( SDL_HINT_VIDEO_FORCE_EGL, "0" );
SDL_SetHint( SDL_HINT_RENDER_DRIVER, NULL );
SDL_SetHint(SDL_HINT_VIDEO_FORCE_EGL, "0");
SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, 0);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 2);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 1);
Expand Down Expand Up @@ -127,6 +209,13 @@ static SDL_bool CreateWindow(Uint32 window_flags, SDL_bool useEGL)
has_videotoolbox_output = SetupVideoToolboxOutput(renderer);
#endif

#ifdef __WIN32__
d3d11_device = SDL_GetRenderD3D11Device(renderer);
if (d3d11_device) {
ID3D11Device_GetImmediateContext(d3d11_device, &d3d11_context);
}
#endif

return SDL_TRUE;
}

Expand Down Expand Up @@ -238,14 +327,19 @@ static SDL_bool SupportedPixelFormat(enum AVPixelFormat format)
return SDL_TRUE;
}
#endif
#ifdef __WIN32__
if (d3d11_device && format == AV_PIX_FMT_D3D11) {
return SDL_TRUE;
}
#endif

if (GetTextureFormat(format) != SDL_PIXELFORMAT_UNKNOWN) {
return SDL_TRUE;
}
return SDL_FALSE;
}

static enum AVPixelFormat GetPixelFormat(AVCodecContext *s, const enum AVPixelFormat *pix_fmts)
static enum AVPixelFormat GetSupportedPixelFormat(AVCodecContext *s, const enum AVPixelFormat *pix_fmts)
{
const enum AVPixelFormat *p;

Expand Down Expand Up @@ -296,29 +390,52 @@ static AVCodecContext *OpenVideoStream(AVFormatContext *ic, int stream, const AV
i = 0;
while (!context->hw_device_ctx &&
(config = avcodec_get_hw_config(codec, i++)) != NULL) {
#if 0
SDL_Log("Found %s hardware acceleration with pixel format %s\n", av_hwdevice_get_type_name(config->device_type), av_get_pix_fmt_name(config->pix_fmt));
#endif

if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) ||
!SupportedPixelFormat(config->pix_fmt)) {
continue;
}

type = AV_HWDEVICE_TYPE_NONE;
while (!context->hw_device_ctx &&
while (!context->hw_device_ctx &&
(type = av_hwdevice_iterate_types(type)) != AV_HWDEVICE_TYPE_NONE) {
if (type != config->device_type) {
continue;
}

result = av_hwdevice_ctx_create(&context->hw_device_ctx, type, NULL, NULL, 0);
if (result < 0) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Couldn't create hardware device context: %s", av_err2str(result));
if (type == AV_HWDEVICE_TYPE_D3D11VA) {
AVD3D11VADeviceContext *device_context;

context->hw_device_ctx = av_hwdevice_ctx_alloc(type);

device_context = (AVD3D11VADeviceContext *)((AVHWDeviceContext *)context->hw_device_ctx->data)->hwctx;
device_context->device = d3d11_device;
ID3D11Device_AddRef(device_context->device);
device_context->device_context = d3d11_context;
ID3D11DeviceContext_AddRef(device_context->device_context);

result = av_hwdevice_ctx_init(context->hw_device_ctx);
if (result < 0) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Couldn't create hardware device context: %s", av_err2str(result));
} else {
SDL_Log("Using %s hardware acceleration with pixel format %s\n", av_hwdevice_get_type_name(config->device_type), av_get_pix_fmt_name(config->pix_fmt));
}
} else {
SDL_Log("Using %s hardware acceleration with pixel format %s\n", av_hwdevice_get_type_name(config->device_type), av_get_pix_fmt_name(config->pix_fmt));
result = av_hwdevice_ctx_create(&context->hw_device_ctx, type, NULL, NULL, 0);
if (result < 0) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Couldn't create hardware device context: %s", av_err2str(result));
} else {
SDL_Log("Using %s hardware acceleration with pixel format %s\n", av_hwdevice_get_type_name(config->device_type), av_get_pix_fmt_name(config->pix_fmt));
}
}
}
}

/* Allow supported hardware accelerated pixel formats */
context->get_format = GetPixelFormat;
context->get_format = GetSupportedPixelFormat;

result = avcodec_open2(context, codec, NULL);
if (result < 0) {
Expand Down Expand Up @@ -478,13 +595,105 @@ static SDL_bool GetTextureForVAAPIFrame(AVFrame *frame, SDL_Texture **texture)
return result;
}

static SDL_bool GetTextureForD3D11Frame(AVFrame *frame, SDL_Texture **texture)
{
#ifdef __WIN32__
ID3D11Texture2D *pTexture = (ID3D11Texture2D *)frame->data[0];
UINT iSliceIndex = (UINT)(uintptr_t)frame->data[1];

D3D11_TEXTURE2D_DESC desc;
SDL_zero(desc);
ID3D11Texture2D_GetDesc(pTexture, &desc);
if (desc.Format != DXGI_FORMAT_NV12) {
SDL_SetError("Unsupported texture format, expected DXGI_FORMAT_NV12, got %d", desc.Format);
return SDL_FALSE;
}

if (!*texture || (UINT)(*texture)->w != desc.Width || (UINT)(*texture)->h != desc.Height) {
if (*texture) {
SDL_DestroyTexture(*texture);
} else {
/* First time set up for NV12 textures */
SetYUVConversionMode(frame);
}

*texture = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_NV12, SDL_TEXTUREACCESS_STATIC, desc.Width, desc.Height);
if (!*texture) {
return SDL_FALSE;
}

/* Set up the resource views for this texture */
struct D3D11_TextureData *pTextureData = (*texture)->driverdata;
if (pTextureData->mainTexture) {
ID3D11Texture2D_Release(pTextureData->mainTexture);
pTextureData->mainTexture = NULL;
}
if (pTextureData->mainTextureResourceView) {
ID3D11ShaderResourceView_Release(pTextureData->mainTextureResourceView);
pTextureData->mainTextureResourceView = NULL;
}
if (pTextureData->mainTextureNV) {
ID3D11Texture2D_Release(pTextureData->mainTextureNV);
pTextureData->mainTextureNV = NULL;
}
if (pTextureData->mainTextureResourceViewNV) {
ID3D11ShaderResourceView_Release(pTextureData->mainTextureResourceViewNV);
pTextureData->mainTextureResourceViewNV = NULL;
}

desc.ArraySize = 1;
desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;

HRESULT result = ID3D11Device_CreateTexture2D(d3d11_device, &desc, NULL, &pTextureData->mainTexture);
if (FAILED(result)) {
SDL_SetError("Couldn't create main texture: 0x%x", result);
return SDL_FALSE;
}

pTextureData->mainTextureNV = pTextureData->mainTexture;
ID3D11Texture2D_AddRef(pTextureData->mainTextureNV);

D3D11_SHADER_RESOURCE_VIEW_DESC resourceViewDesc;
resourceViewDesc.Format = DXGI_FORMAT_R8_UNORM;
resourceViewDesc.ViewDimension = D3D11_SRV_DIMENSION_TEXTURE2D;
resourceViewDesc.Texture2D.MostDetailedMip = 0;
resourceViewDesc.Texture2D.MipLevels = 1;
result = ID3D11Device_CreateShaderResourceView(d3d11_device,
(ID3D11Resource *)pTextureData->mainTexture,
&resourceViewDesc,
&pTextureData->mainTextureResourceView);
if (FAILED(result)) {
SDL_SetError("Couldn't create main texture view: 0x%x", result);
return SDL_FALSE;
}

resourceViewDesc.Format = DXGI_FORMAT_R8G8_UNORM;
result = ID3D11Device_CreateShaderResourceView(d3d11_device,
(ID3D11Resource *)pTextureData->mainTexture,
&resourceViewDesc,
&pTextureData->mainTextureResourceViewNV);
if (FAILED(result)) {
SDL_SetError("Couldn't create secondary texture view: 0x%x", result);
return SDL_FALSE;
}
}

ID3D11DeviceContext_CopySubresourceRegion(d3d11_context, (ID3D11Resource *)(*texture)->driverdata->mainTexture, 0, 0, 0, 0, (ID3D11Resource *)pTexture, iSliceIndex, NULL);
return SDL_TRUE;
#else
return SDL_FALSE;
#endif
}

static SDL_bool GetTextureForFrame(AVFrame *frame, SDL_Texture **texture)
{
switch (frame->format) {
case AV_PIX_FMT_VAAPI:
return GetTextureForVAAPIFrame(frame, texture);
case AV_PIX_FMT_DRM_PRIME:
return GetTextureForDRMFrame(frame, texture);
case AV_PIX_FMT_D3D11:
return GetTextureForD3D11Frame(frame, texture);
default:
return GetTextureForMemoryFrame(frame, texture);
}
Expand Down Expand Up @@ -708,11 +917,25 @@ int main(int argc, char *argv[])
#endif
#ifdef HAVE_EGL
/* Try to create an EGL compatible window for DRM hardware frame support */
CreateWindow(window_flags, SDL_TRUE);
if (!window) {
CreateWindowAndRenderer(window_flags, "opengles2");
}
#endif
if (!window && !CreateWindow(window_flags, SDL_FALSE)) {
return_code = 2;
goto quit;
#ifdef __APPLE__
if (!window) {
CreateWindowAndRenderer(window_flags, "metal");
}
#endif
#ifdef __WIN32__
if (!window) {
CreateWindowAndRenderer(window_flags, "direct3d11");
}
#endif
if (!window) {
if (!CreateWindowAndRenderer(window_flags, NULL)) {
return_code = 2;
goto quit;
}
}

if (SDL_SetWindowTitle(window, file) < 0) {
Expand Down Expand Up @@ -876,6 +1099,16 @@ int main(int argc, char *argv[])
quit:
#ifdef __APPLE__
CleanupVideoToolboxOutput();
#endif
#ifdef __WIN32__
if (d3d11_context) {
ID3D11DeviceContext_Release(d3d11_device);
d3d11_context = NULL;
}
if (d3d11_device) {
ID3D11Device_Release(d3d11_device);
d3d11_device = NULL;
}
#endif
SDL_free(positions);
SDL_free(velocities);
Expand Down

0 comments on commit b32fe2c

Please sign in to comment.