diff options
author | Piotr Srebrny <piotr.srebrny@qt.io> | 2021-12-10 20:25:08 +0100 |
---|---|---|
committer | Qt Cherry-pick Bot <cherrypick_bot@qt-project.org> | 2021-12-14 22:31:28 +0000 |
commit | 68066a2826f37dd711d1d250fbea15d907ce8668 (patch) | |
tree | e3f890c2beabf1512db2627975d5898740f2a676 | |
parent | 0273113863d9cad133470697bde19048979c8447 (diff) | |
download | qtmultimedia-68066a2826f37dd711d1d250fbea15d907ce8668.tar.gz |
Enable zero copy for WMF with OpenGL display
This patch provides texture handles of decoded video frames to
OpenGL RHI renderer. It uses the OpenGL WGL_NV_DX_interop
extension to create GL_TEXTURE_2D handle from IDirect3DSurface9.
Change-Id: I7ebe89ac1d0b6872ccde32705ddd77e55d1741ab
Reviewed-by: Lars Knoll <lars.knoll@qt.io>
(cherry picked from commit 28ede6c2188d0d636e8087efee7a53112ac2648f)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>
-rw-r--r-- | src/multimedia/platform/windows/evr/evrd3dpresentengine.cpp | 330 | ||||
-rw-r--r-- | src/multimedia/platform/windows/evr/evrd3dpresentengine_p.h | 19 |
2 files changed, 275 insertions, 74 deletions
diff --git a/src/multimedia/platform/windows/evr/evrd3dpresentengine.cpp b/src/multimedia/platform/windows/evr/evrd3dpresentengine.cpp index 002d52f3f..6090be6d6 100644 --- a/src/multimedia/platform/windows/evr/evrd3dpresentengine.cpp +++ b/src/multimedia/platform/windows/evr/evrd3dpresentengine.cpp @@ -53,6 +53,11 @@ #include <private/qrhi_p.h> #include <private/qrhid3d11_p.h> +#if QT_CONFIG(opengl) +# include <qopenglcontext.h> +# include <qopenglfunctions.h> +# include <qoffscreensurface.h> +#endif QT_BEGIN_NAMESPACE @@ -62,11 +67,9 @@ class IMFSampleVideoBuffer: public QAbstractVideoBuffer { public: IMFSampleVideoBuffer(QWindowsIUPointer<IDirect3DDevice9Ex> device, - IMFSample *sample, - QWindowsIUPointer<ID3D11Texture2D> d2d11tex) - : QAbstractVideoBuffer(d2d11tex ? QVideoFrame::RhiTextureHandle : QVideoFrame::NoHandle) + IMFSample *sample, QVideoFrame::HandleType type = QVideoFrame::NoHandle) + : QAbstractVideoBuffer(type) , m_device(device) - , m_d2d11tex(d2d11tex) , m_mapMode(QVideoFrame::NotMapped) { sample->AddRef(); @@ -80,76 +83,193 @@ public: } QVideoFrame::MapMode mapMode() const override { return m_mapMode; } - MapData map(QVideoFrame::MapMode mode) override; - void unmap() override; - void mapTextures() override {} - quint64 textureHandle(int /*plane*/) const override { return quint64(m_d2d11tex.get()); } + MapData map(QVideoFrame::MapMode mode) override + { + if (!m_sample || m_mapMode != QVideoFrame::NotMapped || mode != QVideoFrame::ReadOnly) + return {}; + + D3DSURFACE_DESC desc; + if (m_memSurface) { + if (FAILED(m_memSurface->GetDesc(&desc))) + return {}; -private: + } else { + QWindowsIUPointer<IMFMediaBuffer> buffer; + HRESULT hr = m_sample->GetBufferByIndex(0, buffer.address()); + if (FAILED(hr)) + return {}; + + QWindowsIUPointer<IDirect3DSurface9> surface; + hr = MFGetService(buffer.get(), MR_BUFFER_SERVICE, IID_IDirect3DSurface9, (void **)(surface.address())); + if (FAILED(hr)) + return {}; + + if (FAILED(surface->GetDesc(&desc))) + return {}; + + if (FAILED(m_device->CreateOffscreenPlainSurface(desc.Width, desc.Height, desc.Format, D3DPOOL_SYSTEMMEM, m_memSurface.address(), nullptr))) + return {}; + + if (FAILED(m_device->GetRenderTargetData(surface.get(), m_memSurface.get()))) { + m_memSurface.reset(); + return {}; + } + } + + D3DLOCKED_RECT rect; + if (FAILED(m_memSurface->LockRect(&rect, NULL, mode == QVideoFrame::ReadOnly ? D3DLOCK_READONLY : 0))) + return {}; + + m_mapMode = mode; + + MapData mapData; + mapData.nPlanes = 1; + mapData.bytesPerLine[0] = (int)rect.Pitch; + mapData.data[0] = reinterpret_cast<uchar *>(rect.pBits); + mapData.size[0] = (int)(rect.Pitch * desc.Height); + return mapData; + } + + void unmap() override + { + if (m_mapMode == QVideoFrame::NotMapped) + return; + + m_mapMode = QVideoFrame::NotMapped; + if (m_memSurface) + m_memSurface->UnlockRect(); + } + +protected: QWindowsIUPointer<IDirect3DDevice9Ex> m_device; - QWindowsIUPointer<ID3D11Texture2D> m_d2d11tex; QWindowsIUPointer<IMFSample> m_sample; + +private: QWindowsIUPointer<IDirect3DSurface9> m_memSurface; QVideoFrame::MapMode m_mapMode; }; -IMFSampleVideoBuffer::MapData IMFSampleVideoBuffer::map(QVideoFrame::MapMode mode) +class D3D11TextureVideoBuffer: public IMFSampleVideoBuffer { - if (!m_sample || m_mapMode != QVideoFrame::NotMapped || mode != QVideoFrame::ReadOnly) - return {}; +public: + D3D11TextureVideoBuffer(QWindowsIUPointer<IDirect3DDevice9Ex> device, IMFSample *sample, + QWindowsIUPointer<ID3D11Texture2D> d2d11tex) + : IMFSampleVideoBuffer(device, sample, QVideoFrame::RhiTextureHandle) + , m_d2d11tex(d2d11tex) + {} - D3DSURFACE_DESC desc; - if (m_memSurface) { - if (FAILED(m_memSurface->GetDesc(&desc))) - return {}; + quint64 textureHandle(int plane) const override + { + return plane == 0 ? quint64(m_d2d11tex.get()) : 0; + } + +private: + QWindowsIUPointer<ID3D11Texture2D> m_d2d11tex; +}; + +#if QT_CONFIG(opengl) +class OpenGlVideoBuffer: public IMFSampleVideoBuffer +{ +public: + OpenGlVideoBuffer(QWindowsIUPointer<IDirect3DDevice9Ex> device, IMFSample *sample, + const WglNvDxInterop &wglNvDxInterop, HANDLE sharedHandle) + : IMFSampleVideoBuffer(device, sample, QVideoFrame::RhiTextureHandle) + , m_sharedHandle(sharedHandle) + , m_wgl(wglNvDxInterop) + {} + + ~OpenGlVideoBuffer() override + { + if (m_d3dglHandle) { + if (m_glHandle) { + if (!m_wgl.wglDXUnlockObjectsNV(m_d3dglHandle, 1, &m_glHandle)) + qWarning() << "Failed to unlock OpenGL texture"; + if (!m_wgl.wglDXUnregisterObjectNV(m_d3dglHandle, m_glHandle)) + qWarning() << "Failed to unregister OpenGL texture"; + + if (QOpenGLContext::currentContext()) { + QOpenGLFunctions *funcs = QOpenGLContext::currentContext()->functions(); + if (funcs) + funcs->glDeleteTextures(1, &m_glTextureName); + else + qWarning() << "Could not delete texture, OpenGL context functions missing"; + } else { + qWarning() << "Could not delete texture, OpenGL context missing"; + } + } + + if (!m_wgl.wglDXCloseDeviceNV(m_d3dglHandle)) + qWarning() << "Failed to close D3D-GL device"; + } + } + + void mapTextures() override + { + if (!QOpenGLContext::currentContext()) + return; + + if (m_d3dglHandle) + return; - } else { QWindowsIUPointer<IMFMediaBuffer> buffer; HRESULT hr = m_sample->GetBufferByIndex(0, buffer.address()); if (FAILED(hr)) - return {}; + return; QWindowsIUPointer<IDirect3DSurface9> surface; hr = MFGetService(buffer.get(), MR_BUFFER_SERVICE, IID_IDirect3DSurface9, (void **)(surface.address())); if (FAILED(hr)) - return {}; - - if (FAILED(surface->GetDesc(&desc))) - return {}; + return; - if (FAILED(m_device->CreateOffscreenPlainSurface(desc.Width, desc.Height, desc.Format, D3DPOOL_SYSTEMMEM, m_memSurface.address(), nullptr))) - return {}; + hr = surface->GetContainer(IID_IDirect3DTexture9, (void **)m_texture.address()); + if (FAILED(hr)) + return; - if (FAILED(m_device->GetRenderTargetData(surface.get(), m_memSurface.get()))) { - m_memSurface.reset(); - return {}; + m_d3dglHandle = m_wgl.wglDXOpenDeviceNV(m_device.get()); + if (!m_d3dglHandle) { + qWarning() << "Failed to open D3D device"; + return; } - } - - D3DLOCKED_RECT rect; - if (FAILED(m_memSurface->LockRect(&rect, NULL, mode == QVideoFrame::ReadOnly ? D3DLOCK_READONLY : 0))) - return {}; - m_mapMode = mode; + m_wgl.wglDXSetResourceShareHandleNV(m_texture.get(), m_sharedHandle); + + QOpenGLFunctions *funcs = QOpenGLContext::currentContext()->functions(); + if (funcs) { + funcs->glGenTextures(1, &m_glTextureName); + m_glHandle = m_wgl.wglDXRegisterObjectNV(m_d3dglHandle, m_texture.get(), m_glTextureName, + GL_TEXTURE_2D, WglNvDxInterop::WGL_ACCESS_READ_ONLY_NV); + if (m_glHandle) { + if (m_wgl.wglDXLockObjectsNV(m_d3dglHandle, 1, &m_glHandle)) + return; + + qWarning() << "Failed to lock OpenGL texture"; + m_wgl.wglDXUnregisterObjectNV(m_d3dglHandle, m_glHandle); + } else { + qWarning() << "Could not register D3D9 texture in OpenGL"; + } - MapData mapData; - mapData.nPlanes = 1; - mapData.bytesPerLine[0] = (int)rect.Pitch; - mapData.data[0] = reinterpret_cast<uchar *>(rect.pBits); - mapData.size[0] = (int)(rect.Pitch * desc.Height); - return mapData; -} + funcs->glDeleteTextures(1, &m_glTextureName); + m_glTextureName = 0; + } else { + qWarning() << "Failed generate texture names, OpenGL context functions missing"; + } + } -void IMFSampleVideoBuffer::unmap() -{ - if (m_mapMode == QVideoFrame::NotMapped) - return; + quint64 textureHandle(int plane) const override + { + return plane == 0 ? quint64(m_glTextureName) : 0; + } - m_mapMode = QVideoFrame::NotMapped; - if (m_memSurface) - m_memSurface->UnlockRect(); -} +private: + GLuint m_glTextureName = 0; + HANDLE m_d3dglHandle = nullptr; + HANDLE m_glHandle = nullptr; + HANDLE m_sharedHandle = nullptr; + WglNvDxInterop m_wgl; + QWindowsIUPointer<IDirect3DTexture9> m_texture; +}; +#endif D3DPresentEngine::D3DPresentEngine(QVideoSink *sink) : m_deviceResetToken(0) @@ -199,6 +319,62 @@ HRESULT D3DPresentEngine::initializeD3D() return hr; } +static bool findD3D11AdapterID(QRhi &rhi, IDirect3D9Ex *D3D9, UINT &adapterID) +{ + auto nh = static_cast<const QRhiD3D11NativeHandles*>(rhi.nativeHandles()); + if (D3D9 && nh) { + for (auto i = 0u; i < D3D9->GetAdapterCount(); ++i) { + LUID luid = {}; + D3D9->GetAdapterLUID(i, &luid); + if (luid.LowPart == nh->adapterLuidLow && luid.HighPart == nh->adapterLuidHigh) { + adapterID = i; + return true; + } + } + } + + return false; +} + +#if QT_CONFIG(opengl) +template <typename T> +static bool getProc(const QOpenGLContext *ctx, T &fn, const char *fName) +{ + fn = reinterpret_cast<T>(ctx->getProcAddress(fName)); + return fn != nullptr; +} + +static bool readWglNvDxInteropProc(WglNvDxInterop &f) +{ + QScopedPointer<QOffscreenSurface> surface(new QOffscreenSurface); + surface->create(); + QScopedPointer<QOpenGLContext> ctx(new QOpenGLContext); + ctx->create(); + ctx->makeCurrent(surface.get()); + + auto wglGetExtensionsStringARB = reinterpret_cast<const char* (WINAPI* )(HDC)> + (ctx->getProcAddress("wglGetExtensionsStringARB")); + if (!wglGetExtensionsStringARB) { + qCDebug(qLcEvrD3DPresentEngine) << "WGL extensions missing (no wglGetExtensionsStringARB function)"; + return false; + } + + auto dc = ::GetDC(::GetShellWindow()); + if (!strstr(wglGetExtensionsStringARB(dc), "WGL_NV_DX_interop")) { + qCDebug(qLcEvrD3DPresentEngine) << "WGL_NV_DX_interop missing"; + return false; + } + + return getProc(ctx.get(), f.wglDXOpenDeviceNV, "wglDXOpenDeviceNV") + && getProc(ctx.get(), f.wglDXCloseDeviceNV, "wglDXCloseDeviceNV") + && getProc(ctx.get(), f.wglDXSetResourceShareHandleNV, "wglDXSetResourceShareHandleNV") + && getProc(ctx.get(), f.wglDXRegisterObjectNV, "wglDXRegisterObjectNV") + && getProc(ctx.get(), f.wglDXUnregisterObjectNV, "wglDXUnregisterObjectNV") + && getProc(ctx.get(), f.wglDXLockObjectsNV, "wglDXLockObjectsNV") + && getProc(ctx.get(), f.wglDXUnlockObjectsNV, "wglDXUnlockObjectsNV"); +} +#endif + HRESULT D3DPresentEngine::createD3DDevice() { if (!m_D3D9 || !m_devices) @@ -206,26 +382,19 @@ HRESULT D3DPresentEngine::createD3DDevice() m_useTextureRendering = false; UINT adapterID = 0; - if (m_sink->rhi()) { - if (m_sink->rhi()->backend() == QRhi::D3D11) { - auto nh = static_cast<const QRhiD3D11NativeHandles*>(m_sink->rhi()->nativeHandles()); - if (nh) { - for (auto i = 0u; i < m_D3D9->GetAdapterCount(); ++i) { - LUID luid = {}; - m_D3D9->GetAdapterLUID(i, &luid); - if (luid.LowPart == nh->adapterLuidLow && luid.HighPart == nh->adapterLuidHigh) { - adapterID = i; - m_useTextureRendering = true; - break; - } - } - } + QRhi *rhi = m_sink ? m_sink->rhi() : nullptr; + if (rhi) { + if (rhi->backend() == QRhi::D3D11) { + m_useTextureRendering = findD3D11AdapterID(*rhi, m_D3D9.get(), adapterID); +#if QT_CONFIG(opengl) + } else if (rhi->backend() == QRhi::OpenGLES2) { + m_useTextureRendering = readWglNvDxInteropProc(m_wglNvDxInterop); +#endif } else { qCDebug(qLcEvrD3DPresentEngine) << "Not supported RHI backend type"; } } else { qCDebug(qLcEvrD3DPresentEngine) << "No RHI associated with this sink"; - adapterID = 0; } if (!m_useTextureRendering) @@ -404,20 +573,33 @@ QVideoFrame D3DPresentEngine::makeVideoFrame(IMFSample *sample) if (p.first == sample) sharedHandle = p.second; - QWindowsIUPointer<ID3D11Texture2D> d3d11tex; - if (m_useTextureRendering && sharedHandle && m_sink->rhi()) { - auto nh = static_cast<const QRhiD3D11NativeHandles*>(m_sink->rhi()->nativeHandles()); - if (nh) { - auto dev = reinterpret_cast<ID3D11Device *>(nh->dev); - if (dev) { - HRESULT hr = dev->OpenSharedResource(sharedHandle, __uuidof(ID3D11Texture2D), (void**)(d3d11tex.address())); - if (FAILED(hr)) - qWarning() << "Failed to obtain D3D11Texture2D from D3D9Texture2D handle"; + QAbstractVideoBuffer *vb = nullptr; + QRhi *rhi = m_sink ? m_sink->rhi() : nullptr; + if (m_useTextureRendering && sharedHandle && rhi) { + if (rhi->backend() == QRhi::D3D11) { + auto nh = static_cast<const QRhiD3D11NativeHandles*>(rhi->nativeHandles()); + if (nh) { + auto dev = reinterpret_cast<ID3D11Device *>(nh->dev); + if (dev) { + QWindowsIUPointer<ID3D11Texture2D> d3d11tex; + HRESULT hr = dev->OpenSharedResource(sharedHandle, __uuidof(ID3D11Texture2D), (void**)(d3d11tex.address())); + if (SUCCEEDED(hr)) + vb = new D3D11TextureVideoBuffer(m_device, sample, d3d11tex); + else + qWarning() << "Failed to obtain D3D11Texture2D from D3D9Texture2D handle"; + } } +#if QT_CONFIG(opengl) + } else if (rhi->backend() == QRhi::OpenGLES2) { + vb = new OpenGlVideoBuffer(m_device, sample, m_wglNvDxInterop, sharedHandle); +#endif } } - QVideoFrame frame(new IMFSampleVideoBuffer(m_device, sample, d3d11tex), m_surfaceFormat); + if (!vb) + vb = new IMFSampleVideoBuffer(m_device, sample); + + QVideoFrame frame(vb, m_surfaceFormat); // WMF uses 100-nanosecond units, Qt uses microseconds LONGLONG startTime = 0; diff --git a/src/multimedia/platform/windows/evr/evrd3dpresentengine_p.h b/src/multimedia/platform/windows/evr/evrd3dpresentengine_p.h index 1ab0af26c..0f031fa04 100644 --- a/src/multimedia/platform/windows/evr/evrd3dpresentengine_p.h +++ b/src/multimedia/platform/windows/evr/evrd3dpresentengine_p.h @@ -108,6 +108,22 @@ private: #endif // MAYBE_ANGLE +#if QT_CONFIG(opengl) +#include <qopengl.h> +struct WglNvDxInterop { + HANDLE (WINAPI* wglDXOpenDeviceNV) (void* dxDevice); + BOOL (WINAPI* wglDXCloseDeviceNV) (HANDLE hDevice); + HANDLE (WINAPI* wglDXRegisterObjectNV) (HANDLE hDevice, void *dxObject, GLuint name, GLenum type, GLenum access); + BOOL (WINAPI* wglDXSetResourceShareHandleNV) (void *dxResource, HANDLE shareHandle); + BOOL (WINAPI* wglDXLockObjectsNV) (HANDLE hDevice, GLint count, HANDLE *hObjects); + BOOL (WINAPI* wglDXUnlockObjectsNV) (HANDLE hDevice, GLint count, HANDLE *hObjects); + BOOL (WINAPI* wglDXUnregisterObjectNV) (HANDLE hDevice, HANDLE hObject); + + static const int WGL_ACCESS_READ_ONLY_NV = 0; +}; + +#endif + class D3DPresentEngine { Q_DISABLE_COPY(D3DPresentEngine) @@ -147,6 +163,9 @@ private: class QVideoSink *m_sink = nullptr; bool m_useTextureRendering = false; +#if QT_CONFIG(opengl) + WglNvDxInterop m_wglNvDxInterop; +#endif #ifdef MAYBE_ANGLE unsigned int updateTexture(IDirect3DSurface9 *src); |