From 36649887e8ba8841b45522a772cf83b5c5130fca Mon Sep 17 00:00:00 2001 From: Fabian Bumberger Date: Fri, 21 Mar 2014 16:02:48 +0100 Subject: QNX: Remove most of the CPU overhead for video rendering This patch uses the GL_OES_EGL_image extension to create a OpenGL Texture handle for a libscreen pixmap. If the extension is not available it uses the "old" technique as fallback where the image data is copied into a QImage. This reduces the CPU load by more than 70% and allows HD videos to be played jitter-free. Task-number: QTBUG-37752 Change-Id: I4cad22c39390e4cf9eb5be5f0bfe446544a11b9e Reviewed-by: Bernd Weimer Reviewed-by: Sean Harmer --- src/plugins/qnx/camera/bbcamerasession.cpp | 2 +- src/plugins/qnx/common/windowgrabber.cpp | 173 +++++++++++++++------ src/plugins/qnx/common/windowgrabber.h | 29 +++- .../mmrendererplayervideorenderercontrol.cpp | 81 +++++++++- .../mmrendererplayervideorenderercontrol.h | 4 +- .../qdeclarativevideooutput_render.cpp | 10 ++ 6 files changed, 236 insertions(+), 63 deletions(-) diff --git a/src/plugins/qnx/camera/bbcamerasession.cpp b/src/plugins/qnx/camera/bbcamerasession.cpp index 77ba71493..618aca533 100644 --- a/src/plugins/qnx/camera/bbcamerasession.cpp +++ b/src/plugins/qnx/camera/bbcamerasession.cpp @@ -140,7 +140,7 @@ BbCameraSession::BbCameraSession(QObject *parent) connect(this, SIGNAL(captureModeChanged(QCamera::CaptureModes)), SLOT(updateReadyForCapture())); connect(m_orientationHandler, SIGNAL(orientationChanged(int)), SLOT(deviceOrientationChanged(int))); - connect(m_windowGrabber, SIGNAL(frameGrabbed(QImage)), SLOT(viewfinderFrameGrabbed(QImage))); + connect(m_windowGrabber, SIGNAL(frameGrabbed(QImage, int)), SLOT(viewfinderFrameGrabbed(QImage))); } BbCameraSession::~BbCameraSession() diff --git a/src/plugins/qnx/common/windowgrabber.cpp b/src/plugins/qnx/common/windowgrabber.cpp index 5ed54b87e..3f6eaca41 100644 --- a/src/plugins/qnx/common/windowgrabber.cpp +++ b/src/plugins/qnx/common/windowgrabber.cpp @@ -47,6 +47,8 @@ #include #include +#include + #ifdef Q_OS_BLACKBERRY #include #include @@ -57,13 +59,15 @@ QT_BEGIN_NAMESPACE WindowGrabber::WindowGrabber(QObject *parent) : QObject(parent), - m_screenBuffer(0), m_screenBufferWidth(-1), m_screenBufferHeight(-1), m_active(false), m_screenContextInitialized(false), - m_screenPixmapInitialized(false), - m_screenPixmapBufferInitialized(false) + m_screenPixmapBuffersInitialized(false), + m_currentFrame(0), + m_eglImageSupported(false), + m_eglImagesInitialized(false), + m_eglImageCheck(false) { // grab the window frame with 60 frames per second m_timer.setInterval(1000/60); @@ -76,6 +80,11 @@ WindowGrabber::WindowGrabber(QObject *parent) WindowGrabber::~WindowGrabber() { QCoreApplication::eventDispatcher()->removeNativeEventFilter(this); + if (eglImagesInitialized()) { + glDeleteTextures(2, imgTextures); + eglDestroyImageKHR(eglGetDisplay(EGL_DEFAULT_DISPLAY), img[0]); + eglDestroyImageKHR(eglGetDisplay(EGL_DEFAULT_DISPLAY), img[1]); + } } void WindowGrabber::setFrameRate(int frameRate) @@ -83,6 +92,37 @@ void WindowGrabber::setFrameRate(int frameRate) m_timer.setInterval(1000/frameRate); } +void WindowGrabber::createEglImages() +{ + // Do nothing if either egl images are not supported, the screen context is not valid + // or the images are already created + if (!eglImageSupported() || !m_screenContextInitialized || eglImagesInitialized()) + return; + + glGenTextures(2, imgTextures); + glBindTexture(GL_TEXTURE_2D, imgTextures[0]); + img[0] = eglCreateImageKHR(eglGetDisplay(EGL_DEFAULT_DISPLAY), EGL_NO_CONTEXT, + EGL_NATIVE_PIXMAP_KHR, + m_screenPixmaps[0], + 0); + glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, img[0]); + + glBindTexture(GL_TEXTURE_2D, imgTextures[1]); + img[1] = eglCreateImageKHR(eglGetDisplay(EGL_DEFAULT_DISPLAY), EGL_NO_CONTEXT, + EGL_NATIVE_PIXMAP_KHR, + m_screenPixmaps[1], + 0); + + glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, img[1]); + + if (img[0] == 0 || img[1] == 0) { + qWarning() << "Failed to create KHR images" << img[0] << img[1] << strerror(errno) << errno; + m_eglImageSupported = false; + } else { + m_eglImagesInitialized = true; + } +} + void WindowGrabber::setWindowId(const QByteArray &windowId) { m_windowId = windowId; @@ -90,6 +130,9 @@ void WindowGrabber::setWindowId(const QByteArray &windowId) void WindowGrabber::start() { + if (m_active) + return; + int result = 0; #ifdef Q_OS_BLACKBERRY_TABLET @@ -124,30 +167,21 @@ void WindowGrabber::start() m_screenContextInitialized = true; } - result = screen_create_pixmap(&m_screenPixmap, m_screenContext); + result = screen_create_pixmap(&m_screenPixmaps[0], m_screenContext); + result = screen_create_pixmap(&m_screenPixmaps[1], m_screenContext); if (result != 0) { cleanup(); - qWarning() << "WindowGrabber: cannot create pixmap:" << strerror(errno); + qWarning() << "WindowGrabber: cannot create pixmaps:" << strerror(errno); return; - } else { - m_screenPixmapInitialized = true; } - const int usage = SCREEN_USAGE_READ | SCREEN_USAGE_NATIVE; - result = screen_set_pixmap_property_iv(m_screenPixmap, SCREEN_PROPERTY_USAGE, &usage); - if (result != 0) { - cleanup(); - qWarning() << "WindowGrabber: cannot set pixmap usage:" << strerror(errno); - return; - } + const int usage = SCREEN_USAGE_NATIVE; + result = screen_set_pixmap_property_iv(m_screenPixmaps[0], SCREEN_PROPERTY_USAGE, &usage); + result |= screen_set_pixmap_property_iv(m_screenPixmaps[1], SCREEN_PROPERTY_USAGE, &usage); - const int format = SCREEN_FORMAT_RGBA8888; - result = screen_set_pixmap_property_iv(m_screenPixmap, SCREEN_PROPERTY_FORMAT, &format); - if (result != 0) { - cleanup(); - qWarning() << "WindowGrabber: cannot set pixmap format:" << strerror(errno); - return; - } + const int format = SCREEN_FORMAT_RGBX8888; + screen_set_pixmap_property_iv(m_screenPixmaps[0], SCREEN_PROPERTY_FORMAT, &format); + screen_set_pixmap_property_iv(m_screenPixmaps[1], SCREEN_PROPERTY_FORMAT, &format); int size[2] = { 0, 0 }; @@ -172,37 +206,51 @@ void WindowGrabber::updateFrameSize() { int size[2] = { m_screenBufferWidth, m_screenBufferHeight }; - int result = screen_set_pixmap_property_iv(m_screenPixmap, SCREEN_PROPERTY_BUFFER_SIZE, size); - if (result != 0) { - cleanup(); - qWarning() << "WindowGrabber: cannot set pixmap size:" << strerror(errno); - return; - } + screen_set_pixmap_property_iv(m_screenPixmaps[0], SCREEN_PROPERTY_BUFFER_SIZE, size); + if (eglImageSupported()) + screen_set_pixmap_property_iv(m_screenPixmaps[1], SCREEN_PROPERTY_BUFFER_SIZE, size); + + int result = screen_create_pixmap_buffer(m_screenPixmaps[0]); + if (eglImageSupported()) + result |= screen_create_pixmap_buffer(m_screenPixmaps[1]); - result = screen_create_pixmap_buffer(m_screenPixmap); if (result != 0) { cleanup(); qWarning() << "WindowGrabber: cannot create pixmap buffer:" << strerror(errno); return; + } else { + m_screenPixmapBuffersInitialized = true; + } + + result = screen_get_pixmap_property_pv(m_screenPixmaps[0], SCREEN_PROPERTY_RENDER_BUFFERS, + (void**)&m_screenPixmapBuffers[0]); + if (eglImageSupported()) { + result |= screen_get_pixmap_property_pv(m_screenPixmaps[1], SCREEN_PROPERTY_RENDER_BUFFERS, + (void**)&m_screenPixmapBuffers[1]); } - result = screen_get_pixmap_property_pv(m_screenPixmap, SCREEN_PROPERTY_RENDER_BUFFERS, (void**)&m_screenPixmapBuffer); if (result != 0) { cleanup(); qWarning() << "WindowGrabber: cannot get pixmap buffer:" << strerror(errno); return; - } else { - m_screenPixmapBufferInitialized = true; } - result = screen_get_buffer_property_pv(m_screenPixmapBuffer, SCREEN_PROPERTY_POINTER, (void**)&m_screenBuffer); + result = screen_get_buffer_property_pv(m_screenPixmapBuffers[0], SCREEN_PROPERTY_POINTER, + (void**)&m_screenBuffers[0]); + if (eglImageSupported()) { + result |= screen_get_buffer_property_pv(m_screenPixmapBuffers[1], SCREEN_PROPERTY_POINTER, + (void**)&m_screenBuffers[1]); + } + if (result != 0) { cleanup(); qWarning() << "WindowGrabber: cannot get pixmap buffer pointer:" << strerror(errno); return; } - result = screen_get_buffer_property_iv(m_screenPixmapBuffer, SCREEN_PROPERTY_STRIDE, &m_screenBufferStride); + result = screen_get_buffer_property_iv(m_screenPixmapBuffers[0], SCREEN_PROPERTY_STRIDE, + &m_screenBufferStride); + if (result != 0) { cleanup(); qWarning() << "WindowGrabber: cannot get pixmap buffer stride:" << strerror(errno); @@ -310,8 +358,40 @@ QByteArray WindowGrabber::windowGroupId() const return QByteArray(groupIdData); } +bool WindowGrabber::eglImageSupported() +{ + return m_eglImageSupported; +} + +void WindowGrabber::checkForEglImageExtension() +{ + QOpenGLContext *m_context = QOpenGLContext::currentContext(); + if (!m_context) //Should not happen, because we are called from the render thread + return; + + QByteArray eglExtensions = QByteArray(eglQueryString(eglGetDisplay(EGL_DEFAULT_DISPLAY), + EGL_EXTENSIONS)); + m_eglImageSupported = m_context->hasExtension(QByteArrayLiteral("GL_OES_EGL_image")) + && eglExtensions.contains(QByteArrayLiteral("EGL_KHR_image")); + + m_eglImageCheck = true; +} + +bool WindowGrabber::eglImagesInitialized() +{ + return m_eglImagesInitialized; +} + void WindowGrabber::grab() { + if (!m_eglImageCheck) // We did not check for egl images yet + return; + + if (eglImageSupported()) + m_currentFrame = (m_currentFrame + 1) % 2; + else + m_currentFrame = 0; + int size[2] = { 0, 0 }; int result = screen_get_window_property_iv(m_window, SCREEN_PROPERTY_SOURCE_SIZE, size); @@ -324,40 +404,33 @@ void WindowGrabber::grab() if (m_screenBufferWidth != size[0] || m_screenBufferHeight != size[1]) { // The source viewport size changed, so we have to adapt our buffers - if (m_screenPixmapBufferInitialized) { - screen_destroy_pixmap_buffer(m_screenPixmap); - m_screenPixmapBufferInitialized = false; + if (m_screenPixmapBuffersInitialized) { + screen_destroy_pixmap_buffer(m_screenPixmaps[0]); + if (eglImageSupported()) + screen_destroy_pixmap_buffer(m_screenPixmaps[1]); } m_screenBufferWidth = size[0]; m_screenBufferHeight = size[1]; updateFrameSize(); + m_eglImagesInitialized = false; } const int rect[] = { 0, 0, m_screenBufferWidth, m_screenBufferHeight }; - result = screen_read_window(m_window, m_screenPixmapBuffer, 1, rect, 0); + result = screen_read_window(m_window, m_screenPixmapBuffers[m_currentFrame], 1, rect, 0); if (result != 0) return; - const QImage frame((unsigned char*)m_screenBuffer, m_screenBufferWidth, m_screenBufferHeight, - m_screenBufferStride, QImage::Format_ARGB32); + const QImage frame((unsigned char*)m_screenBuffers[m_currentFrame], m_screenBufferWidth, + m_screenBufferHeight, m_screenBufferStride, QImage::Format_ARGB32); - emit frameGrabbed(frame); + emit frameGrabbed(frame, imgTextures[m_currentFrame]); } void WindowGrabber::cleanup() { - if (m_screenPixmapBufferInitialized) { - screen_destroy_buffer(m_screenPixmapBuffer); - m_screenPixmapBufferInitialized = false; - } - - if (m_screenPixmapInitialized) { - screen_destroy_pixmap(m_screenPixmap); - m_screenPixmapInitialized = false; - } - + //We only need to destroy the context as it frees all resources associated with it if (m_screenContextInitialized) { screen_destroy_context(m_screenContext); m_screenContextInitialized = false; diff --git a/src/plugins/qnx/common/windowgrabber.h b/src/plugins/qnx/common/windowgrabber.h index 7ec4202a2..40351ef25 100644 --- a/src/plugins/qnx/common/windowgrabber.h +++ b/src/plugins/qnx/common/windowgrabber.h @@ -41,6 +41,12 @@ #ifndef WINDOWGRABBER_H #define WINDOWGRABBER_H +#define EGL_EGLEXT_PROTOTYPES = 1 +#define GL_GLEXT_PROTOTYPES = 1 +#include +#include +#include +#include #include #include #include @@ -59,6 +65,8 @@ public: void setFrameRate(int frameRate); + void createEglImages(); + void setWindowId(const QByteArray &windowId); void start(); @@ -73,8 +81,12 @@ public: QByteArray windowGroupId() const; + bool eglImageSupported(); + void checkForEglImageExtension(); + bool eglImagesInitialized(); + signals: - void frameGrabbed(const QImage &frame); + void frameGrabbed(const QImage &frame, int); private slots: void grab(); @@ -89,10 +101,10 @@ private: screen_window_t m_window; screen_context_t m_screenContext; - screen_pixmap_t m_screenPixmap; - screen_buffer_t m_screenPixmapBuffer; + screen_pixmap_t m_screenPixmaps[2]; + screen_buffer_t m_screenPixmapBuffers[2]; - char* m_screenBuffer; + char *m_screenBuffers[2]; int m_screenBufferWidth; int m_screenBufferHeight; @@ -100,8 +112,13 @@ private: bool m_active : 1; bool m_screenContextInitialized : 1; - bool m_screenPixmapInitialized : 1; - bool m_screenPixmapBufferInitialized : 1; + bool m_screenPixmapBuffersInitialized : 1; + int m_currentFrame; + EGLImageKHR img[2]; + GLuint imgTextures[2]; + bool m_eglImageSupported : 1; + bool m_eglImagesInitialized : 1; + bool m_eglImageCheck : 1; // We must not send a grabed frame before this is true }; QT_END_NAMESPACE diff --git a/src/plugins/qnx/mediaplayer/mmrendererplayervideorenderercontrol.cpp b/src/plugins/qnx/mediaplayer/mmrendererplayervideorenderercontrol.cpp index 0abdfec49..b9fe95026 100644 --- a/src/plugins/qnx/mediaplayer/mmrendererplayervideorenderercontrol.cpp +++ b/src/plugins/qnx/mediaplayer/mmrendererplayervideorenderercontrol.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include @@ -59,7 +60,7 @@ MmRendererPlayerVideoRendererControl::MmRendererPlayerVideoRendererControl(QObje , m_context(0) , m_videoId(-1) { - connect(m_windowGrabber, SIGNAL(frameGrabbed(QImage)), SLOT(frameGrabbed(QImage))); + connect(m_windowGrabber, SIGNAL(frameGrabbed(QImage, int)), SLOT(frameGrabbed(QImage, int))); } MmRendererPlayerVideoRendererControl::~MmRendererPlayerVideoRendererControl() @@ -75,6 +76,10 @@ QAbstractVideoSurface *MmRendererPlayerVideoRendererControl::surface() const void MmRendererPlayerVideoRendererControl::setSurface(QAbstractVideoSurface *surface) { m_surface = QPointer(surface); + if (QOpenGLContext::currentContext()) + m_windowGrabber->checkForEglImageExtension(); + else + m_surface->setProperty("_q_GLThreadCallback", QVariant::fromValue(this)); } void MmRendererPlayerVideoRendererControl::attachDisplay(mmr_context_t *context) @@ -139,20 +144,86 @@ void MmRendererPlayerVideoRendererControl::resume() m_windowGrabber->resume(); } -void MmRendererPlayerVideoRendererControl::frameGrabbed(const QImage &frame) +class BBTextureBuffer : public QAbstractVideoBuffer +{ +public: + BBTextureBuffer(int handle) : + QAbstractVideoBuffer(QAbstractVideoBuffer::GLTextureHandle) + { + m_handle = handle; + } + MapMode mapMode() const { + return QAbstractVideoBuffer::ReadWrite; + } + void unmap() { + + } + uchar *map(MapMode mode, int * numBytes, int * bytesPerLine) { + Q_UNUSED(mode); + Q_UNUSED(numBytes); + Q_UNUSED(bytesPerLine); + return 0; + } + QVariant handle() const { + return m_handle; + } +private: + int m_handle; +}; + +void MmRendererPlayerVideoRendererControl::frameGrabbed(const QImage &frame, int handle) { if (m_surface) { if (!m_surface->isActive()) { - m_surface->start(QVideoSurfaceFormat(frame.size(), QVideoFrame::Format_ARGB32)); + if (m_windowGrabber->eglImageSupported()) { + if (QOpenGLContext::currentContext()) + m_windowGrabber->createEglImages(); + else + m_surface->setProperty("_q_GLThreadCallback", QVariant::fromValue(this)); + + m_surface->start(QVideoSurfaceFormat(frame.size(), QVideoFrame::Format_BGR32, + QAbstractVideoBuffer::GLTextureHandle)); + } else { + m_surface->start(QVideoSurfaceFormat(frame.size(), QVideoFrame::Format_ARGB32)); + } } else { if (m_surface->surfaceFormat().frameSize() != frame.size()) { + QAbstractVideoBuffer::HandleType type = m_surface->surfaceFormat().handleType(); m_surface->stop(); - m_surface->start(QVideoSurfaceFormat(frame.size(), QVideoFrame::Format_ARGB32)); + if (type != QAbstractVideoBuffer::NoHandle) { + m_surface->setProperty("_q_GLThreadCallback", QVariant::fromValue(this)); + m_surface->start(QVideoSurfaceFormat(frame.size(), QVideoFrame::Format_BGR32, + QAbstractVideoBuffer::GLTextureHandle)); + } else { + m_surface->start(QVideoSurfaceFormat(frame.size(), QVideoFrame::Format_ARGB32)); + } } } - m_surface->present(frame.copy()); + // Depending on the support of EGL images on the current platform we either pass a texture + // handle or a copy of the image data + if (m_surface->surfaceFormat().handleType() != QAbstractVideoBuffer::NoHandle) { + if (m_windowGrabber->eglImagesInitialized() && + m_surface->property("_q_GLThreadCallback") != 0) + m_surface->setProperty("_q_GLThreadCallback", 0); + + + BBTextureBuffer *textBuffer = new BBTextureBuffer(handle); + QVideoFrame actualFrame(textBuffer, frame.size(), QVideoFrame::Format_BGR32); + m_surface->present(actualFrame); + } else { + m_surface->present(frame.copy()); + } } } +void MmRendererPlayerVideoRendererControl::customEvent(QEvent *e) +{ + // This is running in the render thread (OpenGL enabled) + if (e->type() == QEvent::User) + m_windowGrabber->checkForEglImageExtension(); + else if (e->type() == QEvent::User + 1) + m_windowGrabber->createEglImages(); +} + QT_END_NAMESPACE diff --git a/src/plugins/qnx/mediaplayer/mmrendererplayervideorenderercontrol.h b/src/plugins/qnx/mediaplayer/mmrendererplayervideorenderercontrol.h index 4e271ad5d..5624b464e 100644 --- a/src/plugins/qnx/mediaplayer/mmrendererplayervideorenderercontrol.h +++ b/src/plugins/qnx/mediaplayer/mmrendererplayervideorenderercontrol.h @@ -67,8 +67,10 @@ public: void pause(); void resume(); + void customEvent(QEvent *) Q_DECL_OVERRIDE; + private Q_SLOTS: - void frameGrabbed(const QImage &frame); + void frameGrabbed(const QImage &frame, int); private: QPointer m_surface; diff --git a/src/qtmultimediaquicktools/qdeclarativevideooutput_render.cpp b/src/qtmultimediaquicktools/qdeclarativevideooutput_render.cpp index cd03cd6b8..647732485 100644 --- a/src/qtmultimediaquicktools/qdeclarativevideooutput_render.cpp +++ b/src/qtmultimediaquicktools/qdeclarativevideooutput_render.cpp @@ -201,6 +201,16 @@ QSGNode *QDeclarativeVideoRendererBackend::updatePaintNode(QSGNode *oldNode, obj->event(&ev); } } +#if defined (Q_OS_QNX) // On QNX we need to be called back again for creating the egl images + else { + // Internal mechanism to call back the surface renderer from the QtQuick render thread + QObject *obj = m_surface->property("_q_GLThreadCallback").value(); + if (obj) { + QEvent ev(static_cast(QEvent::User + 1)); + obj->event(&ev); + } + } +#endif if (m_frameChanged) { if (videoNode && videoNode->pixelFormat() != m_frame.pixelFormat()) { -- cgit v1.2.1