Resubmit support for subpixel antialiasing on text in the GL2 engine.

The antialiasing is currently not gamma corrected and is disabled on
OpenGL ES 2.0.

Reviewed-by: Samuel

8 files changed, 186 insertions, 25 deletions

diff --git a/src/gui/kernel/qapplication_win.cpp b/src/gui/kernel/qapplication_win.cpp
index 2b1aaf54ed..6ecd535f9d 100644
--- a/src/gui/kernel/qapplication_win.cpp
+++ b/src/gui/kernel/qapplication_win.cpp
@@ -220,7 +220,7 @@ static void resolveAygLibs()
 #  define FE_FONTSMOOTHINGCLEARTYPE 0x0002
 #endif
 
-bool qt_cleartype_enabled;
+Q_GUI_EXPORT bool qt_cleartype_enabled;
 Q_GUI_EXPORT bool qt_win_owndc_required; // CS_OWNDC is required if we use the GL graphicssystem as default
 
 typedef HCTX (API *PtrWTOpen)(HWND, LPLOGCONTEXT, BOOL);
diff --git a/src/opengl/gl2paintengineex/qglengineshadermanager.cpp b/src/opengl/gl2paintengineex/qglengineshadermanager.cpp
index b3458af371..eceed065b4 100644
--- a/src/opengl/gl2paintengineex/qglengineshadermanager.cpp
+++ b/src/opengl/gl2paintengineex/qglengineshadermanager.cpp
@@ -134,7 +134,8 @@ QGLEngineSharedShaders::QGLEngineSharedShaders(const QGLContext* context)
         code[ShockingPinkSrcFragmentShader] = qglslShockingPinkSrcFragmentShader;
 
         code[MaskFragmentShader] = qglslMaskFragmentShader;
-        code[RgbMaskFragmentShader] = ""; //###
+        code[RgbMaskFragmentShaderPass1] = qglslRgbMaskFragmentShaderPass1;
+        code[RgbMaskFragmentShaderPass2] = qglslRgbMaskFragmentShaderPass2;
         code[RgbMaskWithGammaFragmentShader] = ""; //###
 
         code[MultiplyCompositionModeFragmentShader] = ""; //###
@@ -587,10 +588,15 @@ bool QGLEngineShaderManager::useCorrectShaderProg()
         if (maskType == PixelMask) {
             maskShaderName = QGLEngineSharedShaders::MaskFragmentShader;
             texCoords = true;
-        } else if (maskType == SubPixelMask) {
-            maskShaderName = QGLEngineSharedShaders::RgbMaskFragmentShader;
+        } else if (maskType == SubPixelMaskPass1) {
+            maskShaderName = QGLEngineSharedShaders::RgbMaskFragmentShaderPass1;
+            texCoords = true;
+        } else if (maskType == SubPixelMaskPass2) {
+            maskShaderName = QGLEngineSharedShaders::RgbMaskFragmentShaderPass2;
+            texCoords = true;
         } else if (maskType == SubPixelWithGammaMask) {
             maskShaderName = QGLEngineSharedShaders::RgbMaskWithGammaFragmentShader;
+            texCoords = true;
         } else {
             qCritical("QGLEngineShaderManager::useCorrectShaderProg() - Unknown mask type");
         }
diff --git a/src/opengl/gl2paintengineex/qglengineshadermanager_p.h b/src/opengl/gl2paintengineex/qglengineshadermanager_p.h
index 3c5a5f5f91..47d9a2ad6e 100644
--- a/src/opengl/gl2paintengineex/qglengineshadermanager_p.h
+++ b/src/opengl/gl2paintengineex/qglengineshadermanager_p.h
@@ -142,7 +142,8 @@
 
     Masks are implementations of "qcolorp vec4 applyMask(qcolorp vec4 src)":
         qglslMaskFragmentShader
-        qglslRgbMaskFragmentShader
+        qglslRgbMaskFragmentShaderPass1
+        qglslRgbMaskFragmentShaderPass2
         qglslRgbMaskWithGammaFragmentShader
 
     Composition modes are "qcolorp vec4 compose(qcolorp vec4 src)":
@@ -321,7 +322,8 @@ public:
         ShockingPinkSrcFragmentShader,
 
         MaskFragmentShader,
-        RgbMaskFragmentShader,
+        RgbMaskFragmentShaderPass1,
+        RgbMaskFragmentShaderPass2,
         RgbMaskWithGammaFragmentShader,
 
         MultiplyCompositionModeFragmentShader,
@@ -376,7 +378,7 @@ public:
     QGLEngineShaderManager(QGLContext* context);
     ~QGLEngineShaderManager();
 
-    enum MaskType {NoMask, PixelMask, SubPixelMask, SubPixelWithGammaMask};
+    enum MaskType {NoMask, PixelMask, SubPixelMaskPass1, SubPixelMaskPass2, SubPixelWithGammaMask};
     enum PixelSrcType {
         ImageSrc = Qt::TexturePattern+1,
         NonPremultipliedImageSrc = Qt::TexturePattern+2,
diff --git a/src/opengl/gl2paintengineex/qglengineshadersource_p.h b/src/opengl/gl2paintengineex/qglengineshadersource_p.h
index 6bcf010b30..8ae86d333c 100644
--- a/src/opengl/gl2paintengineex/qglengineshadersource_p.h
+++ b/src/opengl/gl2paintengineex/qglengineshadersource_p.h
@@ -401,6 +401,39 @@ static const char* const qglslMaskFragmentShader = "\
         return src * mask.a; \
     }";
 
+// For source over with subpixel antialiasing, the final color is calculated per component as follows
+// (.a is alpha component, .c is red, green or blue component):
+// alpha = src.a * mask.c * opacity
+// dest.c = dest.c * (1 - alpha) + src.c * alpha
+//
+// In the first pass, calculate: dest.c = dest.c * (1 - alpha) with blend funcs: zero, 1 - source color
+// In the second pass, calculate: dest.c = dest.c + src.c * alpha with blend funcs: one, one
+//
+// If source is a solid color (src is constant), only the first pass is needed, with blend funcs: constant, 1 - source color
+
+// For source composition with subpixel antialiasing, the final color is calculated per component as follows:
+// alpha = src.a * mask.c * opacity
+// dest.c = dest.c * (1 - mask.c) + src.c * alpha
+//
+
+static const char* const qglslRgbMaskFragmentShaderPass1 = "\
+    varying highp   vec2      textureCoords;\
+    uniform lowp    sampler2D maskTexture;\
+    lowp vec4 applyMask(lowp vec4 src) \
+    {\
+        lowp vec4 mask = texture2D(maskTexture, textureCoords); \
+        return src.a * mask; \
+    }";
+
+static const char* const qglslRgbMaskFragmentShaderPass2 = "\
+    varying highp   vec2      textureCoords;\
+    uniform lowp    sampler2D maskTexture;\
+    lowp vec4 applyMask(lowp vec4 src) \
+    {\
+        lowp vec4 mask = texture2D(maskTexture, textureCoords); \
+        return src * mask; \
+    }";
+
 /*
     Left to implement:
         RgbMaskFragmentShader,
diff --git a/src/opengl/gl2paintengineex/qpaintengineex_opengl2.cpp b/src/opengl/gl2paintengineex/qpaintengineex_opengl2.cpp
index 930d181290..ba2a0eaddb 100644
--- a/src/opengl/gl2paintengineex/qpaintengineex_opengl2.cpp
+++ b/src/opengl/gl2paintengineex/qpaintengineex_opengl2.cpp
@@ -90,6 +90,10 @@ static const GLuint QT_IMAGE_TEXTURE_UNIT       = 0; //Can be the same as brush
 static const GLuint QT_MASK_TEXTURE_UNIT        = 1;
 static const GLuint QT_BACKGROUND_TEXTURE_UNIT  = 2;
 
+#ifdef Q_WS_WIN
+extern Q_GUI_EXPORT bool qt_cleartype_enabled;
+#endif
+
 class QGLTextureGlyphCache : public QObject, public QTextureGlyphCache
 {
     Q_OBJECT
@@ -100,6 +104,7 @@ public:
     virtual void createTextureData(int width, int height);
     virtual void resizeTextureData(int width, int height);
     virtual void fillTexture(const Coord &c, glyph_t glyph);
+    virtual int glyphMargin() const;
 
     inline GLuint texture() const { return m_texture; }
 
@@ -290,7 +295,7 @@ void QGLTextureGlyphCache::fillTexture(const Coord &c, glyph_t glyph)
 
     glBindTexture(GL_TEXTURE_2D, m_texture);
     if (mask.format() == QImage::Format_RGB32) {
-        glTexSubImage2D(GL_TEXTURE_2D, 0, c.x, m_height - c.y, maskWidth, maskHeight, GL_BGRA, GL_UNSIGNED_BYTE, mask.bits());
+        glTexSubImage2D(GL_TEXTURE_2D, 0, c.x, c.y, maskWidth, maskHeight, GL_BGRA, GL_UNSIGNED_BYTE, mask.bits());
     } else {
 #ifdef QT_OPENGL_ES2
         glTexSubImage2D(GL_TEXTURE_2D, 0, c.x, c.y, maskWidth, maskHeight, GL_ALPHA, GL_UNSIGNED_BYTE, mask.bits());
@@ -310,6 +315,17 @@ void QGLTextureGlyphCache::fillTexture(const Coord &c, glyph_t glyph)
     }
 }
 
+int QGLTextureGlyphCache::glyphMargin() const
+{
+#if defined(Q_WS_MAC)
+    return 2;
+#elif defined (Q_WS_X11)
+    return 0;
+#else
+    return m_type == QFontEngineGlyphCache::Raster_RGBMask ? 2 : 0;
+#endif
+}
+
 extern QImage qt_imageForBrush(int brushStyle, bool invert);
 
 ////////////////////////////////// Private Methods //////////////////////////////////////////
@@ -1209,6 +1225,12 @@ void QGL2PaintEngineEx::drawTextItem(const QPointF &p, const QTextItem &textItem
     if (ti.fontEngine->fontDef.pixelSize * qSqrt(s->matrix.determinant()) >= 64)
         drawCached = false;
 
+    if (d->glyphCacheType == QFontEngineGlyphCache::Raster_RGBMask
+        && state()->composition_mode != QPainter::CompositionMode_Source
+        && state()->composition_mode != QPainter::CompositionMode_SourceOver) {
+        drawCached = false;
+    }
+
     if (drawCached) {
         d->drawCachedGlyphs(p, ti);
         return;
@@ -1229,11 +1251,12 @@ void QGL2PaintEngineExPrivate::drawCachedGlyphs(const QPointF &p, const QTextIte
 
     QFontEngineGlyphCache::Type glyphType = ti.fontEngine->glyphFormat >= 0
         ? QFontEngineGlyphCache::Type(ti.fontEngine->glyphFormat)
-        : QFontEngineGlyphCache::Raster_A8;
+        : glyphCacheType;
 
     QGLTextureGlyphCache *cache =
         (QGLTextureGlyphCache *) ti.fontEngine->glyphCache(ctx, s->matrix);
-    if (!cache) {
+
+    if (!cache || cache->cacheType() != glyphType) {
         cache = new QGLTextureGlyphCache(ctx, glyphType, s->matrix);
         ti.fontEngine->setGlyphCache(ctx, cache);
     }
@@ -1248,12 +1271,6 @@ void QGL2PaintEngineExPrivate::drawCachedGlyphs(const QPointF &p, const QTextIte
         transferMode(BrushDrawingMode);
     transferMode(TextDrawingMode);
 
-    if (glyphType == QFontEngineGlyphCache::Raster_A8)
-        shaderManager->setMaskType(QGLEngineShaderManager::PixelMask);
-    else if (glyphType == QFontEngineGlyphCache::Raster_RGBMask)
-        shaderManager->setMaskType(QGLEngineShaderManager::SubPixelMask);
-    //### TODO: Gamma correction
-
     int margin = cache->glyphMargin();
 
     GLfloat dx = 1.0 / cache->width();
@@ -1274,10 +1291,96 @@ void QGL2PaintEngineExPrivate::drawCachedGlyphs(const QPointF &p, const QTextIte
         textureCoordinateArray.addRect(QRectF(c.x*dx, c.y*dy, c.w * dx, c.h * dy));
     }
 
+    if (vertexCoordinateArray.data() != oldVertexCoordinateDataPtr)
+        glVertexAttribPointer(QT_VERTEX_COORDS_ATTR, 2, GL_FLOAT, GL_FALSE, 0, vertexCoordinateArray.data());
+    if (textureCoordinateArray.data() != oldTextureCoordinateDataPtr)
+        glVertexAttribPointer(QT_TEXTURE_COORDS_ATTR, 2, GL_FLOAT, GL_FALSE, 0, textureCoordinateArray.data());
+
     QBrush pensBrush = q->state()->pen.brush();
     setBrush(&pensBrush);
 
-    prepareForDraw(false); // Text always causes src pixels to be transparent
+    if (glyphType == QFontEngineGlyphCache::Raster_A8) {
+
+        // Greyscale antialiasing
+
+        shaderManager->setMaskType(QGLEngineShaderManager::PixelMask);
+        prepareForDraw(false); // Text always causes src pixels to be transparent
+    } else if (glyphType == QFontEngineGlyphCache::Raster_RGBMask) {
+
+        // Subpixel antialiasing without gamma correction
+
+        QPainter::CompositionMode compMode = q->state()->composition_mode;
+        Q_ASSERT(compMode == QPainter::CompositionMode_Source
+            || compMode == QPainter::CompositionMode_SourceOver);
+
+        shaderManager->setMaskType(QGLEngineShaderManager::SubPixelMaskPass1);
+
+        if (pensBrush.style() == Qt::SolidPattern) {
+            // Solid patterns can get away with only one pass.
+            QColor c = pensBrush.color();
+            qreal oldOpacity = q->state()->opacity;
+            if (compMode == QPainter::CompositionMode_Source) {
+                c = premultiplyColor(c, q->state()->opacity);
+                q->state()->opacity = 1;
+                opacityUniformDirty = true;
+            }
+
+            compositionModeDirty = false; // I can handle this myself, thank you very much
+            prepareForDraw(false); // Text always causes src pixels to be transparent
+
+            // prepareForDraw() have set the opacity on the current shader, so the opacity state can now be reset.
+            if (compMode == QPainter::CompositionMode_Source) {
+                q->state()->opacity = oldOpacity;
+                opacityUniformDirty = true;
+            }
+
+            glEnable(GL_BLEND);
+            glBlendFunc(GL_CONSTANT_COLOR, GL_ONE_MINUS_SRC_COLOR);
+            glBlendColor(c.redF(), c.greenF(), c.blueF(), c.alphaF());
+        } else {
+            // Other brush styles need two passes.
+
+            qreal oldOpacity = q->state()->opacity;
+            if (compMode == QPainter::CompositionMode_Source) {
+                q->state()->opacity = 1;
+                opacityUniformDirty = true;
+                pensBrush = Qt::white;
+                setBrush(&pensBrush);
+            }
+
+            compositionModeDirty = false; // I can handle this myself, thank you very much
+            prepareForDraw(false); // Text always causes src pixels to be transparent
+            glEnable(GL_BLEND);
+            glBlendFunc(GL_ZERO, GL_ONE_MINUS_SRC_COLOR);
+
+            glActiveTexture(GL_TEXTURE0 + QT_MASK_TEXTURE_UNIT);
+            glBindTexture(GL_TEXTURE_2D, cache->texture());
+            updateTextureFilter(GL_TEXTURE_2D, GL_REPEAT, false);
+
+#ifndef QT_OPENGL_ES_2
+            if (inRenderText)
+                shaderManager->currentProgram()->setUniformValue(location(QGLEngineShaderManager::Depth), zValueForRenderText());
+#endif
+            shaderManager->currentProgram()->setUniformValue(location(QGLEngineShaderManager::MaskTexture), QT_MASK_TEXTURE_UNIT);
+            glDrawArrays(GL_TRIANGLES, 0, 6 * glyphs.size());
+
+            shaderManager->setMaskType(QGLEngineShaderManager::SubPixelMaskPass2);
+
+            if (compMode == QPainter::CompositionMode_Source) {
+                q->state()->opacity = oldOpacity;
+                opacityUniformDirty = true;
+                pensBrush = q->state()->pen.brush();
+                setBrush(&pensBrush);
+            }
+
+            compositionModeDirty = false;
+            prepareForDraw(false); // Text always causes src pixels to be transparent
+            glEnable(GL_BLEND);
+            glBlendFunc(GL_ONE, GL_ONE);
+        }
+        compositionModeDirty = true;
+    }
+    //### TODO: Gamma correction
 
     glActiveTexture(GL_TEXTURE0 + QT_MASK_TEXTURE_UNIT);
     glBindTexture(GL_TEXTURE_2D, cache->texture());
@@ -1287,14 +1390,7 @@ void QGL2PaintEngineExPrivate::drawCachedGlyphs(const QPointF &p, const QTextIte
     if (inRenderText)
         shaderManager->currentProgram()->setUniformValue(location(QGLEngineShaderManager::Depth), zValueForRenderText());
 #endif
-
     shaderManager->currentProgram()->setUniformValue(location(QGLEngineShaderManager::MaskTexture), QT_MASK_TEXTURE_UNIT);
-
-    if (vertexCoordinateArray.data() != oldVertexCoordinateDataPtr)
-        glVertexAttribPointer(QT_VERTEX_COORDS_ATTR, 2, GL_FLOAT, GL_FALSE, 0, vertexCoordinateArray.data());
-    if (textureCoordinateArray.data() != oldTextureCoordinateDataPtr)
-        glVertexAttribPointer(QT_TEXTURE_COORDS_ATTR, 2, GL_FLOAT, GL_FALSE, 0, textureCoordinateArray.data());
-
     glDrawArrays(GL_TRIANGLES, 0, 6 * glyphs.size());
 }
 
@@ -1353,6 +1449,18 @@ bool QGL2PaintEngineEx::begin(QPaintDevice *pdev)
     glDisable(GL_MULTISAMPLE);
 #endif
 
+    d->glyphCacheType = QFontEngineGlyphCache::Raster_A8;
+
+#if !defined(QT_OPENGL_ES_2)
+    if (!d->device->format().alpha()
+#if defined(Q_WS_WIN)
+        && qt_cleartype_enabled
+#endif
+       ) {
+        d->glyphCacheType = QFontEngineGlyphCache::Raster_RGBMask;
+    }
+#endif
+
     return true;
 }
 
diff --git a/src/opengl/gl2paintengineex/qpaintengineex_opengl2_p.h b/src/opengl/gl2paintengineex/qpaintengineex_opengl2_p.h
index a44be909ee..8b09f415df 100644
--- a/src/opengl/gl2paintengineex/qpaintengineex_opengl2_p.h
+++ b/src/opengl/gl2paintengineex/qpaintengineex_opengl2_p.h
@@ -60,6 +60,7 @@
 #include <private/qgl2pexvertexarray_p.h>
 #include <private/qglpaintdevice_p.h>
 #include <private/qglpixmapfilter_p.h>
+#include <private/qfontengine_p.h>
 
 enum EngineMode {
     ImageDrawingMode,
@@ -205,6 +206,7 @@ public:
     int width, height;
     QGLContext *ctx;
     EngineMode mode;
+    QFontEngineGlyphCache::Type glyphCacheType;
 
     mutable QOpenGL2PaintEngineState *last_created_state;
 
diff --git a/src/opengl/qglextensions.cpp b/src/opengl/qglextensions.cpp
index 3699d62ac9..c785d8a38b 100644
--- a/src/opengl/qglextensions.cpp
+++ b/src/opengl/qglextensions.cpp
@@ -391,8 +391,9 @@ bool qt_resolve_version_2_0_functions(QGLContext *ctx)
     if (glStencilOpSeparate)
         return gl2supported;
 
+    glBlendColor = (_glBlendColor) ctx->getProcAddress(QLatin1String("glBlendColor"));
     glStencilOpSeparate = (_glStencilOpSeparate) ctx->getProcAddress(QLatin1String("glStencilOpSeparate"));
-    if (!glStencilOpSeparate)
+    if (!glBlendColor || !glStencilOpSeparate)
         gl2supported = false;
 
     return gl2supported;
diff --git a/src/opengl/qglextensions_p.h b/src/opengl/qglextensions_p.h
index 40840b59c5..3510765231 100644
--- a/src/opengl/qglextensions_p.h
+++ b/src/opengl/qglextensions_p.h
@@ -152,6 +152,7 @@ typedef void (APIENTRY *_glActiveStencilFaceEXT) (GLenum );
 // Needed for GL2 engine:
 typedef void (APIENTRY *_glStencilOpSeparate) (GLenum face, GLenum sfail, GLenum dpfail, GLenum dppass);
 typedef void (APIENTRY *_glActiveTexture) (GLenum);
+typedef void (APIENTRY *_glBlendColor) (GLclampf, GLclampf, GLclampf, GLclampf);
 
 
 // EXT_GL_framebuffer_object
@@ -247,6 +248,7 @@ struct QGLExtensionFuncs
         // Extras for GL2 engine:
         qt_glActiveTexture = 0;
         qt_glStencilOpSeparate = 0;
+        qt_glBlendColor = 0;
 
         qt_glActiveStencilFaceEXT = 0;
         qt_glMultiTexCoord4f = 0;
@@ -360,6 +362,8 @@ struct QGLExtensionFuncs
     // Extras needed for GL2 engine:
     _glActiveTexture qt_glActiveTexture;
     _glStencilOpSeparate qt_glStencilOpSeparate;
+    _glBlendColor qt_glBlendColor;
+
 #endif
 
     // FBOs
@@ -574,6 +578,10 @@ struct QGLExtensionFuncs
 #endif
 
 #ifndef GL_VERSION_1_4
+#define GL_CONSTANT_COLOR 0x8001
+#define GL_ONE_MINUS_CONSTANT_COLOR 0x8002
+#define GL_CONSTANT_ALPHA 0x8003
+#define GL_ONE_MINUS_CONSTANT_ALPHA 0x8004
 #define GL_INCR_WRAP 0x8507
 #define GL_DECR_WRAP 0x8508
 #endif
@@ -717,6 +725,7 @@ struct QGLExtensionFuncs
 
 #if !defined(QT_OPENGL_ES_2)
 #define glStencilOpSeparate QGLContextPrivate::extensionFuncs(ctx).qt_glStencilOpSeparate
+#define glBlendColor QGLContextPrivate::extensionFuncs(ctx).qt_glBlendColor
 #endif
 
 #if defined(QT_OPENGL_ES_2)