gl: add GLSLFilter

this will render video frames into a fbo first. You can use opengl()-
>setUserShader()

src/OpenGLVideo.cpp

@@ -361,6 +361,7 @@ void OpenGLVideo::render(const QRectF &target, const QRectF& roi, const QMatrix4
 {
     DPTR_D(OpenGLVideo);
     Q_ASSERT(d.manager);
+    DYGL(glViewport(0, 0, d.rect.width(), d.rect.height())); // viewport may be set in gpu filters
     const qint64 mt = d.material->type();
     if (d.material_type != mt) {
         qDebug() << "material changed: " << VideoMaterial::typeName(d.material_type) << " => " << VideoMaterial::typeName(mt);

src/QtAV/GLSLFilter.h

@@ -0,0 +1,62 @@
+/******************************************************************************
+    QtAV:  Multimedia framework based on Qt and FFmpeg
+    Copyright (C) 2012-2016 Wang Bin <[email protected]>
+
+*   This file is part of QtAV (from 2016)
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+******************************************************************************/
+
+#ifndef QTAV_GLSLFILTER_H
+#define QTAV_GLSLFILTER_H
+
+#include <QtAV/QtAV_Global.h>
+#include <QtAV/Filter.h>
+
+namespace QtAV {
+class OpenGLVideo;
+class GLSLFilterPrivate;
+class Q_AV_EXPORT GLSLFilter : public VideoFilter
+{
+    Q_OBJECT
+    DPTR_DECLARE_PRIVATE(GLSLFilter)
+    Q_PROPERTY(QSize outputSize READ outputSize WRITE setOutputSize NOTIFY outputSizeChanged)
+public:
+    GLSLFilter(QObject* parent = 0);
+    bool isSupported(VideoFilterContext::Type ct) const  Q_DECL_OVERRIDE {
+        return ct == VideoFilterContext::OpenGL;
+    }
+
+    /*!
+     * \brief opengl
+     * Currently you can only use it to set custom shader OpenGLVideo.setUserShader()
+     */
+    OpenGLVideo* opengl() const;
+    // TODO: set FBO format
+    /*!
+     * \brief outputSize
+     * Output frame size. FBO uses the same size to render. An empty size means using the input frame size
+     * \return
+     */
+    QSize outputSize() const;
+    void setOutputSize(const QSize& value);
+Q_SIGNALS:
+    void outputSizeChanged(const QSize& size);
+protected:
+    GLSLFilter(GLSLFilterPrivate& d, QObject *parent = 0);
+    void process(Statistics* statistics, VideoFrame* frame = 0) Q_DECL_OVERRIDE;
+};
+} //namespace QtAV
+#endif // QTAV_GLSLFILTER_H

src/filter/GLSLFilter.cpp

@@ -0,0 +1,128 @@
+/******************************************************************************
+    QtAV:  Multimedia framework based on Qt and FFmpeg
+    Copyright (C) 2012-2016 Wang Bin <[email protected]>
+
+*   This file is part of QtAV (from 2016)
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+******************************************************************************/
+
+#include "QtAV/GLSLFilter.h"
+#include "QtAV/private/Filter_p.h"
+#include "QtAV/VideoFrame.h"
+#include "utils/OpenGLHelper.h"
+#if QT_VERSION >= QT_VERSION_CHECK(5, 0, 0)
+#include <QtGui/QOpenGLFramebufferObject>
+#else
+#include <QtOpenGL/QGLFramebufferObject>
+#define QOpenGLFramebufferObject QGLFramebufferObject
+#endif
+#include "QtAV/SurfaceInterop.h"
+#include "QtAV/OpenGLVideo.h"
+
+namespace QtAV {
+class GLSLFilterPrivate : public VideoFilterPrivate
+{
+public:
+    GLSLFilterPrivate() : VideoFilterPrivate()
+      , fbo(0)
+    {}
+
+    QOpenGLFramebufferObject *fbo;
+    QSize size;
+    OpenGLVideo glv;
+};
+
+GLSLFilter::GLSLFilter(QObject *parent)
+    : VideoFilter(*new GLSLFilterPrivate(), parent)
+{}
+
+GLSLFilter::GLSLFilter(GLSLFilterPrivate &d, QObject *parent)
+    : VideoFilter(d, parent)
+{}
+
+OpenGLVideo* GLSLFilter::opengl() const
+{
+    return const_cast<OpenGLVideo*>(&d_func().glv);
+}
+
+QSize GLSLFilter::outputSize() const
+{
+    return d_func().size;
+}
+
+void GLSLFilter::setOutputSize(const QSize &value)
+{
+    DPTR_D(GLSLFilter);
+    if (d.size == value)
+        return;
+    d.size = value;
+    Q_EMIT outputSizeChanged(value);
+}
+
+void GLSLFilter::process(Statistics *statistics, VideoFrame *frame)
+{
+    Q_UNUSED(statistics);
+    if (!QOpenGLContext::currentContext()) {
+        qWarning() << "No current gl context for glsl filter: " << this;
+        return;
+    }
+    DPTR_D(GLSLFilter);
+    if (!frame || !*frame)
+        return;
+    // now use the frame size
+    if (d.fbo && !d.size.isEmpty() && d.fbo->size() != d.size) {
+        delete d.fbo;
+        d.fbo = 0;
+    }
+    if (!d.fbo) {
+        d.fbo = new QOpenGLFramebufferObject(outputSize().isEmpty() ? frame->size() : outputSize(), GL_TEXTURE_2D); //TODO: prefer 16bit rgb
+        QOpenGLContext *ctx = const_cast<QOpenGLContext*>(QOpenGLContext::currentContext()); //qt4 returns const
+        d.glv.setOpenGLContext(ctx);
+        d.glv.setProjectionMatrixToRect(QRectF(0, 0, d.fbo->width(), d.fbo->height()));
+    }
+    d.fbo->bind();
+    DYGL(glViewport(0, 0, d.fbo->width(), d.fbo->height()));
+    d.glv.setCurrentFrame(*frame);
+    QMatrix4x4 mat; // flip vertical
+    mat.scale(1, -1);
+    d.glv.render(QRectF(), QRectF(), mat);
+    d.fbo->bindDefault();
+
+    VideoFormat fmt(VideoFormat::Format_RGB32);
+    VideoFrame f(d.fbo->width(), d.fbo->height(), fmt); //
+    f.setBytesPerLine(d.fbo->width()*fmt.bytesPerPixel(), 0);
+    // set interop;
+
+    class GLTextureInterop : public VideoSurfaceInterop
+    {
+        GLuint tex;
+    public:
+        GLTextureInterop(GLuint id) : tex(id) {}
+        // FIXME: texture will be deleted in VideoShader. TODO: VideoShader check whether it's texture owner
+        void* map(SurfaceType, const VideoFormat &, void *handle, int plane)
+        {
+            Q_UNUSED(plane);
+            GLuint* t = reinterpret_cast<GLuint*>(handle);
+            *t = tex;
+            return t;
+        }
+    };
+
+    GLTextureInterop *interop = new GLTextureInterop(d.fbo->texture());
+    f.setMetaData(QStringLiteral("surface_interop"), QVariant::fromValue(VideoSurfaceInteropPtr((interop))));
+    *frame = f;
+}
+} //namespace QtAV

src/libQtAV.pro

@@ -311,6 +311,7 @@ config_gl|config_opengl {
   }
   OTHER_FILES += shaders/planar.f.glsl shaders/rgb.f.glsl
   SDK_HEADERS *= \
+    QtAV/GLSLFilter.h \
     QtAV/OpenGLRendererBase.h \
     QtAV/OpenGLVideo.h \
     QtAV/ConvolutionShader.h \
@@ -321,6 +322,7 @@ config_gl|config_opengl {
     utils/OpenGLHelper.h \
     ShaderManager.h
   SOURCES *= \
+    filter/GLSLFilter.cpp \
     output/video/OpenGLRendererBase.cpp \
     OpenGLVideo.cpp \
     VideoShader.cpp \
@@ -539,7 +541,6 @@ HEADERS *= \
     utils/internal.h \
     output/OutputSet.h \
     QtAV/ColorTransform.h
-
 # from mkspecs/features/qt_module.prf
 # OS X and iOS frameworks
 mac_framework { # from common.pri
@@ -552,7 +553,7 @@ mac_framework { # from common.pri
         FRAMEWORK_HEADERS.path = Headers
 # 5.4(beta) workaround for wrong include path
 # TODO: why <QtCore/qglobal.h> can be found?
-        isEqual(QT_MAJOR_VERSION, 5):greaterThan(QT_MINOR_VERSION, 3)|greaterThan(QT_MAJOR_VERSION, 5): FRAMEWORK_HEADERS.path = Headers/$$MODULE_INCNAME
+        qtAtLeast(5,3): FRAMEWORK_HEADERS.path = Headers/$$MODULE_INCNAME
         FRAMEWORK_PRIVATE_HEADERS.version = Versions
         FRAMEWORK_PRIVATE_HEADERS.files = $$SDK_PRIVATE_HEADERS
         FRAMEWORK_PRIVATE_HEADERS.path = Headers/$$VERSION/$$MODULE_INCNAME/private

src/output/video/VideoRenderer.cpp

@@ -1,5 +1,5 @@
 /******************************************************************************
-    QtAV:  Media play library based on Qt and FFmpeg
+    QtAV:  Multimedia framework based on Qt and FFmpeg
     Copyright (C) 2012-2016 Wang Bin <[email protected]>
 
 *   This file is part of QtAV
@@ -465,7 +465,8 @@ void VideoRenderer::handlePaintEvent()
         //lock is required only when drawing the frame
         QMutexLocker locker(&d.img_mutex);
         Q_UNUSED(locker);
-        if (!d.filters.isEmpty() && d.statistics) {
+        // do not apply filters if d.video_frame is already filtered. e.g. rendering an image and resize window to repaint
+        if (d.video_frame.constBits(0) && !d.filters.isEmpty() && d.statistics) {
             // vo filter will not modify video frame, no lock required
             foreach(Filter* filter, d.filters) {
                 VideoFilter *vf = static_cast<VideoFilter*>(filter);
@@ -478,9 +479,11 @@ void VideoRenderer::handlePaintEvent()
                     continue;
                 // qpainter on video frame always runs on video thread. qpainter on renderer's paint device can work on rendering thread
                 // Here apply filters on frame on video thread, for example, GPU filters
-                if (!vf->context() || vf->context()->type() != VideoFilterContext::OpenGL)
-                    continue;
+
                 //vf->prepareContext(d.filter_context, d.statistics, 0);
+                //if (!vf->context() || vf->context()->type() != VideoFilterContext::OpenGL)
+                if (!vf->isSupported(VideoFilterContext::OpenGL))
+                    continue;
                 vf->apply(d.statistics, &d.video_frame); //painter and paint device are ready, pass video frame is ok.
             }
         }