add a convenience class to extract frame at a given time

src/QtAV/QtAV.h

@@ -55,6 +55,7 @@
 #include <QtAV/VideoDecoderTypes.h>
 #include <QtAV/VideoFormat.h>
 #include <QtAV/VideoFrame.h>
+#include <QtAV/VideoFrameExtractor.h>
 #include <QtAV/VideoRenderer.h>
 #include <QtAV/VideoRendererTypes.h>
 #include <QtAV/VideoOutput.h>

src/QtAV/VideoFrameExtractor.h

@@ -0,0 +1,84 @@
+/******************************************************************************
+    QtAV:  Media play library based on Qt and FFmpeg
+    Copyright (C) 2014 Wang Bin <[email protected]>
+
+*   This file is part of QtAV
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+******************************************************************************/
+
+#ifndef QTAV_VIDEOFRAMEEXTRACTOR_H
+#define QTAV_VIDEOFRAMEEXTRACTOR_H
+
+#include <QtCore/QObject>
+#include <QtAV/VideoFrame.h>
+
+namespace QtAV {
+
+class VideoFrameExtractorPrivate;
+class Q_AV_EXPORT VideoFrameExtractor : public QObject
+{
+    Q_OBJECT
+    DPTR_DECLARE_PRIVATE(VideoFrameExtractor)
+public:
+    explicit VideoFrameExtractor(QObject *parent = 0);
+    void setSource(const QString value);
+    QString source() const;
+    void setAutoExtract(bool value);
+    bool autoExtract() const;
+    /*!
+     * \brief setPrecision
+     * if the difference of the next requested position is less than the value, the
+     * last one is used and not positionChanged() signal to emit.
+     * Default is 500ms.
+     */
+    void setPrecision(int value);
+    int precision() const;
+    void setPosition(qint64 value);
+    qint64 position() const;
+    /*!
+     * \brief frame
+     * \return the last video frame extracted
+     */
+    VideoFrame frame();
+
+signals:
+    void frameExtracted(); // parameter: VideoFrame, bool changed?
+    void sourceChanged();
+    void error(); // clear preview image in a slot
+    void autoExtractChanged();
+    /*!
+     * \brief positionChanged
+     * If not autoExtract, positionChanged() => extract() in a slot
+     */
+    void positionChanged();
+    void precisionChanged();
+
+public slots:
+    /*!
+     * \brief extract
+     * If last extracted frame can be use, use it.
+     * If there is a key frame in [position-precision, position+precision], then the nearest key frame will be extracted.
+     * Otherwise, the given position frame will be extracted.
+     */
+    void extract();
+
+protected:
+    VideoFrameExtractor(VideoFrameExtractorPrivate &d, QObject* parent = 0);
+    DPTR_DECLARE(VideoFrameExtractor)
+};
+
+} //namespace QtAV
+#endif // QTAV_VIDEOFRAMEEXTRACTOR_H

src/VideoFrameExtractor.cpp

@@ -0,0 +1,255 @@
+#include "QtAV/VideoFrameExtractor.h"
+#include <QtCore/QScopedPointer>
+#include "QtAV/VideoCapture.h"
+#include "QtAV/VideoDecoder.h"
+#include "QtAV/AVDemuxer.h"
+#include "QtAV/Packet.h"
+#include "utils/Logger.h"
+
+namespace QtAV {
+
+const int kDefaultPrecision = 500;
+class VideoFrameExtractorPrivate : public DPtrPrivate<VideoFrameExtractor>
+{
+public:
+    VideoFrameExtractorPrivate()
+        : extracted(false)
+        , auto_extract(true)
+        , position(-2*kDefaultPrecision)
+        , precision(kDefaultPrecision)
+        , decoder(0)
+    {
+        codecs << "DXVA" << "CUDA" << "VAAPI" << "VDA" << "Cedarv" << "FFmpeg";
+    }
+    ~VideoFrameExtractorPrivate() {
+        demuxer.close();
+    }
+
+    bool checkAndOpen() {
+        const bool loaded = demuxer.isLoaded(source);
+        if (loaded && decoder)
+            return true;
+        if (decoder) { // new source
+            decoder->close();
+            decoder.reset(0);
+        }
+        if (!loaded) {
+            demuxer.close();
+            if (!demuxer.loadFile(source))
+                return false;
+        }
+        if (codecs.isEmpty())
+            return false;
+        qDebug("creating decoder...");
+        foreach (const QString& c, codecs) {
+            qDebug() << "codec " << c;
+            VideoDecoderId cid = VideoDecoderFactory::id(c.toUtf8().constData());
+            VideoDecoder *vd = VideoDecoderFactory::create(cid);
+            if (!vd)
+                continue;
+            decoder.reset(vd);
+            decoder->setCodecContext(demuxer.videoCodecContext());
+            if (!decoder->prepare()) {
+                decoder.reset(0);
+                continue;
+            }
+            if (!decoder->open()) {
+                decoder.reset(0);
+                continue;
+            }
+            break;
+        }
+        return !!decoder;
+    }
+
+    // return the key frame position
+    bool extractInPrecision(qint64 value, int range) {
+        demuxer.seek(value + range); //
+        const int vstream = demuxer.videoStream();
+        while (!demuxer.atEnd()) {
+            if (!demuxer.readFrame()) {
+                qDebug("!!!!!!read frame error!!!!!!");
+                continue;
+            }
+            if (demuxer.stream() != vstream)
+                continue;
+            qDebug("video packet: %f", demuxer.packet()->pts);
+            if (demuxer.packet()->hasKeyFrame)
+                break;
+        }
+        decoder->flush();
+        const qint64 t_key = qint64(demuxer.packet()->pts * 1000.0);
+        qDebug("delta t = %d, data size: %d", int(value - t_key), demuxer.packet()->data.size());
+        // must decode key frame
+        if (!decoder->decode(demuxer.packet()->data)) {
+            qWarning("!!!!!!!!!decode failed!!!!!!!!");
+            return false;
+        }
+        // seek backward, so value >= t
+        // decode key frame
+        if (int(value - t_key) <= range) {
+            qDebug("!!!!!!!!!use key frame!!!!!!!");
+            frame = decoder->frame();
+            frame.setTimestamp(demuxer.packet()->pts);
+            if (frame.isValid()) {
+                qDebug() << "frame found. format: " <<  frame.format();
+                return true;
+            }
+            // why key frame invalid?
+            qWarning("invalid key frame!!!!!");
+        }
+        // decode at the given position
+        qreal t0 = qreal(value/1000LL);
+        while (!demuxer.atEnd()) {
+            if (!demuxer.readFrame()) {
+                qDebug("!!!!!!----read frame error!!!!!!");
+                continue;
+            }
+            if (demuxer.stream() != vstream) {
+                //qDebug("not video packet");
+                continue;
+            }
+            const qreal t = demuxer.packet()->pts;
+            qDebug("video packet: %f, delta=%lld", t, value - qint64(t*1000.0));
+            if (qint64(t*1000.0) - value > range) { // use last decoded frame
+                qWarning("out of range");
+                return frame.isValid();
+            }
+            if (demuxer.packet()->hasKeyFrame) {
+                qCritical("Internal error. Can not be a key frame!!!!");
+            }
+            // invalid packet?
+            if (!decoder->decode(demuxer.packet()->data)) {
+                qWarning("!!!!!!!!!decode failed!!!!");
+                return false;
+            }
+            // store the last decoded frame because next frame may be out of range
+            frame = decoder->frame();
+            qDebug() << "frame found. format: " <<  frame.format();
+            frame.setTimestamp(t);
+            if (!frame.isValid()) {
+                qDebug("invalid frame!!!");
+                continue;
+            }
+            // TODO: break if t is in (t0-range, t0+range)
+            if (qAbs(value - qint64(t*1000.0)) < range)
+                break;
+            if (t < t0)
+                continue;
+            break;
+        }
+        // now we get the final frame
+        return true;
+    }
+
+    bool extracted;
+    bool auto_extract;
+    qint64 position;
+    int precision;
+    QString source;
+    AVDemuxer demuxer;
+    QScopedPointer<VideoDecoder> decoder;
+    VideoFrame frame;
+    QStringList codecs;
+};
+
+
+VideoFrameExtractor::VideoFrameExtractor(QObject *parent) :
+    QObject(parent)
+{
+}
+
+VideoFrameExtractor::VideoFrameExtractor(VideoFrameExtractorPrivate &d, QObject *parent)
+    : QObject(parent)
+    , DPTR_INIT(&d)
+{}
+
+void VideoFrameExtractor::setSource(const QString value)
+{
+    DPTR_D(VideoFrameExtractor);
+    if (value == d.source)
+        return;
+    d.source = value;
+    emit sourceChanged();
+}
+
+QString VideoFrameExtractor::source() const
+{
+    return d_func().source;
+}
+
+void VideoFrameExtractor::setAutoExtract(bool value)
+{
+    DPTR_D(VideoFrameExtractor);
+    if (d.auto_extract == value)
+        return;
+    d.auto_extract = value;
+    emit autoExtractChanged();
+}
+
+bool VideoFrameExtractor::autoExtract() const
+{
+    return d_func().auto_extract;
+}
+
+// what if >= mediaStopPosition()?
+void VideoFrameExtractor::setPosition(qint64 value)
+{
+    DPTR_D(VideoFrameExtractor);
+    if (qAbs(value - d.position) < precision()) {
+        frameExtracted();
+        return;
+    }
+    d.extracted = false;
+    d.position = value;
+    emit positionChanged();
+    if (!autoExtract())
+        return;
+    qDebug("extractiong...");
+    extract();
+}
+
+qint64 VideoFrameExtractor::position() const
+{
+    return d_func().position;
+}
+
+void VideoFrameExtractor::setPrecision(int value)
+{
+    DPTR_D(VideoFrameExtractor);
+    if (d.precision == value)
+        return;
+    // explain why value (p0) is used but not the actual decoded position (p)
+    // it's key frame finding rule
+    d.precision = value;
+    emit precisionChanged();
+}
+
+int VideoFrameExtractor::precision() const
+{
+    return d_func().precision;
+}
+
+VideoFrame VideoFrameExtractor::frame()
+{
+    return d_func().frame;
+}
+
+void VideoFrameExtractor::extract()
+{
+    DPTR_D(VideoFrameExtractor);
+    if (!d.checkAndOpen()) {
+        emit error();
+        qWarning("can not open decoder....");
+        return; // error handling
+    }
+    qDebug("start to extract");
+    d.extracted = d.extractInPrecision(position(), precision());
+    if (!d.extracted) {
+        emit error();
+        return;
+    }
+    emit frameExtracted();
+}
+
+} //namespace QtAV

src/libQtAV.pro

@@ -316,6 +316,7 @@ SOURCES += \
     codec/video/VideoDecoderFFmpeg.cpp \
     codec/video/VideoDecoderFFmpegHW.cpp \
     VideoThread.cpp \
+    VideoFrameExtractor.cpp \
     CommonTypes.cpp
 
 SDK_HEADERS *= \
@@ -354,6 +355,7 @@ SDK_HEADERS *= \
     QtAV/VideoDecoderFFmpegHW.h \
     QtAV/VideoFormat.h \
     QtAV/VideoFrame.h \
+    QtAV/VideoFrameExtractor.h \
     QtAV/FactoryDefine.h \
     QtAV/Statistics.h \
     QtAV/Subtitle.h \

tests/decoder/main.cpp

@@ -37,10 +37,12 @@ int main(int argc, char *argv[])
     QElapsedTimer timer;
     timer.start();
     int count = 0;
-    VideoFrame frame;
+    int vstream = demux.videoStream();
     while (!demux.atEnd()) {
         if (!demux.readFrame())
             continue;
+        if (demux.stream() != vstream)
+            continue;
         if (dec->decode(demux.packet()->data)) {
             /*
              * TODO: may contains more than 1 frames

tests/extract/extract.pro

@@ -0,0 +1,11 @@
+TEMPLATE = app
+QT += opengl
+greaterThan(QT_MAJOR_VERSION, 4): QT += widgets
+CONFIG -= app_bundle
+
+STATICLINK = 0
+PROJECTROOT = $$PWD/../..
+include($$PROJECTROOT/src/libQtAV.pri)
+preparePaths($$OUT_PWD/../../out)
+
+SOURCES += main.cpp