va: check change before setup(). wang-bin#626

must release hwa resources in setup() to avoid mem leak

src/codec/video/VideoDecoderD3D.cpp

@@ -367,8 +367,6 @@ bool VideoDecoderD3DPrivate::open()
 
 void VideoDecoderD3DPrivate::close()
 {
-    if (codec_ctx)
-        codec_ctx->hwaccel_context = NULL;
     qDeleteAll(surfaces);
     surfaces.clear();
     restore();
@@ -377,18 +375,14 @@ void VideoDecoderD3DPrivate::close()
     destroyDevice();
 }
 
-bool VideoDecoderD3DPrivate::setup(AVCodecContext *avctx)
+void* VideoDecoderD3DPrivate::setup(AVCodecContext *avctx)
 {
     const int w = codedWidth(avctx);
     const int h = codedHeight(avctx);
-    if (avctx->hwaccel_context && surface_width == aligned(w) && surface_height == aligned(h))
-        return true;
     width = avctx->width; // not necessary. set in decode()
     height = avctx->height;
-    codec_ctx->hwaccel_context = NULL;
     releaseUSWC();
     destroyDecoder();
-    avctx->hwaccel_context = NULL;
 
     /* Allocates all surfaces needed for the decoder */
     if (surface_auto) {
@@ -417,17 +411,16 @@ bool VideoDecoderD3DPrivate::setup(AVCodecContext *avctx)
     hw_surfaces.clear();
     surfaces.resize(surface_count);
     if (!createDecoder(codec_ctx->codec_id, w, h, surfaces))
-        return false;
+        return NULL;
     hw_surfaces.resize(surface_count);
     for (int i = 0; i < surfaces.size(); ++i) {
         hw_surfaces[i] = surfaces[i]->getSurface();
     }
     surface_order = 0;
     surface_width = aligned(w);
     surface_height = aligned(h);
-    setupAVVAContext(avctx); //can not use codec_ctx for threaded mode!
     initUSWC(surface_width);
-    return true;
+    return setupAVVAContext(); //can not use codec_ctx for threaded mode!
 }
 
 /* FIXME it is nearly common with VAAPI */

src/codec/video/VideoDecoderD3D.h

@@ -73,7 +73,7 @@ class VideoDecoderD3DPrivate : public VideoDecoderFFmpegHWPrivate
 
     bool open() Q_DECL_OVERRIDE;
     void close() Q_DECL_OVERRIDE;
-    bool setup(AVCodecContext *avctx) Q_DECL_OVERRIDE;
+    void* setup(AVCodecContext *avctx) Q_DECL_OVERRIDE;
     bool getBuffer(void **opaque, uint8_t **data) Q_DECL_OVERRIDE;
     void releaseBuffer(void *opaque, uint8_t *data) Q_DECL_OVERRIDE;
 
@@ -85,7 +85,7 @@ class VideoDecoderD3DPrivate : public VideoDecoderFFmpegHWPrivate
     virtual bool checkDevice() {return true;}
     virtual QVector<GUID> getSupportedCodecs() const = 0;
 
-    virtual void setupAVVAContext(AVCodecContext* avctx) = 0;
+    virtual void* setupAVVAContext() = 0;
     /// create surfaces and decoder. width and height are coded value, maybe not aligned for d3d surface
     /// surfaces count is given, but not allocated
     virtual bool createDecoder(AVCodecID codec, int width, int height, QVector<va_surface_t*>& surf) = 0;

src/codec/video/VideoDecoderD3D11.cpp

@@ -137,7 +137,7 @@ class VideoDecoderD3D11Private Q_DECL_FINAL : public VideoDecoderD3DPrivate
     bool createDecoder(AVCodecID codec_id, int w, int h, QVector<va_surface_t*>& surf) Q_DECL_OVERRIDE;
     void destroyDecoder() Q_DECL_OVERRIDE;
     bool setupSurfaceInterop() Q_DECL_OVERRIDE;
-    void setupAVVAContext(AVCodecContext *avctx) Q_DECL_OVERRIDE;
+    void* setupAVVAContext() Q_DECL_OVERRIDE;
     QVector<GUID> getSupportedCodecs() const Q_DECL_OVERRIDE;
     int fourccFor(const GUID *guid) const Q_DECL_OVERRIDE;
 
@@ -372,9 +372,8 @@ bool VideoDecoderD3D11Private::setupSurfaceInterop()
     return true;
 }
 
-void VideoDecoderD3D11Private::setupAVVAContext(AVCodecContext *avctx)
+void* VideoDecoderD3D11Private::setupAVVAContext()
 {
-    avctx->hwaccel_context = &hw;
     // TODO: FF_DXVA2_WORKAROUND_SCALING_LIST_ZIGZAG
     if (isIntelClearVideo(&codec_guid)) {
 #ifdef FF_DXVA2_WORKAROUND_INTEL_CLEARVIDEO //2014-03-07 - 8b2a130 - lavc 55.50.0 / 55.53.100 - dxva2.h
@@ -389,5 +388,6 @@ void VideoDecoderD3D11Private::setupAVVAContext(AVCodecContext *avctx)
     hw.cfg = &cfg;
     hw.surface_count = hw_surfaces.size();
     hw.surface = (ID3D11VideoDecoderOutputView**)hw_surfaces.constData();
+    return &hw;
 }
 } //namespace QtAV

src/codec/video/VideoDecoderDXVA.cpp

@@ -127,7 +127,7 @@ class VideoDecoderDXVAPrivate Q_DECL_FINAL: public VideoDecoderD3DPrivate
     bool createDecoder(AVCodecID codec_id, int w, int h, QVector<va_surface_t*>& surf) Q_DECL_OVERRIDE;
     void destroyDecoder() Q_DECL_OVERRIDE;
     bool setupSurfaceInterop() Q_DECL_OVERRIDE;
-    void setupAVVAContext(AVCodecContext *avctx) Q_DECL_OVERRIDE;
+    void* setupAVVAContext() Q_DECL_OVERRIDE;
     int fourccFor(const GUID *guid) const Q_DECL_OVERRIDE;
 
     /* DLL */
@@ -424,9 +424,8 @@ void VideoDecoderDXVAPrivate::destroyDecoder()
     SafeRelease(&decoder);
 }
 
-void VideoDecoderDXVAPrivate::setupAVVAContext(AVCodecContext* avctx)
+void* VideoDecoderDXVAPrivate::setupAVVAContext()
 {
-    avctx->hwaccel_context = &hw_ctx;
     // TODO: FF_DXVA2_WORKAROUND_SCALING_LIST_ZIGZAG
     if (isIntelClearVideo(&codec_guid)) {
 #ifdef FF_DXVA2_WORKAROUND_INTEL_CLEARVIDEO //2014-03-07 - 8b2a130 - lavc 55.50.0 / 55.53.100 - dxva2.h
@@ -440,6 +439,7 @@ void VideoDecoderDXVAPrivate::setupAVVAContext(AVCodecContext* avctx)
     hw_ctx.cfg = &cfg;
     hw_ctx.surface_count = hw_surfaces.size();
     hw_ctx.surface = (IDirect3DSurface9**)hw_surfaces.constData();
+    return &hw_ctx;
 }
 
 bool VideoDecoderDXVAPrivate::setupSurfaceInterop()

src/codec/video/VideoDecoderFFmpegHW.cpp

@@ -153,7 +153,7 @@ bool VideoDecoderFFmpegHWPrivate::prepare()
     return true;
 }
 
-AVPixelFormat VideoDecoderFFmpegHWPrivate::getFormat(struct AVCodecContext *p_context, const AVPixelFormat *pi_fmt)
+AVPixelFormat VideoDecoderFFmpegHWPrivate::getFormat(struct AVCodecContext *avctx, const AVPixelFormat *pi_fmt)
 {
     bool can_hwaccel = false;
     for (size_t i = 0; pi_fmt[i] != QTAV_PIX_FMT_C(NONE); i++) {
@@ -172,21 +172,27 @@ AVPixelFormat VideoDecoderFFmpegHWPrivate::getFormat(struct AVCodecContext *p_co
     for (size_t i = 0; pi_fmt[i] != QTAV_PIX_FMT_C(NONE); i++) {
         if (vaPixelFormat() != pi_fmt[i])
             continue;
-        /* We try to call setup when possible to detect errors when possible (later is too late) */
-        if (p_context->width > 0 && p_context->height > 0
-         && !setup(p_context)) {
+        if (hw_w == codedWidth((avctx)) && hw_h == codedHeight(avctx)
+                && hw_profile == avctx->profile // update decoder if profile changed. but now only surfaces are updated
+                && avctx->hwaccel_context)
+            return pi_fmt[i];
+        // TODO: manage uswc here for x86 (surface size is decoder dependent)
+        avctx->hwaccel_context = setup(avctx);
+        if (!avctx->hwaccel_context) {
             qWarning("acceleration setup failure");
             break;
         }
+        hw_w = codedWidth((avctx));
+        hw_h = codedHeight(avctx);
+        hw_profile = avctx->profile;
         qDebug("Using %s for hardware decoding.", qPrintable(description));
-        p_context->draw_horiz_band = NULL; //??
         return pi_fmt[i];
     }
     close();
 end:
     qWarning("hardware acceleration is not available" );
     /* Fallback to default behaviour */
-    return avcodec_default_get_format(p_context, pi_fmt);
+    return avcodec_default_get_format(avctx, pi_fmt);
 }
 
 int VideoDecoderFFmpegHWPrivate::codedWidth(AVCodecContext *avctx) const

src/codec/video/VideoDecoderFFmpegHW_p.h

@@ -44,6 +44,9 @@ class VideoDecoderFFmpegHWPrivate : public VideoDecoderFFmpegBasePrivate
         , get_buffer2(NULL)
         , threads(0)
         , copy_mode(VideoDecoderFFmpegHW::OptimizedCopy)
+        , hw_w(0)
+        , hw_h(0)
+        , hw_profile(0)
     {}
     virtual ~VideoDecoderFFmpegHWPrivate() {} //ctx is 0 now
     bool enableFrameRef() const Q_DECL_OVERRIDE { return false;} //because of ffmpeg_get_va_buffer2?
@@ -60,8 +63,8 @@ class VideoDecoderFFmpegHWPrivate : public VideoDecoderFFmpegBasePrivate
         codec_ctx->reget_buffer = reget_buffer;
 #endif //QTAV_HAVE(AVBUFREF)
     }
-
-    virtual bool setup(AVCodecContext* avctx) = 0;
+    // return hwaccel_context or null
+    virtual void* setup(AVCodecContext* avctx) = 0;
 
     AVPixelFormat getFormat(struct AVCodecContext *p_context, const AVPixelFormat *pi_fmt);
     //TODO: remove opaque
@@ -88,6 +91,9 @@ class VideoDecoderFFmpegHWPrivate : public VideoDecoderFFmpegBasePrivate
     // TODO: flag enable, disable, auto
     VideoDecoderFFmpegHW::CopyMode copy_mode;
     GPUMemCopy gpu_mem;
+
+private:
+    int hw_w, hw_h, hw_profile;
 };
 
 } //namespace QtAV

src/codec/video/VideoDecoderVAAPI.cpp

@@ -226,7 +226,7 @@ class VideoDecoderVAAPIPrivate Q_DECL_FINAL: public VideoDecoderFFmpegHWPrivate,
     void close() Q_DECL_OVERRIDE;
     bool ensureSurfaces(int count, int w, int h, bool discard_old = false);
     bool prepareVAImage(int w, int h);
-    bool setup(AVCodecContext *avctx) Q_DECL_OVERRIDE;
+    void* setup(AVCodecContext *avctx) Q_DECL_OVERRIDE;
     bool getBuffer(void **opaque, uint8_t **data) Q_DECL_OVERRIDE;
     void releaseBuffer(void *opaque, uint8_t *data) Q_DECL_OVERRIDE;
     AVPixelFormat vaPixelFormat() const Q_DECL_OVERRIDE { return QTAV_PIX_FMT_C(VAAPI_VLD); }
@@ -577,7 +577,6 @@ bool VideoDecoderVAAPIPrivate::open()
     }
 #endif //QTAV_HAVE(EGL_CAPI)
 #endif //QT_NO_OPENGL
-    codec_ctx->hwaccel_context = &hw_ctx; //must set before open
     return true;
 }
 
@@ -631,12 +630,12 @@ bool VideoDecoderVAAPIPrivate::prepareVAImage(int w, int h)
     return true;
 }
 
-bool VideoDecoderVAAPIPrivate::setup(AVCodecContext *avctx)
+void* VideoDecoderVAAPIPrivate::setup(AVCodecContext *avctx)
 {
     Q_UNUSED(avctx);
     if (!display || config_id == VA_INVALID_ID) {
         qWarning("va-api is not initialized. display: %p, config_id: %#x", display->get(), config_id);
-        return false;
+        return NULL;
     }
     int surface_count =  nb_surfaces;
     if (surface_count <= 0) {
@@ -652,24 +651,31 @@ bool VideoDecoderVAAPIPrivate::setup(AVCodecContext *avctx)
         if (codec_ctx->active_thread_type & FF_THREAD_FRAME)
             surface_count += codec_ctx->thread_count;
     }
+    releaseUSWC();
+    if (image.image_id != VA_INVALID_ID) {
+        VAWARN(vaDestroyImage(display->get(), image.image_id));
+        image.image_id = VA_INVALID_ID;
+    }
+    if (context_id != VA_INVALID_ID) {
+        VAWARN(vaDestroyContext(display->get(), context_id));
+        context_id = VA_INVALID_ID;
+    }
+    // TODO: config_id reset?
     if (!ensureSurfaces(surface_count, surface_width, surface_height, true))
-        return false;
+        return NULL;
     if (copy_mode != VideoDecoderFFmpegHW::ZeroCopy || OpenGLHelper::isEGL()) { //egl_dma && va_0_38
         // egl needs VAImage too
         if (!prepareVAImage(surface_width, surface_height))
-            return false;
-        // copy-back mode
-        if (copy_mode != VideoDecoderFFmpegHW::ZeroCopy)
-            initUSWC(surface_width);
+            return NULL;
     }
-    context_id = VA_INVALID_ID;
-    VA_ENSURE_TRUE(vaCreateContext(display->get(), config_id, surface_width, surface_height, VA_PROGRESSIVE, surfaces.data(), surfaces.size(), &context_id), false);
+    initUSWC(surface_width);
+    VA_ENSURE_TRUE(vaCreateContext(display->get(), config_id, surface_width, surface_height, VA_PROGRESSIVE, surfaces.data(), surfaces.size(), &context_id), NULL);
     /* Setup the ffmpeg hardware context */
     memset(&hw_ctx, 0, sizeof(hw_ctx));
     hw_ctx.display = display->get();
     hw_ctx.config_id = config_id;
     hw_ctx.context_id = context_id;
-    return true;
+    return &hw_ctx;
 }
 
 void VideoDecoderVAAPIPrivate::close()

src/codec/video/VideoDecoderVDA.cpp

@@ -96,7 +96,7 @@ class VideoDecoderVDAPrivate Q_DECL_FINAL: public VideoDecoderFFmpegHWPrivate
     bool open() Q_DECL_OVERRIDE;
     void close() Q_DECL_OVERRIDE;
 
-    bool setup(AVCodecContext *avctx) Q_DECL_OVERRIDE;
+    void* setup(AVCodecContext *avctx) Q_DECL_OVERRIDE;
     bool getBuffer(void **opaque, uint8_t **data) Q_DECL_OVERRIDE;
     void releaseBuffer(void *opaque, uint8_t *data) Q_DECL_OVERRIDE;
     AVPixelFormat vaPixelFormat() const Q_DECL_OVERRIDE { return QTAV_PIX_FMT_C(VDA_VLD);}
@@ -182,10 +182,10 @@ VideoFrame VideoDecoderVDA::frame()
         }
         void* mapToHost(const VideoFormat &format, void *handle, int plane) {
             Q_UNUSED(plane);
-            CVPixelBufferLockBaseAddress(cvbuf, 0);
+            CVPixelBufferLockBaseAddress(cvbuf, kCVPixelBufferLock_ReadOnly);
             const VideoFormat fmt(cv::format_from_cv(CVPixelBufferGetPixelFormatType(cvbuf)));
             if (!fmt.isValid()) {
-                CVPixelBufferUnlockBaseAddress(cvbuf, 0);
+                CVPixelBufferUnlockBaseAddress(cvbuf, kCVPixelBufferLock_ReadOnly);
                 return NULL;
             }
             const int w = CVPixelBufferGetWidth(cvbuf);
@@ -197,7 +197,7 @@ VideoFrame VideoDecoderVDA::frame()
                 src[i] = (uint8_t*)CVPixelBufferGetBaseAddressOfPlane(cvbuf, i);
                 pitch[i] = CVPixelBufferGetBytesPerRowOfPlane(cvbuf, i);
             }
-            CVPixelBufferUnlockBaseAddress(cvbuf, 0);
+            CVPixelBufferUnlockBaseAddress(cvbuf, kCVPixelBufferLock_ReadOnly);
             //CVPixelBufferRelease(cv_buffer); // release when video frame is destroyed
             VideoFrame frame(VideoFrame::fromGPU(fmt, w, h, h, src, pitch));
             if (fmt != format)
@@ -306,7 +306,7 @@ VideoFrame VideoDecoderVDA::frame()
             src[i] = (uint8_t*)CVPixelBufferGetBaseAddressOfPlane(cv_buffer, i);
             pitch[i] = CVPixelBufferGetBytesPerRowOfPlane(cv_buffer, i);
         }
-        CVPixelBufferUnlockBaseAddress(cv_buffer, 0);
+        CVPixelBufferUnlockBaseAddress(cv_buffer, kCVPixelBufferLock_ReadOnly);
         //CVPixelBufferRelease(cv_buffer); // release when video frame is destroyed
     }
     VideoFrame f;
@@ -346,14 +346,10 @@ VideoDecoderVDA::PixelFormat VideoDecoderVDA::format() const
     return d_func().out_fmt;
 }
 
-bool VideoDecoderVDAPrivate::setup(AVCodecContext *avctx)
+void* VideoDecoderVDAPrivate::setup(AVCodecContext *avctx)
 {
     const int w = codedWidth(avctx);
     const int h = codedHeight(avctx);
-    if (hw_ctx.width == w && hw_ctx.height == h && hw_ctx.decoder) {
-        avctx->hwaccel_context = &hw_ctx;
-        return true;
-    }
     if (hw_ctx.decoder) {
         ff_vda_destroy_decoder(&hw_ctx);
         releaseUSWC();
@@ -367,17 +363,15 @@ bool VideoDecoderVDAPrivate::setup(AVCodecContext *avctx)
     hw_ctx.height = h;
     width = avctx->width; // not necessary. set in decode()
     height = avctx->height;
-    avctx->hwaccel_context = NULL;
     /* create the decoder */
     int status = ff_vda_create_decoder(&hw_ctx, codec_ctx->extradata, codec_ctx->extradata_size);
     if (status) {
         qWarning("Failed to create decoder (%i): %s", status, vda_err_str(status));
-        return false;
+        return NULL;
     }
-    avctx->hwaccel_context = &hw_ctx;
     initUSWC(hw_ctx.width);
     qDebug() << "VDA decoder created. format: " << cv::format_from_cv(out_fmt);
-    return true;
+    return &hw_ctx;
 }
 
 bool VideoDecoderVDAPrivate::getBuffer(void **opaque, uint8_t **data)