contrib: added qsv vvc decoder patches from master and allow usage of…

… sw vvc decoder
sadok · Aug 16, 2024 · f80fcdb · f80fcdb
1 parent 9847a93
commit f80fcdb
Show file tree

Hide file tree

Showing 9 changed files with 539 additions and 0 deletions.
diff --git a/contrib/ffmpeg/A19-qsvdec-use-ffmpeg-default-125-framerate.patch b/contrib/ffmpeg/A19-qsvdec-use-ffmpeg-default-125-framerate.patch
@@ -0,0 +1,33 @@
+From 67fc9b84272a88b5edace5ca25f493c21b02955d Mon Sep 17 00:00:00 2001
+From: Fei Wang <[email protected]>
+Date: Thu, 18 Apr 2024 16:15:09 +0800
+Subject: [PATCH] lavc/qsvdec: Use FFmpeg default 1/25 framerate if can't
+ derive it from bitstream
+
+Fix error:
+$ ffmpeg -hwaccel qsv -i input.h265 -f null -
+...
+[null @ 0x55da1a629200] Application provided invalid, non monotonically
+increasing dts to muxer in stream 0: 3 >= 3
+
+Signed-off-by: Fei Wang <[email protected]>
+---
+ libavcodec/qsvdec.c | 5 +++++
+ 1 file changed, 5 insertions(+)
+
+diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
+index 5528bcdc8cd87..ed0bfe4c8b8e2 100644
+--- a/libavcodec/qsvdec.c
++++ b/libavcodec/qsvdec.c
+@@ -441,6 +441,11 @@ static int qsv_decode_header(AVCodecContext *avctx, QSVContext *q,
+     param->ExtParam    = q->ext_buffers;
+     param->NumExtParam = q->nb_ext_buffers;
+
++    if (param->mfx.FrameInfo.FrameRateExtN == 0 || param->mfx.FrameInfo.FrameRateExtD == 0) {
++        param->mfx.FrameInfo.FrameRateExtN = 25;
++        param->mfx.FrameInfo.FrameRateExtD = 1;
++    }
++
+ #if QSV_VERSION_ATLEAST(1, 34)
+     if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 1, 34) && avctx->codec_id == AV_CODEC_ID_AV1)
+         param->mfx.FilmGrain = (avctx->export_side_data & AV_CODEC_EXPORT_DATA_FILM_GRAIN) ? 0 : param->mfx.FilmGrain;
diff --git a/contrib/ffmpeg/A20-qsvdec-use-coded_wh.patch b/contrib/ffmpeg/A20-qsvdec-use-coded_wh.patch
@@ -0,0 +1,71 @@
+From 01c7f68f7aecafba64037ce47a757a058306c8d3 Mon Sep 17 00:00:00 2001
+From: Fei Wang <[email protected]>
+Date: Mon, 20 May 2024 10:05:53 +0800
+Subject: [PATCH] lavc/qsvdec: Use coded_w/h for frame resolution when use
+ system memory
+
+Fix output mismatch when decode clip with crop(conf_win_*offset in
+syntax) info by using system memory:
+
+$ ffmpeg -c:v hevc_qsv -i conf_win_offet.bit -y out.yuv
+
+Signed-off-by: Fei Wang <[email protected]>
+---
+ libavcodec/qsvdec.c | 16 ++++++++--------
+ 1 file changed, 8 insertions(+), 8 deletions(-)
+
+diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
+index 1895698c3caea..768968fd7bca0 100644
+--- a/libavcodec/qsvdec.c
++++ b/libavcodec/qsvdec.c
+@@ -137,26 +137,26 @@ static int qsv_get_continuous_buffer(AVCodecContext *avctx, AVFrame *frame,
+     if (ret < 0)
+         return ret;
+
+-    frame->width       = avctx->width;
+-    frame->height      = avctx->height;
++    frame->width       = avctx->coded_width;
++    frame->height      = avctx->coded_height;
+
+     switch (avctx->pix_fmt) {
+     case AV_PIX_FMT_NV12:
+-        frame->linesize[0] = FFALIGN(avctx->width, 128);
++        frame->linesize[0] = FFALIGN(avctx->coded_width, 128);
+         break;
+     case AV_PIX_FMT_P010:
+     case AV_PIX_FMT_P012:
+     case AV_PIX_FMT_YUYV422:
+-        frame->linesize[0] = 2 * FFALIGN(avctx->width, 128);
++        frame->linesize[0] = 2 * FFALIGN(avctx->coded_width, 128);
+         break;
+     case AV_PIX_FMT_Y210:
+     case AV_PIX_FMT_VUYX:
+     case AV_PIX_FMT_XV30:
+     case AV_PIX_FMT_Y212:
+-        frame->linesize[0] = 4 * FFALIGN(avctx->width, 128);
++        frame->linesize[0] = 4 * FFALIGN(avctx->coded_width, 128);
+         break;
+     case AV_PIX_FMT_XV36:
+-        frame->linesize[0] = 8 * FFALIGN(avctx->width, 128);
++        frame->linesize[0] = 8 * FFALIGN(avctx->coded_width, 128);
+         break;
+     default:
+         av_log(avctx, AV_LOG_ERROR, "Unsupported pixel format.\n");
+@@ -173,7 +173,7 @@ static int qsv_get_continuous_buffer(AVCodecContext *avctx, AVFrame *frame,
+         avctx->pix_fmt == AV_PIX_FMT_P012) {
+         frame->linesize[1] = frame->linesize[0];
+         frame->data[1] = frame->data[0] +
+-            frame->linesize[0] * FFALIGN(avctx->height, 64);
++            frame->linesize[0] * FFALIGN(avctx->coded_height, 64);
+     }
+
+     ret = ff_attach_decode_data(frame);
+@@ -413,7 +413,7 @@ static int qsv_decode_init_context(AVCodecContext *avctx, QSVContext *q, mfxVide
+     q->frame_info = param->mfx.FrameInfo;
+
+     if (!avctx->hw_frames_ctx) {
+-        ret = av_image_get_buffer_size(avctx->pix_fmt, FFALIGN(avctx->width, 128), FFALIGN(avctx->height, 64), 1);
++        ret = av_image_get_buffer_size(avctx->pix_fmt, FFALIGN(avctx->coded_width, 128), FFALIGN(avctx->coded_height, 64), 1);
+         if (ret < 0)
+             return ret;
+         q->pool = av_buffer_pool_init(ret, av_buffer_allocz);
diff --git a/contrib/ffmpeg/A21-qsvdec-update-hdr-side-data-on-output-avframe.patch b/contrib/ffmpeg/A21-qsvdec-update-hdr-side-data-on-output-avframe.patch
@@ -0,0 +1,87 @@
+From 473e84ad62a05e83bba5e00b4073980ee171a5bd Mon Sep 17 00:00:00 2001
+From: Haihao Xiang <[email protected]>
+Date: Wed, 22 May 2024 12:15:59 +0800
+Subject: [PATCH] lavc/qsvdec: update HDR side data on output AVFrame for AV1
+ decoding
+
+The SDK may provide HDR metadata for HDR streams via mfxExtBuffer
+attached on output mfxFrameSurface1
+
+Signed-off-by: Haihao Xiang <[email protected]>
+---
+ libavcodec/qsvdec.c | 48 ++++++++++++++++++++++++++++++++++++++++++++-
+ 1 file changed, 47 insertions(+), 1 deletion(-)
+
+diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
+index df0d49bc10208..7741baff06ced 100644
+--- a/libavcodec/qsvdec.c
++++ b/libavcodec/qsvdec.c
+@@ -538,7 +538,8 @@ static int alloc_frame(AVCodecContext *avctx, QSVContext *q, QSVFrame *frame)
+ #endif
+
+ #if QSV_VERSION_ATLEAST(1, 35)
+-    if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 1, 35) && avctx->codec_id == AV_CODEC_ID_HEVC) {
++    if ((QSV_RUNTIME_VERSION_ATLEAST(q->ver, 1, 35) && avctx->codec_id == AV_CODEC_ID_HEVC) ||
++        (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 9) && avctx->codec_id == AV_CODEC_ID_AV1)) {
+         frame->mdcv.Header.BufferId = MFX_EXTBUFF_MASTERING_DISPLAY_COLOUR_VOLUME;
+         frame->mdcv.Header.BufferSz = sizeof(frame->mdcv);
+         // The data in mdcv is valid when this flag is 1
+@@ -742,6 +743,45 @@ static int qsv_export_hdr_side_data(AVCodecContext *avctx, mfxExtMasteringDispla
+     return 0;
+ }
+
++static int qsv_export_hdr_side_data_av1(AVCodecContext *avctx, mfxExtMasteringDisplayColourVolume *mdcv,
++                                        mfxExtContentLightLevelInfo *clli, AVFrame *frame)
++{
++    if (mdcv->InsertPayloadToggle) {
++        AVMasteringDisplayMetadata *mastering = av_mastering_display_metadata_create_side_data(frame);
++        const int chroma_den   = 1 << 16;
++        const int max_luma_den = 1 << 8;
++        const int min_luma_den = 1 << 14;
++
++        if (!mastering)
++            return AVERROR(ENOMEM);
++
++        for (int i = 0; i < 3; i++) {
++            mastering->display_primaries[i][0] = av_make_q(mdcv->DisplayPrimariesX[i], chroma_den);
++            mastering->display_primaries[i][1] = av_make_q(mdcv->DisplayPrimariesY[i], chroma_den);
++        }
++
++        mastering->white_point[0] = av_make_q(mdcv->WhitePointX, chroma_den);
++        mastering->white_point[1] = av_make_q(mdcv->WhitePointY, chroma_den);
++
++        mastering->max_luminance = av_make_q(mdcv->MaxDisplayMasteringLuminance, max_luma_den);
++        mastering->min_luminance = av_make_q(mdcv->MinDisplayMasteringLuminance, min_luma_den);
++
++        mastering->has_luminance = 1;
++        mastering->has_primaries = 1;
++    }
++
++    if (clli->InsertPayloadToggle) {
++        AVContentLightMetadata *light = av_content_light_metadata_create_side_data(frame);
++        if (!light)
++            return AVERROR(ENOMEM);
++
++        light->MaxCLL  = clli->MaxContentLightLevel;
++        light->MaxFALL = clli->MaxPicAverageLightLevel;
++    }
++
++    return 0;
++}
++
+ #endif
+
+ static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
+@@ -874,6 +914,12 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
+             if (ret < 0)
+                 return ret;
+         }
++
++        if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 9) && avctx->codec_id == AV_CODEC_ID_AV1) {
++            ret = qsv_export_hdr_side_data_av1(avctx, &aframe.frame->mdcv, &aframe.frame->clli, frame);
++            if (ret < 0)
++                return ret;
++        }
+ #endif
+
+         frame->repeat_pict =
diff --git a/contrib/ffmpeg/A22-qsvdec-require-dynamic-frame-pool.patch b/contrib/ffmpeg/A22-qsvdec-require-dynamic-frame-pool.patch
@@ -0,0 +1,132 @@
+From a00cfc6c2461d9a2525b01e64fe140615c5d5f34 Mon Sep 17 00:00:00 2001
+From: Haihao Xiang <[email protected]>
+Date: Wed, 8 May 2024 14:03:14 +0800
+Subject: [PATCH] lavc/qsvdec: require a dynamic frame pool if possible
+
+This allows a downstream element stores more frames from qsv decoders
+and fixes error in get_buffer().
+
+$ ffmpeg -hwaccel qsv -hwaccel_output_format qsv -i input.mp4 -vf
+reverse -f null -
+
+[vist#0:0/h264 @ 0x562248f12c50] Decoding error: Cannot allocate memory
+[h264_qsv @ 0x562248f66b10] get_buffer() failed
+
+Signed-off-by: Haihao Xiang <[email protected]>
+---
+ libavcodec/qsvdec.c | 57 +++++++++++++++++++++++++++++++++++----------
+ 1 file changed, 45 insertions(+), 12 deletions(-)
+
+diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
+index a51ddace622fe..df0d49bc10208 100644
+--- a/libavcodec/qsvdec.c
++++ b/libavcodec/qsvdec.c
+@@ -42,6 +42,7 @@
+ #include "libavutil/imgutils.h"
+ #include "libavutil/film_grain_params.h"
+ #include "libavutil/mastering_display_metadata.h"
++#include "libavutil/avassert.h"
+
+ #include "avcodec.h"
+ #include "codec_internal.h"
+@@ -68,6 +69,8 @@ static const AVRational mfx_tb = { 1, 90000 };
+     AV_NOPTS_VALUE : pts_tb.num ? \
+     av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts)
+
++#define MFX_IMPL_VIA_MASK(impl) (0x0f00 & (impl))
++
+ typedef struct QSVAsyncFrame {
+     mfxSyncPoint *sync;
+     QSVFrame     *frame;
+@@ -77,6 +80,7 @@ typedef struct QSVContext {
+     // the session used for decoding
+     mfxSession session;
+     mfxVersion ver;
++    mfxHandleType handle_type;
+
+     // the session we allocated internally, in case the caller did not provide
+     // one
+@@ -183,6 +187,7 @@ static int qsv_init_session(AVCodecContext *avctx, QSVContext *q, mfxSession ses
+                             AVBufferRef *hw_frames_ref, AVBufferRef *hw_device_ref)
+ {
+     int ret;
++    mfxIMPL impl;
+
+     if (q->gpu_copy == MFX_GPUCOPY_ON &&
+         !(q->iopattern & MFX_IOPATTERN_OUT_SYSTEM_MEMORY)) {
+@@ -240,27 +245,52 @@ static int qsv_init_session(AVCodecContext *avctx, QSVContext *q, mfxSession ses
+         q->session = q->internal_qs.session;
+     }
+
+-    if (MFXQueryVersion(q->session, &q->ver) != MFX_ERR_NONE) {
+-        av_log(avctx, AV_LOG_ERROR, "Error querying the session version. \n");
+-        q->session = NULL;
++    if (MFXQueryIMPL(q->session, &impl) == MFX_ERR_NONE) {
++        switch (MFX_IMPL_VIA_MASK(impl)) {
++        case MFX_IMPL_VIA_VAAPI:
++            q->handle_type = MFX_HANDLE_VA_DISPLAY;
++            break;
+
+-        if (q->internal_qs.session) {
+-            MFXClose(q->internal_qs.session);
+-            q->internal_qs.session = NULL;
+-        }
++        case MFX_IMPL_VIA_D3D11:
++            q->handle_type = MFX_HANDLE_D3D11_DEVICE;
++            break;
++
++        case MFX_IMPL_VIA_D3D9:
++            q->handle_type = MFX_HANDLE_D3D9_DEVICE_MANAGER;
++            break;
+
+-        if (q->internal_qs.loader) {
+-            MFXUnload(q->internal_qs.loader);
+-            q->internal_qs.loader = NULL;
++        default:
++            av_assert0(!"should not reach here");
+         }
++    } else {
++        av_log(avctx, AV_LOG_ERROR, "Error querying the implementation. \n");
++        goto fail;
++    }
+
+-        return AVERROR_EXTERNAL;
++    if (MFXQueryVersion(q->session, &q->ver) != MFX_ERR_NONE) {
++        av_log(avctx, AV_LOG_ERROR, "Error querying the session version. \n");
++        goto fail;
+     }
+
+     /* make sure the decoder is uninitialized */
+     MFXVideoDECODE_Close(q->session);
+
+     return 0;
++
++fail:
++    q->session = NULL;
++
++    if (q->internal_qs.session) {
++        MFXClose(q->internal_qs.session);
++        q->internal_qs.session = NULL;
++    }
++
++    if (q->internal_qs.loader) {
++        MFXUnload(q->internal_qs.loader);
++        q->internal_qs.loader = NULL;
++    }
++
++    return AVERROR_EXTERNAL;
+ }
+
+ static int qsv_decode_preinit(AVCodecContext *avctx, QSVContext *q, enum AVPixelFormat pix_fmt, mfxVideoParam *param)
+@@ -310,7 +340,10 @@ static int qsv_decode_preinit(AVCodecContext *avctx, QSVContext *q, enum AVPixel
+         hwframes_ctx->height            = FFALIGN(avctx->coded_height, 32);
+         hwframes_ctx->format            = AV_PIX_FMT_QSV;
+         hwframes_ctx->sw_format         = avctx->sw_pix_fmt;
+-        hwframes_ctx->initial_pool_size = q->suggest_pool_size + 16 + avctx->extra_hw_frames;
++        if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 9) && q->handle_type != MFX_HANDLE_D3D9_DEVICE_MANAGER)
++            hwframes_ctx->initial_pool_size = 0;
++        else
++            hwframes_ctx->initial_pool_size = q->suggest_pool_size + 16 + avctx->extra_hw_frames;
+         frames_hwctx->frame_type        = MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET;
+
+         ret = av_hwframe_ctx_init(avctx->hw_frames_ctx);
diff --git a/contrib/ffmpeg/A23-qsvdec-fix-keyframes.patch b/contrib/ffmpeg/A23-qsvdec-fix-keyframes.patch
@@ -0,0 +1,40 @@
+From dbdd9ccded9018718bad6df14b56bdd4b7e327f1 Mon Sep 17 00:00:00 2001
+From: Haihao Xiang <[email protected]>
+Date: Wed, 22 May 2024 12:31:53 +0800
+Subject: [PATCH] lavc/qsvdec: fix keyframes
+
+MFX_FRAMETYPE_IDR is ORed to the frame type for AVC and HEVC keyframes,
+and MFX_FRAMETYPE_I is taken as keyframe flag for other codecs when
+getting the output surface from the SDK, hence we may mark the output
+frame as keyframe accordingly.
+
+Signed-off-by: Haihao Xiang <[email protected]>
+---
+ libavcodec/qsvdec.c | 10 ++++++++--
+ 1 file changed, 8 insertions(+), 2 deletions(-)
+
+diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
+index 768968fd7bca0..f2cd6ae05c0a9 100644
+--- a/libavcodec/qsvdec.c
++++ b/libavcodec/qsvdec.c
+@@ -931,12 +931,18 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
+         frame->flags |= AV_FRAME_FLAG_INTERLACED *
+             !(outsurf->Info.PicStruct & MFX_PICSTRUCT_PROGRESSIVE);
+         frame->pict_type = ff_qsv_map_pictype(aframe.frame->dec_info.FrameType);
+-        //Key frame is IDR frame is only suitable for H264. For HEVC, IRAPs are key frames.
+-        if (avctx->codec_id == AV_CODEC_ID_H264) {
++
++        if (avctx->codec_id == AV_CODEC_ID_H264 ||
++            avctx->codec_id == AV_CODEC_ID_HEVC) {
+             if (aframe.frame->dec_info.FrameType & MFX_FRAMETYPE_IDR)
+                 frame->flags |= AV_FRAME_FLAG_KEY;
+             else
+                 frame->flags &= ~AV_FRAME_FLAG_KEY;
++        } else {
++            if (aframe.frame->dec_info.FrameType & MFX_FRAMETYPE_I)
++                frame->flags |= AV_FRAME_FLAG_KEY;
++            else
++                frame->flags &= ~AV_FRAME_FLAG_KEY;
+         }
+         frame->crop_left   = outsurf->Info.CropX;
+         frame->crop_top    = outsurf->Info.CropY;