Backed out 3 changesets (bug 1660336) for causing build bustages in v…

…aapi. CLOSED TREE

Backed out changeset f53408903b5e (bug 1660336)
Backed out changeset 450b4f240ff5 (bug 1660336)
Backed out changeset 7694d9e2424b (bug 1660336)

dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp

@@ -11,9 +11,6 @@
 #include "mozilla/Types.h"
 #include "PlatformDecoderModule.h"
 #include "prlink.h"
-#ifdef MOZ_WAYLAND
-#  include "gfxPlatformGtk.h"
-#endif
 
 #define AV_LOG_DEBUG 48
 #define AV_LOG_INFO 32
@@ -256,52 +253,6 @@ void FFmpegLibWrapper::Unlink() {
   PodZero(this);
 }
 
-#ifdef MOZ_WAYLAND
-void FFmpegLibWrapper::LinkVAAPILibs() {
-  if (gfxPlatformGtk::GetPlatform()->UseHardwareVideoDecoding()) {
-    PRLibSpec lspec;
-    lspec.type = PR_LibSpec_Pathname;
-
-    if (gfxPlatformGtk::GetPlatform()->UseDRMVAAPIDisplay()) {
-      const char* libDrm = "libva-drm.so.2";
-      lspec.value.pathname = libDrm;
-      mVALibDrm = PR_LoadLibraryWithFlags(lspec, PR_LD_NOW | PR_LD_LOCAL);
-      if (!mVALibDrm) {
-        FFMPEG_LOG("VA-API support: Missing or old %s library.\n", libDrm);
-      }
-    } else {
-      if (gfxPlatformGtk::GetPlatform()->IsWaylandDisplay()) {
-        const char* libWayland = "libva-wayland.so.2";
-        lspec.value.pathname = libWayland;
-        mVALibWayland = PR_LoadLibraryWithFlags(lspec, PR_LD_NOW | PR_LD_LOCAL);
-        if (!mVALibWayland) {
-          FFMPEG_LOG("VA-API support: Missing or old %s library.\n",
-                     libWayland);
-        }
-      } else {
-        FFMPEG_LOG("VA-API X11 display is not implemented.\n");
-      }
-    }
-
-    if (mVALibWayland || mVALibDrm) {
-      const char* lib = "libva.so.2";
-      lspec.value.pathname = lib;
-      mVALib = PR_LoadLibraryWithFlags(lspec, PR_LD_NOW | PR_LD_LOCAL);
-      // Don't use libva when it's missing vaExportSurfaceHandle.
-      if (mVALib && !PR_FindSymbol(mVALib, "vaExportSurfaceHandle")) {
-        PR_UnloadLibrary(mVALib);
-        mVALib = nullptr;
-      }
-      if (!mVALib) {
-        FFMPEG_LOG("VA-API support: Missing or old %s library.\n", lib);
-      }
-    }
-  } else {
-    FFMPEG_LOG("VA-API FFmpeg is disabled by platform");
-  }
-}
-#endif
-
 #ifdef MOZ_WAYLAND
 bool FFmpegLibWrapper::IsVAAPIAvailable() {
 #  define VA_FUNC_LOADED(func) (func != nullptr)

dom/media/platforms/ffmpeg/FFmpegLibWrapper.h

@@ -56,7 +56,6 @@ struct MOZ_ONLY_USED_TO_AVOID_STATIC_CONSTRUCTORS FFmpegLibWrapper {
 #ifdef MOZ_WAYLAND
   // Check if mVALib are available and we can use HW decode.
   bool IsVAAPIAvailable();
-  void LinkVAAPILibs();
 #endif
 
   // indicate the version of libavcodec linked to.

dom/media/platforms/ffmpeg/FFmpegRuntimeLinker.cpp

@@ -57,7 +57,50 @@ bool FFmpegRuntimeLinker::Init() {
   }
 
 #ifdef MOZ_WAYLAND
-  sLibAV.LinkVAAPILibs();
+  if (gfxPlatformGtk::GetPlatform()->UseHardwareVideoDecoding()) {
+    PRLibSpec lspec;
+    lspec.type = PR_LibSpec_Pathname;
+
+    if (gfxPlatformGtk::GetPlatform()->UseDRMVAAPIDisplay()) {
+      const char* libDrm = "libva-drm.so.2";
+      lspec.value.pathname = libDrm;
+      sLibAV.mVALibDrm =
+          PR_LoadLibraryWithFlags(lspec, PR_LD_NOW | PR_LD_LOCAL);
+      if (!sLibAV.mVALibDrm) {
+        FFMPEG_LOG("VA-API support: Missing or old %s library.\n", libDrm);
+      }
+    } else {
+      if (gfxPlatformGtk::GetPlatform()->IsWaylandDisplay()) {
+        const char* libWayland = "libva-wayland.so.2";
+        lspec.value.pathname = libWayland;
+        sLibAV.mVALibWayland =
+            PR_LoadLibraryWithFlags(lspec, PR_LD_NOW | PR_LD_LOCAL);
+        if (!sLibAV.mVALibWayland) {
+          FFMPEG_LOG("VA-API support: Missing or old %s library.\n",
+                     libWayland);
+        }
+      } else {
+        FFMPEG_LOG("VA-API X11 display is not implemented.\n");
+      }
+    }
+
+    if (sLibAV.mVALibWayland || sLibAV.mVALibDrm) {
+      const char* lib = "libva.so.2";
+      lspec.value.pathname = lib;
+      sLibAV.mVALib = PR_LoadLibraryWithFlags(lspec, PR_LD_NOW | PR_LD_LOCAL);
+      // Don't use libva when it's missing vaExportSurfaceHandle.
+      if (sLibAV.mVALib &&
+          !PR_FindSymbol(sLibAV.mVALib, "vaExportSurfaceHandle")) {
+        PR_UnloadLibrary(sLibAV.mVALib);
+        sLibAV.mVALib = nullptr;
+      }
+      if (!sLibAV.mVALib) {
+        FFMPEG_LOG("VA-API support: Missing or old %s library.\n", lib);
+      }
+    }
+  } else {
+    FFMPEG_LOG("VA-API FFmpeg is disabled by platform");
+  }
 #endif
 
   // While going through all possible libs, this status will be updated with a

dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp

@@ -125,8 +125,8 @@ static AVPixelFormat ChooseVAAPIPixelFormat(AVCodecContext* aCodecContext,
   return AV_PIX_FMT_NONE;
 }
 
-DMABufSurfaceWrapper<LIBAV_VER>::DMABufSurfaceWrapper(DMABufSurface* aSurface,
-                                                      FFmpegLibWrapper* aLib)
+DMABufSurfaceWrapper::DMABufSurfaceWrapper(DMABufSurface* aSurface,
+                                           FFmpegLibWrapper* aLib)
     : mSurface(aSurface),
       mLib(aLib),
       mAVHWFramesContext(nullptr),
@@ -139,8 +139,8 @@ DMABufSurfaceWrapper<LIBAV_VER>::DMABufSurfaceWrapper(DMABufSurface* aSurface,
              mSurface->GetUID());
 }
 
-void DMABufSurfaceWrapper<LIBAV_VER>::LockVAAPIData(
-    AVCodecContext* aAVCodecContext, AVFrame* aAVFrame) {
+void DMABufSurfaceWrapper::LockVAAPIData(AVCodecContext* aAVCodecContext,
+                                         AVFrame* aAVFrame) {
   FFMPEG_LOG("DMABufSurfaceWrapper: VAAPI locking dmabuf surface UID = %d",
              mSurface->GetUID());
   if (aAVCodecContext && aAVFrame) {
@@ -149,7 +149,7 @@ void DMABufSurfaceWrapper<LIBAV_VER>::LockVAAPIData(
   }
 }
 
-void DMABufSurfaceWrapper<LIBAV_VER>::ReleaseVAAPIData() {
+void DMABufSurfaceWrapper::ReleaseVAAPIData() {
   FFMPEG_LOG("DMABufSurfaceWrapper: VAAPI releasing dmabuf surface UID = %d",
              mSurface->GetUID());
   if (mHWAVBuffer && mAVHWFramesContext) {
@@ -159,7 +159,7 @@ void DMABufSurfaceWrapper<LIBAV_VER>::ReleaseVAAPIData() {
   mSurface->ReleaseSurface();
 }
 
-DMABufSurfaceWrapper<LIBAV_VER>::~DMABufSurfaceWrapper() {
+DMABufSurfaceWrapper::~DMABufSurfaceWrapper() {
   FFMPEG_LOG("DMABufSurfaceWrapper: deleting dmabuf surface UID = %d",
              mSurface->GetUID());
   ReleaseVAAPIData();
@@ -182,14 +182,7 @@ AVCodec* FFmpegVideoDecoder<LIBAV_VER>::FindVAAPICodec() {
   return nullptr;
 }
 
-template <int V>
-class VAAPIDisplayHolder {};
-
-template <>
-class VAAPIDisplayHolder<LIBAV_VER>;
-
-template <>
-class VAAPIDisplayHolder<LIBAV_VER> {
+class VAAPIDisplayHolder {
  public:
   VAAPIDisplayHolder(FFmpegLibWrapper* aLib, VADisplay aDisplay)
       : mLib(aLib), mDisplay(aDisplay){};
@@ -201,8 +194,7 @@ class VAAPIDisplayHolder<LIBAV_VER> {
 };
 
 static void VAAPIDisplayReleaseCallback(struct AVHWDeviceContext* hwctx) {
-  auto displayHolder =
-      static_cast<VAAPIDisplayHolder<LIBAV_VER>*>(hwctx->user_opaque);
+  auto displayHolder = static_cast<VAAPIDisplayHolder*>(hwctx->user_opaque);
   delete displayHolder;
 }
 
@@ -243,7 +235,7 @@ bool FFmpegVideoDecoder<LIBAV_VER>::CreateVAAPIDeviceContext() {
     }
   }
 
-  hwctx->user_opaque = new VAAPIDisplayHolder<LIBAV_VER>(mLib, mDisplay);
+  hwctx->user_opaque = new VAAPIDisplayHolder(mLib, mDisplay);
   hwctx->free = VAAPIDisplayReleaseCallback;
 
   int major, minor;
@@ -718,7 +710,7 @@ void FFmpegVideoDecoder<LIBAV_VER>::ReleaseUnusedVAAPIFrames() {
   }
 }
 
-DMABufSurfaceWrapper<LIBAV_VER>*
+DMABufSurfaceWrapper*
 FFmpegVideoDecoder<LIBAV_VER>::GetUnusedDMABufSurfaceWrapper() {
   int len = mDMABufSurfaces.Length();
   for (int i = 0; i < len; i++) {
@@ -777,8 +769,7 @@ MediaResult FFmpegVideoDecoder<LIBAV_VER>::CreateImageDMABuf(
 
   RefPtr<DMABufSurfaceYUV> surface;
 
-  DMABufSurfaceWrapper<LIBAV_VER>* surfaceWrapper =
-      GetUnusedDMABufSurfaceWrapper();
+  DMABufSurfaceWrapper* surfaceWrapper = GetUnusedDMABufSurfaceWrapper();
   if (!surfaceWrapper) {
     if (mVAAPIDeviceContext) {
       surface = DMABufSurfaceYUV::CreateYUVSurface(vaDesc);

dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h

@@ -55,14 +55,7 @@ namespace mozilla {
 // We own the DMABufSurface underlying GPU data and we use it for
 // repeated rendering of video frames.
 //
-template <int V>
-class DMABufSurfaceWrapper {};
-
-template <>
-class DMABufSurfaceWrapper<LIBAV_VER>;
-
-template <>
-class DMABufSurfaceWrapper<LIBAV_VER> final {
+class DMABufSurfaceWrapper final {
  public:
   DMABufSurfaceWrapper(DMABufSurface* aSurface, FFmpegLibWrapper* aLib);
   ~DMABufSurfaceWrapper();
@@ -169,7 +162,7 @@ class FFmpegVideoDecoder<LIBAV_VER>
                                 MediaDataDecoder::DecodedData& aResults);
 
   void ReleaseUnusedVAAPIFrames();
-  DMABufSurfaceWrapper<LIBAV_VER>* GetUnusedDMABufSurfaceWrapper();
+  DMABufSurfaceWrapper* GetUnusedDMABufSurfaceWrapper();
   void ReleaseDMABufSurfaces();
 #endif
 
@@ -187,7 +180,7 @@ class FFmpegVideoDecoder<LIBAV_VER>
   const bool mDisableHardwareDecoding;
   VADisplay mDisplay;
   bool mUseDMABufSurfaces;
-  nsTArray<DMABufSurfaceWrapper<LIBAV_VER>> mDMABufSurfaces;
+  nsTArray<DMABufSurfaceWrapper> mDMABufSurfaces;
 #endif
   RefPtr<KnowsCompositor> mImageAllocator;
   RefPtr<ImageContainer> mImageContainer;

dom/media/platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.cpp

@@ -64,10 +64,6 @@ bool FFVPXRuntimeLinker::Init() {
   MOZ_ASSERT(NS_IsMainThread());
   sLinkStatus = LinkStatus_FAILED;
 
-#ifdef MOZ_WAYLAND
-  sFFVPXLib.LinkVAAPILibs();
-#endif
-
   // We retrieve the path of the lgpllibs library as this is where mozavcodec
   // and mozavutil libs are located.
   PathString lgpllibsname = GetLibraryName(nullptr, "lgpllibs");

media/ffvpx/FILES

@@ -131,11 +131,6 @@
 ./libavcodec/thread.h
 ./libavcodec/unary.h
 ./libavcodec/utils.c
-./libavcodec/vaapi.h
-./libavcodec/vaapi_decode.h
-./libavcodec/vaapi_decode.c
-./libavcodec/vaapi_vp8.c
-./libavcodec/vaapi_vp9.c
 ./libavcodec/version.h
 ./libavcodec/videodsp.c
 ./libavcodec/videodsp.h
@@ -268,11 +263,7 @@
 ./libavutil/frame.h
 ./libavutil/hwcontext.c
 ./libavutil/hwcontext.h
-./libavutil/hwcontext_drm.h
-./libavutil/hwcontext_drm.c
 ./libavutil/hwcontext_internal.h
-./libavutil/hwcontext_vaapi.h
-./libavutil/hwcontext_vaapi.c
 ./libavutil/imgutils.c
 ./libavutil/imgutils.h
 ./libavutil/imgutils_internal.h

media/ffvpx/README_MOZILLA

@@ -56,5 +56,3 @@ $ for i in `cat $PATH_CENTRAL/media/ffvpx/FILES`; do git diff $REV_LASTSYNC HEAD
 Then apply patch.diff on the ffvpx tree.
 
 Compilation will reveal if any files are missing.
-
-Apply linux-vaapi-build.patch patch to enable build VA-API support for Linux.

media/ffvpx/config_common.h

@@ -18,14 +18,4 @@
 #define CONFIG_RDFT 1
 #endif
 
-#ifdef MOZ_WAYLAND
-#define CONFIG_VAAPI 1
-#define CONFIG_VP8_VAAPI_HWACCEL 1
-#define CONFIG_VP9_VAAPI_HWACCEL 1
-#else
-#define CONFIG_VAAPI 0
-#define CONFIG_VP8_VAAPI_HWACCEL 0
-#define CONFIG_VP9_VAAPI_HWACCEL 0
-#endif
-
 #endif

media/ffvpx/config_unix32.h

@@ -524,6 +524,7 @@
 #define CONFIG_FFNVCODEC 0
 #define CONFIG_NVDEC 0
 #define CONFIG_NVENC 0
+#define CONFIG_VAAPI 0
 #define CONFIG_VDPAU 0
 #define CONFIG_VIDEOTOOLBOX 0
 #define CONFIG_V4L2_M2M 0

media/ffvpx/config_unix64.h

@@ -524,6 +524,7 @@
 #define CONFIG_FFNVCODEC 0
 #define CONFIG_NVDEC 0
 #define CONFIG_NVENC 0
+#define CONFIG_VAAPI 0
 #define CONFIG_VDPAU 0
 #define CONFIG_VIDEOTOOLBOX 0
 #define CONFIG_V4L2_M2M 1

media/ffvpx/defaults_disabled.h

@@ -1706,6 +1706,7 @@
 #define CONFIG_VP8_V4L2M2M_DECODER 0
 #define CONFIG_VP8_V4L2M2M_ENCODER 0
 #define CONFIG_VP8_VAAPI_ENCODER 0
+#define CONFIG_VP8_VAAPI_HWACCEL 0
 #define CONFIG_VP9_CUVID_DECODER 0
 #define CONFIG_VP9_D3D11VA2_HWACCEL 0
 #define CONFIG_VP9_D3D11VA_HWACCEL 0
@@ -1718,6 +1719,7 @@
 #define CONFIG_VP9_SUPERFRAME_BSF 0
 #define CONFIG_VP9_V4L2M2M_DECODER 0
 #define CONFIG_VP9_VAAPI_ENCODER 0
+#define CONFIG_VP9_VAAPI_HWACCEL 0
 #define CONFIG_VPK_DEMUXER 0
 #define CONFIG_VPLAYER_DECODER 0
 #define CONFIG_VPLAYER_DEMUXER 0

media/ffvpx/ffvpxcommon.mozbuild

@@ -71,9 +71,5 @@ elif not CONFIG['RELEASE_OR_BETA']:
     # Enable fast assertions in opt builds of Nightly and Aurora.
     DEFINES['ASSERT_LEVEL'] = 1
 
-if CONFIG['MOZ_WAYLAND']:
-    CFLAGS += CONFIG['MOZ_WAYLAND_CFLAGS']
-    CXXFLAGS += CONFIG['MOZ_WAYLAND_CFLAGS']
-
 # Add libFuzzer configuration directives
 include('/tools/fuzzing/libfuzzer-config.mozbuild')

media/ffvpx/libavcodec/avcodec.symbols

@@ -28,11 +28,6 @@ av_get_pcm_codec
 av_get_profile_name
 av_grow_packet
 av_hwaccel_next
-av_hwdevice_ctx_init
-av_hwdevice_ctx_alloc
-av_hwdevice_ctx_create_derived
-av_hwframe_transfer_get_formats
-av_hwframe_ctx_alloc
 av_init_packet
 av_lockmgr_register
 av_new_packet
@@ -98,7 +93,6 @@ avcodec_free_context
 avcodec_get_class
 avcodec_get_context_defaults3
 avcodec_get_frame_class
-avcodec_get_hw_config
 avcodec_get_name
 avcodec_get_subtitle_rect_class
 avcodec_get_type

media/ffvpx/libavcodec/moz.build

@@ -96,12 +96,6 @@ if not CONFIG['MOZ_FFVPX_AUDIOONLY']:
         'vp9prob.c',
         'vp9recon.c'
     ]
-    if CONFIG['MOZ_WAYLAND']:
-        SOURCES += [
-            'vaapi_decode.c',
-            'vaapi_vp8.c',
-            'vaapi_vp9.c',
-        ]
 
 if CONFIG['MOZ_LIBAV_FFT']:
     SOURCES += [