obs-ffmpeg: Use Libva in FFmpeg VA-API

Libva is directly used to check if DRI devices support H264 encoding.

cmake/Modules/FindLibva.cmake

@@ -0,0 +1,49 @@
+# * Try to find Libva, once done this will define
+#
+# * LIBVA_FOUND - system has Libva
+# * LIBVA_INCLUDE_DIRS - the Libva include directory
+# * LIBVA_LIBRARIES - the libraries needed to use Libva
+# * LIBVA_DEFINITIONS - Compiler switches required for using Libva
+
+# Use pkg-config to get the directories and then use these values in the
+# find_path() and find_library() calls
+
+find_package(PkgConfig QUIET)
+if(PKG_CONFIG_FOUND)
+  pkg_check_modules(_LIBVA libva)
+endif()
+
+find_path(
+  LIBVA_INCLUDE_DIR
+  NAMES va.h
+  HINTS ${_LIBVA_INCLUDE_DIRS}
+  PATHS /usr/include /usr/local/include /opt/local/include
+  PATH_SUFFIXES va/)
+
+find_library(
+  LIBVA_LIB
+  NAMES ${_LIBVA_LIBRARIES} libva
+  HINTS ${_LIBVA_LIBRARY_DIRS}
+  PATHS /usr/lib /usr/local/lib /opt/local/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Libva REQUIRED_VARS LIBVA_LIB
+                                                      LIBVA_INCLUDE_DIR)
+mark_as_advanced(LIBVA_INCLUDE_DIR LIBVA_LIB)
+
+if(LIBVA_FOUND)
+  set(LIBVA_INCLUDE_DIRS ${LIBVA_INCLUDE_DIR})
+  set(LIBVA_LIBRARIES ${LIBVA_LIB})
+
+  if(NOT TARGET Libva::va)
+    if(IS_ABSOLUTE "${LIBVA_LIBRARIES}")
+      add_library(Libva::va UNKNOWN IMPORTED)
+      set_target_properties(Libva::va PROPERTIES IMPORTED_LOCATION
+                                                 "${LIBVA_LIBRARIES}")
+    else()
+      add_library(Libva::va INTERFACE IMPORTED)
+      set_target_properties(Libva::va PROPERTIES IMPORTED_LIBNAME
+                                                 "${LIBVA_LIBRARIES}")
+    endif()
+  endif()
+endif()

plugins/obs-ffmpeg/CMakeLists.txt

@@ -119,9 +119,11 @@ if(OS_WINDOWS)
             obs-ffmpeg.rc)
 
 elseif(OS_POSIX AND NOT OS_MACOS)
+  find_package(Libva REQUIRED)
   find_package(Libpci REQUIRED)
-  target_sources(obs-ffmpeg PRIVATE obs-ffmpeg-vaapi.c)
-  target_link_libraries(obs-ffmpeg PRIVATE LIBPCI::LIBPCI)
+  target_sources(obs-ffmpeg PRIVATE obs-ffmpeg-vaapi.c vaapi-utils.c
+                                    vaapi-utils.h)
+  target_link_libraries(obs-ffmpeg PRIVATE Libva::va LIBPCI::LIBPCI)
 endif()
 
 setup_plugin_target(obs-ffmpeg)

plugins/obs-ffmpeg/obs-ffmpeg-vaapi.c

@@ -38,10 +38,11 @@
 
 #include <pci/pci.h>
 
+#include "vaapi-utils.h"
 #include "obs-ffmpeg-formats.h"
 
 #define do_log(level, format, ...)                          \
-	blog(level, "[FFMPEG VAAPI encoder: '%s'] " format, \
+	blog(level, "[FFmpeg VAAPI encoder: '%s'] " format, \
 	     obs_encoder_get_name(enc->encoder), ##__VA_ARGS__)
 
 #define warn(format, ...) do_log(LOG_WARNING, format, ##__VA_ARGS__)
@@ -77,7 +78,7 @@ struct vaapi_encoder {
 static const char *vaapi_getname(void *unused)
 {
 	UNUSED_PARAMETER(unused);
-	return "FFMPEG VAAPI H.264";
+	return "FFmpeg VAAPI H.264";
 }
 
 static inline bool valid_format(enum video_format format)
@@ -511,18 +512,77 @@ static void set_visible(obs_properties_t *ppts, const char *name, bool visible)
 
 static void vaapi_defaults(obs_data_t *settings)
 {
-	obs_data_set_default_string(settings, "vaapi_device",
-				    "/dev/dri/renderD128");
+	const char *device = vaapi_get_h264_default_device();
+
+	obs_data_set_default_string(settings, "vaapi_device", device);
 	obs_data_set_default_int(settings, "profile",
 				 FF_PROFILE_H264_CONSTRAINED_BASELINE);
 	obs_data_set_default_int(settings, "level", 40);
 	obs_data_set_default_int(settings, "bitrate", 2500);
 	obs_data_set_default_int(settings, "keyint_sec", 0);
 	obs_data_set_default_int(settings, "bf", 0);
 	obs_data_set_default_int(settings, "rendermode", 0);
-	obs_data_set_default_string(settings, "rate_control", "CBR");
 	obs_data_set_default_int(settings, "qp", 20);
 	obs_data_set_default_int(settings, "maxrate", 0);
+
+	int drm_fd = -1;
+	VADisplay va_dpy = vaapi_open_device(&drm_fd, device, "vaapi_defaults");
+	if (!va_dpy)
+		return;
+
+	if (vaapi_device_rc_supported(VAProfileH264ConstrainedBaseline, va_dpy,
+				      VA_RC_CBR, device))
+		obs_data_set_default_string(settings, "rate_control", "CBR");
+	else if (vaapi_device_rc_supported(VAProfileH264ConstrainedBaseline,
+					   va_dpy, VA_RC_VBR, device))
+		obs_data_set_default_string(settings, "rate_control", "VBR");
+	else
+		obs_data_set_default_string(settings, "rate_control", "CQP");
+
+	vaapi_close_device(&drm_fd, va_dpy);
+}
+
+static bool vaapi_device_modified(obs_properties_t *ppts, obs_property_t *p,
+				  obs_data_t *settings)
+{
+	UNUSED_PARAMETER(p);
+
+	const char *device = obs_data_get_string(settings, "vaapi_device");
+	int drm_fd = -1;
+	VADisplay va_dpy =
+		vaapi_open_device(&drm_fd, device, "vaapi_device_modified");
+	int profile = obs_data_get_int(settings, "profile");
+	obs_property_t *rc_p = obs_properties_get(ppts, "rate_control");
+
+	obs_property_list_clear(rc_p);
+
+	if (!va_dpy || !vaapi_display_h264_supported(va_dpy, device))
+		goto fail;
+
+	switch (profile) {
+	case FF_PROFILE_H264_CONSTRAINED_BASELINE:
+		profile = VAProfileH264ConstrainedBaseline;
+		break;
+	case FF_PROFILE_H264_MAIN:
+		profile = VAProfileH264Main;
+		break;
+	case FF_PROFILE_H264_HIGH:
+		profile = VAProfileH264High;
+		break;
+	}
+
+	if (vaapi_device_rc_supported(profile, va_dpy, VA_RC_CBR, device))
+		obs_property_list_add_string(rc_p, "CBR (default)", "CBR");
+
+	if (vaapi_device_rc_supported(profile, va_dpy, VA_RC_VBR, device))
+		obs_property_list_add_string(rc_p, "VBR", "VBR");
+
+	if (vaapi_device_rc_supported(profile, va_dpy, VA_RC_CQP, device))
+		obs_property_list_add_string(rc_p, "CQP", "CQP");
+
+fail:
+	vaapi_close_device(&drm_fd, va_dpy);
+	return true;
 }
 
 static bool rate_control_modified(obs_properties_t *ppts, obs_property_t *p,
@@ -617,6 +677,10 @@ static obs_properties_t *vaapi_properties(void *unused)
 				bool name_found = get_device_name_from_pci(
 					pacc, pci_slot, namebuf,
 					sizeof(namebuf));
+
+				if (!vaapi_device_h264_supported(path))
+					continue;
+
 				if (!name_found)
 					obs_property_list_add_string(list, path,
 								     path);
@@ -640,13 +704,19 @@ static obs_properties_t *vaapi_properties(void *unused)
 					blog(LOG_DEBUG,
 					     "obs-ffmpeg-vaapi: A format truncation may have occurred."
 					     " This can be ignored since it is quite improbable.");
+
+				if (!vaapi_device_h264_supported(path))
+					continue;
+
 				obs_property_list_add_string(list, card, path);
 			} else {
 				break;
 			}
 		}
 	}
 
+	obs_property_set_modified_callback(list, vaapi_device_modified);
+
 	list = obs_properties_add_list(props, "profile",
 				       obs_module_text("Profile"),
 				       OBS_COMBO_TYPE_LIST,
@@ -656,6 +726,8 @@ static obs_properties_t *vaapi_properties(void *unused)
 	obs_property_list_add_int(list, "Main", FF_PROFILE_H264_MAIN);
 	obs_property_list_add_int(list, "High", FF_PROFILE_H264_HIGH);
 
+	obs_property_set_modified_callback(list, vaapi_device_modified);
+
 	list = obs_properties_add_list(props, "level", obs_module_text("Level"),
 				       OBS_COMBO_TYPE_LIST,
 				       OBS_COMBO_FORMAT_INT);
@@ -674,9 +746,6 @@ static obs_properties_t *vaapi_properties(void *unused)
 				       obs_module_text("RateControl"),
 				       OBS_COMBO_TYPE_LIST,
 				       OBS_COMBO_FORMAT_STRING);
-	obs_property_list_add_string(list, "CBR (default)", "CBR");
-	obs_property_list_add_string(list, "CQP", "CQP");
-	obs_property_list_add_string(list, "VBR", "VBR");
 
 	obs_property_set_modified_callback(list, rate_control_modified);
 

plugins/obs-ffmpeg/obs-ffmpeg.c

@@ -14,6 +14,13 @@
 #include "jim-nvenc.h"
 #endif
 
+#if !defined(_WIN32) && !defined(__APPLE__) && \
+	LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 27, 100)
+#include "vaapi-utils.h"
+
+#define LIBAVUTIL_VAAPI_AVAILABLE
+#endif
+
 OBS_DECLARE_MODULE()
 OBS_MODULE_USE_DEFAULT_LOCALE("obs-ffmpeg", "en-US")
 MODULE_EXPORT const char *obs_module_description(void)
@@ -36,10 +43,6 @@ extern struct obs_encoder_info hevc_nvenc_encoder_info;
 extern struct obs_encoder_info svt_av1_encoder_info;
 extern struct obs_encoder_info aom_av1_encoder_info;
 
-#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 27, 100)
-#define LIBAVUTIL_VAAPI_AVAILABLE
-#endif
-
 #ifdef LIBAVUTIL_VAAPI_AVAILABLE
 extern struct obs_encoder_info vaapi_encoder_info;
 #endif
@@ -320,10 +323,16 @@ static bool nvenc_supported(bool *out_h264, bool *out_hevc, bool *out_av1)
 #endif
 
 #ifdef LIBAVUTIL_VAAPI_AVAILABLE
-static bool vaapi_supported(void)
+static bool h264_vaapi_supported(void)
 {
 	const AVCodec *vaenc = avcodec_find_encoder_by_name("h264_vaapi");
-	return !!vaenc;
+
+	if (!vaenc)
+		return false;
+
+	/* NOTE: If default device is NULL, it means there is no device
+	 * that support H264. */
+	return vaapi_get_h264_default_device() != NULL;
 }
 #endif
 
@@ -403,10 +412,18 @@ bool obs_module_load(void)
 	amf_load();
 #endif
 
-#if !defined(_WIN32) && defined(LIBAVUTIL_VAAPI_AVAILABLE)
-	if (vaapi_supported()) {
-		blog(LOG_INFO, "FFMPEG VAAPI supported");
+#ifdef LIBAVUTIL_VAAPI_AVAILABLE
+	const char *libva_env = getenv("LIBVA_DRIVER_NAME");
+	if (!!libva_env)
+		blog(LOG_WARNING,
+		     "LIBVA_DRIVER_NAME variable is set,"
+		     " this could prevent FFmpeg VAAPI from working correctly");
+
+	if (h264_vaapi_supported()) {
+		blog(LOG_INFO, "FFmpeg VAAPI H264 encoding supported");
 		obs_register_encoder(&vaapi_encoder_info);
+	} else {
+		blog(LOG_INFO, "FFmpeg VAAPI H264 encoding not supported");
 	}
 #endif
 #endif