Merge pull request wtsnz#43 from wtsnz/feature/mac-hw-decoding

Add macOS hardware decoding

data/locale/en-US.ini

@@ -1 +1,5 @@
-OBSIOSCamera.Title="iOS Camera"
+OBSIOSCamera.Title="iOS Camera"
+OBSIOSCamera.Settings.Latency="Latency"
+OBSIOSCamera.Settings.Latency.Normal="Normal"
+OBSIOSCamera.Settings.Latency.Low="Low"
+OBSIOSCamera.Settings.UseHardwareDecoder="Enable Hardware Decoder"

deps/portal/src/Protocol.cpp

@@ -92,6 +92,9 @@ namespace portal
                 // Remove the data from buffer
                 buffer.erase(buffer.begin(), buffer.begin() + sizeof(PortalFrame) + frame.payloadSize);
 
+                // Attempt to parse another packet
+                processData(nullptr, 0);
+
             } else {
 
                 // We haven't got the data for the packet just yet, so wait for next time!

src/Thread.hpp

@@ -21,6 +21,7 @@
 
 #include <stdio.h>
 #include <thread>
+#include <atomic>
 
 class Thread
 {
@@ -36,14 +37,14 @@ class Thread
     virtual void* run() = 0;
 
     bool shouldStop() {
-        return mShouldStop;
+        return mShouldStop.load();
     }
 
 private:
 
     std::thread *mThread;
-    bool mRunning;
-    bool mShouldStop;
+    std::atomic<bool> mRunning;
+    std::atomic<bool> mShouldStop;
 };
 
 #endif /* Thread_hpp */

src/VideoToolboxVideoDecoder.cpp

@@ -30,6 +30,11 @@ VideoToolboxDecoder::VideoToolboxDecoder()
     memset(&frame, 0, sizeof(frame));
 }
 
+VideoToolboxDecoder::~VideoToolboxDecoder()
+{
+    this->Shutdown();
+}
+
 void VideoToolboxDecoder::Init()
 {
     // Start the thread.
@@ -48,21 +53,24 @@ void VideoToolboxDecoder::Drain()
 
 void VideoToolboxDecoder::Shutdown()
 {
-    this->join();
+    mQueue.stop();
 
     if (mSession != NULL) {
         VTDecompressionSessionInvalidate(mSession);
     }
 
     mSession = NULL;
+
+    this->join();
 }
 
 void *VideoToolboxDecoder::run() {
 
-    for (int i = 0;; i++) {
-
+    while (shouldStop() == false) {
         PacketItem *item = (PacketItem *)mQueue.remove();
-        this->processPacketItem(item);
+        if (item != NULL) {
+            this->processPacketItem(item);
+        }
         delete item;
     }
 
@@ -238,15 +246,10 @@ void VideoToolboxDecoder::processPacketItem(PacketItem *packetItem)
                                   &sampleSize,
                                   &sampleBuffer);
 
-    // set some values of the sample buffer's attachments
-    //    CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, YES);
-    //    CFMutableDictionaryRef dict = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0);
-    //    CFDictionarySetValue(dict, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue);
-
-    VTDecodeFrameFlags flags = kVTDecodeFrame_1xRealTimePlayback;
+    VTDecodeFrameFlags flags = 0;
     VTDecodeInfoFlags flagOut;
 
-    long long now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+    auto now = os_gettime_ns();
 
     VTDecompressionSessionDecodeFrame(mSession, sampleBuffer, flags,
                                       (void*)now, &flagOut);
@@ -354,8 +357,7 @@ bool VideoToolboxDecoder::update_frame(obs_source_t *capture, obs_source_frame *
     // video_format    format = format_from_subtype(fourcc);
     CMVideoDimensions dims = CMVideoFormatDescriptionGetDimensions(formatDesc);
 
-    long long now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
-    frame->timestamp = now;
+    frame->timestamp = os_gettime_ns();
     frame->width    = dims.width;
     frame->height   = dims.height;
     frame->format   = VIDEO_FORMAT_BGRA;

src/VideoToolboxVideoDecoder.h

@@ -19,25 +19,20 @@
 #include <obs.h>
 #include <chrono>
 #include <vector>
+#include <util/platform.h>
 
 #include <VideoToolbox/VideoToolbox.h>
 
 #include "Queue.hpp"
 #include "Thread.hpp"
 #include "VideoDecoder.h"
 
-//class VideoToolboxDecoderCallback {
-//public:
-//    virtual ~VideoToolboxDecoderCallback() {}
-//
-//    virtual void VideoToolboxDecodedFrame(CVPixelBufferRef aImage, CMVideoFormatDescriptionRef formatDescription) = 0;
-//};
-
 class VideoToolboxDecoder: public VideoDecoder, private Thread
 {
 public:
     VideoToolboxDecoder();
-
+    ~VideoToolboxDecoder();
+
     void Init() override;
 
     void Input(std::vector<char> packet, int type, int tag) override;
@@ -53,7 +48,6 @@ class VideoToolboxDecoder: public VideoDecoder, private Thread
     // The OBS Source to update.
     obs_source_t *source;
 
-
 private:
 
     void *run() override; // Thread

src/ffmpeg-decode.c

@@ -23,7 +23,6 @@ int ffmpeg_decode_init(struct ffmpeg_decode *decode, enum AVCodecID id)
 {
     int ret;
 
-    avcodec_register_all();
     memset(decode, 0, sizeof(*decode));
 
     decode->codec = avcodec_find_decoder(id);
@@ -42,6 +41,7 @@ int ffmpeg_decode_init(struct ffmpeg_decode *decode, enum AVCodecID id)
     if (decode->codec->capabilities & CODEC_CAP_TRUNC)
         decode->decoder->flags |= CODEC_FLAG_TRUNC;
 
+    decode->decoder->flags |= AV_CODEC_FLAG_LOW_DELAY;
     decode->decoder->flags2 = AV_CODEC_FLAG2_CHUNKS;
 
     return 0;
@@ -233,8 +233,6 @@ bool ffmpeg_decode_video(struct ffmpeg_decode *decode,
     packet.data = decode->packet_buffer;
     packet.size = (int)size;
     packet.pts = *ts;
-    //    packet.flags |= CODEC_FLAG_TRUNCATED; /* We may send incomplete frames */
-    //    packet.flags |= CODEC_FLAG2_CHUNKS;
 
     if (decode->codec->id == AV_CODEC_ID_H264 && obs_avc_keyframe(data, size))
     {

src/obs-ios-camera-plugin.cpp

@@ -22,7 +22,7 @@
 OBS_DECLARE_MODULE()
 OBS_MODULE_USE_DEFAULT_LOCALE("obs-ios-camera-plugin", "en-US")
 
-#define IOS_CAMERA_PLUGIN_VERSION "2.5.2"
+#define IOS_CAMERA_PLUGIN_VERSION "2.6.0"
 
 extern void RegisterIOSCameraSource();
 

src/obs-ios-camera-source.cpp

@@ -24,12 +24,19 @@
 
 #include "FFMpegVideoDecoder.h"
 #include "FFMpegAudioDecoder.h"
-//#include "VideoToolboxVideoDecoder.h"
+#ifdef __APPLE__
+    #include "VideoToolboxVideoDecoder.h"
+#endif
 
 #define TEXT_INPUT_NAME obs_module_text("OBSIOSCamera.Title")
 
 #define SETTING_DEVICE_UUID "setting_device_uuid"
 #define SETTING_DEVICE_UUID_NONE_VALUE "null"
+#define SETTING_PROP_LATENCY "latency"
+#define SETTING_PROP_LATENCY_NORMAL 0
+#define SETTING_PROP_LATENCY_LOW 1
+
+#define SETTING_PROP_HARDWARE_DECODER "setting_use_hw_decoder"
 
 class IOSCameraInput: public portal::PortalDelegate
 {
@@ -43,7 +50,12 @@ class IOSCameraInput: public portal::PortalDelegate
 
     std::shared_ptr<portal::Portal> sharedPortal;
     portal::Portal portal;
-    FFMpegVideoDecoder videoDecoder;
+
+    VideoDecoder *videoDecoder;
+#ifdef __APPLE__
+    VideoToolboxDecoder videoToolboxVideoDecoder;
+#endif
+    FFMpegVideoDecoder ffmpegVideoDecoder;
     FFMpegAudioDecoder audioDecoder;
 
     IOSCameraInput(obs_source_t *source_, obs_data_t *settings)
@@ -64,13 +76,18 @@ class IOSCameraInput: public portal::PortalDelegate
         auto portalReference = std::shared_ptr<portal::Portal>(&portal, null_deleter);
         sharedPortal = portalReference;
 
-        videoDecoder.source = source;
-        videoDecoder.Init();
+#ifdef __APPLE__
+        videoToolboxVideoDecoder.source = source;
+        videoToolboxVideoDecoder.Init();
+#endif
+
+        ffmpegVideoDecoder.source = source;
+        ffmpegVideoDecoder.Init();
 
         audioDecoder.source = source;
         audioDecoder.Init();
 
-        obs_source_set_async_unbuffered(source, true);
+        videoDecoder = &ffmpegVideoDecoder;
 
         loadSettings(settings);
         active = true;
@@ -147,9 +164,8 @@ class IOSCameraInput: public portal::PortalDelegate
         try
         {
             switch (type) {
-
                 case 101: // Video Packet
-                    this->videoDecoder.Input(packet, type, tag);
+                    this->videoDecoder->Input(packet, type, tag);
                     break;
                 case 102: // Audio Packet
                     this->audioDecoder.Input(packet, type, tag);
@@ -313,13 +329,31 @@ static obs_properties_t *GetIOSCameraProperties(void *data)
     obs_properties_add_button(ppts, "setting_refresh_devices", "Refresh Devices", refresh_devices);
     obs_properties_add_button(ppts, "setting_button_connect_to_device", "Reconnect to Device", reconnect_to_device);
 
+    obs_property_t* latency_modes = obs_properties_add_list(ppts, SETTING_PROP_LATENCY, obs_module_text("OBSIOSCamera.Settings.Latency"), OBS_COMBO_TYPE_LIST, OBS_COMBO_FORMAT_INT);
+
+    obs_property_list_add_int(latency_modes,
+        obs_module_text("OBSIOSCamera.Settings.Latency.Normal"),
+        SETTING_PROP_LATENCY_NORMAL);
+    obs_property_list_add_int(latency_modes,
+        obs_module_text("OBSIOSCamera.Settings.Latency.Low"),
+        SETTING_PROP_LATENCY_LOW);
+
+#ifdef __APPLE__
+    obs_properties_add_bool(ppts, SETTING_PROP_HARDWARE_DECODER,
+        obs_module_text("OBSIOSCamera.Settings.UseHardwareDecoder"));
+#endif
+
     return ppts;
 }
 
 
 static void GetIOSCameraDefaults(obs_data_t *settings)
 {
     obs_data_set_default_string(settings, SETTING_DEVICE_UUID, "");
+    obs_data_set_default_int(settings, SETTING_PROP_LATENCY, SETTING_PROP_LATENCY_LOW);
+#ifdef __APPLE__
+    obs_data_set_default_bool(settings, SETTING_PROP_HARDWARE_DECODER, false);
+#endif
 }
 
 static void SaveIOSCameraInput(void *data, obs_data_t *settings)
@@ -338,6 +372,21 @@ static void UpdateIOSCameraInput(void *data, obs_data_t *settings)
     // Connect to the device
     auto uuid = obs_data_get_string(settings, SETTING_DEVICE_UUID);
     input->connectToDevice(uuid, false);
+
+    const bool is_unbuffered =
+        (obs_data_get_int(settings, SETTING_PROP_LATENCY) == SETTING_PROP_LATENCY_LOW);
+    obs_source_set_async_unbuffered(input->source, is_unbuffered);
+
+#ifdef __APPLE__
+    bool useHardwareDecoder = obs_data_get_bool(settings, SETTING_PROP_HARDWARE_DECODER);
+
+    if (useHardwareDecoder) {
+        input->videoDecoder = &input->videoToolboxVideoDecoder;
+    } else {
+        input->videoDecoder = &input->ffmpegVideoDecoder;
+    }
+#endif
+
 }
 
 void RegisterIOSCameraSource()