improving WebM & Ogg encoders

FrameCapturer/Assets/UTJ/FrameCapturer/Scripts/AudioRecorder.cs

@@ -8,6 +8,7 @@ namespace UTJ.FrameCapturer
 {
     [AddComponentMenu("UTJ/FrameCapturer/Audio Recorder")]
     [RequireComponent(typeof(AudioListener))]
+    [ExecuteInEditMode]
     public class AudioRecorder : RecorderBase
     {
         #region fields

FrameCapturer/Assets/UTJ/FrameCapturer/Scripts/Encoder/AudioEncoder.cs

@@ -8,7 +8,7 @@ namespace UTJ.FrameCapturer
     [Serializable]
     public class AudioEncoderConfigs
     {
-        public AudioEncoder.Type format = AudioEncoder.Type.Wave;
+        public AudioEncoder.Type format = AudioEncoder.Type.Flac;
         public fcAPI.fcWaveConfig waveEncoderSettings = fcAPI.fcWaveConfig.default_value;
         public fcAPI.fcOggConfig oggEncoderSettings = fcAPI.fcOggConfig.default_value;
         public fcAPI.fcFlacConfig flacEncoderSettings = fcAPI.fcFlacConfig.default_value;

FrameCapturer/Assets/UTJ/FrameCapturer/Scripts/Encoder/WebMEncoder.cs

@@ -14,7 +14,6 @@ public class WebMEncoder : MovieEncoder
         public override void Initialize(object config, string outPath)
         {
             m_config = (fcAPI.fcWebMConfig)config;
-            m_config.videoTargetFramerate = 60;
             m_config.audioSampleRate = AudioSettings.outputSampleRate;
             m_config.audioNumChannels = fcAPI.fcGetNumAudioChannels();
             m_ctx = fcAPI.fcWebMCreateContext(ref m_config);

Plugin/fccore/Encoder/Audio/fcOggContext.cpp

@@ -10,24 +10,6 @@
 #endif
 #include "vorbis/vorbisenc.h"
 
-class fcOggWriter
-{
-public:
-    fcOggWriter(fcStream *s);
-    ~fcOggWriter();
-    void write(ogg_packet& packet);
-    void flush();
-    void pageOut();
-
-private:
-    fcStream *m_stream = nullptr;
-    ogg_stream_state m_ogstream;
-    ogg_page         m_ogpage;
-    ogg_packet       m_ogpacket;
-
-};
-using fcOggWriterPtr = std::unique_ptr<fcOggWriter>;
-
 
 class fcOggContext : public fcIOggContext
 {
@@ -45,63 +27,20 @@ class fcOggContext : public fcIOggContext
 
 private:
     fcOggConfig m_conf;
-    std::vector<fcOggWriterPtr> m_writers;
+    std::vector<fcStream*> m_streams;
 
     TaskQueue           m_tasks;
     AudioBuffers        m_buffers;
+    Buffer              m_header_data;
 
     vorbis_info         m_vo_info;
     vorbis_comment      m_vo_comment;
     vorbis_dsp_state    m_vo_dsp;
     vorbis_block        m_vo_block;
-    ogg_packet          m_og_header;
-    ogg_packet          m_og_header_comm;
-    ogg_packet          m_og_header_code;
-};
 
-
-
-fcOggWriter::fcOggWriter(fcStream *s)
-    : m_stream(s)
-{
-    static int s_serial = 0;
-    m_stream->addRef();
-    ogg_stream_init(&m_ogstream, s_serial++);
-}
-
-fcOggWriter::~fcOggWriter()
-{
-    ogg_stream_clear(&m_ogstream);
-    m_stream->release();
-}
-
-void fcOggWriter::write(ogg_packet& packet)
-{
-    ogg_stream_packetin(&m_ogstream, &packet);
-}
-
-void fcOggWriter::flush()
-{
-    for (;;) {
-        int result = ogg_stream_flush(&m_ogstream, &m_ogpage);
-        if (result == 0)break;
-        m_stream->write(m_ogpage.header, m_ogpage.header_len);
-        m_stream->write(m_ogpage.body, m_ogpage.body_len);
-    }
-}
-
-void fcOggWriter::pageOut()
-{
-    bool eos = false;
-    while (!eos) {
-        int result = ogg_stream_pageout(&m_ogstream, &m_ogpage);
-        if (result == 0)break;
-        m_stream->write(m_ogpage.header, m_ogpage.header_len);
-        m_stream->write(m_ogpage.body, m_ogpage.body_len);
-
-        if (ogg_page_eos(&m_ogpage))eos = true;
-    }
-}
+    ogg_stream_state    m_ogstream;
+    ogg_page            m_ogpage;
+};
 
 
 
@@ -122,7 +61,25 @@ fcOggContext::fcOggContext(const fcOggConfig& conf)
     vorbis_comment_init(&m_vo_comment);
     vorbis_analysis_init(&m_vo_dsp, &m_vo_info);
     vorbis_block_init(&m_vo_dsp, &m_vo_block);
-    vorbis_analysis_headerout(&m_vo_dsp, &m_vo_comment, &m_og_header, &m_og_header_comm, &m_og_header_code);
+
+    static int s_serial = 0;
+    ogg_stream_init(&m_ogstream, ++s_serial);
+
+    ogg_packet og_header, og_header_comm, og_header_code;
+    vorbis_analysis_headerout(&m_vo_dsp, &m_vo_comment, &og_header, &og_header_comm, &og_header_code);
+    ogg_stream_packetin(&m_ogstream, &og_header);
+    ogg_stream_packetin(&m_ogstream, &og_header_comm);
+    ogg_stream_packetin(&m_ogstream, &og_header_code);
+    {
+        // make ogg header data
+        BufferStream hs(m_header_data);
+        for (;;) {
+            int result = ogg_stream_flush(&m_ogstream, &m_ogpage);
+            if (result == 0) break;
+            hs.write(m_ogpage.header, m_ogpage.header_len);
+            hs.write(m_ogpage.body, m_ogpage.body_len);
+        }
+    }
 
     for (int i = 0; i < m_conf.max_tasks; ++i) {
         m_buffers.emplace();
@@ -138,8 +95,9 @@ fcOggContext::~fcOggContext()
         }
     });
     m_tasks.wait();
-    m_writers.clear();
+    for (auto s : m_streams) { s->release(); }
 
+    ogg_stream_clear(&m_ogstream);
     vorbis_block_clear(&m_vo_block);
     vorbis_dsp_clear(&m_vo_dsp);
     vorbis_comment_clear(&m_vo_comment);
@@ -148,12 +106,10 @@ fcOggContext::~fcOggContext()
 
 void fcOggContext::addOutputStream(fcStream *s)
 {
-    auto writer = new fcOggWriter(s);
-    writer->write(m_og_header);
-    writer->write(m_og_header_comm);
-    writer->write(m_og_header_code);
-    writer->flush();
-    m_writers.emplace_back(writer);
+    if (!s) { return; }
+    s->addRef();
+    m_streams.push_back(s);
+    s->write(m_header_data.data(), m_header_data.size());
 }
 
 bool fcOggContext::addSamples(const float *samples, int num_samples)
@@ -191,9 +147,15 @@ void fcOggContext::pageOut()
 
         ogg_packet packet;
         while (vorbis_bitrate_flushpacket(&m_vo_dsp, &packet) == 1) {
-            for (auto& w : m_writers) {
-                w->write(packet);
-                w->pageOut();
+            ogg_stream_packetin(&m_ogstream, &packet);
+            for (;;) {
+                int result = ogg_stream_pageout(&m_ogstream, &m_ogpage);
+                if (result == 0) { break; }
+                for (auto& w : m_streams) {
+                    w->write(m_ogpage.header, m_ogpage.header_len);
+                    w->write(m_ogpage.body, m_ogpage.body_len);
+                }
+                if (ogg_page_eos(&m_ogpage)) { break; }
             }
         }
     }

Plugin/fccore/Encoder/WebM/fcVorbisEncoder.cpp

@@ -37,7 +37,6 @@ class fcVorbisEncoder : public fcIWebMAudioEncoder
 };
 
 
-
 fcVorbisEncoder::fcVorbisEncoder(const fcVorbisEncoderConfig& conf)
     : m_conf(conf)
 {
@@ -54,22 +53,22 @@ fcVorbisEncoder::fcVorbisEncoder(const fcVorbisEncoderConfig& conf)
     vorbis_analysis_init(&m_vo_dsp, &m_vo_info);
     vorbis_block_init(&m_vo_dsp, &m_vo_block);
 
-
     {
         // get codec private data
-        ogg_packet ident, comment, setup;
-        vorbis_analysis_headerout(&m_vo_dsp, &m_vo_comment, &ident, &comment, &setup);
+        ogg_packet header, comment, code;
+        vorbis_analysis_headerout(&m_vo_dsp, &m_vo_comment, &header, &comment, &code);
 
-        m_codec_private.resize(ident.bytes + comment.bytes + setup.bytes + 3);
+        m_codec_private.resize(header.bytes + comment.bytes + code.bytes + 3);
         auto* it = (uint8_t*)m_codec_private.data();
         *it++ = 2;
-        *it++ = (uint8_t)(ident.bytes);
+        *it++ = (uint8_t)(header.bytes);
         *it++ = (uint8_t)(comment.bytes);
-        memcpy(it, ident.packet, ident.bytes);
-        it += ident.bytes;
+
+        memcpy(it, header.packet, header.bytes);
+        it += header.bytes;
         memcpy(it, comment.packet, comment.bytes);
         it += comment.bytes;
-        memcpy(it, setup.packet, setup.bytes);
+        memcpy(it, code.packet, code.bytes);
     }
 }
 

Plugin/fccore/Encoder/WebM/fcWebMContext.cpp

@@ -129,11 +129,7 @@ fcWebMContext::~fcWebMContext()
 void fcWebMContext::addOutputStream(fcStream *s)
 {
     if (!s) { return; }
-
-    auto *writer = fcCreateWebMWriter(s, m_conf);
-    if (m_video_encoder) { writer->setVideoEncoderInfo(*m_video_encoder); }
-    if (m_audio_encoder) { writer->setAudioEncoderInfo(*m_audio_encoder); }
-    m_writers.emplace_back(writer);
+    m_writers.emplace_back(fcCreateWebMWriter(s, m_conf, m_video_encoder.get(), m_audio_encoder.get()));
 }
 
 
@@ -211,7 +207,7 @@ bool fcWebMContext::addAudioSamples(const float *samples, int num_samples)
     m_audio_tasks.run([this, buf]() {
         if (m_audio_encoder->encode(m_audio_frame, buf->data(), buf->size())) {
             eachStreams([&](fcIWebMWriter& writer) {
-                writer.AddAudioSamples(m_audio_frame);
+                writer.addAudioSamples(m_audio_frame);
             });
             m_audio_frame.clear();
         }
@@ -226,7 +222,7 @@ void fcWebMContext::flushAudio()
     m_audio_tasks.run([this]() {
         if (m_audio_encoder->flush(m_audio_frame)) {
             eachStreams([&](fcIWebMWriter& writer) {
-                writer.AddAudioSamples(m_audio_frame);
+                writer.addAudioSamples(m_audio_frame);
             });
             m_audio_frame.clear();
         }

Plugin/fccore/Encoder/WebM/fcWebMInternal.h

@@ -50,13 +50,15 @@ class fcIWebMEncoderInfo
 class fcIWebMVideoEncoder : public fcIWebMEncoderInfo
 {
 public:
+    virtual ~fcIWebMVideoEncoder() {}
     virtual bool encode(fcWebMFrameData& dst, const void *image, fcPixelFormat fmt, fcTime timestamp, bool force_keyframe = false) = 0;
     virtual bool flush(fcWebMFrameData& dst) = 0;
 };
 
 class fcIWebMAudioEncoder : public fcIWebMEncoderInfo
 {
 public:
+    virtual ~fcIWebMAudioEncoder() {}
     virtual bool encode(fcWebMFrameData& dst, const float *samples, size_t num_samples) = 0;
     virtual bool flush(fcWebMFrameData& dst) = 0;
 };

Plugin/fccore/Encoder/WebM/fcWebMWriter.cpp

@@ -29,13 +29,10 @@ class fcMkvStream : public mkvmuxer::IMkvWriter
 class fcWebMWriter : public fcIWebMWriter
 {
 public:
-    fcWebMWriter(BinaryStream *stream, const fcWebMConfig &conf);
+    fcWebMWriter(BinaryStream *stream, const fcWebMConfig &conf, const fcIWebMEncoderInfo* vinfo, const fcIWebMEncoderInfo* ainfo);
     ~fcWebMWriter() override;
-    void setVideoEncoderInfo(const fcIWebMEncoderInfo& info) override;
-    void setAudioEncoderInfo(const fcIWebMEncoderInfo& info) override;
-
     void addVideoFrame(const fcWebMFrameData& buf) override;
-    void AddAudioSamples(const fcWebMFrameData& buf) override;
+    void addAudioSamples(const fcWebMFrameData& buf) override;
 
     void writeOut(double timestamp);
 
@@ -57,20 +54,47 @@ class fcWebMWriter : public fcIWebMWriter
 };
 
 
-fcWebMWriter::fcWebMWriter(BinaryStream *stream, const fcWebMConfig &conf)
+fcWebMWriter::fcWebMWriter(
+    BinaryStream *stream, const fcWebMConfig &conf,
+    const fcIWebMEncoderInfo* vinfo, const fcIWebMEncoderInfo* ainfo)
     : m_conf(conf)
     , m_stream(new fcMkvStream(stream))
 {
     m_segment.Init(m_stream.get());
     m_segment.set_mode(mkvmuxer::Segment::kFile);
     m_segment.set_estimate_file_duration(true);
-    if (conf.video) {
+    if (conf.video && vinfo) {
         m_video_track_id = m_segment.AddVideoTrack(conf.video_width, conf.video_height, 0);
+        auto track = dynamic_cast<mkvmuxer::VideoTrack*>(m_segment.GetTrackByNumber(m_video_track_id));
+        track->set_codec_id(vinfo->getMatroskaCodecID());
+
+        mkvmuxer::Colour color;
+        color.set_bits_per_channel(10);
+        const uint64_t horizontal_subsampling = 1;
+        const uint64_t vertical_subsampling = 1;
+        color.set_chroma_subsampling_horz(horizontal_subsampling);
+        color.set_chroma_subsampling_vert(vertical_subsampling);
+        track->SetColour(color);
+        track->set_display_width(conf.video_width);
+        track->set_display_height(conf.video_height);
+        track->set_frame_rate(conf.video_target_framerate);
+
+        m_segment.CuesTrack(m_video_track_id);
     }
-    if(conf.audio) {
+    if (conf.audio && ainfo) {
         m_audio_track_id = m_segment.AddAudioTrack(conf.audio_sample_rate, conf.audio_num_channels, 0);
-    }
+        auto track = dynamic_cast<mkvmuxer::AudioTrack*>(m_segment.GetTrackByNumber(m_audio_track_id));
+        track->set_codec_id(ainfo->getMatroskaCodecID());
+
+        const auto& cp = ainfo->getCodecPrivate();
+        if (!cp.empty()) {
+            track->SetCodecPrivate((const uint8_t*)cp.data(), cp.size());
+        }
 
+        if (!conf.video) {
+            m_segment.CuesTrack(m_audio_track_id);
+        }
+    }
     auto info = m_segment.GetSegmentInfo();
     info->set_writing_app("Unity Movie Recorder");
 }
@@ -83,25 +107,6 @@ fcWebMWriter::~fcWebMWriter()
     m_segment.Finalize();
 }
 
-void fcWebMWriter::setVideoEncoderInfo(const fcIWebMEncoderInfo& info)
-{
-    auto track = dynamic_cast<mkvmuxer::VideoTrack*>(m_segment.GetTrackByNumber(m_video_track_id));
-    if (track) {
-        track->set_codec_id(info.getMatroskaCodecID());
-    }
-}
-
-void fcWebMWriter::setAudioEncoderInfo(const fcIWebMEncoderInfo& info)
-{
-    auto track = dynamic_cast<mkvmuxer::AudioTrack*>(m_segment.GetTrackByNumber(m_audio_track_id));
-    if (track) {
-        track->set_codec_id(info.getMatroskaCodecID());
-
-        const auto& cp = info.getCodecPrivate();
-        track->SetCodecPrivate((const uint8_t*)cp.data(), cp.size());
-    }
-}
-
 void fcWebMWriter::addVideoFrame(const fcWebMFrameData& frame)
 {
     if (m_video_track_id == 0 || frame.data.empty()) { return; }
@@ -119,7 +124,7 @@ void fcWebMWriter::addVideoFrame(const fcWebMFrameData& frame)
     writeOut(m_timestamp_video_last - 1.0);
 }
 
-void fcWebMWriter::AddAudioSamples(const fcWebMFrameData& frame)
+void fcWebMWriter::addAudioSamples(const fcWebMFrameData& frame)
 {
     if (m_audio_track_id == 0 || frame.data.empty()) { return; }
 
@@ -163,9 +168,10 @@ void fcWebMWriter::writeOut(double timestamp_)
 }
 
 
-fcIWebMWriter* fcCreateWebMWriter(BinaryStream *stream, const fcWebMConfig &conf)
+fcIWebMWriter* fcCreateWebMWriter(
+    BinaryStream *stream, const fcWebMConfig &conf, const fcIWebMEncoderInfo* vinfo, const fcIWebMEncoderInfo* ainfo)
 {
-    return new fcWebMWriter(stream, conf);
+    return new fcWebMWriter(stream, conf, vinfo, ainfo);
 }
 
 #endif // fcSupportWebM