feat: route audio from CEF

commit 7e2f134
Author: Julian Waller <[email protected]>
Date:   Fri Jan 5 16:36:53 2024 +0000

    chore: format

commit 91e9954
Author: Julian Waller <[email protected]>
Date:   Fri Jan 5 16:34:23 2024 +0000

    wip: rework frame flow

commit 0ce2bfd
Author: Julian Waller <[email protected]>
Date:   Fri Jan 5 16:11:02 2024 +0000

    wip: inspired by #16

src/modules/ffmpeg/CMakeLists.txt

@@ -5,6 +5,7 @@ set(SOURCES
 	producer/av_producer.cpp
 	producer/av_input.cpp
 	util/av_util.cpp
+	util/audio_resampler.cpp
 	producer/ffmpeg_producer.cpp
 	consumer/ffmpeg_consumer.cpp
 
@@ -15,6 +16,7 @@ set(HEADERS
 	producer/av_producer.h
 	producer/av_input.h
 	util/av_util.h
+	util/audio_resampler.h
 	producer/ffmpeg_producer.h
 	consumer/ffmpeg_consumer.h
 

src/modules/ffmpeg/util/audio_resampler.cpp

@@ -0,0 +1,38 @@
+#include "audio_resampler.h"
+#include "av_assert.h"
+
+extern "C" {
+#include <libavutil/samplefmt.h>
+#include <libswresample/swresample.h>
+}
+
+namespace caspar::ffmpeg {
+
+AudioResampler::AudioResampler(int64_t sample_rate, AVSampleFormat in_sample_fmt)
+    : ctx(std::shared_ptr<SwrContext>(swr_alloc_set_opts(nullptr,
+                                                         AV_CH_LAYOUT_7POINT1,
+                                                         AV_SAMPLE_FMT_S32,
+                                                         sample_rate,
+                                                         AV_CH_LAYOUT_7POINT1,
+                                                         in_sample_fmt,
+                                                         sample_rate,
+                                                         0,
+                                                         nullptr),
+                                      [](SwrContext* ptr) { swr_free(&ptr); }))
+{
+    if (!ctx)
+        FF_RET(AVERROR(ENOMEM), "swr_alloc_set_opts");
+
+    FF_RET(swr_init(ctx.get()), "swr_init");
+}
+
+caspar::array<int32_t> AudioResampler::convert(int frames, const void** src)
+{
+    auto result = caspar::array<int32_t>(frames * 8 * sizeof(int32_t));
+    auto ptr    = result.data();
+    auto ret    = swr_convert(ctx.get(), (uint8_t**)&ptr, frames, reinterpret_cast<const uint8_t**>(src), frames);
+
+    return result;
+}
+
+}; // namespace caspar::ffmpeg

src/modules/ffmpeg/util/audio_resampler.h

@@ -0,0 +1,27 @@
+#include <common/array.h>
+#include <memory>
+
+#pragma once
+
+extern "C" {
+#include <libavutil/samplefmt.h>
+}
+
+struct SwrContext;
+
+namespace caspar::ffmpeg {
+
+class AudioResampler
+{
+    std::shared_ptr<SwrContext> ctx;
+
+  public:
+    AudioResampler(int64_t sample_rate, AVSampleFormat in_sample_fmt);
+
+    AudioResampler(const AudioResampler&)            = delete;
+    AudioResampler& operator=(const AudioResampler&) = delete;
+
+    caspar::array<int32_t> convert(int frames, const void** src);
+};
+
+}; // namespace caspar::ffmpeg

src/modules/html/CMakeLists.txt

@@ -25,7 +25,9 @@ target_include_directories(html PRIVATE
     ..
     ../..
     ${CEF_INCLUDE_PATH}
-    )
+	${FFMPEG_INCLUDE_PATH}
+)
+target_link_libraries(html ffmpeg)
 
 set_target_properties(html PROPERTIES FOLDER modules)
 source_group(sources\\producer producer/*)

src/modules/html/producer/html_producer.cpp

@@ -60,13 +60,83 @@
 #include <queue>
 #include <utility>
 
+#include <ffmpeg/util/audio_resampler.h>
+
 #include "../html.h"
 
 namespace caspar { namespace html {
 
+inline std::int_least64_t now()
+{
+    return std::chrono::duration_cast<std::chrono::milliseconds>(
+               std::chrono::high_resolution_clock::now().time_since_epoch())
+        .count();
+}
+
+struct presentation_frame
+{
+    std::int_least64_t timestamp = now();
+    core::draw_frame   frame     = core::draw_frame::empty();
+    bool               has_video = false;
+    bool               has_audio = false;
+
+    explicit presentation_frame(core::draw_frame video = {})
+    {
+        if (video) {
+            frame     = std::move(video);
+            has_video = true;
+        }
+    }
+
+    presentation_frame(presentation_frame&& other) noexcept
+        : timestamp(other.timestamp)
+        , frame(std::move(other.frame))
+    {
+    }
+
+    presentation_frame(const presentation_frame&)            = delete;
+    presentation_frame& operator=(const presentation_frame&) = delete;
+
+    presentation_frame& operator=(presentation_frame&& rhs)
+    {
+        timestamp = rhs.timestamp;
+        frame     = std::move(rhs.frame);
+        return *this;
+    }
+
+    ~presentation_frame() {}
+
+    void add_audio(core::mutable_frame audio)
+    {
+        if (has_audio)
+            return;
+        has_audio = true;
+
+        if (frame) {
+            frame = core::draw_frame::over(frame, core::draw_frame(std::move(audio)));
+        } else {
+            frame = core::draw_frame(std::move(audio));
+        }
+    }
+
+    void add_video(core::draw_frame video)
+    {
+        if (has_video)
+            return;
+        has_video = true;
+
+        if (frame) {
+            frame = core::draw_frame::over(frame, std::move(video));
+        } else {
+            frame = std::move(video);
+        }
+    }
+};
+
 class html_client
     : public CefClient
     , public CefRenderHandler
+    , public CefAudioHandler
     , public CefLifeSpanHandler
     , public CefLoadHandler
     , public CefDisplayHandler
@@ -80,15 +150,18 @@ class html_client
     caspar::timer                       paint_timer_;
     caspar::timer                       test_timer_;
 
-    spl::shared_ptr<core::frame_factory>                        frame_factory_;
-    core::video_format_desc                                     format_desc_;
-    bool                                                        gpu_enabled_;
-    tbb::concurrent_queue<std::wstring>                         javascript_before_load_;
-    std::atomic<bool>                                           loaded_;
-    std::queue<std::pair<std::int_least64_t, core::draw_frame>> frames_;
-    mutable std::mutex                                          frames_mutex_;
-    const size_t                                                frames_max_size_ = 4;
-    std::atomic<bool>                                           closing_;
+    spl::shared_ptr<core::frame_factory> frame_factory_;
+    core::video_format_desc              format_desc_;
+    bool                                 gpu_enabled_;
+    tbb::concurrent_queue<std::wstring>  javascript_before_load_;
+    std::atomic<bool>                    loaded_;
+    std::queue<presentation_frame>       frames_;
+    core::draw_frame                     last_generated_frame_;
+    mutable std::mutex                   frames_mutex_;
+    const size_t                         frames_max_size_ = 4;
+    std::atomic<bool>                    closing_;
+
+    std::unique_ptr<ffmpeg::AudioResampler> audioResampler_;
 
     core::draw_frame   last_frame_;
     std::int_least64_t last_frame_time_;
@@ -167,15 +240,15 @@ class html_client
 
                 // Check if the sole buffered frame is too young to have a partner field generated (with a tolerance)
                 auto time_per_frame           = (1000 * 1.5) / format_desc_.fps;
-                auto front_frame_is_too_young = (now_time - frames_.front().first) < time_per_frame;
+                auto front_frame_is_too_young = (now_time - frames_.front().timestamp) < time_per_frame;
 
                 if (follows_gap_in_frames && front_frame_is_too_young) {
                     return false;
                 }
             }
 
-            last_frame_time_ = frames_.front().first;
-            last_frame_      = std::move(frames_.front().second);
+            last_frame_time_ = frames_.front().timestamp;
+            last_frame_      = std::move(frames_.front().frame);
             frames_.pop();
 
             graph_->set_value("buffered-frames", (double)frames_.size() / frames_max_size_);
@@ -190,12 +263,13 @@ class html_client
     {
         if (!try_pop(field)) {
             graph_->set_tag(diagnostics::tag_severity::SILENT, "late-frame");
+            return core::draw_frame::still(last_frame_);
+        } else {
+            return last_frame_;
         }
-
-        return last_frame_;
     }
 
-    core::draw_frame last_frame() const { return last_frame_; }
+    core::draw_frame last_frame() const { return core::draw_frame::still(last_frame_); }
 
     bool is_ready() const
     {
@@ -245,13 +319,6 @@ class html_client
     }
 
   private:
-    std::int_least64_t now()
-    {
-        return std::chrono::duration_cast<std::chrono::milliseconds>(
-                   std::chrono::high_resolution_clock::now().time_since_epoch())
-            .count();
-    }
-
     void GetViewRect(CefRefPtr<CefBrowser> browser, CefRect& rect) override
     {
         CASPAR_ASSERT(CefCurrentlyOn(TID_UI));
@@ -302,7 +369,10 @@ class html_client
         {
             std::lock_guard<std::mutex> lock(frames_mutex_);
 
-            frames_.push(std::make_pair(now(), core::draw_frame(std::move(frame))));
+            core::draw_frame new_frame = core::draw_frame(std::move(frame));
+            last_generated_frame_      = new_frame;
+
+            frames_.push(presentation_frame(std::move(new_frame)));
             while (frames_.size() > 4) {
                 frames_.pop();
                 graph_->set_tag(diagnostics::tag_severity::WARNING, "dropped-frame");
@@ -353,6 +423,8 @@ class html_client
 
     CefRefPtr<CefRenderHandler> GetRenderHandler() override { return this; }
 
+    CefRefPtr<CefAudioHandler> GetAudioHandler() override { return this; }
+
     CefRefPtr<CefLifeSpanHandler> GetLifeSpanHandler() override { return this; }
 
     CefRefPtr<CefLoadHandler> GetLoadHandler() override { return this; }
@@ -378,7 +450,7 @@ class html_client
 
             {
                 std::lock_guard<std::mutex> lock(frames_mutex_);
-                frames_.push(std::make_pair(now(), core::draw_frame::empty()));
+                frames_.push(presentation_frame());
             }
 
             {
@@ -399,6 +471,51 @@ class html_client
         return false;
     }
 
+    bool GetAudioParameters(CefRefPtr<CefBrowser> browser, CefAudioParameters& params) override
+    {
+        params.channel_layout    = CEF_CHANNEL_LAYOUT_7_1;
+        params.sample_rate       = format_desc_.audio_sample_rate;
+        params.frames_per_buffer = format_desc_.audio_cadence[0];
+        return format_desc_.audio_cadence.size() == 1; // TODO - handle 59.94
+    }
+
+    void OnAudioStreamStarted(CefRefPtr<CefBrowser> browser, const CefAudioParameters& params, int channels) override
+    {
+        audioResampler_ = std::make_unique<ffmpeg::AudioResampler>(params.sample_rate, AV_SAMPLE_FMT_FLTP);
+    }
+    void OnAudioStreamPacket(CefRefPtr<CefBrowser> browser, const float** data, int samples, int64_t pts) override
+    {
+        if (!audioResampler_)
+            return;
+
+        auto audio       = audioResampler_->convert(samples, reinterpret_cast<const void**>(data));
+        auto audio_frame = core::mutable_frame(this, {}, std::move(audio), core::pixel_format_desc());
+
+        {
+            std::lock_guard<std::mutex> lock(frames_mutex_);
+            if (frames_.empty()) {
+                presentation_frame wrapped_frame(last_generated_frame_);
+                wrapped_frame.add_audio(std::move(audio_frame));
+
+                frames_.push(std::move(wrapped_frame));
+            } else {
+                if (!frames_.back().has_audio) {
+                    frames_.back().add_audio(std::move(audio_frame));
+                } else {
+                    presentation_frame wrapped_frame(last_generated_frame_);
+                    wrapped_frame.add_audio(std::move(audio_frame));
+                    frames_.push(std::move(wrapped_frame));
+                }
+            }
+        }
+    }
+    void OnAudioStreamStopped(CefRefPtr<CefBrowser> browser) override { audioResampler_ = nullptr; }
+    void OnAudioStreamError(CefRefPtr<CefBrowser> browser, const CefString& message) override
+    {
+        CASPAR_LOG(info) << "[html_producer] OnAudioStreamError: \"" << message.ToString() << "\"";
+        audioResampler_ = nullptr;
+    }
+
     void do_execute_javascript(const std::wstring& javascript)
     {
         html::begin_invoke([=] {