From d65539010499bb98ca73c0a99b958d702f60a6ec Mon Sep 17 00:00:00 2001 From: Niklas Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Wed, 4 Oct 2023 06:51:17 +0000 Subject: [PATCH] capture audio from CEF --- src/core/frame/frame.cpp | 1 + src/core/frame/frame.h | 3 +- src/modules/ffmpeg/CMakeLists.txt | 2 + src/modules/ffmpeg/util/audio_resampler.cpp | 38 ++++ src/modules/ffmpeg/util/audio_resampler.h | 25 +++ src/modules/html/CMakeLists.txt | 3 +- src/modules/html/producer/html_producer.cpp | 183 ++++++++++++++------ 7 files changed, 197 insertions(+), 58 deletions(-) create mode 100644 src/modules/ffmpeg/util/audio_resampler.cpp create mode 100644 src/modules/ffmpeg/util/audio_resampler.h diff --git a/src/core/frame/frame.cpp b/src/core/frame/frame.cpp index b9da23fece..8634b10d25 100644 --- a/src/core/frame/frame.cpp +++ b/src/core/frame/frame.cpp @@ -82,6 +82,7 @@ const array& mutable_frame::image_data(std::size_t index) const { const array& mutable_frame::audio_data() const { return impl_->audio_data_; } array& mutable_frame::image_data(std::size_t index) { return impl_->image_data_.at(index); } array& mutable_frame::audio_data() { return impl_->audio_data_; } +void mutable_frame::set_audio_data(caspar::array&& audio) { impl_->audio_data_ = std::move(audio); } std::size_t mutable_frame::width() const { return impl_->desc_.planes.at(0).width; } std::size_t mutable_frame::height() const { return impl_->desc_.planes.at(0).height; } const frame_geometry& mutable_frame::geometry() const { return impl_->geometry_; } diff --git a/src/core/frame/frame.h b/src/core/frame/frame.h index 2f166a3878..49818d50bf 100644 --- a/src/core/frame/frame.h +++ b/src/core/frame/frame.h @@ -30,7 +30,7 @@ class mutable_frame final ~mutable_frame(); mutable_frame& operator=(const mutable_frame&) = delete; - mutable_frame& operator =(mutable_frame&& other); + mutable_frame& operator=(mutable_frame&& other); void swap(mutable_frame& other); @@ -41,6 +41,7 @@ class mutable_frame final array& audio_data(); const array& audio_data() const; + void set_audio_data(caspar::array&& audio); std::size_t width() const; diff --git a/src/modules/ffmpeg/CMakeLists.txt b/src/modules/ffmpeg/CMakeLists.txt index aed946c268..2c05a01ab5 100644 --- a/src/modules/ffmpeg/CMakeLists.txt +++ b/src/modules/ffmpeg/CMakeLists.txt @@ -5,6 +5,7 @@ set(SOURCES producer/av_producer.cpp producer/av_input.cpp util/av_util.cpp + util/audio_resampler.cpp producer/ffmpeg_producer.cpp consumer/ffmpeg_consumer.cpp ffmpeg.cpp @@ -14,6 +15,7 @@ set(HEADERS producer/av_producer.h producer/av_input.h util/av_util.h + util/audio_resampler.h producer/ffmpeg_producer.h consumer/ffmpeg_consumer.h ffmpeg.h diff --git a/src/modules/ffmpeg/util/audio_resampler.cpp b/src/modules/ffmpeg/util/audio_resampler.cpp new file mode 100644 index 0000000000..823685499f --- /dev/null +++ b/src/modules/ffmpeg/util/audio_resampler.cpp @@ -0,0 +1,38 @@ +#include "audio_resampler.h" +#include "av_assert.h" + +extern "C" { +#include +#include +} + +namespace caspar { namespace ffmpeg { + +AudioResampler::AudioResampler(int64_t sample_rate, AVSampleFormat in_sample_fmt) + : ctx(std::shared_ptr(swr_alloc_set_opts(nullptr, + AV_CH_LAYOUT_7POINT1, + AV_SAMPLE_FMT_S32, + sample_rate, + AV_CH_LAYOUT_7POINT1, + in_sample_fmt, + sample_rate, + 0, + nullptr), + [](SwrContext* ptr) { swr_free(&ptr); })) +{ + if (!ctx) + FF_RET(AVERROR(ENOMEM), "swr_alloc_set_opts"); + + FF_RET(swr_init(ctx.get()), "swr_init"); +} + +caspar::array AudioResampler::convert(int frames, const void** src) +{ + auto result = caspar::array(frames * 8 * sizeof(int32_t)); + auto ptr = result.data(); + auto ret = swr_convert(ctx.get(), (uint8_t**)&ptr, frames, reinterpret_cast(src), frames); + + return result; +} + +}}; // namespace caspar::ffmpeg \ No newline at end of file diff --git a/src/modules/ffmpeg/util/audio_resampler.h b/src/modules/ffmpeg/util/audio_resampler.h new file mode 100644 index 0000000000..2712dee1f8 --- /dev/null +++ b/src/modules/ffmpeg/util/audio_resampler.h @@ -0,0 +1,25 @@ +#include +#include + +extern "C" { +#include +} + +struct SwrContext; + +namespace caspar { namespace ffmpeg { + +class AudioResampler +{ + std::shared_ptr ctx; + + public: + AudioResampler(int64_t sample_rate, AVSampleFormat in_sample_fmt); + + AudioResampler(const AudioResampler&) = delete; + AudioResampler& operator=(const AudioResampler&) = delete; + + caspar::array convert(int frames, const void** src); +}; + +}}; // namespace caspar::ffmpeg \ No newline at end of file diff --git a/src/modules/html/CMakeLists.txt b/src/modules/html/CMakeLists.txt index fc07ca2cfa..289c856ddd 100644 --- a/src/modules/html/CMakeLists.txt +++ b/src/modules/html/CMakeLists.txt @@ -21,6 +21,7 @@ include_directories(../..) include_directories(${BOOST_INCLUDE_PATH}) include_directories(${TBB_INCLUDE_PATH}) include_directories(${CEF_INCLUDE_PATH}) +include_directories(${FFMPEG_INCLUDE_PATH}) set_target_properties(html PROPERTIES FOLDER modules) source_group(sources\\producer producer/*) @@ -29,7 +30,7 @@ source_group(sources ./*) target_link_libraries(html common core - + ffmpeg ${CEF_BIN_PATH}/libcef.so ${CEF_BIN_PATH}/libcef_dll_wrapper.a ) diff --git a/src/modules/html/producer/html_producer.cpp b/src/modules/html/producer/html_producer.cpp index 016ba59b79..12cb696476 100644 --- a/src/modules/html/producer/html_producer.cpp +++ b/src/modules/html/producer/html_producer.cpp @@ -40,6 +40,8 @@ #include #include +#include + #include #include #include @@ -73,28 +75,24 @@ struct presentation_frame { core::mutable_frame frame; int64_t audio_pts; - int64_t video_pts; presentation_frame() : frame(core::mutable_frame(nullptr, std::vector>(), caspar::array(), core::pixel_format_desc())) - , video_pts(0) , audio_pts(0) { } - presentation_frame(core::mutable_frame&& frame, int64_t video_pts = 0) + presentation_frame(core::mutable_frame&& frame) : frame(std::move(frame)) - , video_pts(video_pts) , audio_pts(0) { } presentation_frame(presentation_frame&& other) : frame(std::move(other.frame)) - , video_pts(other.video_pts) , audio_pts(other.audio_pts) { } @@ -105,24 +103,58 @@ struct presentation_frame presentation_frame& operator=(presentation_frame&& rhs) { frame = std::move(rhs.frame); - video_pts = rhs.video_pts; audio_pts = rhs.audio_pts; return *this; } - presentation_frame clone_video(caspar::spl::shared_ptr frame_factory, void* tag) - { - auto new_frame = frame_factory->create_frame(tag, frame.pixel_format_desc()); - auto src = reinterpret_cast(frame.image_data(0).begin()); - auto dst = reinterpret_cast(new_frame.image_data(0).begin()); - std::memcpy(dst, src, new_frame.image_data(0).size()); + ~presentation_frame() {} - return presentation_frame(std::move(new_frame), video_pts); + void set_audio(caspar::array&& data, int64_t pts) + { + frame.set_audio_data(std::move(data)); + audio_pts = pts; + } + void set_video(core::mutable_frame&& data) + { + auto audio = std::move(frame.audio_data()); + frame = std::move(data); + set_audio(std::move(audio), audio_pts); } + bool has_audio() const { return audio_pts != 0; } + bool has_video() const { return frame.pixel_format_desc().format != core::pixel_format::invalid; } + bool is_empty() const { return !has_audio() && !has_video(); } +}; - ~presentation_frame() {} +struct video_frame_data +{ + int width; + int height; + caspar::array data; - bool is_empty() { return frame.pixel_format_desc().format == core::pixel_format::invalid; } + video_frame_data() = delete; + video_frame_data(int width, int height, const uint8_t* src) + : width(width) + , height(height) + , data(width * height * 4) + { + memcpy(data.begin(), src, width * height * 4); + } + video_frame_data(const video_frame_data& other) + : width(other.width) + , height(other.height) + , data(other.data.size()) + { + memcpy(data.begin(), other.data.begin(), other.data.size()); + } + + video_frame_data& operator=(const video_frame_data& other) + { + width = other.width; + height = other.height; + data = caspar::array(other.data.size()); + memcpy(data.begin(), other.data.begin(), other.data.size()); + return *this; + } }; class html_client @@ -133,13 +165,15 @@ class html_client , public CefLoadHandler , public CefDisplayHandler { - std::wstring url_; - spl::shared_ptr graph_; - core::monitor::state state_; - mutable std::mutex state_mutex_; - caspar::timer tick_timer_; - caspar::timer frame_timer_; - caspar::timer paint_timer_; + std::wstring url_; + spl::shared_ptr graph_; + core::monitor::state state_; + mutable std::mutex state_mutex_; + caspar::timer tick_timer_; + caspar::timer frame_timer_; + caspar::timer paint_timer_; + std::unique_ptr audioResampler_; + std::shared_ptr last_video_frame_; spl::shared_ptr frame_factory_; core::video_format_desc format_desc_; @@ -222,18 +256,18 @@ class html_client } } - bool OnBeforePopup(CefRefPtr browser, - CefRefPtr frame, - const CefString& target_url, - const CefString& target_frame_name, - cef_window_open_disposition_t target_disposition, - bool user_gesture, - const CefPopupFeatures& popupFeatures, - CefWindowInfo& windowInfo, - CefRefPtr& client, - CefBrowserSettings& settings, + bool OnBeforePopup(CefRefPtr browser, + CefRefPtr frame, + const CefString& target_url, + const CefString& target_frame_name, + cef_window_open_disposition_t target_disposition, + bool user_gesture, + const CefPopupFeatures& popupFeatures, + CefWindowInfo& windowInfo, + CefRefPtr& client, + CefBrowserSettings& settings, CefRefPtr& dict, - bool* no_javascript_access) override + bool* no_javascript_access) override { // This blocks popup windows from opening, as they dont make sense and hit an exception in get_browser_host upon // closing @@ -261,6 +295,21 @@ class html_client rect = CefRect(0, 0, format_desc_.square_width, format_desc_.square_height); } + core::mutable_frame create_filled_frame(int width, int height, const void* data) + { + core::pixel_format_desc pixel_desc; + pixel_desc.format = core::pixel_format::bgra; + pixel_desc.planes.push_back(core::pixel_format_desc::plane(width, height, 4)); + auto frame = frame_factory_->create_frame(this, pixel_desc); + auto dst = frame.image_data(0).begin(); + std::memcpy(dst, data, width * height * 4); + return frame; + } + core::mutable_frame create_filled_frame(const video_frame_data& video_frame) + { + return create_filled_frame(video_frame.width, video_frame.height, video_frame.data.begin()); + } + void OnPaint(CefRefPtr browser, PaintElementType type, const RectList& dirtyRects, @@ -278,19 +327,18 @@ class html_client if (type != PET_VIEW) return; - core::pixel_format_desc pixel_desc; - pixel_desc.format = core::pixel_format::bgra; - pixel_desc.planes.push_back(core::pixel_format_desc::plane(width, height, 4)); - - auto frame = frame_factory_->create_frame(this, pixel_desc); - auto src = (char*)buffer; - auto dst = reinterpret_cast(frame.image_data(0).begin()); - std::memcpy(dst, src, width * height * 4); + auto frame = create_filled_frame(width, height, buffer); + auto frame_data = std::make_shared(width, height, (const uint8_t*)buffer); { std::lock_guard lock(frames_mutex_); + last_video_frame_ = frame_data; + if (!frames_.empty() && !frames_.back().has_video()) { + frames_.back().set_video(std::move(frame)); + } else { + frames_.push(presentation_frame(std::move(frame))); + } - frames_.push(presentation_frame(std::move(frame))); while (frames_.size() > 8) { frames_.pop(); graph_->set_tag(diagnostics::tag_severity::WARNING, "dropped-frame"); @@ -387,23 +435,46 @@ class html_client return false; } - void OnAudioStreamStarted(CefRefPtr browser, const CefAudioParameters& params_, int channels_) + bool GetAudioParameters(CefRefPtr browser, CefAudioParameters& params) { + params.channel_layout = CEF_CHANNEL_LAYOUT_7_1; + params.sample_rate = format_desc_.audio_sample_rate; + params.frames_per_buffer = format_desc_.audio_cadence[0]; + return format_desc_.audio_cadence.size() == 1; } - void OnAudioStreamPacket(CefRefPtr browser, const float** data, int frames, int64_t pts) + + void OnAudioStreamStarted(CefRefPtr browser, const CefAudioParameters& params, int channels) { - const uint8_t** pcm = (const uint8_t**)data; - int speaker_count = 8; + audioResampler_ = std::make_unique(params.sample_rate, AV_SAMPLE_FMT_FLTP); } - void OnAudioStreamStopped(CefRefPtr browser) + void OnAudioStreamPacket(CefRefPtr browser, const float** data, int frames, int64_t pts) { + if (audioResampler_) { + auto audio = audioResampler_->convert(frames, reinterpret_cast(data)); + { + std::lock_guard lock(frames_mutex_); + if (frames_.empty()) { + if (last_video_frame_) { + frames_.push(presentation_frame(create_filled_frame(*last_video_frame_))); + } else + frames_.push(presentation_frame()); + } + + if (!frames_.back().has_audio()) { + frames_.back().set_audio(std::move(audio), pts); + } else { + auto frame = presentation_frame(create_filled_frame(*last_video_frame_)); + frame.set_audio(std::move(audio), pts); + frames_.push(std::move(frame)); + } + } + } } + void OnAudioStreamStopped(CefRefPtr browser) { audioResampler_ = nullptr; } void OnAudioStreamError(CefRefPtr browser, const CefString& message) { - } - bool GetAudioParameters(CefRefPtr browser, CefAudioParameters& params) - { - return false; + CASPAR_LOG(info) << "[html_producer] OnAudioStreamError: \"" << message.ToString() << "\""; + audioResampler_ = nullptr; } void invoke_requested_animation_frames() @@ -487,14 +558,14 @@ class html_producer : public core::frame_producer client_ = new html_client(frame_factory, graph_, format_desc, shared_texture_enable, url_); CefWindowInfo window_info; - window_info.bounds.width = format_desc.square_width; - window_info.bounds.height = format_desc.square_height; + window_info.bounds.width = format_desc.square_width; + window_info.bounds.height = format_desc.square_height; window_info.windowless_rendering_enabled = true; - window_info.shared_texture_enabled = shared_texture_enable; + window_info.shared_texture_enabled = shared_texture_enable; CefBrowserSettings browser_settings; - browser_settings.webgl = enable_gpu ? cef_state_t::STATE_ENABLED : cef_state_t::STATE_DISABLED; - double fps = format_desc.fps; + browser_settings.webgl = enable_gpu ? cef_state_t::STATE_ENABLED : cef_state_t::STATE_DISABLED; + double fps = format_desc.fps; browser_settings.windowless_frame_rate = int(ceil(fps)); CefBrowserHost::CreateBrowser(window_info, client_.get(), url, browser_settings, nullptr, nullptr); });