Skip to content

Commit

Permalink
Use v210 as 10bit pixel format on decklink
Browse files Browse the repository at this point in the history
  • Loading branch information
@niklaspandersson @deadbeef84
niklaspandersson authored and deadbeef84 committed Jun 5, 2024
1 parent 13b68c1 commit a918bf7
Show file tree
Hide file tree
Showing 4 changed files with 129 additions and 30 deletions.
100 changes: 83 additions & 17 deletions src/modules/decklink/consumer/decklink_consumer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,25 +222,47 @@ class decklink_frame
, public IDeckLinkVideoFrameMetadataExtensions
{
core::video_format_desc format_desc_;
BMDPixelFormat pix_fmt_;
std::shared_ptr<void> data_;
std::atomic<int> ref_count_{0};
int nb_samples_;
const bool hdr_;
core::color_space color_space_;
hdr_meta_configuration hdr_metadata_;
BMDFrameFlags flags_;
BMDPixelFormat pix_fmt_;

public:
decklink_frame(std::shared_ptr<void> data, core::video_format_desc format_desc, int nb_samples, bool hdr, core::color_space color_space, const hdr_meta_configuration& hdr_metadata)
decklink_frame(core::video_format_desc format_desc,
int nb_samples,
bool hdr,
core::color_space color_space,
const hdr_meta_configuration& hdr_metadata,
BMDPixelFormat pix_fmt,
std::shared_ptr<void> data)
: format_desc_(std::move(format_desc))
, pix_fmt_(pix_fmt)
, data_(std::move(data))
, nb_samples_(nb_samples)
, hdr_(hdr)
, color_space_(color_space)
, hdr_metadata_(hdr_metadata)
, flags_(hdr ? bmdFrameFlagDefault | bmdFrameContainsHDRMetadata : bmdFrameFlagDefault)
{
}

decklink_frame(core::video_format_desc format_desc,
int nb_samples,
bool hdr,
core::color_space color_space,
const hdr_meta_configuration& hdr_metadata)
: format_desc_(std::move(format_desc))
, pix_fmt_(get_pixel_format(hdr))
, data_(allocate_frame_data(format_desc, pix_fmt_))
, nb_samples_(nb_samples)
, hdr_(hdr)
, color_space_(color_space)
, hdr_metadata_(hdr_metadata)
, flags_(hdr ? bmdFrameFlagDefault | bmdFrameContainsHDRMetadata : bmdFrameFlagDefault)
{
}

Expand All @@ -255,7 +277,7 @@ class decklink_frame
#else
REFIID iunknown = IID_IUnknown;
#endif
HRESULT result = E_NOINTERFACE;
HRESULT result = E_NOINTERFACE;

if (ppv == nullptr)
return E_INVALIDARG;
Expand Down Expand Up @@ -292,9 +314,12 @@ class decklink_frame

// IDecklinkVideoFrame

long STDMETHODCALLTYPE GetWidth() override { return static_cast<long>(format_desc_.width); }
long STDMETHODCALLTYPE GetHeight() override { return static_cast<long>(format_desc_.height); }
long STDMETHODCALLTYPE GetRowBytes() override { return static_cast<long>(get_row_bytes(format_desc_, hdr_)); }
long STDMETHODCALLTYPE GetWidth() override { return static_cast<long>(format_desc_.width); }
long STDMETHODCALLTYPE GetHeight() override { return static_cast<long>(format_desc_.height); }
long STDMETHODCALLTYPE GetRowBytes() override
{
return static_cast<long>(get_row_bytes(pix_fmt_, format_desc_.width));
}
BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat() override { return pix_fmt_; }
BMDFrameFlags STDMETHODCALLTYPE GetFlags() override { return flags_; }

Expand Down Expand Up @@ -338,7 +363,7 @@ class decklink_frame
HRESULT STDMETHODCALLTYPE GetFloat(BMDDeckLinkFrameMetadataID metadataID, double* value)
{
const auto color_space = (color_space_ == core::color_space::bt2020) ? &REC_2020 : &REC_709;
HRESULT result = S_OK;
HRESULT result = S_OK;

switch (metadataID) {
case bmdDeckLinkFrameMetadataHDRDisplayPrimariesRedX:
Expand Down Expand Up @@ -438,7 +463,7 @@ struct decklink_secondary_port final : public IDeckLinkVideoOutputCallback
const core::video_format_desc decklink_format_desc_;
com_ptr<IDeckLinkDisplayMode> mode_ = get_display_mode(output_,
decklink_format_desc_.format,
get_pixel_format(config_.hdr),
config_.hdr ? bmdFormat10BitRGBXLE : bmdFormat8BitBGRA,
bmdSupportedVideoModeDefault,
config_.hdr);

Expand Down Expand Up @@ -555,7 +580,13 @@ struct decklink_secondary_port final : public IDeckLinkVideoOutputCallback
void schedule_next_video(std::shared_ptr<void> image_data, int nb_samples, BMDTimeValue display_time)
{
auto packed_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(
new decklink_frame(std::move(image_data), decklink_format_desc_, nb_samples, config_.hdr, core::color_space::bt709, config_.hdr_meta));
new decklink_frame(decklink_format_desc_,
nb_samples,
config_.hdr,
core::color_space::bt709,
config_.hdr_meta,
config_.hdr ? bmdFormat10BitRGBXLE : bmdFormat8BitBGRA,
std::move(image_data)));
if (FAILED(output_->ScheduleVideoFrame(get_raw(packed_frame),
display_time,
decklink_format_desc_.duration,
Expand Down Expand Up @@ -624,6 +655,8 @@ struct decklink_consumer final : public IDeckLinkVideoOutputCallback
bmdSupportedVideoModeDefault,
config_.hdr);

com_ptr<IDeckLinkVideoConversion> video_conversion_;

std::atomic<bool> abort_request_{false};

public:
Expand All @@ -640,6 +673,10 @@ struct decklink_consumer final : public IDeckLinkVideoOutputCallback
graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));

if (config_.hdr) {
video_conversion_ = create_video_converter();
}

if (config.duplex != configuration::duplex_t::default_duplex) {
set_duplex(iface_cast<IDeckLinkAttributes_v10_11>(decklink_),
iface_cast<IDeckLinkConfiguration_v10_11>(decklink_),
Expand Down Expand Up @@ -708,11 +745,12 @@ struct decklink_consumer final : public IDeckLinkVideoOutputCallback
nb_samples);
}

std::shared_ptr<void> image_data = allocate_frame_data(decklink_format_desc_, config_.hdr);
std::shared_ptr<void> rgb_image_data =
allocate_frame_data(decklink_format_desc_, config_.hdr ? bmdFormat10BitRGBXLE : bmdFormat8BitBGRA);

schedule_next_video(image_data, nb_samples, video_scheduled_, config_.hdr_meta.default_color_space);
schedule_next_video(rgb_image_data, nb_samples, video_scheduled_, config_.hdr_meta.default_color_space);
for (auto& context : secondary_port_contexts_) {
context->schedule_next_video(image_data, 0, video_scheduled_);
context->schedule_next_video(rgb_image_data, 0, video_scheduled_);
}

video_scheduled_ += decklink_format_desc_.duration;
Expand Down Expand Up @@ -932,7 +970,8 @@ struct decklink_consumer final : public IDeckLinkVideoOutputCallback
mode_->GetFieldDominance(),
config_.hdr);

schedule_next_video(image_data, nb_samples, video_display_time, frame1.pixel_format_desc().color_space);
schedule_next_video(
image_data, nb_samples, video_display_time, frame1.pixel_format_desc().color_space);

if (config_.embedded_audio) {
schedule_next_audio(std::move(audio_data), nb_samples);
Expand Down Expand Up @@ -992,12 +1031,39 @@ struct decklink_consumer final : public IDeckLinkVideoOutputCallback
audio_scheduled_ += nb_samples; // TODO - what if there are too many/few samples in this frame?
}

void schedule_next_video(std::shared_ptr<void> image_data, int nb_samples, BMDTimeValue display_time, core::color_space color_space)
void schedule_next_video(std::shared_ptr<void> image_data,
int nb_samples,
BMDTimeValue display_time,
core::color_space color_space)
{
auto fill_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(
new decklink_frame(std::move(image_data), decklink_format_desc_, nb_samples, config_.hdr, color_space, config_.hdr_meta));
auto rgb_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(
new decklink_frame(decklink_format_desc_,
nb_samples,
config_.hdr,
color_space,
config_.hdr_meta,
config_.hdr ? bmdFormat10BitRGBXLE : bmdFormat8BitBGRA,
std::move(image_data)));

if (config_.hdr) {
auto yuv_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(
new decklink_frame(decklink_format_desc_, nb_samples, config_.hdr, color_space, config_.hdr_meta));

if (FAILED(video_conversion_->ConvertFrame(get_raw(rgb_frame), get_raw(yuv_frame)))) {
CASPAR_LOG(warning) << print() << L" Failed to convert video frame.";
}

if (FAILED(output_->ScheduleVideoFrame(get_raw(yuv_frame),
display_time,
decklink_format_desc_.duration,
decklink_format_desc_.time_scale))) {
CASPAR_LOG(error) << print() << L" Failed to schedule primary video.";
}
return;
}

if (FAILED(output_->ScheduleVideoFrame(
get_raw(fill_frame), display_time, decklink_format_desc_.duration, decklink_format_desc_.time_scale))) {
get_raw(rgb_frame), display_time, decklink_format_desc_.duration, decklink_format_desc_.time_scale))) {
CASPAR_LOG(error) << print() << L" Failed to schedule primary video.";
}
}
Expand Down
36 changes: 25 additions & 11 deletions src/modules/decklink/consumer/frame.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,26 @@

namespace caspar { namespace decklink {

BMDPixelFormat get_pixel_format(bool hdr) { return hdr ? bmdFormat10BitRGBXLE : bmdFormat8BitBGRA; }
int get_row_bytes(const core::video_format_desc& format_desc, bool hdr)
BMDPixelFormat get_pixel_format(bool hdr) { return hdr ? bmdFormat10BitYUV : bmdFormat8BitBGRA; }

int get_row_bytes(BMDPixelFormat pix_fmt, int width)
{
return hdr ? ((format_desc.width + 63) / 64) * 256 : format_desc.width * 4;
switch (pix_fmt) {
case bmdFormat10BitYUV:
return ((width + 47) / 48) * 128;
case bmdFormat10BitRGBXLE:
return ((width + 63) / 64) * 256;
default:
break;
}

return width * 4;
}

std::shared_ptr<void> allocate_frame_data(const core::video_format_desc& format_desc, bool hdr)
std::shared_ptr<void> allocate_frame_data(const core::video_format_desc& format_desc, BMDPixelFormat pix_fmt)
{
auto alignment = hdr ? 256 : 64;
auto size = hdr ? get_row_bytes(format_desc, hdr) * format_desc.height : format_desc.size;
auto alignment = 256;
auto size = get_row_bytes(pix_fmt, format_desc.width) * format_desc.height;
return create_aligned_buffer(size, alignment);
}

Expand Down Expand Up @@ -74,17 +84,20 @@ void convert_frame(const core::video_format_desc& channel_format_desc,
// Pack eight byte R16G16B16A16 pixels as four byte 10bit RGB R10G10B10XX
const int NUM_THREADS = 4;
auto rows_per_thread = decklink_format_desc.height / NUM_THREADS;
size_t byte_count_line = get_row_bytes(decklink_format_desc, hdr);
size_t byte_count_line = get_row_bytes(bmdFormat10BitRGBXLE, decklink_format_desc.width);
tbb::parallel_for(0, NUM_THREADS, [&](int i) {
auto end = (i + 1) * rows_per_thread;
for (int y = firstLine + i * rows_per_thread; y < end; y += decklink_format_desc.field_count) {
auto dest = reinterpret_cast<uint32_t*>(image_data.get()) + (long long)y * byte_count_line / 4;
for (int x = 0; x < decklink_format_desc.width; x += 1) {
auto src = reinterpret_cast<const uint16_t*>(
frame.image_data(0).data() + (long long)y * decklink_format_desc.width * 8 + x * 8);
uint16_t blue = src[0] >> 6;
uint16_t green = src[1] >> 6;
uint16_t red = src[2] >> 6;

// Scale down to 10 bit and convert to video range to get a valid
// v210 value after the decklink conversion
uint32_t blue = (src[0] >> 6) * 876 / 1024 + 64;
uint32_t green = (src[1] >> 6) * 876 / 1024 + 64;
uint32_t red = (src[2] >> 6) * 876 / 1024 + 64;
dest[x] = ((uint32_t)(red) << 22) + ((uint32_t)(green) << 12) + ((uint32_t)(blue) << 2);
}
}
Expand Down Expand Up @@ -175,7 +188,8 @@ std::shared_ptr<void> convert_frame_for_port(const core::video_format_desc& chan
BMDFieldDominance field_dominance,
bool hdr)
{
std::shared_ptr<void> image_data = allocate_frame_data(decklink_format_desc, hdr);
std::shared_ptr<void> image_data =
allocate_frame_data(decklink_format_desc, hdr ? bmdFormat10BitRGBXLE : bmdFormat8BitBGRA);

if (field_dominance != bmdProgressiveFrame) {
convert_frame(channel_format_desc,
Expand Down
4 changes: 2 additions & 2 deletions src/modules/decklink/consumer/frame.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,9 @@
namespace caspar { namespace decklink {

BMDPixelFormat get_pixel_format(bool hdr);
int get_row_bytes(const core::video_format_desc& format_desc, bool hdr);
int get_row_bytes(BMDPixelFormat pix_fmt, int width);

std::shared_ptr<void> allocate_frame_data(const core::video_format_desc& format_desc, bool hdr);
std::shared_ptr<void> allocate_frame_data(const core::video_format_desc& format_desc, BMDPixelFormat pix_fmt);

std::shared_ptr<void> convert_frame_for_port(const core::video_format_desc& channel_format_desc,
const core::video_format_desc& decklink_format_desc,
Expand Down
19 changes: 19 additions & 0 deletions src/modules/decklink/decklink_api.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,15 @@ static com_ptr<IDeckLinkIterator> create_iterator()
return pDecklinkIterator;
}

static com_ptr<IDeckLinkVideoConversion> create_video_converter()
{
CComPtr<IDeckLinkIterator> pVideoConversion_;
if (FAILED(pVideoConversion_.CoCreateInstance(CLSID_CDeckLinkVideoConversion)))
CASPAR_THROW_EXCEPTION(not_supported() << msg_info("Could not create video converter."));

return pVideoConversion_;
}

template <typename I, typename T>
static com_iface_ptr<I> iface_cast(const com_ptr<T>& ptr, bool optional = false)
{
Expand Down Expand Up @@ -164,6 +173,16 @@ static com_ptr<IDeckLinkIterator> create_iterator()
return wrap_raw<com_ptr>(iterator, true);
}

static com_ptr<IDeckLinkVideoConversion> create_video_converter()
{
IDeckLinkVideoConversion* converter = CreateVideoConversionInstance();

if (converter == nullptr)
CASPAR_THROW_EXCEPTION(not_supported() << msg_info("Could not create video converter."));

return wrap_raw<com_ptr>(converter, true);
}

template <typename T>
static REFIID iface_id()
{
Expand Down

0 comments on commit a918bf7

Please sign in to comment.