Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement SEI parsing for QSV decoders #31

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
+782 −434
Open

Implement SEI parsing for QSV decoders #31

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4e9adcd
avcodec/hevcdec: factor out ff_hevc_set_set_to_frame
softworkz Oct 25, 2022
51b234c
avcodec/h264dec: make h264_export_frame_props() accessible
softworkz Oct 25, 2022
61626eb
avcodec/qsvdec: Implement SEI parsing for QSV decoders
softworkz Oct 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion libavcodec/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ OBJS-$(CONFIG_MSS34DSP) += mss34dsp.o
OBJS-$(CONFIG_PIXBLOCKDSP) += pixblockdsp.o
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

On the FFmpeg mailing list, "Xiang, Haihao" wrote (reply to this):

On Tue, 2022-10-25 at 04:03 +0000, softworkz wrote:
> From: softworkz <[email protected]>
> 
> Signed-off-by: softworkz <[email protected]>
> ---
>  libavcodec/Makefile |   2 +-
>  libavcodec/qsvdec.c | 321 ++++++++++++++++++++++++++++++++++++++++++++
>  2 files changed, 322 insertions(+), 1 deletion(-)
> 
> diff --git a/libavcodec/Makefile b/libavcodec/Makefile
> index 90c7f113a3..cbddbb0ace 100644
> --- a/libavcodec/Makefile
> +++ b/libavcodec/Makefile
> @@ -146,7 +146,7 @@ OBJS-$(CONFIG_MSS34DSP)                += mss34dsp.o
>  OBJS-$(CONFIG_PIXBLOCKDSP)             += pixblockdsp.o
>  OBJS-$(CONFIG_QPELDSP)                 += qpeldsp.o
>  OBJS-$(CONFIG_QSV)                     += qsv.o
> -OBJS-$(CONFIG_QSVDEC)                  += qsvdec.o
> +OBJS-$(CONFIG_QSVDEC)                  += qsvdec.o h264_sei.o hevc_sei.o
>  OBJS-$(CONFIG_QSVENC)                  += qsvenc.o
>  OBJS-$(CONFIG_RANGECODER)              += rangecoder.o
>  OBJS-$(CONFIG_RDFT)                    += rdft.o
> diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> index 73405b5747..467a248224 100644
> --- a/libavcodec/qsvdec.c
> +++ b/libavcodec/qsvdec.c
> @@ -41,6 +41,7 @@
>  #include "libavutil/time.h"
>  #include "libavutil/imgutils.h"
>  #include "libavutil/film_grain_params.h"
> +#include <libavutil/reverse.h>
>  
>  #include "avcodec.h"
>  #include "codec_internal.h"
> @@ -49,6 +50,9 @@
>  #include "hwconfig.h"
>  #include "qsv.h"
>  #include "qsv_internal.h"
> +#include "h264_sei.h"
> +#include "hevc_ps.h"
> +#include "hevc_sei.h"
>  
>  #if QSV_ONEVPL
>  #include <mfxdispatcher.h>
> @@ -66,6 +70,8 @@ static const AVRational mfx_tb = { 1, 90000 };
>      AV_NOPTS_VALUE : pts_tb.num ? \
>      av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts)
>  
> +#define PAYLOAD_BUFFER_SIZE 65535
> +
>  typedef struct QSVAsyncFrame {
>      mfxSyncPoint *sync;
>      QSVFrame     *frame;
> @@ -107,6 +113,9 @@ typedef struct QSVContext {
>  
>      mfxExtBuffer **ext_buffers;
>      int         nb_ext_buffers;
> +
> +    mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE];
> +    AVBufferRef *a53_buf_ref;
>  } QSVContext;
>  
>  static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
> @@ -628,6 +637,299 @@ static int qsv_export_film_grain(AVCodecContext *avctx,
> mfxExtAV1FilmGrainParam
>  }
>  #endif
>  
> +static int find_start_offset(mfxU8 data[4])
> +{
> +    if (data[0] == 0 && data[1] == 0 && data[2] == 1)
> +        return 3;
> +
> +    if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1)
> +        return 4;
> +
> +    return 0;
> +}
> +
> +static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q, AVFrame* out)
> +{
> +    H264SEIContext sei = { 0 };
> +    GetBitContext gb = { 0 };
> +    mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize =
> sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> +    mfxU64 ts;
> +    int ret;
> +
> +    while (1) {
> +        int start;
> +        memset(payload.Data, 0, payload.BufSize);
> +
> +        ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
> +        if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> +            av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on
> GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer),
> payload.BufSize);
> +            return 0;
> +        }
> +        if (ret != MFX_ERR_NONE)
> +            return ret;
> +
> +        if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
> +            break;
> +
> +        start = find_start_offset(payload.Data);
> +
> +        switch (payload.Type) {
> +            case SEI_TYPE_BUFFERING_PERIOD:
> +            case SEI_TYPE_PIC_TIMING:
> +                continue;
> +        }
> +
> +        if (init_get_bits(&gb, &payload.Data[start], payload.NumBit - start *
> 8) < 0)
> +            av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader
> SEI type: %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> +        else {
> +            ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx);
> +
> +            if (ret < 0)
> +                av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type:
> %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> +            else
> +                av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d  Numbits
> %d\n", payload.Type, payload.NumBit);
> +        }
> +    }
> +
> +    if (out)
> +        return ff_h264_set_sei_to_frame(avctx, &sei, out, NULL, 0);
> +
> +    return 0;
> +}
> +
> +static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q, QSVFrame*
> out)
> +{
> +    HEVCSEI sei = { 0 };
> +    HEVCParamSets ps = { 0 };
> +    GetBitContext gb = { 0 };
> +    mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize =
> sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> +    mfxFrameSurface1 *surface = &out->surface;
> +    mfxU64 ts;
> +    int ret, has_logged = 0;
> +
> +    while (1) {
> +        int start;
> +        memset(payload.Data, 0, payload.BufSize);
> +
> +        ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
> +        if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> +            av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on
> GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer),
> payload.BufSize);
> +            return 0;
> +        }
> +        if (ret != MFX_ERR_NONE)
> +            return ret;
> +
> +        if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
> +            break;
> +
> +        if (!has_logged) {
> +            has_logged = 1;
> +            av_log(avctx, AV_LOG_VERBOSE, "--------------------------------
> ---------\n");
> +            av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI - payload
> timestamp: %llu - surface timestamp: %llu\n", ts, surface->Data.TimeStamp);
> +        }
> +
> +        if (ts != surface->Data.TimeStamp) {
> +            av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp (%llu) does
> not match surface timestamp: (%llu)\n", ts, surface->Data.TimeStamp);
> +        }
> +
> +        start = find_start_offset(payload.Data);
> +
> +        av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d  Numbits
> %3d  Start: %d\n", payload.Type, payload.NumBit, start);
> +
> +        switch (payload.Type) {
> +            case SEI_TYPE_BUFFERING_PERIOD:
> +            case SEI_TYPE_PIC_TIMING:
> +                continue;
> +            case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME:
> +                // There seems to be a bug in MSDK
> +                payload.NumBit -= 8;
> +
> +                break;
> +            case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO:
> +                // There seems to be a bug in MSDK
> +                payload.NumBit = 48;
> +
> +                break;
> +            case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35:
> +                // There seems to be a bug in MSDK
> +                if (payload.NumBit == 552)
> +                    payload.NumBit = 528;
> +                break;
> +        }
> +
> +        if (init_get_bits(&gb, &payload.Data[start], payload.NumBit - start *
> 8) < 0)
> +            av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader
> SEI type: %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> +        else {
> +            ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps,
> HEVC_NAL_SEI_PREFIX);
> +
> +            if (ret < 0)
> +                av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type:
> %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> +            else
> +                av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d  Numbits
> %d\n", payload.Type, payload.NumBit);
> +        }
> +    }
> +
> +    if (has_logged) {
> +        av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n");
> +    }
> +
> +    if (out && out->frame)
> +        return ff_hevc_set_sei_to_frame(avctx, &sei, out->frame, avctx-
> >framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps->bit_depth_chroma);

I got segfault when trying your patchset, 

Thread 1 "ffmpeg" received signal SIGSEGV, Segmentation fault.
0x00007ffff67c0497 in parse_sei_hevc (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288, out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777
777             return ff_hevc_set_sei_to_frame(avctx, &sei, out->frame, avctx->framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps->bit_depth_chroma);
(gdb) bt
#0  0x00007ffff67c0497 in parse_sei_hevc (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288, out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777
#1  0x00007ffff67c1afe in qsv_decode (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288, frame=frame@entry=0x5555556df740, got_frame=got_frame@entry=0x7fffffffd6bc,
    avpkt=avpkt@entry=0x555555635398) at libavcodec/qsvdec.c:1020

BTW the SDK provides support for hevc HDR metadata, we needn't parse SEI payload
in qsvdec and may get the corresponding info from the SDK, see 
https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html 

Thanks
Haihao


> +
> +    return 0;
> +}
> +
> +#define A53_MAX_CC_COUNT 2000
> +
> +static int mpeg_decode_a53_cc(AVCodecContext *avctx, QSVContext *s,
> +                              const uint8_t *p, int buf_size)
> +{
> +    if (buf_size >= 6 &&
> +        p[0] == 'G' && p[1] == 'A' && p[2] == '9' && p[3] == '4' &&
> +        p[4] == 3 && (p[5] & 0x40)) {
> +        /* extract A53 Part 4 CC data */
> +        unsigned cc_count = p[5] & 0x1f;
> +        if (cc_count > 0 && buf_size >= 7 + cc_count * 3) {
> +            const uint64_t old_size = s->a53_buf_ref ? s->a53_buf_ref->size :
> 0;
> +            const uint64_t new_size = (old_size + cc_count
> +                                            * UINT64_C(3));
> +            int ret;
> +
> +            if (new_size > 3*A53_MAX_CC_COUNT)
> +                return AVERROR(EINVAL);
> +
> +            ret = av_buffer_realloc(&s->a53_buf_ref, new_size);
> +            if (ret >= 0)
> +                memcpy(s->a53_buf_ref->data + old_size, p + 7, cc_count *
> UINT64_C(3));
> +
> +            avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS;
> +        }
> +        return 1;
> +    } else if (buf_size >= 2 && p[0] == 0x03 && (p[1]&0x7f) == 0x01) {
> +        /* extract SCTE-20 CC data */
> +        GetBitContext gb;
> +        unsigned cc_count = 0;
> +        int ret;
> +
> +        init_get_bits8(&gb, p + 2, buf_size - 2);
> +        cc_count = get_bits(&gb, 5);
> +        if (cc_count > 0) {
> +            uint64_t old_size = s->a53_buf_ref ? s->a53_buf_ref->size : 0;
> +            uint64_t new_size = (old_size + cc_count * UINT64_C(3));
> +            if (new_size > 3 * A53_MAX_CC_COUNT)
> +                return AVERROR(EINVAL);
> +
> +            ret = av_buffer_realloc(&s->a53_buf_ref, new_size);
> +            if (ret >= 0) {
> +                uint8_t field, cc1, cc2;
> +                uint8_t *cap = s->a53_buf_ref->data;
> +
> +                memset(s->a53_buf_ref->data + old_size, 0, cc_count * 3);
> +                for (unsigned i = 0; i < cc_count && get_bits_left(&gb) >=
> 26; i++) {
> +                    skip_bits(&gb, 2); // priority
> +                    field = get_bits(&gb, 2);
> +                    skip_bits(&gb, 5); // line_offset
> +                    cc1 = get_bits(&gb, 8);
> +                    cc2 = get_bits(&gb, 8);
> +                    skip_bits(&gb, 1); // marker
> +
> +                    if (!field) { // forbidden
> +                        cap[0] = cap[1] = cap[2] = 0x00;
> +                    } else {
> +                        field = (field == 2 ? 1 : 0);
> +                        ////if (!s1->mpeg_enc_ctx.top_field_first) field =
> !field;
> +                        cap[0] = 0x04 | field;
> +                        cap[1] = ff_reverse[cc1];
> +                        cap[2] = ff_reverse[cc2];
> +                    }
> +                    cap += 3;
> +                }
> +            }
> +            avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS;
> +        }
> +        return 1;
> +    } else if (buf_size >= 11 && p[0] == 'C' && p[1] == 'C' && p[2] == 0x01
> && p[3] == 0xf8) {
> +        int cc_count = 0;
> +        int i, ret;
> +        // There is a caption count field in the data, but it is often
> +        // incorrect.  So count the number of captions present.
> +        for (i = 5; i + 6 <= buf_size && ((p[i] & 0xfe) == 0xfe); i += 6)
> +            cc_count++;
> +        // Transform the DVD format into A53 Part 4 format
> +        if (cc_count > 0) {
> +            int old_size = s->a53_buf_ref ? s->a53_buf_ref->size : 0;
> +            uint64_t new_size = (old_size + cc_count
> +                                            * UINT64_C(6));
> +            if (new_size > 3*A53_MAX_CC_COUNT)
> +                return AVERROR(EINVAL);
> +
> +            ret = av_buffer_realloc(&s->a53_buf_ref, new_size);
> +            if (ret >= 0) {
> +                uint8_t field1 = !!(p[4] & 0x80);
> +                uint8_t *cap = s->a53_buf_ref->data;
> +                p += 5;
> +                for (i = 0; i < cc_count; i++) {
> +                    cap[0] = (p[0] == 0xff && field1) ? 0xfc : 0xfd;
> +                    cap[1] = p[1];
> +                    cap[2] = p[2];
> +                    cap[3] = (p[3] == 0xff && !field1) ? 0xfc : 0xfd;
> +                    cap[4] = p[4];
> +                    cap[5] = p[5];
> +                    cap += 6;
> +                    p += 6;
> +                }
> +            }
> +            avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS;
> +        }
> +        return 1;
> +    }
> +    return 0;
> +}
> +
> +static int parse_sei_mpeg12(AVCodecContext* avctx, QSVContext* q, AVFrame*
> out)
> +{
> +    mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize =
> sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> +    mfxU64 ts;
> +    int ret;
> +
> +    while (1) {
> +        int start;
> +
> +        memset(payload.Data, 0, payload.BufSize);
> +        ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
> +        if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> +            av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on
> GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer),
> payload.BufSize);
> +            return 0;
> +        }
> +        if (ret != MFX_ERR_NONE)
> +            return ret;
> +
> +        if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
> +            break;
> +
> +        start = find_start_offset(payload.Data);
> +
> +        start++;
> +
> +        mpeg_decode_a53_cc(avctx, q, &payload.Data[start],
> (int)((payload.NumBit + 7) / 8) - start);
> +
> +        av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d  Numbits %d start %d
> -> %.s\n", payload.Type, payload.NumBit, start, (char
> *)(&payload.Data[start]));
> +    }
> +
> +    if (!out)
> +        return 0;
> +
> +    if (q->a53_buf_ref) {
> +
> +        AVFrameSideData *sd = av_frame_new_side_data_from_buf(out,
> AV_FRAME_DATA_A53_CC, q->a53_buf_ref);
> +        if (!sd)
> +            av_buffer_unref(&q->a53_buf_ref);
> +        q->a53_buf_ref = NULL;
> +    }
> +
> +    return 0;
> +}
> +
>  static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
>                        AVFrame *frame, int *got_frame,
>                        const AVPacket *avpkt)
> @@ -664,6 +966,8 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext
> *q,
>                                                insurf, &outsurf, sync);
>          if (ret == MFX_WRN_DEVICE_BUSY)
>              av_usleep(500);
> +        else if (avctx->codec_id == AV_CODEC_ID_MPEG2VIDEO)
> +            parse_sei_mpeg12(avctx, q, NULL);
>  
>      } while (ret == MFX_WRN_DEVICE_BUSY || ret == MFX_ERR_MORE_SURFACE);
>  
> @@ -705,6 +1009,23 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext
> *q,
>              return AVERROR_BUG;
>          }
>  
> +        switch (avctx->codec_id) {
> +        case AV_CODEC_ID_MPEG2VIDEO:
> +            ret = parse_sei_mpeg12(avctx, q, out_frame->frame);
> +            break;
> +        case AV_CODEC_ID_H264:
> +            ret = parse_sei_h264(avctx, q, out_frame->frame);
> +            break;
> +        case AV_CODEC_ID_HEVC:
> +            ret = parse_sei_hevc(avctx, q, out_frame);
> +            break;
> +        default:
> +            ret = 0;
> +        }
> +
> +        if (ret < 0)
> +            av_log(avctx, AV_LOG_ERROR, "Error parsing SEI data: %d\n", ret);
> +
>          out_frame->queued += 1;
>  
>          aframe = (QSVAsyncFrame){ sync, out_frame };
_______________________________________________
ffmpeg-devel mailing list
[email protected]
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
[email protected] with subject "unsubscribe".

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

On the FFmpeg mailing list, Soft Works wrote (reply to this):



> -----Original Message-----
> From: Xiang, Haihao <[email protected]>
> Sent: Monday, November 21, 2022 3:45 AM
> To: [email protected]
> Cc: [email protected]; [email protected]; haihao.xiang-at-
> [email protected]; [email protected]
> Subject: Re: [FFmpeg-devel] [PATCH v6 3/3] avcodec/qsvdec: Implement
> SEI parsing for QSV decoders
> 
> On Tue, 2022-10-25 at 04:03 +0000, softworkz wrote:
> > From: softworkz <[email protected]>
> >
> > Signed-off-by: softworkz <[email protected]>
> > ---
> >  libavcodec/Makefile |   2 +-
> >  libavcodec/qsvdec.c | 321
> ++++++++++++++++++++++++++++++++++++++++++++
> >  2 files changed, 322 insertions(+), 1 deletion(-)
> >
> > diff --git a/libavcodec/Makefile b/libavcodec/Makefile
> > index 90c7f113a3..cbddbb0ace 100644
> > --- a/libavcodec/Makefile
> > +++ b/libavcodec/Makefile
> > @@ -146,7 +146,7 @@ OBJS-$(CONFIG_MSS34DSP)                +=
> mss34dsp.o
> >  OBJS-$(CONFIG_PIXBLOCKDSP)             += pixblockdsp.o
> >  OBJS-$(CONFIG_QPELDSP)                 += qpeldsp.o
> >  OBJS-$(CONFIG_QSV)                     += qsv.o
> > -OBJS-$(CONFIG_QSVDEC)                  += qsvdec.o
> > +OBJS-$(CONFIG_QSVDEC)                  += qsvdec.o h264_sei.o
> hevc_sei.o
> >  OBJS-$(CONFIG_QSVENC)                  += qsvenc.o
> >  OBJS-$(CONFIG_RANGECODER)              += rangecoder.o
> >  OBJS-$(CONFIG_RDFT)                    += rdft.o
> > diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> > index 73405b5747..467a248224 100644
> > --- a/libavcodec/qsvdec.c
> > +++ b/libavcodec/qsvdec.c
> > @@ -41,6 +41,7 @@
> >  #include "libavutil/time.h"
> >  #include "libavutil/imgutils.h"
> >  #include "libavutil/film_grain_params.h"
> > +#include <libavutil/reverse.h>
> >
> >  #include "avcodec.h"
> >  #include "codec_internal.h"
> > @@ -49,6 +50,9 @@
> >  #include "hwconfig.h"
> >  #include "qsv.h"
> >  #include "qsv_internal.h"
> > +#include "h264_sei.h"
> > +#include "hevc_ps.h"
> > +#include "hevc_sei.h"
> >
> >  #if QSV_ONEVPL
> >  #include <mfxdispatcher.h>
> > @@ -66,6 +70,8 @@ static const AVRational mfx_tb = { 1, 90000 };
> >      AV_NOPTS_VALUE : pts_tb.num ? \
> >      av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts)
> >
> > +#define PAYLOAD_BUFFER_SIZE 65535
> > +
> >  typedef struct QSVAsyncFrame {
> >      mfxSyncPoint *sync;
> >      QSVFrame     *frame;
> > @@ -107,6 +113,9 @@ typedef struct QSVContext {
> >
> >      mfxExtBuffer **ext_buffers;
> >      int         nb_ext_buffers;
> > +
> > +    mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE];
> > +    AVBufferRef *a53_buf_ref;
> >  } QSVContext;
> >
> >  static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
> > @@ -628,6 +637,299 @@ static int
> qsv_export_film_grain(AVCodecContext *avctx,
> > mfxExtAV1FilmGrainParam
> >  }
> >  #endif
> >
> > +static int find_start_offset(mfxU8 data[4])
> > +{
> > +    if (data[0] == 0 && data[1] == 0 && data[2] == 1)
> > +        return 3;
> > +
> > +    if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] ==
> 1)
> > +        return 4;
> > +
> > +    return 0;
> > +}
> > +
> > +static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q,
> AVFrame* out)
> > +{
> > +    H264SEIContext sei = { 0 };
> > +    GetBitContext gb = { 0 };
> > +    mfxPayload payload = { 0, .Data = &q->payload_buffer[0],
> .BufSize =
> > sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> > +    mfxU64 ts;
> > +    int ret;
> > +
> > +    while (1) {
> > +        int start;
> > +        memset(payload.Data, 0, payload.BufSize);
> > +
> > +        ret = MFXVideoDECODE_GetPayload(q->session, &ts,
> &payload);
> > +        if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > +            av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient
> buffer on
> > GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q-
> >payload_buffer),
> > payload.BufSize);
> > +            return 0;
> > +        }
> > +        if (ret != MFX_ERR_NONE)
> > +            return ret;
> > +
> > +        if (payload.NumBit == 0 || payload.NumBit >=
> payload.BufSize * 8)
> > +            break;
> > +
> > +        start = find_start_offset(payload.Data);
> > +
> > +        switch (payload.Type) {
> > +            case SEI_TYPE_BUFFERING_PERIOD:
> > +            case SEI_TYPE_PIC_TIMING:
> > +                continue;
> > +        }
> > +
> > +        if (init_get_bits(&gb, &payload.Data[start],
> payload.NumBit - start *
> > 8) < 0)
> > +            av_log(avctx, AV_LOG_ERROR, "Error initializing
> bitstream reader
> > SEI type: %d  Numbits %d error: %d\n", payload.Type,
> payload.NumBit, ret);
> > +        else {
> > +            ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx);
> > +
> > +            if (ret < 0)
> > +                av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI
> type:
> > %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> > +            else
> > +                av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d
> Numbits
> > %d\n", payload.Type, payload.NumBit);
> > +        }
> > +    }
> > +
> > +    if (out)
> > +        return ff_h264_set_sei_to_frame(avctx, &sei, out, NULL,
> 0);
> > +
> > +    return 0;
> > +}
> > +
> > +static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q,
> QSVFrame*
> > out)
> > +{
> > +    HEVCSEI sei = { 0 };
> > +    HEVCParamSets ps = { 0 };
> > +    GetBitContext gb = { 0 };
> > +    mfxPayload payload = { 0, .Data = &q->payload_buffer[0],
> .BufSize =
> > sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> > +    mfxFrameSurface1 *surface = &out->surface;
> > +    mfxU64 ts;
> > +    int ret, has_logged = 0;
> > +
> > +    while (1) {
> > +        int start;
> > +        memset(payload.Data, 0, payload.BufSize);
> > +
> > +        ret = MFXVideoDECODE_GetPayload(q->session, &ts,
> &payload);
> > +        if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > +            av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient
> buffer on
> > GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q-
> >payload_buffer),
> > payload.BufSize);
> > +            return 0;
> > +        }
> > +        if (ret != MFX_ERR_NONE)
> > +            return ret;
> > +
> > +        if (payload.NumBit == 0 || payload.NumBit >=
> payload.BufSize * 8)
> > +            break;
> > +
> > +        if (!has_logged) {
> > +            has_logged = 1;
> > +            av_log(avctx, AV_LOG_VERBOSE, "-----------------------
> ---------
> > ---------\n");
> > +            av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI -
> payload
> > timestamp: %llu - surface timestamp: %llu\n", ts, surface-
> >Data.TimeStamp);
> > +        }
> > +
> > +        if (ts != surface->Data.TimeStamp) {
> > +            av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp
> (%llu) does
> > not match surface timestamp: (%llu)\n", ts, surface-
> >Data.TimeStamp);
> > +        }
> > +
> > +        start = find_start_offset(payload.Data);
> > +
> > +        av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d
> Numbits
> > %3d  Start: %d\n", payload.Type, payload.NumBit, start);
> > +
> > +        switch (payload.Type) {
> > +            case SEI_TYPE_BUFFERING_PERIOD:
> > +            case SEI_TYPE_PIC_TIMING:
> > +                continue;
> > +            case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME:
> > +                // There seems to be a bug in MSDK
> > +                payload.NumBit -= 8;
> > +
> > +                break;
> > +            case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO:
> > +                // There seems to be a bug in MSDK
> > +                payload.NumBit = 48;
> > +
> > +                break;
> > +            case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35:
> > +                // There seems to be a bug in MSDK
> > +                if (payload.NumBit == 552)
> > +                    payload.NumBit = 528;
> > +                break;
> > +        }
> > +
> > +        if (init_get_bits(&gb, &payload.Data[start],
> payload.NumBit - start *
> > 8) < 0)
> > +            av_log(avctx, AV_LOG_ERROR, "Error initializing
> bitstream reader
> > SEI type: %d  Numbits %d error: %d\n", payload.Type,
> payload.NumBit, ret);
> > +        else {
> > +            ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps,
> > HEVC_NAL_SEI_PREFIX);
> > +
> > +            if (ret < 0)
> > +                av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI
> type:
> > %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> > +            else
> > +                av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d
> Numbits
> > %d\n", payload.Type, payload.NumBit);
> > +        }
> > +    }
> > +
> > +    if (has_logged) {
> > +        av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n");
> > +    }
> > +
> > +    if (out && out->frame)
> > +        return ff_hevc_set_sei_to_frame(avctx, &sei, out->frame,
> avctx-
> > >framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps-
> >bit_depth_chroma);
> 
> I got segfault when trying your patchset,
> 
> Thread 1 "ffmpeg" received signal SIGSEGV, Segmentation fault.
> 0x00007ffff67c0497 in parse_sei_hevc
> (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288,
> out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777
> 777             return ff_hevc_set_sei_to_frame(avctx, &sei, out-
> >frame, avctx->framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps-
> >bit_depth_chroma);
> (gdb) bt
> #0  0x00007ffff67c0497 in parse_sei_hevc
> (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288,
> out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777
> #1  0x00007ffff67c1afe in qsv_decode
> (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288,
> frame=frame@entry=0x5555556df740,
> got_frame=got_frame@entry=0x7fffffffd6bc,
>     avpkt=avpkt@entry=0x555555635398) at libavcodec/qsvdec.c:1020


> BTW the SDK provides support for hevc HDR metadata, we needn't parse
> SEI payload
> in qsvdec and may get the corresponding info from the SDK, see
> https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html

I know. I was the one who had requested this to be added to MSDK :-)

But it's just one small part of SEI information, it's limited to
the latest MSDK versions and I'm not sure whether it's working
as reliably as this implementation. This would need to be tested.

You should still have access to the repo with the test files for 
demoing the offset problems (which my patchset is working around).

But it's also about dynamic HDR data, dovi data, etc. - which 
MSDK doesn't provide, so this single bit of SEI data doesn't 
help much.

Best regards,
softworkz



_______________________________________________
ffmpeg-devel mailing list
[email protected]
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
[email protected] with subject "unsubscribe".

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

On the FFmpeg mailing list, "Xiang, Haihao" wrote (reply to this):

On Mon, 2022-11-21 at 15:58 +0000, Soft Works wrote:
> > -----Original Message-----
> > From: Xiang, Haihao <[email protected]>
> > Sent: Monday, November 21, 2022 3:45 AM
> > To: [email protected]
> > Cc: [email protected]; [email protected]; haihao.xiang-at-
> > [email protected]; [email protected]
> > Subject: Re: [FFmpeg-devel] [PATCH v6 3/3] avcodec/qsvdec: Implement
> > SEI parsing for QSV decoders
> > 
> > On Tue, 2022-10-25 at 04:03 +0000, softworkz wrote:
> > > From: softworkz <[email protected]>
> > > 
> > > Signed-off-by: softworkz <[email protected]>
> > > ---
> > >  libavcodec/Makefile |   2 +-
> > >  libavcodec/qsvdec.c | 321
> > ++++++++++++++++++++++++++++++++++++++++++++
> > >  2 files changed, 322 insertions(+), 1 deletion(-)
> > > 
> > > diff --git a/libavcodec/Makefile b/libavcodec/Makefile
> > > index 90c7f113a3..cbddbb0ace 100644
> > > --- a/libavcodec/Makefile
> > > +++ b/libavcodec/Makefile
> > > @@ -146,7 +146,7 @@ OBJS-$(CONFIG_MSS34DSP)                +=
> > mss34dsp.o
> > >  OBJS-$(CONFIG_PIXBLOCKDSP)             += pixblockdsp.o
> > >  OBJS-$(CONFIG_QPELDSP)                 += qpeldsp.o
> > >  OBJS-$(CONFIG_QSV)                     += qsv.o
> > > -OBJS-$(CONFIG_QSVDEC)                  += qsvdec.o
> > > +OBJS-$(CONFIG_QSVDEC)                  += qsvdec.o h264_sei.o
> > hevc_sei.o
> > >  OBJS-$(CONFIG_QSVENC)                  += qsvenc.o
> > >  OBJS-$(CONFIG_RANGECODER)              += rangecoder.o
> > >  OBJS-$(CONFIG_RDFT)                    += rdft.o
> > > diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> > > index 73405b5747..467a248224 100644
> > > --- a/libavcodec/qsvdec.c
> > > +++ b/libavcodec/qsvdec.c
> > > @@ -41,6 +41,7 @@
> > >  #include "libavutil/time.h"
> > >  #include "libavutil/imgutils.h"
> > >  #include "libavutil/film_grain_params.h"
> > > +#include <libavutil/reverse.h>
> > > 
> > >  #include "avcodec.h"
> > >  #include "codec_internal.h"
> > > @@ -49,6 +50,9 @@
> > >  #include "hwconfig.h"
> > >  #include "qsv.h"
> > >  #include "qsv_internal.h"
> > > +#include "h264_sei.h"
> > > +#include "hevc_ps.h"
> > > +#include "hevc_sei.h"
> > > 
> > >  #if QSV_ONEVPL
> > >  #include <mfxdispatcher.h>
> > > @@ -66,6 +70,8 @@ static const AVRational mfx_tb = { 1, 90000 };
> > >      AV_NOPTS_VALUE : pts_tb.num ? \
> > >      av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts)
> > > 
> > > +#define PAYLOAD_BUFFER_SIZE 65535
> > > +
> > >  typedef struct QSVAsyncFrame {
> > >      mfxSyncPoint *sync;
> > >      QSVFrame     *frame;
> > > @@ -107,6 +113,9 @@ typedef struct QSVContext {
> > > 
> > >      mfxExtBuffer **ext_buffers;
> > >      int         nb_ext_buffers;
> > > +
> > > +    mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE];
> > > +    AVBufferRef *a53_buf_ref;
> > >  } QSVContext;
> > > 
> > >  static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
> > > @@ -628,6 +637,299 @@ static int
> > qsv_export_film_grain(AVCodecContext *avctx,
> > > mfxExtAV1FilmGrainParam
> > >  }
> > >  #endif
> > > 
> > > +static int find_start_offset(mfxU8 data[4])
> > > +{
> > > +    if (data[0] == 0 && data[1] == 0 && data[2] == 1)
> > > +        return 3;
> > > +
> > > +    if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] ==
> > 1)
> > > +        return 4;
> > > +
> > > +    return 0;
> > > +}
> > > +
> > > +static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q,
> > AVFrame* out)
> > > +{
> > > +    H264SEIContext sei = { 0 };
> > > +    GetBitContext gb = { 0 };
> > > +    mfxPayload payload = { 0, .Data = &q->payload_buffer[0],
> > .BufSize =
> > > sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> > > +    mfxU64 ts;
> > > +    int ret;
> > > +
> > > +    while (1) {
> > > +        int start;
> > > +        memset(payload.Data, 0, payload.BufSize);
> > > +
> > > +        ret = MFXVideoDECODE_GetPayload(q->session, &ts,
> > &payload);
> > > +        if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > > +            av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient
> > buffer on
> > > GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q-
> > > payload_buffer),
> > > payload.BufSize);
> > > +            return 0;
> > > +        }
> > > +        if (ret != MFX_ERR_NONE)
> > > +            return ret;
> > > +
> > > +        if (payload.NumBit == 0 || payload.NumBit >=
> > payload.BufSize * 8)
> > > +            break;
> > > +
> > > +        start = find_start_offset(payload.Data);
> > > +
> > > +        switch (payload.Type) {
> > > +            case SEI_TYPE_BUFFERING_PERIOD:
> > > +            case SEI_TYPE_PIC_TIMING:
> > > +                continue;
> > > +        }
> > > +
> > > +        if (init_get_bits(&gb, &payload.Data[start],
> > payload.NumBit - start *
> > > 8) < 0)
> > > +            av_log(avctx, AV_LOG_ERROR, "Error initializing
> > bitstream reader
> > > SEI type: %d  Numbits %d error: %d\n", payload.Type,
> > payload.NumBit, ret);
> > > +        else {
> > > +            ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx);
> > > +
> > > +            if (ret < 0)
> > > +                av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI
> > type:
> > > %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> > > +            else
> > > +                av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d
> > Numbits
> > > %d\n", payload.Type, payload.NumBit);
> > > +        }
> > > +    }
> > > +
> > > +    if (out)
> > > +        return ff_h264_set_sei_to_frame(avctx, &sei, out, NULL,
> > 0);
> > > +
> > > +    return 0;
> > > +}
> > > +
> > > +static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q,
> > QSVFrame*
> > > out)
> > > +{
> > > +    HEVCSEI sei = { 0 };
> > > +    HEVCParamSets ps = { 0 };
> > > +    GetBitContext gb = { 0 };
> > > +    mfxPayload payload = { 0, .Data = &q->payload_buffer[0],
> > .BufSize =
> > > sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> > > +    mfxFrameSurface1 *surface = &out->surface;
> > > +    mfxU64 ts;
> > > +    int ret, has_logged = 0;
> > > +
> > > +    while (1) {
> > > +        int start;
> > > +        memset(payload.Data, 0, payload.BufSize);
> > > +
> > > +        ret = MFXVideoDECODE_GetPayload(q->session, &ts,
> > &payload);
> > > +        if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > > +            av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient
> > buffer on
> > > GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q-
> > > payload_buffer),
> > > payload.BufSize);
> > > +            return 0;
> > > +        }
> > > +        if (ret != MFX_ERR_NONE)
> > > +            return ret;
> > > +
> > > +        if (payload.NumBit == 0 || payload.NumBit >=
> > payload.BufSize * 8)
> > > +            break;
> > > +
> > > +        if (!has_logged) {
> > > +            has_logged = 1;
> > > +            av_log(avctx, AV_LOG_VERBOSE, "-----------------------
> > ---------
> > > ---------\n");
> > > +            av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI -
> > payload
> > > timestamp: %llu - surface timestamp: %llu\n", ts, surface-
> > > Data.TimeStamp);
> > > +        }
> > > +
> > > +        if (ts != surface->Data.TimeStamp) {
> > > +            av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp
> > (%llu) does
> > > not match surface timestamp: (%llu)\n", ts, surface-
> > > Data.TimeStamp);
> > > +        }
> > > +
> > > +        start = find_start_offset(payload.Data);
> > > +
> > > +        av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d
> > Numbits
> > > %3d  Start: %d\n", payload.Type, payload.NumBit, start);
> > > +
> > > +        switch (payload.Type) {
> > > +            case SEI_TYPE_BUFFERING_PERIOD:
> > > +            case SEI_TYPE_PIC_TIMING:
> > > +                continue;
> > > +            case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME:
> > > +                // There seems to be a bug in MSDK
> > > +                payload.NumBit -= 8;
> > > +
> > > +                break;
> > > +            case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO:
> > > +                // There seems to be a bug in MSDK
> > > +                payload.NumBit = 48;
> > > +
> > > +                break;
> > > +            case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35:
> > > +                // There seems to be a bug in MSDK
> > > +                if (payload.NumBit == 552)
> > > +                    payload.NumBit = 528;
> > > +                break;
> > > +        }
> > > +
> > > +        if (init_get_bits(&gb, &payload.Data[start],
> > payload.NumBit - start *
> > > 8) < 0)
> > > +            av_log(avctx, AV_LOG_ERROR, "Error initializing
> > bitstream reader
> > > SEI type: %d  Numbits %d error: %d\n", payload.Type,
> > payload.NumBit, ret);
> > > +        else {
> > > +            ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps,
> > > HEVC_NAL_SEI_PREFIX);
> > > +
> > > +            if (ret < 0)
> > > +                av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI
> > type:
> > > %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> > > +            else
> > > +                av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d
> > Numbits
> > > %d\n", payload.Type, payload.NumBit);
> > > +        }
> > > +    }
> > > +
> > > +    if (has_logged) {
> > > +        av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n");
> > > +    }
> > > +
> > > +    if (out && out->frame)
> > > +        return ff_hevc_set_sei_to_frame(avctx, &sei, out->frame,
> > avctx-
> > > > framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps-
> > > bit_depth_chroma);
> > 
> > I got segfault when trying your patchset,
> > 
> > Thread 1 "ffmpeg" received signal SIGSEGV, Segmentation fault.
> > 0x00007ffff67c0497 in parse_sei_hevc
> > (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288,
> > out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777
> > 777             return ff_hevc_set_sei_to_frame(avctx, &sei, out-
> > > frame, avctx->framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps-
> > > bit_depth_chroma);
> > (gdb) bt
> > #0  0x00007ffff67c0497 in parse_sei_hevc
> > (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288,
> > out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777
> > #1  0x00007ffff67c1afe in qsv_decode
> > (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288,
> > frame=frame@entry=0x5555556df740,
> > got_frame=got_frame@entry=0x7fffffffd6bc,
> >     avpkt=avpkt@entry=0x555555635398) at libavcodec/qsvdec.c:1020
> > BTW the SDK provides support for hevc HDR metadata, we needn't parse
> > SEI payload
> > in qsvdec and may get the corresponding info from the SDK, see
> > https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html
> 
> I know. I was the one who had requested this to be added to MSDK :-)
> 
> But it's just one small part of SEI information, it's limited to
> the latest MSDK versions and I'm not sure whether it's working
> as reliably as this implementation. This would need to be tested.
> 
> You should still have access to the repo with the test files for 
> demoing the offset problems (which my patchset is working around).


I worked out a patchset (https://github.com/intel-media-ci/ffmpeg/pull/518),
including https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html an
d others. I may use your command (with some changes) to convert all HDR clips in
your repo to SDR clips except one clip (w/wo my changes, there is a GPU hang
issue with this clip). 

We may only do a small update to support AV1 HDR in the future if applying 
https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html.

Note the command below doesn't work with all test files in your repo with your
patchset v6.

$ ffmpeg -hwaccel qsv -c:v hevc_qsv -i input.mp4 -f null -

Thanks
Haihao

> 
> But it's also about dynamic HDR data, dovi data, etc. - which 
> MSDK doesn't provide, so this single bit of SEI data doesn't 
> help much.
> 
> Best regards,
> softworkz
> 
> 
> 
_______________________________________________
ffmpeg-devel mailing list
[email protected]
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
[email protected] with subject "unsubscribe".

OBJS-$(CONFIG_QPELDSP) += qpeldsp.o
OBJS-$(CONFIG_QSV) += qsv.o
OBJS-$(CONFIG_QSVDEC) += qsvdec.o
OBJS-$(CONFIG_QSVDEC) += qsvdec.o h264_sei.o hevc_sei.o
OBJS-$(CONFIG_QSVENC) += qsvenc.o
OBJS-$(CONFIG_RANGECODER) += rangecoder.o
OBJS-$(CONFIG_RDFT) += rdft.o
Expand Down
197 changes: 197 additions & 0 deletions libavcodec/h264_sei.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@
#include "h264_ps.h"
#include "h264_sei.h"
#include "sei.h"
#include "libavutil/display.h"
#include "libavutil/film_grain_params.h"
#include "libavutil/stereo3d.h"
#include "libavutil/timecode.h"

#define AVERROR_PS_NOT_FOUND FFERRTAG(0xF8,'?','P','S')

Expand Down Expand Up @@ -587,3 +591,196 @@ const char *ff_h264_sei_stereo_mode(const H264SEIFramePacking *h)
return NULL;
}
}

int ff_h264_set_sei_to_frame(AVCodecContext *avctx, H264SEIContext *sei, AVFrame *out, const SPS *sps, uint64_t seed)
{
if (sei->frame_packing.present &&
sei->frame_packing.arrangement_type <= 6 &&
sei->frame_packing.content_interpretation_type > 0 &&
sei->frame_packing.content_interpretation_type < 3) {
H264SEIFramePacking *fp = &sei->frame_packing;
AVStereo3D *stereo = av_stereo3d_create_side_data(out);
if (stereo) {
switch (fp->arrangement_type) {
case H264_SEI_FPA_TYPE_CHECKERBOARD:
stereo->type = AV_STEREO3D_CHECKERBOARD;
break;
case H264_SEI_FPA_TYPE_INTERLEAVE_COLUMN:
stereo->type = AV_STEREO3D_COLUMNS;
break;
case H264_SEI_FPA_TYPE_INTERLEAVE_ROW:
stereo->type = AV_STEREO3D_LINES;
break;
case H264_SEI_FPA_TYPE_SIDE_BY_SIDE:
if (fp->quincunx_sampling_flag)
stereo->type = AV_STEREO3D_SIDEBYSIDE_QUINCUNX;
else
stereo->type = AV_STEREO3D_SIDEBYSIDE;
break;
case H264_SEI_FPA_TYPE_TOP_BOTTOM:
stereo->type = AV_STEREO3D_TOPBOTTOM;
break;
case H264_SEI_FPA_TYPE_INTERLEAVE_TEMPORAL:
stereo->type = AV_STEREO3D_FRAMESEQUENCE;
break;
case H264_SEI_FPA_TYPE_2D:
stereo->type = AV_STEREO3D_2D;
break;
}

if (fp->content_interpretation_type == 2)
stereo->flags = AV_STEREO3D_FLAG_INVERT;

if (fp->arrangement_type == H264_SEI_FPA_TYPE_INTERLEAVE_TEMPORAL) {
if (fp->current_frame_is_frame0_flag)
stereo->view = AV_STEREO3D_VIEW_LEFT;
else
stereo->view = AV_STEREO3D_VIEW_RIGHT;
}
}
}

if (sei->display_orientation.present &&
(sei->display_orientation.anticlockwise_rotation ||
sei->display_orientation.hflip ||
sei->display_orientation.vflip)) {
H264SEIDisplayOrientation *o = &sei->display_orientation;
double angle = o->anticlockwise_rotation * 360 / (double) (1 << 16);
AVFrameSideData *rotation = av_frame_new_side_data(out,
AV_FRAME_DATA_DISPLAYMATRIX,
sizeof(int32_t) * 9);
if (rotation) {
/* av_display_rotation_set() expects the angle in the clockwise
* direction, hence the first minus.
* The below code applies the flips after the rotation, yet
* the H.2645 specs require flipping to be applied first.
* Because of R O(phi) = O(-phi) R (where R is flipping around
* an arbitatry axis and O(phi) is the proper rotation by phi)
* we can create display matrices as desired by negating
* the degree once for every flip applied. */
angle = -angle * (1 - 2 * !!o->hflip) * (1 - 2 * !!o->vflip);
av_display_rotation_set((int32_t *)rotation->data, angle);
av_display_matrix_flip((int32_t *)rotation->data,
o->hflip, o->vflip);
}
}

if (sei->afd.present) {
AVFrameSideData *sd = av_frame_new_side_data(out, AV_FRAME_DATA_AFD,
sizeof(uint8_t));

if (sd) {
*sd->data = sei->afd.active_format_description;
sei->afd.present = 0;
}
}

if (sei->a53_caption.buf_ref) {
H264SEIA53Caption *a53 = &sei->a53_caption;

AVFrameSideData *sd = av_frame_new_side_data_from_buf(out, AV_FRAME_DATA_A53_CC, a53->buf_ref);
if (!sd)
av_buffer_unref(&a53->buf_ref);
a53->buf_ref = NULL;

avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS;
}

for (int i = 0; i < sei->unregistered.nb_buf_ref; i++) {
H264SEIUnregistered *unreg = &sei->unregistered;

if (unreg->buf_ref[i]) {
AVFrameSideData *sd = av_frame_new_side_data_from_buf(out,
AV_FRAME_DATA_SEI_UNREGISTERED,
unreg->buf_ref[i]);
if (!sd)
av_buffer_unref(&unreg->buf_ref[i]);
unreg->buf_ref[i] = NULL;
}
}
sei->unregistered.nb_buf_ref = 0;

if (sps && sei->film_grain_characteristics.present) {
H264SEIFilmGrainCharacteristics *fgc = &sei->film_grain_characteristics;
AVFilmGrainParams *fgp = av_film_grain_params_create_side_data(out);
if (!fgp)
return AVERROR(ENOMEM);

fgp->type = AV_FILM_GRAIN_PARAMS_H274;
fgp->seed = seed;

fgp->codec.h274.model_id = fgc->model_id;
if (fgc->separate_colour_description_present_flag) {
fgp->codec.h274.bit_depth_luma = fgc->bit_depth_luma;
fgp->codec.h274.bit_depth_chroma = fgc->bit_depth_chroma;
fgp->codec.h274.color_range = fgc->full_range + 1;
fgp->codec.h274.color_primaries = fgc->color_primaries;
fgp->codec.h274.color_trc = fgc->transfer_characteristics;
fgp->codec.h274.color_space = fgc->matrix_coeffs;
} else {
fgp->codec.h274.bit_depth_luma = sps->bit_depth_luma;
fgp->codec.h274.bit_depth_chroma = sps->bit_depth_chroma;
if (sps->video_signal_type_present_flag)
fgp->codec.h274.color_range = sps->full_range + 1;
else
fgp->codec.h274.color_range = AVCOL_RANGE_UNSPECIFIED;
if (sps->colour_description_present_flag) {
fgp->codec.h274.color_primaries = sps->color_primaries;
fgp->codec.h274.color_trc = sps->color_trc;
fgp->codec.h274.color_space = sps->colorspace;
} else {
fgp->codec.h274.color_primaries = AVCOL_PRI_UNSPECIFIED;
fgp->codec.h274.color_trc = AVCOL_TRC_UNSPECIFIED;
fgp->codec.h274.color_space = AVCOL_SPC_UNSPECIFIED;
}
}
fgp->codec.h274.blending_mode_id = fgc->blending_mode_id;
fgp->codec.h274.log2_scale_factor = fgc->log2_scale_factor;

memcpy(&fgp->codec.h274.component_model_present, &fgc->comp_model_present_flag,
sizeof(fgp->codec.h274.component_model_present));
memcpy(&fgp->codec.h274.num_intensity_intervals, &fgc->num_intensity_intervals,
sizeof(fgp->codec.h274.num_intensity_intervals));
memcpy(&fgp->codec.h274.num_model_values, &fgc->num_model_values,
sizeof(fgp->codec.h274.num_model_values));
memcpy(&fgp->codec.h274.intensity_interval_lower_bound, &fgc->intensity_interval_lower_bound,
sizeof(fgp->codec.h274.intensity_interval_lower_bound));
memcpy(&fgp->codec.h274.intensity_interval_upper_bound, &fgc->intensity_interval_upper_bound,
sizeof(fgp->codec.h274.intensity_interval_upper_bound));
memcpy(&fgp->codec.h274.comp_model_value, &fgc->comp_model_value,
sizeof(fgp->codec.h274.comp_model_value));

fgc->present = !!fgc->repetition_period;

avctx->properties |= FF_CODEC_PROPERTY_FILM_GRAIN;
}

if (sei->picture_timing.timecode_cnt > 0) {
uint32_t *tc_sd;
char tcbuf[AV_TIMECODE_STR_SIZE];

AVFrameSideData *tcside = av_frame_new_side_data(out,
AV_FRAME_DATA_S12M_TIMECODE,
sizeof(uint32_t)*4);
if (!tcside)
return AVERROR(ENOMEM);

tc_sd = (uint32_t*)tcside->data;
tc_sd[0] = sei->picture_timing.timecode_cnt;

for (int i = 0; i < tc_sd[0]; i++) {
int drop = sei->picture_timing.timecode[i].dropframe;
int hh = sei->picture_timing.timecode[i].hours;
int mm = sei->picture_timing.timecode[i].minutes;
int ss = sei->picture_timing.timecode[i].seconds;
int ff = sei->picture_timing.timecode[i].frame;

tc_sd[i + 1] = av_timecode_get_smpte(avctx->framerate, drop, hh, mm, ss, ff);
av_timecode_make_smpte_tc_string2(tcbuf, avctx->framerate, tc_sd[i + 1], 0, 0);
av_dict_set(&out->metadata, "timecode", tcbuf, 0);
}
sei->picture_timing.timecode_cnt = 0;
}

return 0;
}
2 changes: 2 additions & 0 deletions libavcodec/h264_sei.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,4 +221,6 @@ const char *ff_h264_sei_stereo_mode(const H264SEIFramePacking *h);
int ff_h264_sei_process_picture_timing(H264SEIPictureTiming *h, const SPS *sps,
void *logctx);

int ff_h264_set_sei_to_frame(AVCodecContext *avctx, H264SEIContext *sei, AVFrame *out, const SPS *sps, uint64_t seed);

#endif /* AVCODEC_H264_SEI_H */
Loading