Skip to content

Commit

Permalink
Merge pull request #201 from kaltura/master-hevc-ngnx-rtmp-module
Browse files Browse the repository at this point in the history
add support for HEVC to rtmp.
  • Loading branch information
igorshevach authored May 7, 2024
2 parents 79f318f + b05730d commit bcd0e8f
Show file tree
Hide file tree
Showing 12 changed files with 663 additions and 76 deletions.
77 changes: 76 additions & 1 deletion nginx-kmp-rtmp-module/src/ngx_kmp_rtmp_encoder.c
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@
#define NGX_RTMP_AVC_SEQUENCE_HEADER 0
#define NGX_RTMP_AVC_NALU 1

#define NGX_RTMP_EXT_HEADER_SIZE 5 /* (frame | packet type), fourcc */
#define NGX_RTMP_EXT_HEADER_SIZE_NALU 8 /* + pts_delay */
#define NGX_RTMP_EXT_SEQUENCE_HEADER 0
#define NGX_RTMP_EXT_NALU 1

/* audio */
#define NGX_RTMP_AAC_HEADER_SIZE 2 /* sound_info + packet_type */

Expand All @@ -52,7 +57,7 @@


#define NGX_RTMP_FRAME_HEADER_MAX_SIZE (NGX_RTMP_HEADER_0_SIZE \
+ NGX_RTMP_EXT_TIMESTAMP_SIZE + NGX_RTMP_AVC_HEADER_SIZE)
+ NGX_RTMP_EXT_TIMESTAMP_SIZE + NGX_RTMP_EXT_HEADER_SIZE_NALU)


#define ngx_kmp_rtmp_chunk_count(mlen, chunk_size) \
Expand Down Expand Up @@ -892,6 +897,66 @@ ngx_kmp_rtmp_encoder_avc_sequence_write(u_char *p,
}


static u_char *
ngx_kmp_rtmp_encoder_ext_header_write(u_char *p, u_char packet_type,
u_char key_frame, uint32_t fourcc, uint32_t pts_delay)
{
u_char frame_type;

frame_type = key_frame ? NGX_RTMP_FRAME_TYPE_KEY
: NGX_RTMP_FRAME_TYPE_INTER;

*p++ = 0x80 | (frame_type << 4) | packet_type;

p = ngx_copy(p, &fourcc, sizeof(fourcc));

if (packet_type == NGX_RTMP_EXT_NALU) {
ngx_kmp_rtmp_amf_write_be24(p, pts_delay);
}

return p;
}


size_t
ngx_kmp_rtmp_encoder_ext_sequence_get_size(ngx_kmp_rtmp_stream_ctx_t *sc,
ngx_str_t *extra_data)
{
size_t mlen;

mlen = NGX_RTMP_EXT_HEADER_SIZE + extra_data->len;

return NGX_RTMP_HEADER_0_SIZE + mlen
+ (ngx_kmp_rtmp_chunk_count(mlen, sc->chunk_size) - 1)
* NGX_RTMP_HEADER_3_SIZE;
}


u_char *
ngx_kmp_rtmp_encoder_ext_sequence_write(u_char *p,
ngx_kmp_rtmp_stream_ctx_t *sc, uint32_t fourcc, ngx_str_t *extra_data)
{
u_char *body;
ngx_kmp_rtmp_header_t h;

ngx_memzero(&h, sizeof(h));
h.csid = sc->csid;
h.mlen = NGX_RTMP_EXT_HEADER_SIZE + extra_data->len;
h.type = NGX_RTMP_MSG_VIDEO;
h.msid = sc->msid;

p = ngx_kmp_rtmp_encoder_write_header(p, &h);

body = p;
p = ngx_kmp_rtmp_encoder_ext_header_write(p, NGX_RTMP_EXT_SEQUENCE_HEADER,
1, fourcc, 0);
ngx_memcpy(p, extra_data->data, extra_data->len);

return ngx_kmp_rtmp_encoder_add_chunk_headers(body, h.mlen, sc->chunk_size,
h.csid);
}


static u_char *
ngx_kmp_rtmp_encoder_aac_header_write(u_char *p, u_char sound_info,
u_char packet_type)
Expand Down Expand Up @@ -1005,6 +1070,7 @@ ngx_kmp_rtmp_encoder_frame_write(ngx_kmp_rtmp_stream_ctx_t *sc,
h.timestamp = NGX_RTMP_EXT_TIMESTAMP;
}

/* TODO: add support for "vp09" and "av01" */
switch (codec_id) {

case KMP_CODEC_VIDEO_H264:
Expand All @@ -1015,6 +1081,15 @@ ngx_kmp_rtmp_encoder_frame_write(ngx_kmp_rtmp_stream_ctx_t *sc,
frame->flags & KMP_FRAME_FLAG_KEY, frame->pts_delay);
break;

case KMP_CODEC_VIDEO_H265:
header_size = NGX_RTMP_EXT_HEADER_SIZE_NALU;
h.type = NGX_RTMP_MSG_VIDEO;

p = ngx_kmp_rtmp_encoder_ext_header_write(p, NGX_RTMP_EXT_NALU,
frame->flags & KMP_FRAME_FLAG_KEY, NGX_RTMP_EXT_FOURCC_HVC1,
frame->pts_delay);
break;

case KMP_CODEC_AUDIO_MP3:
header_size = 1;
h.type = NGX_RTMP_MSG_AUDIO;
Expand Down
7 changes: 7 additions & 0 deletions nginx-kmp-rtmp-module/src/ngx_kmp_rtmp_encoder.h
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@

#define NGX_KMP_RTMP_TIMESCALE (1000)

#define NGX_RTMP_EXT_FOURCC_HVC1 0x31637668


typedef ngx_int_t (*ngx_kmp_rtmp_write_pt)(void *data, void *buf, size_t size);

Expand Down Expand Up @@ -95,6 +97,11 @@ size_t ngx_kmp_rtmp_encoder_avc_sequence_get_size(
u_char *ngx_kmp_rtmp_encoder_avc_sequence_write(u_char *p,
ngx_kmp_rtmp_stream_ctx_t *sc, ngx_str_t *extra_data);

size_t ngx_kmp_rtmp_encoder_ext_sequence_get_size(
ngx_kmp_rtmp_stream_ctx_t *sc, ngx_str_t *extra_data);
u_char *ngx_kmp_rtmp_encoder_ext_sequence_write(u_char *p,
ngx_kmp_rtmp_stream_ctx_t *sc, uint32_t fourcc, ngx_str_t *extra_data);

size_t ngx_kmp_rtmp_encoder_aac_sequence_get_size(
ngx_kmp_rtmp_stream_ctx_t *sc, ngx_str_t *extra_data);
u_char *ngx_kmp_rtmp_encoder_aac_sequence_write(u_char *p,
Expand Down
20 changes: 18 additions & 2 deletions nginx-kmp-rtmp-module/src/ngx_kmp_rtmp_stream.c
Original file line number Diff line number Diff line change
Expand Up @@ -185,9 +185,17 @@ ngx_kmp_rtmp_stream_write_meta(ngx_kmp_rtmp_stream_t *stream)

size = ngx_kmp_rtmp_encoder_metadata_get_size(&stream->ctx, &meta);

if (meta.mi[KMP_MEDIA_VIDEO].codec_id == KMP_CODEC_VIDEO_H264) {
switch (meta.mi[KMP_MEDIA_VIDEO].codec_id) {

case KMP_CODEC_VIDEO_H264:
size += ngx_kmp_rtmp_encoder_avc_sequence_get_size(
&stream->ctx, &extra_data[KMP_MEDIA_VIDEO]);
break;

case KMP_CODEC_VIDEO_H265:
size += ngx_kmp_rtmp_encoder_ext_sequence_get_size(
&stream->ctx, &extra_data[KMP_MEDIA_VIDEO]);
break;
}

if (meta.mi[KMP_MEDIA_AUDIO].codec_id == KMP_CODEC_AUDIO_AAC) {
Expand All @@ -204,9 +212,17 @@ ngx_kmp_rtmp_stream_write_meta(ngx_kmp_rtmp_stream_t *stream)

p = ngx_kmp_rtmp_encoder_metadata_write(start, &stream->ctx, &meta);

if (meta.mi[KMP_MEDIA_VIDEO].codec_id == KMP_CODEC_VIDEO_H264) {
switch (meta.mi[KMP_MEDIA_VIDEO].codec_id) {

case KMP_CODEC_VIDEO_H264:
p = ngx_kmp_rtmp_encoder_avc_sequence_write(p, &stream->ctx,
&extra_data[KMP_MEDIA_VIDEO]);
break;

case KMP_CODEC_VIDEO_H265:
p = ngx_kmp_rtmp_encoder_ext_sequence_write(p, &stream->ctx,
NGX_RTMP_EXT_FOURCC_HVC1, &extra_data[KMP_MEDIA_VIDEO]);
break;
}

if (meta.mi[KMP_MEDIA_AUDIO].codec_id == KMP_CODEC_AUDIO_AAC) {
Expand Down
1 change: 1 addition & 0 deletions nginx-kmp-rtmp-module/src/ngx_kmp_rtmp_track.c
Original file line number Diff line number Diff line change
Expand Up @@ -316,6 +316,7 @@ ngx_kmp_rtmp_validate_media_info(ngx_log_t *log, kmp_media_info_t *media_info)
switch (media_info->codec_id) {

case KMP_CODEC_VIDEO_H264:
case KMP_CODEC_VIDEO_H265:
break;

default:
Expand Down
134 changes: 101 additions & 33 deletions nginx-rtmp-kmp-module/src/ngx_rtmp_kmp_track.c
Original file line number Diff line number Diff line change
Expand Up @@ -162,8 +162,19 @@ ngx_rtmp_kmp_track_send_media_info(ngx_kmp_out_track_t *track,
switch (media_info->media_type) {

case KMP_MEDIA_VIDEO:
/* KMP video codec ids match NGX_RTMP_VIDEO_XXX */
media_info->codec_id = codec_ctx->video_codec_id;
switch (codec_ctx->video_codec_id) {

case NGX_RTMP_CODEC_FOURCC_HVC1:
case NGX_RTMP_CODEC_FOURCC_HEV1:
media_info->codec_id = KMP_CODEC_VIDEO_H265;
break;

default:
/* KMP video codec ids match NGX_RTMP_VIDEO_XXX */
media_info->codec_id = codec_ctx->video_codec_id;
break;
}

media_info->bitrate = codec_ctx->video_data_rate * 1000;

media_info->u.video.width = codec_ctx->width;
Expand Down Expand Up @@ -208,17 +219,15 @@ ngx_rtmp_kmp_track_init_frame(ngx_kmp_out_track_t *track,
{
u_char frame_info;
u_char packet_type;
u_char codec_id;
u_char comp_time[3];
int32_t pts_delay;
uint32_t codec_id;
uint32_t rtmpscale;
ngx_int_t rc;
ngx_flag_t ext_header;
ngx_flag_t has_pts_delay;
ngx_rtmp_kmp_track_ctx_t *ctx = track->ctx;

struct {
u_char packet_type;
u_char comp_time[3];
} avc_header;

*sequence_header = 0;

rc = ngx_rtmp_kmp_copy(&track->log, &frame_info, src, 1, in);
Expand All @@ -239,42 +248,101 @@ ngx_rtmp_kmp_track_init_frame(ngx_kmp_out_track_t *track,
switch (h->type) {

case NGX_RTMP_MSG_VIDEO:
codec_id = frame_info & 0x0f;
ext_header = (frame_info & NGX_RTMP_EXT_HEADER_MASK) ? 1 : 0;
if (!ext_header) {

if ((frame_info >> 4) == NGX_RTMP_KEY_FRAME) {
frame->f.flags |= KMP_FRAME_FLAG_KEY;
}
codec_id = frame_info & 0x0f;
if (codec_id != NGX_RTMP_VIDEO_H264) {
break;
}

rc = ngx_rtmp_kmp_copy(&track->log, &packet_type, src,
sizeof(packet_type), in);
if (rc != NGX_OK) {
ngx_log_error(NGX_LOG_NOTICE, &track->log, 0,
"ngx_rtmp_kmp_track_init_frame: "
"failed to read packet_type");
ngx_kmp_out_track_set_error_reason(track, "rtmp_bad_data");
return NGX_ERROR;
}

frame->header.data_size -= sizeof(packet_type);

has_pts_delay = 1;

if (packet_type == NGX_RTMP_AVC_SEQUENCE_HEADER) {
*sequence_header = 1;
}

} else {

frame_info &= ~NGX_RTMP_EXT_HEADER_MASK;
packet_type = (frame_info & 0x0f);

rc = ngx_rtmp_kmp_copy(&track->log, &codec_id, src,
sizeof(codec_id), in);
if (rc != NGX_OK) {
ngx_log_error(NGX_LOG_NOTICE, &track->log, 0,
"ngx_rtmp_kmp_track_init_frame: "
"failed to read extended codec_id");
ngx_kmp_out_track_set_error_reason(track, "rtmp_bad_data");
return NGX_ERROR;
}

frame->header.data_size -= sizeof(codec_id);

switch (codec_id) {

case NGX_RTMP_CODEC_FOURCC_HVC1:
case NGX_RTMP_CODEC_FOURCC_HEV1:
has_pts_delay = packet_type == NGX_RTMP_PKT_TYPE_CODED_FRAMES;
break;

default:
has_pts_delay = 0;
}

if (packet_type == NGX_RTMP_PKT_TYPE_SEQUENCE_START) {
*sequence_header = 1;
}

if (codec_id != NGX_RTMP_VIDEO_H264) {
break;
}

rc = ngx_rtmp_kmp_copy(&track->log, &avc_header, src,
sizeof(avc_header), in);
if (rc != NGX_OK) {
ngx_log_error(NGX_LOG_NOTICE, &track->log, 0,
"ngx_rtmp_kmp_track_init_frame: failed to read avc header");
ngx_kmp_out_track_set_error_reason(track, "rtmp_bad_data");
return NGX_ERROR;
if ((frame_info >> 4) == NGX_RTMP_KEY_FRAME) {
frame->f.flags |= KMP_FRAME_FLAG_KEY;
}

frame->header.data_size -= sizeof(avc_header);
if (has_pts_delay) {
rc = ngx_rtmp_kmp_copy(&track->log, &comp_time, src,
sizeof(comp_time), in);
if (rc != NGX_OK) {
ngx_log_error(NGX_LOG_NOTICE, &track->log, 0,
"ngx_rtmp_kmp_track_init_frame: failed to read comp_time");
ngx_kmp_out_track_set_error_reason(track, "rtmp_bad_data");
return NGX_ERROR;
}

if (avc_header.packet_type == NGX_RTMP_AVC_SEQUENCE_HEADER) {
*sequence_header = 1;
}
frame->header.data_size -= sizeof(comp_time);

pts_delay =
(avc_header.comp_time[0] << 16) |
(avc_header.comp_time[1] << 8) |
avc_header.comp_time[2];
pts_delay =
(comp_time[0] << 16) |
(comp_time[1] << 8) |
comp_time[2];

/* sign extend */
if (pts_delay & 0x800000) {
pts_delay |= 0xff000000;
}

/* sign extend */
if (pts_delay & 0x800000) {
pts_delay |= 0xff000000;
frame->f.pts_delay = pts_delay * rtmpscale;
}

frame->f.pts_delay = pts_delay * rtmpscale;
ngx_log_debug5(NGX_LOG_DEBUG_RTMP, &track->log, 0,
"ngx_rtmp_kmp_track_init_frame: %s rtmp, codec_id: 0x%uxD, "
"data_size: %uD, packet_type: %uD, pts_delay: %D",
ext_header ? "extended" : "", codec_id,
frame->header.data_size, (uint32_t) packet_type,
frame->f.pts_delay);
break;

case NGX_RTMP_MSG_AUDIO:
Expand Down
7 changes: 0 additions & 7 deletions nginx-rtmp-module/src/ngx_rtmp.h
Original file line number Diff line number Diff line change
Expand Up @@ -628,13 +628,6 @@ ngx_rtmp_get_video_frame_type(ngx_chain_t *in)
}


static ngx_inline ngx_int_t
ngx_rtmp_is_codec_header(ngx_chain_t *in)
{
return in->buf->pos + 1 < in->buf->last && in->buf->pos[1] == 0;
}


extern ngx_rtmp_bandwidth_t ngx_rtmp_bw_out;
extern ngx_rtmp_bandwidth_t ngx_rtmp_bw_in;

Expand Down
Loading

0 comments on commit bcd0e8f

Please sign in to comment.