From 1be800b54459c03ea2a63d49239b3be2acd60a3d Mon Sep 17 00:00:00 2001 From: niyatim23 Date: Thu, 18 Apr 2024 15:50:34 -0700 Subject: [PATCH 1/7] gst-sample-fix --- README.md | 8 ++++++++ samples/kvsWebRTCClientMasterGstSample.c | 2 -- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e2351eedbe..4f04433c9c 100644 --- a/README.md +++ b/README.md @@ -309,6 +309,14 @@ To run: ./samples/kvsWebrtcClientViewerGstSample ``` +##### Note: +Our GStreamer samples leverage [MatroskaMux](https://gstreamer.freedesktop.org/documentation/matroska/matroskamux.html?gi-language=c) to receive media from its peer and save it to a file. However, MatroskaMux is designed for scenarios where the media's format remains constant throughout streaming. Unfortunately, this is not the case while using certain browsers as peers with ([AWS KVS JS SDK](https://awslabs.github.io/amazon-kinesis-video-streams-webrtc-sdk-js/examples/index.html)), so receiving media from the browsers andf writing it to filesink via MatroskaMux is not supported by our existing GStreamer samples. When the media's format changes mid-streaming (referred to as "caps changes"), MatroskaMux encounters limitations and is unable to handle these changes, resulting in an error message like: + +```shell +matroskamux matroska-mux.c:1134:gst_matroska_mux_video_pad_setcaps: error: Caps changes are not supported by Matroska +``` +To address this issue, users need to adapt the pipeline to utilize components capable of managing dynamic changes in media formats. This might involve integrating different muxers or customizing the pipeline to handle caps changes effectively. + #### Sample: Generating sample frames ##### H264 diff --git a/samples/kvsWebRTCClientMasterGstSample.c b/samples/kvsWebRTCClientMasterGstSample.c index 8fc74c2bd9..81bd74845e 100644 --- a/samples/kvsWebRTCClientMasterGstSample.c +++ b/samples/kvsWebRTCClientMasterGstSample.c @@ -326,8 +326,6 @@ INT32 main(INT32 argc, CHAR* argv[]) pSampleConfiguration->videoSource = sendGstreamerAudioVideo; pSampleConfiguration->mediaType = SAMPLE_STREAMING_VIDEO_ONLY; - pSampleConfiguration->receiveAudioVideoSource = receiveGstreamerAudioVideo; - #ifdef ENABLE_DATA_CHANNEL pSampleConfiguration->onDataChannel = onDataChannel; #endif From dc84de846112473f701f61c8d4ba5f4adc0b5947 Mon Sep 17 00:00:00 2001 From: niyatim23 Date: Thu, 18 Apr 2024 16:00:57 -0700 Subject: [PATCH 2/7] modify the message --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4f04433c9c..1067bbda88 100644 --- a/README.md +++ b/README.md @@ -309,8 +309,8 @@ To run: ./samples/kvsWebrtcClientViewerGstSample ``` -##### Note: -Our GStreamer samples leverage [MatroskaMux](https://gstreamer.freedesktop.org/documentation/matroska/matroskamux.html?gi-language=c) to receive media from its peer and save it to a file. However, MatroskaMux is designed for scenarios where the media's format remains constant throughout streaming. Unfortunately, this is not the case while using certain browsers as peers with ([AWS KVS JS SDK](https://awslabs.github.io/amazon-kinesis-video-streams-webrtc-sdk-js/examples/index.html)), so receiving media from the browsers andf writing it to filesink via MatroskaMux is not supported by our existing GStreamer samples. When the media's format changes mid-streaming (referred to as "caps changes"), MatroskaMux encounters limitations and is unable to handle these changes, resulting in an error message like: +##### Known issues: +Our GStreamer samples leverage [MatroskaMux](https://gstreamer.freedesktop.org/documentation/matroska/matroskamux.html?gi-language=c) to receive media from its peer and save it to a file. However, MatroskaMux is designed for scenarios where the media's format remains constant throughout streaming. When the media's format changes mid-streaming (referred to as "caps changes"), MatroskaMux encounters limitations, its behavior cannot be predicted and it may be unable to handle these changes, resulting in an error message like: ```shell matroskamux matroska-mux.c:1134:gst_matroska_mux_video_pad_setcaps: error: Caps changes are not supported by Matroska @@ -333,12 +333,12 @@ gst-launch-1.0 videotestsrc pattern=ball num-buffers=1500 ! timeoverlay ! videoc ###### ADTS LC ```shell -gst-launch-1.0 audiotestsrc num-buffers=1500 ! audioconvert ! audioresample ! faac ! capsfilter caps=audio/mpeg,mpegversion=4,stream-format=adts,base-profile=lc,channels=2,rate=48000 ! multifilesink location="sample-%03d.aac" index=1 +gst-launch-1.0 audiotestsrc num-buffers=1500 ! audioconvert ! audioresample ! faac ! capsfilter caps=audio/mpeg,mpegversion=4,stream-format=adts,base-profile=lc,channels=2,rate=16000 ! multifilesink location="sample-%03d.aac" index=1 ``` ###### RAW LC ```shell -gst-launch-1.0 audiotestsrc num-buffers=1500 ! audioconvert ! audioresample ! faac ! capsfilter caps=audio/mpeg,mpegversion=4,stream-format=raw,base-profile=lc,channels=2,rate=44100 ! multifilesink location="sample-%03d.aac" index=1 +gst-launch-1.0 audiotestsrc num-buffers=1500 ! audioconvert ! audioresample ! faac ! capsfilter caps=audio/mpeg,mpegversion=4,stream-format=raw,base-profile=lc,channels=2,rate=16000 ! multifilesink location="sample-%03d.aac" index=1 ``` ### Viewing Master Samples From 6cddc1e1b0ad270e25e767c0698a865cb2bc7844 Mon Sep 17 00:00:00 2001 From: niyatim23 Date: Thu, 18 Apr 2024 16:04:56 -0700 Subject: [PATCH 3/7] fix windows --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1de2a7e4e5..bc3e6b90c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -584,6 +584,7 @@ jobs: - name: Install dependencies shell: powershell run: | + choco install pkgconfiglite choco install gstreamer --version=1.16.3 choco install gstreamer-devel --version=1.16.3 curl.exe -o C:\tools\pthreads-w32-2-9-1-release.zip ftp://sourceware.org/pub/pthreads-win32/pthreads-w32-2-9-1-release.zip From 55528d6e684368075bb1292dff37d1c218d72167 Mon Sep 17 00:00:00 2001 From: niyatim23 Date: Thu, 18 Apr 2024 16:06:47 -0700 Subject: [PATCH 4/7] set to null --- samples/kvsWebRTCClientMasterGstSample.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/samples/kvsWebRTCClientMasterGstSample.c b/samples/kvsWebRTCClientMasterGstSample.c index 81bd74845e..3569a5b89b 100644 --- a/samples/kvsWebRTCClientMasterGstSample.c +++ b/samples/kvsWebRTCClientMasterGstSample.c @@ -326,6 +326,8 @@ INT32 main(INT32 argc, CHAR* argv[]) pSampleConfiguration->videoSource = sendGstreamerAudioVideo; pSampleConfiguration->mediaType = SAMPLE_STREAMING_VIDEO_ONLY; + pSampleConfiguration->receiveAudioVideoSource = NULL; + #ifdef ENABLE_DATA_CHANNEL pSampleConfiguration->onDataChannel = onDataChannel; #endif From 5c5e8ffcf07eddc5edc191b582a5338e4ca7b1e6 Mon Sep 17 00:00:00 2001 From: niyatim23 Date: Thu, 18 Apr 2024 20:35:30 -0700 Subject: [PATCH 5/7] fix pts and eos in sample --- samples/GstAudioVideoReceiver.c | 69 ++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/samples/GstAudioVideoReceiver.c b/samples/GstAudioVideoReceiver.c index a750a95b2e..02e4c0d010 100644 --- a/samples/GstAudioVideoReceiver.c +++ b/samples/GstAudioVideoReceiver.c @@ -3,6 +3,9 @@ #include #include +static UINT64 presentationTsIncrement = 0; +static BOOL eos = FALSE; + // This function is a callback for the transceiver for every single video frame it receives // It writes these frames to a buffer and pushes it to the `appsrcVideo` element of the // GStreamer pipeline created in `receiveGstreamerAudioVideo`. Any logic to modify / discard the frames would go here @@ -16,22 +19,28 @@ VOID onGstVideoFrameReady(UINT64 customData, PFrame pFrame) CHK_ERR(appsrcVideo != NULL, STATUS_NULL_ARG, "appsrcVideo is null"); CHK_ERR(pFrame != NULL, STATUS_NULL_ARG, "Video frame is null"); - buffer = gst_buffer_new_allocate(NULL, pFrame->size, NULL); - CHK_ERR(buffer != NULL, STATUS_NULL_ARG, "Buffer allocation failed"); + if (!eos) { + buffer = gst_buffer_new_allocate(NULL, pFrame->size, NULL); + CHK_ERR(buffer != NULL, STATUS_NULL_ARG, "Buffer allocation failed"); + + DLOGV("Video frame size: %d, presentationTs: %llu", pFrame->size, presentationTsIncrement); + + GST_BUFFER_DTS(buffer) = presentationTsIncrement; + GST_BUFFER_PTS(buffer) = presentationTsIncrement; + GST_BUFFER_DURATION(buffer) = gst_util_uint64_scale(1, GST_SECOND, DEFAULT_FPS_VALUE); + presentationTsIncrement += gst_util_uint64_scale(1, GST_SECOND, DEFAULT_FPS_VALUE); - DLOGV("Frame size: %d, presentationTs: %llu", pFrame->size, pFrame->presentationTs); - GST_BUFFER_PTS(buffer) = pFrame->presentationTs; - GST_BUFFER_DURATION(buffer) = gst_util_uint64_scale(1, GST_SECOND, DEFAULT_FPS_VALUE); - if (gst_buffer_fill(buffer, 0, pFrame->frameData, pFrame->size) != pFrame->size) { - DLOGE("Buffer fill did not complete correctly"); + if (gst_buffer_fill(buffer, 0, pFrame->frameData, pFrame->size) != pFrame->size) { + DLOGE("Buffer fill did not complete correctly"); + gst_buffer_unref(buffer); + return; + } + g_signal_emit_by_name(appsrcVideo, "push-buffer", buffer, &ret); + if (ret != GST_FLOW_OK) { + DLOGE("Error pushing buffer: %s", gst_flow_get_name(ret)); + } gst_buffer_unref(buffer); - return; - } - g_signal_emit_by_name(appsrcVideo, "push-buffer", buffer, &ret); - if (ret != GST_FLOW_OK) { - DLOGE("Error pushing buffer: %s", gst_flow_get_name(ret)); } - gst_buffer_unref(buffer); CleanUp: return; @@ -50,25 +59,27 @@ VOID onGstAudioFrameReady(UINT64 customData, PFrame pFrame) CHK_ERR(appsrcAudio != NULL, STATUS_NULL_ARG, "appsrcAudio is null"); CHK_ERR(pFrame != NULL, STATUS_NULL_ARG, "Audio frame is null"); - buffer = gst_buffer_new_allocate(NULL, pFrame->size, NULL); - CHK_ERR(buffer != NULL, STATUS_NULL_ARG, "Buffer allocation failed"); + if (!eos) { + buffer = gst_buffer_new_allocate(NULL, pFrame->size, NULL); + CHK_ERR(buffer != NULL, STATUS_NULL_ARG, "Buffer allocation failed"); - DLOGV("Audio Frame size: %d, presentationTs: %llu", pFrame->size, pFrame->presentationTs); - GST_BUFFER_PTS(buffer) = pFrame->presentationTs; + DLOGV("Audio frame size: %d, presentationTs: %llu", pFrame->size, presentationTsIncrement); - // Recalculate the byte-rate if not using the default values - GST_BUFFER_DURATION(buffer) = gst_util_uint64_scale(pFrame->size, GST_SECOND, DEFAULT_AUDIO_OPUS_BYTE_RATE); - if (gst_buffer_fill(buffer, 0, pFrame->frameData, pFrame->size) != pFrame->size) { - DLOGE("Buffer fill did not complete correctly"); + GST_BUFFER_DTS(buffer) = presentationTsIncrement; + GST_BUFFER_PTS(buffer) = presentationTsIncrement; + GST_BUFFER_DURATION(buffer) = gst_util_uint64_scale(pFrame->size, GST_SECOND, DEFAULT_AUDIO_OPUS_BYTE_RATE); + + if (gst_buffer_fill(buffer, 0, pFrame->frameData, pFrame->size) != pFrame->size) { + DLOGE("Buffer fill did not complete correctly"); + gst_buffer_unref(buffer); + return; + } + g_signal_emit_by_name(appsrcAudio, "push-buffer", buffer, &ret); + if (ret != GST_FLOW_OK) { + DLOGE("Error pushing buffer: %s", gst_flow_get_name(ret)); + } gst_buffer_unref(buffer); - return; } - g_signal_emit_by_name(appsrcAudio, "push-buffer", buffer, &ret); - if (ret != GST_FLOW_OK) { - DLOGE("Error pushing buffer: %s", gst_flow_get_name(ret)); - } - gst_buffer_unref(buffer); - CleanUp: return; } @@ -78,7 +89,9 @@ VOID onGstAudioFrameReady(UINT64 customData, PFrame pFrame) VOID onSampleStreamingSessionShutdown(UINT64 customData, PSampleStreamingSession pSampleStreamingSession) { (void) (pSampleStreamingSession); + eos = TRUE; GstElement* pipeline = (GstElement*) customData; + GstAppSrc *appsrcVideo = NULL, *appsrcAudio = NULL; gst_element_send_event(pipeline, gst_event_new_eos()); } From d080c064dd2311d62a2ce898da82b9101882107e Mon Sep 17 00:00:00 2001 From: niyatim23 Date: Thu, 18 Apr 2024 20:36:43 -0700 Subject: [PATCH 6/7] unused var removed --- samples/GstAudioVideoReceiver.c | 1 - 1 file changed, 1 deletion(-) diff --git a/samples/GstAudioVideoReceiver.c b/samples/GstAudioVideoReceiver.c index 02e4c0d010..c961d5895f 100644 --- a/samples/GstAudioVideoReceiver.c +++ b/samples/GstAudioVideoReceiver.c @@ -91,7 +91,6 @@ VOID onSampleStreamingSessionShutdown(UINT64 customData, PSampleStreamingSession (void) (pSampleStreamingSession); eos = TRUE; GstElement* pipeline = (GstElement*) customData; - GstAppSrc *appsrcVideo = NULL, *appsrcAudio = NULL; gst_element_send_event(pipeline, gst_event_new_eos()); } From 42310eab454af82e8b85844bc099789669b0aca3 Mon Sep 17 00:00:00 2001 From: niyatim23 Date: Mon, 22 Apr 2024 12:55:56 -0700 Subject: [PATCH 7/7] move null init --- samples/Common.c | 1 + samples/kvsWebRTCClientMasterGstSample.c | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/Common.c b/samples/Common.c index 295c8c0c46..64e9739180 100644 --- a/samples/Common.c +++ b/samples/Common.c @@ -880,6 +880,7 @@ STATUS createSampleConfiguration(PCHAR channelName, SIGNALING_CHANNEL_ROLE_TYPE pSampleConfiguration->trickleIce = trickleIce; pSampleConfiguration->useTurn = useTurn; pSampleConfiguration->enableSendingMetricsToViewerViaDc = FALSE; + pSampleConfiguration->receiveAudioVideoSource = NULL; pSampleConfiguration->channelInfo.version = CHANNEL_INFO_CURRENT_VERSION; pSampleConfiguration->channelInfo.pChannelName = channelName; diff --git a/samples/kvsWebRTCClientMasterGstSample.c b/samples/kvsWebRTCClientMasterGstSample.c index 3569a5b89b..4cbe8e2b9f 100644 --- a/samples/kvsWebRTCClientMasterGstSample.c +++ b/samples/kvsWebRTCClientMasterGstSample.c @@ -326,7 +326,6 @@ INT32 main(INT32 argc, CHAR* argv[]) pSampleConfiguration->videoSource = sendGstreamerAudioVideo; pSampleConfiguration->mediaType = SAMPLE_STREAMING_VIDEO_ONLY; - pSampleConfiguration->receiveAudioVideoSource = NULL; #ifdef ENABLE_DATA_CHANNEL pSampleConfiguration->onDataChannel = onDataChannel;