From 3f01606360db87e6b488d6c9d2d6215deee3e08c Mon Sep 17 00:00:00 2001 From: volodymyr-bondarenko85 <131799099+volodymyr-bondarenko85@users.noreply.github.com> Date: Wed, 22 Nov 2023 19:34:27 +0200 Subject: [PATCH] =?UTF-8?q?KUX-1021:=20RN=5FMulticast=5FSTB=20-=20UDP=20/?= =?UTF-8?q?=20Multicast=20zapping=20time=20is=20over=205=20s=E2=80=A6=20(#?= =?UTF-8?q?812)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * KUX-1021: RN_Multicast_STB - UDP / Multicast zapping time is over 5 seconds --- .../audio/KMediaCodecAudioRenderer.java | 216 ++++++++++++++++++ .../video/KMediaCodecVideoRenderer.java | 72 ++++++ ...rerFirstFrameWhenStartedEventListener.java | 6 + .../main/java/com/kaltura/playkit/Player.java | 8 + .../SpeedAdjustedRenderersFactory.java | 140 ++++++++++++ .../playkit/player/BaseExoplayerView.java | 3 +- .../kaltura/playkit/player/ExoPlayerView.java | 48 +++- .../playkit/player/ExoPlayerWrapper.java | 33 ++- .../playkit/player/MulticastSettings.java | 63 +++++ .../playkit/player/PlayerSettings.java | 10 + 10 files changed, 587 insertions(+), 12 deletions(-) create mode 100644 playkit/src/main/java/com/kaltura/android/exoplayer2/audio/KMediaCodecAudioRenderer.java create mode 100644 playkit/src/main/java/com/kaltura/android/exoplayer2/video/KMediaCodecVideoRenderer.java create mode 100644 playkit/src/main/java/com/kaltura/android/exoplayer2/video/KVideoRendererFirstFrameWhenStartedEventListener.java create mode 100644 playkit/src/main/java/com/kaltura/playkit/SpeedAdjustedRenderersFactory.java diff --git a/playkit/src/main/java/com/kaltura/android/exoplayer2/audio/KMediaCodecAudioRenderer.java b/playkit/src/main/java/com/kaltura/android/exoplayer2/audio/KMediaCodecAudioRenderer.java new file mode 100644 index 000000000..10bb43a58 --- /dev/null +++ b/playkit/src/main/java/com/kaltura/android/exoplayer2/audio/KMediaCodecAudioRenderer.java @@ -0,0 +1,216 @@ +package com.kaltura.android.exoplayer2.audio; + +import static com.kaltura.android.exoplayer2.audio.DefaultAudioSink.DEFAULT_PLAYBACK_SPEED; +import static java.lang.Math.max; +import static java.lang.Math.min; + +import android.content.Context; +import android.media.MediaCodec; +import android.os.Handler; + +import androidx.annotation.Nullable; + +import com.kaltura.android.exoplayer2.ExoPlaybackException; +import com.kaltura.android.exoplayer2.Format; +import com.kaltura.android.exoplayer2.PlaybackParameters; +import com.kaltura.android.exoplayer2.decoder.DecoderInputBuffer; +import com.kaltura.android.exoplayer2.mediacodec.MediaCodecAdapter; +import com.kaltura.android.exoplayer2.mediacodec.MediaCodecSelector; +import com.kaltura.playkit.PKLog; + +import java.lang.reflect.Field; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class KMediaCodecAudioRenderer extends MediaCodecAudioRenderer { + + private static final String ALLOW_FIRST_BUFFER_POSITION_DISCONTINUITY_FIELD_NAME = "allowFirstBufferPositionDiscontinuity"; + + private static final String CURRENT_POSITION_US_FIELD_NAME = "currentPositionUs"; + + private static final String DECRYPT_ONLY_CODEC_FORMAT_FIELD_NAME = "decryptOnlyCodecFormat"; + + private static final long DEFAULT_MAX_AUDIO_GAP_THRESHOLD = 3_000_000L; + + private static final boolean DEFAULT_USE_CONTINUOUS_SPEED_ADJUSTMENT = false; + + private static final float DEFAULT_MAX_SPEED_FACTOR = 4.0f; + + private static final float DEFAULT_SPEED_STEP = 3.0f; + + private static final long DEFAULT_MAX_AV_GAP = 600_000L; + + private final long maxAudioGapThreshold; + + private final boolean useContinuousSpeedAdjustment; + + private final float maxSpeedFactor; + + private final float speedStep; + + private final long maxAVGap; + + private static final PKLog log = PKLog.get("KMediaCodecAudioRenderer"); + + private boolean speedAdjustedAfterPositionReset = false; + + public KMediaCodecAudioRenderer(Context context, + MediaCodecAdapter.Factory codecAdapterFactory, + MediaCodecSelector mediaCodecSelector, + boolean enableDecoderFallback, + @Nullable Handler eventHandler, + @Nullable AudioRendererEventListener eventListener, + AudioSink audioSink) { + this(context, + codecAdapterFactory, + mediaCodecSelector, + enableDecoderFallback, + eventHandler, + eventListener, + audioSink, + DEFAULT_MAX_AUDIO_GAP_THRESHOLD, + DEFAULT_MAX_SPEED_FACTOR, + DEFAULT_SPEED_STEP, + DEFAULT_MAX_AV_GAP, + DEFAULT_USE_CONTINUOUS_SPEED_ADJUSTMENT); + } + + public KMediaCodecAudioRenderer(Context context, + MediaCodecAdapter.Factory codecAdapterFactory, + MediaCodecSelector mediaCodecSelector, + boolean enableDecoderFallback, + @Nullable Handler eventHandler, + @Nullable AudioRendererEventListener eventListener, + AudioSink audioSink, + long maxAudioGapThreshold, + float maxSpeedFactor, + float speedStep, + long maxAVGap, + boolean useContinuousSpeedAdjustment) { + super(context, codecAdapterFactory, mediaCodecSelector, enableDecoderFallback, eventHandler, eventListener, audioSink); + this.maxAudioGapThreshold = maxAudioGapThreshold; + this.maxSpeedFactor = maxSpeedFactor; + this.speedStep = speedStep; + this.maxAVGap = maxAVGap; + this.useContinuousSpeedAdjustment = useContinuousSpeedAdjustment; + log.d("KMediaCodecAudioRenderer", "getSpeedGap()=" + getMaxAVGap() + + ", getSpeedFactor()=" + getMaxSpeedFactor() + + ", getSpeedStep()=" + getSpeedStep() + + ", continuousSpeedAdjustment=" + getContinuousSpeedAdjustment()); + } + + protected long getMaxAudioGapThreshold() { + return maxAudioGapThreshold; + } + + protected float getMaxSpeedFactor() { + return maxSpeedFactor; + } + + protected float getSpeedStep() { + return speedStep; + } + + protected long getMaxAVGap() { + return maxAVGap; + } + + protected boolean getContinuousSpeedAdjustment() { + return useContinuousSpeedAdjustment; + } + + @Override + protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException { + super.onPositionReset(positionUs, joining); + speedAdjustedAfterPositionReset = false; + } + + @Override + protected void onQueueInputBuffer(DecoderInputBuffer buffer) { + try { + Field allowFirstBufferPositionDiscontinuityField = Objects.requireNonNull( + getClass().getSuperclass()) + .getDeclaredField(ALLOW_FIRST_BUFFER_POSITION_DISCONTINUITY_FIELD_NAME); + allowFirstBufferPositionDiscontinuityField.setAccessible(true); + Field currentPositionUsField = Objects.requireNonNull( + getClass().getSuperclass()) + .getDeclaredField(CURRENT_POSITION_US_FIELD_NAME); + currentPositionUsField.setAccessible(true); + if (allowFirstBufferPositionDiscontinuityField.getBoolean(this) && !buffer.isDecodeOnly()) { + log.d("KMediaCodecAudioRenderer", "A/V start buffers gap measured: " + + Math.abs(buffer.timeUs - currentPositionUsField.getLong(this)) + " uS"); + if (Math.abs(buffer.timeUs - currentPositionUsField.getLong(this)) > getMaxAudioGapThreshold()) { + currentPositionUsField.setLong(this, buffer.timeUs); + } + allowFirstBufferPositionDiscontinuityField.setBoolean(this, false); + } + } catch (NoSuchFieldException | IllegalArgumentException | IllegalAccessException | NullPointerException e) { + log.e("KMediaCodecAudioRenderer", "Error subclassing audio renderer: " + e.getMessage()); + // Fallback to superclass in case something goes wrong + super.onQueueInputBuffer(buffer); + } + } + + @Override + protected boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs, @Nullable MediaCodecAdapter codec, @Nullable ByteBuffer buffer, int bufferIndex, int bufferFlags, int sampleCount, long bufferPresentationTimeUs, boolean isDecodeOnlyBuffer, boolean isLastBuffer, Format format) throws ExoPlaybackException { + Format decryptOnlyCodecFormat = null; + try { + Field decryptOnlyCodeFormatField = Objects.requireNonNull( + getClass().getSuperclass()).getDeclaredField(DECRYPT_ONLY_CODEC_FORMAT_FIELD_NAME); + decryptOnlyCodeFormatField.setAccessible(true); + decryptOnlyCodecFormat = (Format)decryptOnlyCodeFormatField.get(this); + } catch (NoSuchFieldException | IllegalArgumentException | IllegalAccessException | NullPointerException e) { + log.e("KMediaCodecAudioRenderer", "Error getting decryptOnlyCodecFormat: " + e.getMessage()); + } + + if ((decryptOnlyCodecFormat != null + && (bufferFlags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) || isDecodeOnlyBuffer) { + return super.processOutputBuffer( + positionUs, + elapsedRealtimeUs, + codec, + buffer, + bufferIndex, + bufferFlags, + sampleCount, + bufferPresentationTimeUs, + isDecodeOnlyBuffer, + isLastBuffer, + format); + } + + if (!speedAdjustedAfterPositionReset || getContinuousSpeedAdjustment()) { + log.d("KMediaCodecAudioRenderer", "currentSpeed=" + getPlaybackParameters().speed + + ", bufferPresentationTimeUs=" + bufferPresentationTimeUs + + ", positionUs=" + positionUs); + if (bufferPresentationTimeUs - positionUs > getMaxAVGap() + && getPlaybackParameters().speed < getMaxSpeedFactor()) { + float newSpeed = getPlaybackParameters().speed + getSpeedStep(); + newSpeed = min(newSpeed, getMaxSpeedFactor()); + log.d("KMediaCodecAudioRenderer", "Setting speed to " + newSpeed); + setPlaybackParameters(new PlaybackParameters(newSpeed)); + } else if (getPlaybackParameters().speed != DEFAULT_PLAYBACK_SPEED) { + float newSpeed = getPlaybackParameters().speed - getSpeedStep(); + newSpeed = max(newSpeed, DEFAULT_PLAYBACK_SPEED); + log.d("KMediaCodecAudioRenderer", "Setting speed to " + newSpeed); + setPlaybackParameters(new PlaybackParameters(newSpeed)); + if (newSpeed == DEFAULT_PLAYBACK_SPEED) { + speedAdjustedAfterPositionReset = true; + } + } + } + + return super.processOutputBuffer( + positionUs, + elapsedRealtimeUs, + codec, + buffer, + bufferIndex, + bufferFlags, + sampleCount, + bufferPresentationTimeUs, + isDecodeOnlyBuffer, + isLastBuffer, + format); + } +} diff --git a/playkit/src/main/java/com/kaltura/android/exoplayer2/video/KMediaCodecVideoRenderer.java b/playkit/src/main/java/com/kaltura/android/exoplayer2/video/KMediaCodecVideoRenderer.java new file mode 100644 index 000000000..29e0663ba --- /dev/null +++ b/playkit/src/main/java/com/kaltura/android/exoplayer2/video/KMediaCodecVideoRenderer.java @@ -0,0 +1,72 @@ +package com.kaltura.android.exoplayer2.video; + +import android.content.Context; +import android.os.Handler; +import android.os.Looper; + +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; + +import com.kaltura.android.exoplayer2.ExoPlaybackException; +import com.kaltura.android.exoplayer2.mediacodec.MediaCodecAdapter; +import com.kaltura.android.exoplayer2.mediacodec.MediaCodecSelector; +import com.kaltura.playkit.PKLog; + +public class KMediaCodecVideoRenderer extends MediaCodecVideoRenderer{ + + private boolean renderedFirstFrameAfterResetAfterReady = false; + + private boolean shouldNotifyRenderedFirstFrameAfterStarted = false; + + private static final PKLog log = PKLog.get("KMediaCodecVideoRenderer"); + + @Nullable private KVideoRendererFirstFrameWhenStartedEventListener rendererFirstFrameWhenStartedEventListener; + + public KMediaCodecVideoRenderer(Context context, + MediaCodecAdapter.Factory codecAdapterFactory, + MediaCodecSelector mediaCodecSelector, + long allowedJoiningTimeMs, + boolean enableDecoderFallback, + @Nullable Handler eventHandler, + @Nullable VideoRendererEventListener eventListener, + int maxDroppedFramesToNotify, + KVideoRendererFirstFrameWhenStartedEventListener rendererFirstFrameWhenStartedEventListener) { + super(context, codecAdapterFactory, mediaCodecSelector, allowedJoiningTimeMs, enableDecoderFallback, eventHandler, eventListener, maxDroppedFramesToNotify); + this.rendererFirstFrameWhenStartedEventListener = rendererFirstFrameWhenStartedEventListener; + } + + @Override + void maybeNotifyRenderedFirstFrame() { + super.maybeNotifyRenderedFirstFrame(); + if (this.shouldNotifyRenderedFirstFrameAfterStarted) { + log.d("KMediaCodecVideoRenderer", "maybeNotifyRenderedFirstFrame"); + this.shouldNotifyRenderedFirstFrameAfterStarted = false; + new Handler(Looper.getMainLooper()).post(() -> { + if (rendererFirstFrameWhenStartedEventListener != null) { + log.d("KMediaCodecVideoRenderer", "onRenderedFirstFrameWhenStarted"); + rendererFirstFrameWhenStartedEventListener.onRenderedFirstFrameWhenStarted(); + } + }); + } + } + + @Override + protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException { + log.d("KMediaCodecVideoRenderer", "onPositionReset() called with: positionUs = [" + positionUs + "], joining = [" + joining + "]"); + super.onPositionReset(positionUs, joining); + this.renderedFirstFrameAfterResetAfterReady = false; + this.shouldNotifyRenderedFirstFrameAfterStarted = false; + } + + @RequiresApi(21) + @Override + protected void renderOutputBufferV21(MediaCodecAdapter codec, int index, long presentationTimeUs, long releaseTimeNs) { + if(getState() == STATE_STARTED) { + if(!this.renderedFirstFrameAfterResetAfterReady) { + this.renderedFirstFrameAfterResetAfterReady = true; + this.shouldNotifyRenderedFirstFrameAfterStarted = true; + } + } + super.renderOutputBufferV21(codec, index, presentationTimeUs, releaseTimeNs); + } +} diff --git a/playkit/src/main/java/com/kaltura/android/exoplayer2/video/KVideoRendererFirstFrameWhenStartedEventListener.java b/playkit/src/main/java/com/kaltura/android/exoplayer2/video/KVideoRendererFirstFrameWhenStartedEventListener.java new file mode 100644 index 000000000..a8bf40bd1 --- /dev/null +++ b/playkit/src/main/java/com/kaltura/android/exoplayer2/video/KVideoRendererFirstFrameWhenStartedEventListener.java @@ -0,0 +1,6 @@ +package com.kaltura.android.exoplayer2.video; + +public interface KVideoRendererFirstFrameWhenStartedEventListener { + default void onRenderedFirstFrameWhenStarted() { + } +} diff --git a/playkit/src/main/java/com/kaltura/playkit/Player.java b/playkit/src/main/java/com/kaltura/playkit/Player.java index 15cfd2382..6a41f68fa 100644 --- a/playkit/src/main/java/com/kaltura/playkit/Player.java +++ b/playkit/src/main/java/com/kaltura/playkit/Player.java @@ -337,6 +337,14 @@ interface Settings { */ Settings setHandleAudioFocus(boolean handleAudioFocus); + /** + * Set shutterStaysOnRenderedFirstFrame - Whether shutter view being hide on first frame rendered + * + * @param shutterStaysOnRenderedFirstFrame + * @return - Player Settings + */ + Settings setShutterStaysOnRenderedFirstFrame(boolean shutterStaysOnRenderedFirstFrame); + /** * Set preference to choose internal subtitles over external subtitles (Only in the case if the same language is present * in both Internal and External subtitles) - Default is true (Internal is preferred) diff --git a/playkit/src/main/java/com/kaltura/playkit/SpeedAdjustedRenderersFactory.java b/playkit/src/main/java/com/kaltura/playkit/SpeedAdjustedRenderersFactory.java new file mode 100644 index 000000000..c51cacb78 --- /dev/null +++ b/playkit/src/main/java/com/kaltura/playkit/SpeedAdjustedRenderersFactory.java @@ -0,0 +1,140 @@ +/* + * ============================================================================ + * Copyright (C) 2023 Kaltura Inc. + * + * Licensed under the AGPLv3 license, unless a different license for a + * particular library is specified in the applicable library path. + * + * You may obtain a copy of the License at + * https://www.gnu.org/licenses/agpl-3.0.html + * ============================================================================ + */ + +package com.kaltura.playkit; + +import android.content.Context; +import android.os.Handler; + +import androidx.annotation.NonNull; + +import com.kaltura.android.exoplayer2.DefaultRenderersFactory; +import com.kaltura.android.exoplayer2.Renderer; +import com.kaltura.android.exoplayer2.audio.AudioRendererEventListener; +import com.kaltura.android.exoplayer2.audio.AudioSink; +import com.kaltura.android.exoplayer2.audio.KMediaCodecAudioRenderer; +import com.kaltura.android.exoplayer2.audio.MediaCodecAudioRenderer; +import com.kaltura.android.exoplayer2.mediacodec.MediaCodecSelector; +import com.kaltura.android.exoplayer2.video.KMediaCodecVideoRenderer; +import com.kaltura.android.exoplayer2.video.KVideoRendererFirstFrameWhenStartedEventListener; +import com.kaltura.android.exoplayer2.video.MediaCodecVideoRenderer; +import com.kaltura.android.exoplayer2.video.VideoRendererEventListener; +import com.kaltura.playkit.player.PlayerSettings; + +import java.util.ArrayList; + +/** + * Utility class, providing a mechanism for creating renderers factory, which in its turn + * creates Audio/Video renderers, which are capable to adjust playback speed, in case when + * there's a big gap between Audio and Video streams buffers position at playback startup. + * Also, Video renderer takes a callback interface for providing notification once playback + * actually begins (i.e. in addition to onFirstFrameRendered, when no playback is actually + * happening yet). + * Speed adjustment behavioral values may be provided inside the {@link com.kaltura.playkit.player.PlayerSettings} + * instance passed into factory method + * Currently this mechanism is used only for multicast streams + */ +public class SpeedAdjustedRenderersFactory { + public static DefaultRenderersFactory createSpeedAdjustedRenderersFactory( + Context context, + PlayerSettings playerSettings, + KVideoRendererFirstFrameWhenStartedEventListener rendererFirstFrameWhenStartedEventListener + ) { + return new DefaultRenderersFactory(context) { + @Override + protected void buildAudioRenderers(@NonNull Context context, + int extensionRendererMode, + @NonNull MediaCodecSelector mediaCodecSelector, + boolean enableDecoderFallback, + @NonNull AudioSink audioSink, + @NonNull Handler eventHandler, + @NonNull AudioRendererEventListener eventListener, + @NonNull ArrayList out) { + ArrayList renderersArrayList = new ArrayList<>(); + super.buildAudioRenderers(context, + extensionRendererMode, + mediaCodecSelector, + enableDecoderFallback, + audioSink, + eventHandler, + eventListener, + renderersArrayList); + for (Renderer renderer : renderersArrayList) { + if (renderer instanceof MediaCodecAudioRenderer) { + if (playerSettings.getMulticastSettings() != null) { + out.add(new KMediaCodecAudioRenderer( + context, + getCodecAdapterFactory(), + mediaCodecSelector, + enableDecoderFallback, + eventHandler, + eventListener, + audioSink, + playerSettings.getMulticastSettings().getExperimentalMaxAudioGapThreshold(), + playerSettings.getMulticastSettings().getExperimentalMaxSpeedFactor(), + playerSettings.getMulticastSettings().getExperimentalSpeedStep(), + playerSettings.getMulticastSettings().getExperimentalAVGapForSpeedAdjustment(), + playerSettings.getMulticastSettings().getExperimentalContinuousSpeedAdjustment())); + } else { + out.add(new KMediaCodecAudioRenderer( + context, + getCodecAdapterFactory(), + mediaCodecSelector, + enableDecoderFallback, + eventHandler, + eventListener, + audioSink)); + } + } else { + out.add(renderer); + } + } + } + + @Override + protected void buildVideoRenderers(@NonNull Context context, + int extensionRendererMode, + @NonNull MediaCodecSelector mediaCodecSelector, + boolean enableDecoderFallback, + @NonNull Handler eventHandler, + @NonNull VideoRendererEventListener eventListener, + long allowedVideoJoiningTimeMs, + @NonNull ArrayList out) { + ArrayList renderersArrayList = new ArrayList<>(); + super.buildVideoRenderers(context, + extensionRendererMode, + mediaCodecSelector, + enableDecoderFallback, + eventHandler, + eventListener, + allowedVideoJoiningTimeMs, + renderersArrayList); + for (Renderer renderer : renderersArrayList) { + if (renderer instanceof MediaCodecVideoRenderer) { + out.add(new KMediaCodecVideoRenderer( + context, + this.getCodecAdapterFactory(), + mediaCodecSelector, + allowedVideoJoiningTimeMs, + enableDecoderFallback, + eventHandler, + eventListener, + MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY, + rendererFirstFrameWhenStartedEventListener)); + } else { + out.add(renderer); + } + } + } + }; + } +} diff --git a/playkit/src/main/java/com/kaltura/playkit/player/BaseExoplayerView.java b/playkit/src/main/java/com/kaltura/playkit/player/BaseExoplayerView.java index 2314ef5b3..d5a70647e 100644 --- a/playkit/src/main/java/com/kaltura/playkit/player/BaseExoplayerView.java +++ b/playkit/src/main/java/com/kaltura/playkit/player/BaseExoplayerView.java @@ -5,8 +5,9 @@ import com.kaltura.android.exoplayer2.ExoPlayer; import com.kaltura.android.exoplayer2.ui.SubtitleView; +import com.kaltura.android.exoplayer2.video.KVideoRendererFirstFrameWhenStartedEventListener; -public abstract class BaseExoplayerView extends PlayerView { +public abstract class BaseExoplayerView extends PlayerView implements KVideoRendererFirstFrameWhenStartedEventListener { public BaseExoplayerView(Context context) { super(context); diff --git a/playkit/src/main/java/com/kaltura/playkit/player/ExoPlayerView.java b/playkit/src/main/java/com/kaltura/playkit/player/ExoPlayerView.java index 41403c22a..827e4dcde 100644 --- a/playkit/src/main/java/com/kaltura/playkit/player/ExoPlayerView.java +++ b/playkit/src/main/java/com/kaltura/playkit/player/ExoPlayerView.java @@ -61,22 +61,46 @@ class ExoPlayerView extends BaseExoplayerView { private PKSubtitlePosition subtitleViewPosition; private boolean isVideoViewVisible; private List lastReportedCues; + private boolean shutterStaysOnRenderedFirstFrame; - ExoPlayerView(Context context) { - this(context, null); + private boolean usingSpeedAdjustedRenderer; + + ExoPlayerView(Context context, boolean shutterStaysOnRenderedFirstFrame) { + this(context, null, shutterStaysOnRenderedFirstFrame); } - ExoPlayerView(Context context, AttributeSet attrs) { - this(context, attrs, 0); + ExoPlayerView(Context context, AttributeSet attrs, boolean shutterStaysOnRenderedFirstFrame) { + this(context, attrs, 0, shutterStaysOnRenderedFirstFrame); } - ExoPlayerView(Context context, AttributeSet attrs, int defStyleAttr) { + ExoPlayerView(Context context, AttributeSet attrs, int defStyleAttr, boolean shutterStaysOnRenderedFirstFrame) { super(context, attrs, defStyleAttr); componentListener = new ComponentListener(); playerEventListener = getPlayerEventListener(); initContentFrame(); initSubtitleLayout(); initPosterView(); + setShutterStaysOnRenderedFirstFrame(shutterStaysOnRenderedFirstFrame); + } + + public void setShutterStaysOnRenderedFirstFrame(boolean shutterStaysOnRenderedFirstFrame) { + this.shutterStaysOnRenderedFirstFrame = shutterStaysOnRenderedFirstFrame; + } + + public void setUsingSpeedAdjustedRenderer(boolean usingSpeedAdjustedRenderer) { + this.usingSpeedAdjustedRenderer = usingSpeedAdjustedRenderer; + } + + @Override + public void onRenderedFirstFrameWhenStarted() { + log.d("onRenderedFirstFrameWhenStarted"); + if (shutterView != null) { + shutterView.setVisibility(INVISIBLE); + } + } + + private boolean shouldHideShutterView() { + return shutterView != null && !(shutterStaysOnRenderedFirstFrame && usingSpeedAdjustedRenderer); } @NonNull @@ -85,11 +109,18 @@ private Player.Listener getPlayerEventListener() { @Override public void onPlaybackStateChanged(int playbackState) { switch (playbackState) { + case Player.STATE_IDLE: + if (shutterStaysOnRenderedFirstFrame && usingSpeedAdjustedRenderer) { + if (shutterView != null) { + shutterView.setVisibility(VISIBLE); + } + } + break; case Player.STATE_READY: if (player != null && player.getPlayWhenReady()) { log.d("ExoPlayerView READY. playWhenReady => true"); - if (shutterView != null) { + if (shouldHideShutterView()) { shutterView.setVisibility(INVISIBLE); } } @@ -97,7 +128,6 @@ public void onPlaybackStateChanged(int playbackState) { case Player.STATE_BUFFERING: case Player.STATE_ENDED: - case Player.STATE_IDLE: default: break; } @@ -106,7 +136,7 @@ public void onPlaybackStateChanged(int playbackState) { @Override public void onIsPlayingChanged(boolean isPlaying) { log.d("ExoPlayerView onIsPlayingChanged isPlaying = " + isPlaying); - if (isPlaying && shutterView != null) { + if (isPlaying && shouldHideShutterView()) { shutterView.setVisibility(INVISIBLE); } } @@ -375,7 +405,7 @@ public void onVideoSizeChanged(@NonNull VideoSize videoSize) { @Override public void onRenderedFirstFrame() { - if (shutterView != null) { + if (shouldHideShutterView()) { shutterView.setVisibility(GONE); } } diff --git a/playkit/src/main/java/com/kaltura/playkit/player/ExoPlayerWrapper.java b/playkit/src/main/java/com/kaltura/playkit/player/ExoPlayerWrapper.java index 341019beb..32370b976 100644 --- a/playkit/src/main/java/com/kaltura/playkit/player/ExoPlayerWrapper.java +++ b/playkit/src/main/java/com/kaltura/playkit/player/ExoPlayerWrapper.java @@ -96,6 +96,7 @@ import com.kaltura.playkit.PlaybackInfo; import com.kaltura.playkit.PlayerEvent; import com.kaltura.playkit.PlayerState; +import com.kaltura.playkit.SpeedAdjustedRenderersFactory; import com.kaltura.playkit.Utils; import com.kaltura.playkit.drm.DeferredDrmSessionManager; import com.kaltura.playkit.drm.DrmCallback; @@ -161,6 +162,7 @@ public interface LoadControlStrategy { private boolean isSeeking; private boolean useTextureView; + private boolean useSpeedAdjustingRenderer; private boolean isSurfaceSecured; private boolean shouldGetTracksInfo; private boolean preferredLanguageWasSelected; @@ -190,7 +192,7 @@ public interface LoadControlStrategy { private Cache downloadCache; ExoPlayerWrapper(Context context, PlayerSettings playerSettings, PlayerView rootPlayerView) { - this(context, new ExoPlayerView(context), playerSettings, rootPlayerView); + this(context, new ExoPlayerView(context, playerSettings.isShutterStaysOnRenderedFirstFrame()), playerSettings, rootPlayerView); } ExoPlayerWrapper(Context context, BaseExoplayerView exoPlayerView, PlayerSettings settings, PlayerView rootPlayerView) { @@ -239,7 +241,12 @@ public void onBandwidthSample(int elapsedMs, long bytes, long bitrate) { private void initializePlayer() { DefaultTrackSelector trackSelector = initializeTrackSelector(); - DefaultRenderersFactory renderersFactory = new DefaultRenderersFactory(context); + if (exoPlayerView instanceof ExoPlayerView) { + ((ExoPlayerView)exoPlayerView).setUsingSpeedAdjustedRenderer(this.useSpeedAdjustingRenderer); + } + DefaultRenderersFactory renderersFactory = this.useSpeedAdjustingRenderer + ? SpeedAdjustedRenderersFactory.createSpeedAdjustedRenderersFactory(context, playerSettings, exoPlayerView) + : new DefaultRenderersFactory(context); renderersFactory.setAllowedVideoJoiningTimeMs(playerSettings.getLoadControlBuffers().getAllowedVideoJoiningTimeMs()); renderersFactory.setEnableDecoderFallback(playerSettings.enableDecoderFallback()); @@ -1108,14 +1115,23 @@ public void load(PKMediaSourceConfig mediaSourceConfig) { if (player == null) { this.useTextureView = playerSettings.useTextureView(); this.isSurfaceSecured = playerSettings.isSurfaceSecured(); + this.useSpeedAdjustingRenderer = shouldUseSpeedAdjustingRenderer(mediaSourceConfig.mediaSource.getMediaFormat()); initializePlayer(); } else { // for change media case need to verify if surface swap is needed maybeChangePlayerRenderView(); + + // for change speed adjustment case need to verify if re-init is required + maybeReInitPlayerOnSpeedAdjustmentChange(mediaSourceConfig.mediaSource.getMediaFormat()); } preparePlayer(mediaSourceConfig); } + private boolean shouldUseSpeedAdjustingRenderer(PKMediaFormat format) { + return format == PKMediaFormat.udp + && playerSettings.getMulticastSettings().getExperimentalAdjustSpeedOnNegativePosition(); + } + private boolean isBehindLiveWindow(PlaybackException e) { return e.errorCode == PlaybackException.ERROR_CODE_BEHIND_LIVE_WINDOW; } @@ -1135,6 +1151,15 @@ private void maybeChangePlayerRenderView() { exoPlayerView.setVideoSurfaceProperties(playerSettings.useTextureView(), playerSettings.isSurfaceSecured(), playerSettings.isVideoViewHidden()); } + private void maybeReInitPlayerOnSpeedAdjustmentChange(PKMediaFormat format) { + boolean useSpeedAdjustingRenderer = shouldUseSpeedAdjustingRenderer(format); + if (useSpeedAdjustingRenderer != this.useSpeedAdjustingRenderer) { + destroyPlayer(); + initializePlayer(); + } + this.useSpeedAdjustingRenderer = useSpeedAdjustingRenderer; + } + @Override public PlayerView getView() { return exoPlayerView; @@ -1347,6 +1372,10 @@ public void destroy() { log.v("destroy"); closeProfilerSession(); removeCustomLoadErrorPolicy(); + destroyPlayer(); + } + + private void destroyPlayer() { if (assertPlayerIsNotNull("destroy()")) { player.release(); } diff --git a/playkit/src/main/java/com/kaltura/playkit/player/MulticastSettings.java b/playkit/src/main/java/com/kaltura/playkit/player/MulticastSettings.java index e5b33e0f8..3203ba2d5 100644 --- a/playkit/src/main/java/com/kaltura/playkit/player/MulticastSettings.java +++ b/playkit/src/main/java/com/kaltura/playkit/player/MulticastSettings.java @@ -15,6 +15,21 @@ public class MulticastSettings { private ExtractorMode extractorMode = ExtractorMode.MODE_MULTI_PMT; //The desired value of the first adjusted sample timestamp in microseconds - for no offset give MAX_LONG private long firstSampleTimestampUs; + // experimental value to control whether to adjust speed if we get negative position on udp media load time + private boolean experimentalAdjustSpeedOnNegativePosition = false; + // experimental value to control whether adjusting speed (in case if gap) should happen only once on start + // or should be adjusted all the time + private boolean experimentalContinuousSpeedAdjustment = false; + // experimental value to control maximum audio gap between first audio and video buffers, which should be + // treated as normal one + private long experimentalMaxAudioGapThreshold = 3_000_000L; + // experimental value to control maximum playback speed used during speed adjustment + private float experimentalMaxSpeedFactor = 4.0f; + // experimental value to control speed adjustment step during speed adjustment + private float experimentalSpeedStep = 3.0f; + // experimental value to control maximum gap between presented audio and video buffers + // when the speed adjustment should not be used + private long experimentalAVGapForSpeedAdjustment = 600_000L; enum ExtractorMode { MODE_MULTI_PMT(0), @@ -87,4 +102,52 @@ public MulticastSettings setFirstSampleTimestampUs(long firstSampleTimestampUs) this.firstSampleTimestampUs = firstSampleTimestampUs; return this; } + + public boolean getExperimentalAdjustSpeedOnNegativePosition() { + return experimentalAdjustSpeedOnNegativePosition; + } + + public boolean getExperimentalContinuousSpeedAdjustment() { + return experimentalContinuousSpeedAdjustment; + } + + public long getExperimentalMaxAudioGapThreshold() { + return experimentalMaxAudioGapThreshold; + } + + public float getExperimentalMaxSpeedFactor() { + return experimentalMaxSpeedFactor; + } + + public float getExperimentalSpeedStep() { + return experimentalSpeedStep; + } + + public long getExperimentalAVGapForSpeedAdjustment() { + return experimentalAVGapForSpeedAdjustment; + } + + public void setExperimentalAdjustSpeedOnNegativePosition(boolean experimentalAdjustSpeedOnNegativePosition) { + this.experimentalAdjustSpeedOnNegativePosition = experimentalAdjustSpeedOnNegativePosition; + } + + public void setExperimentalContinuousSpeedAdjustment(boolean experimentalContinuousSpeedAdjustment) { + this.experimentalContinuousSpeedAdjustment = experimentalContinuousSpeedAdjustment; + } + + public void setExperimentalMaxAudioGapThreshold(long experimentalMaxAudioGapThreshold) { + this.experimentalMaxAudioGapThreshold = experimentalMaxAudioGapThreshold; + } + + public void setExperimentalMaxSpeedFactor(float experimentalMaxSpeedFactor) { + this.experimentalMaxSpeedFactor = experimentalMaxSpeedFactor; + } + + public void setExperimentalSpeedStep(float experimentalSpeedStep) { + this.experimentalSpeedStep = experimentalSpeedStep; + } + + public void setExperimentalAVGapForSpeedAdjustment(long experimentalAVGapForSpeedAdjustment) { + this.experimentalAVGapForSpeedAdjustment = experimentalAVGapForSpeedAdjustment; + } } diff --git a/playkit/src/main/java/com/kaltura/playkit/player/PlayerSettings.java b/playkit/src/main/java/com/kaltura/playkit/player/PlayerSettings.java index d5666b242..61469315d 100644 --- a/playkit/src/main/java/com/kaltura/playkit/player/PlayerSettings.java +++ b/playkit/src/main/java/com/kaltura/playkit/player/PlayerSettings.java @@ -37,6 +37,7 @@ public class PlayerSettings implements Player.Settings { private boolean isTunneledAudioPlayback; private boolean handleAudioBecomingNoisyEnabled; private boolean handleAudioFocus; + private boolean shutterStaysOnRenderedFirstFrame; // Flag helping to check if client app wants to use a single player instance at a time // Only if IMA plugin is there then only this flag is set to true. @@ -194,6 +195,10 @@ public boolean isHandleAudioFocus() { return handleAudioFocus; } + public boolean isShutterStaysOnRenderedFirstFrame() { + return shutterStaysOnRenderedFirstFrame; + } + public PKSubtitlePreference getSubtitlePreference() { return subtitlePreference; } @@ -416,6 +421,11 @@ public Player.Settings setHandleAudioFocus(boolean handleAudioFocus) { return this; } + public Player.Settings setShutterStaysOnRenderedFirstFrame(boolean shutterStaysOnRenderedFirstFrame) { + this.shutterStaysOnRenderedFirstFrame = shutterStaysOnRenderedFirstFrame; + return this; + } + @Override public Player.Settings setSubtitlePreference(PKSubtitlePreference subtitlePreference) { if (subtitlePreference == null) {