ExoPlayer: Implement media segment support

app/src/main/assets/native/MediaSegmentsPlugin.js

@@ -0,0 +1,32 @@
+export class MediaSegmentsPlugin {
+    SETTING_PREFIX = 'segmentTypeAction';
+
+    constructor({ events, appSettings, dashboard }) {
+        this.appSettings = appSettings;
+        this.dashboard = dashboard;
+
+        events.on(appSettings, 'change', (_, name) => this.onSettingsChanged(name));
+    }
+
+    getSettingId(type) {
+        return `${this.SETTING_PREFIX}__${type}`;
+    }
+
+    getSettingValue(id) {
+        var userId = this.dashboard.getCurrentUserId();
+
+        return this.appSettings.get(id, userId);
+    }
+
+    // Update media segment action
+    onSettingsChanged(name) {
+        if (name.startsWith(this.SETTING_PREFIX)) {
+            var type = name.slice(this.SETTING_PREFIX.length + 2);
+            var action = this.getSettingValue(this.getSettingId(type));
+
+            if (type != null && action != null) {
+                MediaSegments.setSegmentTypeAction(type, action);
+            }
+        }
+    }
+}

app/src/main/assets/native/nativeshell.js

@@ -18,7 +18,8 @@ const features = [
 const plugins = [
     'NavigationPlugin',
     'ExoPlayerPlugin',
-    'ExternalPlayerPlugin'
+    'ExternalPlayerPlugin',
+    'MediaSegmentsPlugin'
 ];
 
 // Add plugin loaders

app/src/main/java/org/jellyfin/mobile/app/ApiClientController.kt

@@ -8,7 +8,6 @@ import org.jellyfin.mobile.data.entity.ServerEntity
 import org.jellyfin.sdk.Jellyfin
 import org.jellyfin.sdk.api.client.ApiClient
 import org.jellyfin.sdk.model.DeviceInfo
-import org.jellyfin.sdk.model.serializer.toUUID
 
 class ApiClientController(
     private val appPreferences: AppPreferences,
@@ -27,7 +26,7 @@ class ApiClientController(
         appPreferences.currentServerId = withContext(Dispatchers.IO) {
             serverDao.getServerByHostname(hostname)?.id ?: serverDao.insert(hostname)
         }
-        apiClient.baseUrl = hostname
+        apiClient.update(baseUrl = hostname)
     }
 
     suspend fun setupUser(serverId: Long, userId: String, accessToken: String) {
@@ -69,19 +68,21 @@ class ApiClientController(
     }
 
     private fun configureApiClientServer(server: ServerEntity?) {
-        apiClient.baseUrl = server?.hostname
+        apiClient.update(baseUrl = server?.hostname)
     }
 
     private fun configureApiClientUser(userId: String, accessToken: String) {
-        apiClient.userId = userId.toUUID()
-        apiClient.accessToken = accessToken
-        // Append user id to device id to ensure uniqueness across sessions
-        apiClient.deviceInfo = baseDeviceInfo.copy(id = baseDeviceInfo.id + userId)
+        apiClient.update(
+            accessToken = accessToken,
+            // Append user id to device id to ensure uniqueness across sessions
+            deviceInfo = baseDeviceInfo.copy(id = baseDeviceInfo.id + userId),
+        )
     }
 
     private fun resetApiClientUser() {
-        apiClient.userId = null
-        apiClient.accessToken = null
-        apiClient.deviceInfo = baseDeviceInfo
+        apiClient.update(
+            accessToken = null,
+            deviceInfo = baseDeviceInfo,
+        )
     }
 }

app/src/main/java/org/jellyfin/mobile/app/AppModule.kt

@@ -1,6 +1,7 @@
 package org.jellyfin.mobile.app
 
 import android.content.Context
+import androidx.core.net.toUri
 import coil.ImageLoader
 import com.google.android.exoplayer2.ext.cronet.CronetDataSource
 import com.google.android.exoplayer2.extractor.DefaultExtractorsFactory
@@ -11,8 +12,10 @@ import com.google.android.exoplayer2.source.ProgressiveMediaSource
 import com.google.android.exoplayer2.source.SingleSampleMediaSource
 import com.google.android.exoplayer2.source.hls.HlsMediaSource
 import com.google.android.exoplayer2.upstream.DataSource
+import com.google.android.exoplayer2.upstream.DataSpec
 import com.google.android.exoplayer2.upstream.DefaultDataSource
 import com.google.android.exoplayer2.upstream.DefaultHttpDataSource
+import com.google.android.exoplayer2.upstream.ResolvingDataSource
 import com.google.android.exoplayer2.util.Util
 import kotlinx.coroutines.channels.Channel
 import okhttp3.OkHttpClient
@@ -24,6 +27,7 @@ import org.jellyfin.mobile.events.ActivityEventHandler
 import org.jellyfin.mobile.player.audio.car.LibraryBrowser
 import org.jellyfin.mobile.player.deviceprofile.DeviceProfileBuilder
 import org.jellyfin.mobile.player.interaction.PlayerEvent
+import org.jellyfin.mobile.player.mediasegments.MediaSegmentRepository
 import org.jellyfin.mobile.player.qualityoptions.QualityOptionsProvider
 import org.jellyfin.mobile.player.source.MediaSourceResolver
 import org.jellyfin.mobile.player.ui.PlayerFragment
@@ -34,6 +38,8 @@ import org.jellyfin.mobile.utils.isLowRamDevice
 import org.jellyfin.mobile.webapp.RemoteVolumeProvider
 import org.jellyfin.mobile.webapp.WebViewFragment
 import org.jellyfin.mobile.webapp.WebappFunctionChannel
+import org.jellyfin.sdk.api.client.ApiClient
+import org.jellyfin.sdk.api.client.util.AuthorizationHeaderBuilder
 import org.koin.android.ext.koin.androidApplication
 import org.koin.androidx.fragment.dsl.fragment
 import org.koin.androidx.viewmodel.dsl.viewModel
@@ -77,10 +83,12 @@ val applicationModule = module {
     single { MediaSourceResolver(get()) }
     single { DeviceProfileBuilder(get()) }
     single { QualityOptionsProvider() }
+    single { MediaSegmentRepository() }
 
     // ExoPlayer factories
     single<DataSource.Factory> {
         val context: Context = get()
+        val apiClient: ApiClient = get()
 
         val provider = CronetProvider.getAllProviders(context).firstOrNull { provider: CronetProvider ->
             (provider.name == CronetProvider.PROVIDER_NAME_APP_PACKAGED) && provider.isEnabled
@@ -102,7 +110,27 @@ val applicationModule = module {
             }
         }
 
-        DefaultDataSource.Factory(context, baseDataSourceFactory)
+        val dataSourceFactory = DefaultDataSource.Factory(context, baseDataSourceFactory)
+
+        // Add authorization header. This is needed as we don't pass the
+        // access token in the URL for Android Auto.
+        ResolvingDataSource.Factory(dataSourceFactory) { dataSpec: DataSpec ->
+            // Only send authorization header if URI matches the jellyfin server
+            val baseUrlAuthority = apiClient.baseUrl?.toUri()?.authority
+
+            if (dataSpec.uri.authority == baseUrlAuthority) {
+                val authorizationHeaderString = AuthorizationHeaderBuilder.buildHeader(
+                    clientName = apiClient.clientInfo.name,
+                    clientVersion = apiClient.clientInfo.version,
+                    deviceId = apiClient.deviceInfo.id,
+                    deviceName = apiClient.deviceInfo.name,
+                    accessToken = apiClient.accessToken,
+                )
+
+                dataSpec.withRequestHeaders(hashMapOf("Authorization" to authorizationHeaderString))
+            } else
+                dataSpec
+        }
     }
     single<MediaSource.Factory> {
         val context: Context = get()

app/src/main/java/org/jellyfin/mobile/app/AppPreferences.kt

@@ -5,9 +5,12 @@ import android.content.SharedPreferences
 import android.os.Environment
 import android.view.WindowManager.LayoutParams.BRIGHTNESS_OVERRIDE_NONE
 import androidx.core.content.edit
+import org.jellyfin.mobile.player.mediasegments.MediaSegmentAction
+import org.jellyfin.mobile.player.mediasegments.toMediaSegmentActionsString
 import org.jellyfin.mobile.settings.ExternalPlayerPackage
 import org.jellyfin.mobile.settings.VideoPlayerType
 import org.jellyfin.mobile.utils.Constants
+import org.jellyfin.sdk.model.api.MediaSegmentType
 import java.io.File
 
 class AppPreferences(context: Context) {
@@ -90,6 +93,19 @@ class AppPreferences(context: Context) {
             }
         }
 
+    /**
+     * The actions to take for each media segment type. Managed by the MediaSegmentRepository.
+     */
+    var mediaSegmentActions: String
+        get() = sharedPreferences.getString(
+            Constants.PREF_MEDIA_SEGMENT_ACTIONS,
+            mapOf(
+                MediaSegmentType.INTRO to MediaSegmentAction.ASK_TO_SKIP,
+                MediaSegmentType.OUTRO to MediaSegmentAction.ASK_TO_SKIP,
+            ).toMediaSegmentActionsString(),
+        )!!
+        set(value) = sharedPreferences.edit { putString(Constants.PREF_MEDIA_SEGMENT_ACTIONS, value) }
+
     val musicNotificationAlwaysDismissible: Boolean
         get() = sharedPreferences.getBoolean(Constants.PREF_MUSIC_NOTIFICATION_ALWAYS_DISMISSIBLE, false)
 

app/src/main/java/org/jellyfin/mobile/bridge/MediaSegments.kt

@@ -0,0 +1,35 @@
+package org.jellyfin.mobile.bridge
+
+import android.content.Context
+import android.webkit.JavascriptInterface
+import org.jellyfin.mobile.player.mediasegments.MediaSegmentAction
+import org.jellyfin.mobile.player.mediasegments.MediaSegmentRepository
+import org.jellyfin.sdk.model.api.MediaSegmentType
+import org.koin.core.component.KoinComponent
+import org.koin.core.component.get
+
+@Suppress("unused")
+class MediaSegments(private val context: Context) : KoinComponent {
+    private val mediaSegmentRepository: MediaSegmentRepository = get()
+
+    @JavascriptInterface
+    fun setSegmentTypeAction(typeString: String, actionString: String) {
+        val type: MediaSegmentType = when(typeString) {
+            "Intro" -> MediaSegmentType.INTRO
+            "Outro" -> MediaSegmentType.OUTRO
+            "Preview" -> MediaSegmentType.PREVIEW
+            "Recap" -> MediaSegmentType.RECAP
+            "Commercial" -> MediaSegmentType.COMMERCIAL
+            else -> return
+        }
+
+        val action: MediaSegmentAction = when(actionString) {
+            "None" -> MediaSegmentAction.NOTHING
+            "Skip" -> MediaSegmentAction.SKIP
+            "AskToSkip" -> MediaSegmentAction.ASK_TO_SKIP
+            else -> return
+        }
+
+        mediaSegmentRepository.setDefaultSegmentTypeAction(type, action)
+    }
+}

app/src/main/java/org/jellyfin/mobile/player/PlayerViewModel.kt

@@ -38,6 +38,8 @@ import org.jellyfin.mobile.player.interaction.PlayerEvent
 import org.jellyfin.mobile.player.interaction.PlayerLifecycleObserver
 import org.jellyfin.mobile.player.interaction.PlayerMediaSessionCallback
 import org.jellyfin.mobile.player.interaction.PlayerNotificationHelper
+import org.jellyfin.mobile.player.mediasegments.MediaSegmentAction
+import org.jellyfin.mobile.player.mediasegments.MediaSegmentRepository
 import org.jellyfin.mobile.player.queue.QueueManager
 import org.jellyfin.mobile.player.source.JellyfinMediaSource
 import org.jellyfin.mobile.player.ui.DecoderType
@@ -47,7 +49,9 @@ import org.jellyfin.mobile.utils.Constants
 import org.jellyfin.mobile.utils.Constants.SUPPORTED_VIDEO_PLAYER_PLAYBACK_ACTIONS
 import org.jellyfin.mobile.utils.applyDefaultAudioAttributes
 import org.jellyfin.mobile.utils.applyDefaultLocalAudioAttributes
+import org.jellyfin.mobile.utils.extensions.end
 import org.jellyfin.mobile.utils.extensions.scaleInRange
+import org.jellyfin.mobile.utils.extensions.start
 import org.jellyfin.mobile.utils.extensions.width
 import org.jellyfin.mobile.utils.getVolumeLevelPercent
 import org.jellyfin.mobile.utils.getVolumeRange
@@ -65,7 +69,9 @@ import org.jellyfin.sdk.api.operations.DisplayPreferencesApi
 import org.jellyfin.sdk.api.operations.HlsSegmentApi
 import org.jellyfin.sdk.api.operations.PlayStateApi
 import org.jellyfin.sdk.api.operations.UserApi
+import org.jellyfin.sdk.model.api.MediaSegmentDto
 import org.jellyfin.sdk.model.api.PlayMethod
+import org.jellyfin.sdk.model.api.PlaybackOrder
 import org.jellyfin.sdk.model.api.PlaybackProgressInfo
 import org.jellyfin.sdk.model.api.PlaybackStartInfo
 import org.jellyfin.sdk.model.api.PlaybackStopInfo
@@ -95,6 +101,7 @@ class PlayerViewModel(application: Application) : AndroidViewModel(application),
     val queueManager = QueueManager(this)
     val mediaSourceOrNull: JellyfinMediaSource?
         get() = queueManager.currentMediaSourceOrNull
+    private val mediaSegmentRepository = MediaSegmentRepository()
 
     // ExoPlayer
     private val _player = MutableLiveData<ExoPlayer?>()
@@ -263,6 +270,9 @@ class PlayerViewModel(application: Application) : AndroidViewModel(application),
 
         val startTime = jellyfinMediaSource.startTimeMs
         if (startTime > 0) player.seekTo(startTime)
+
+        applyMediaSegments(jellyfinMediaSource)
+
         player.playWhenReady = playWhenReady
 
         mediaSession.setMetadata(jellyfinMediaSource.toMediaMetadata())
@@ -319,6 +329,7 @@ class PlayerViewModel(application: Application) : AndroidViewModel(application),
                     positionTicks = mediaSource.startTimeMs * Constants.TICKS_PER_MILLISECOND,
                     volumeLevel = audioManager.getVolumeLevelPercent(),
                     repeatMode = RepeatMode.REPEAT_NONE,
+                    playbackOrder = PlaybackOrder.DEFAULT,
                 ),
             )
         } catch (e: ApiClientException) {
@@ -347,6 +358,7 @@ class PlayerViewModel(application: Application) : AndroidViewModel(application),
                         positionTicks = playbackPositionMillis * Constants.TICKS_PER_MILLISECOND,
                         volumeLevel = (currentVolume - volumeRange.first) * Constants.PERCENT_MAX / volumeRange.width,
                         repeatMode = RepeatMode.REPEAT_NONE,
+                        playbackOrder = PlaybackOrder.DEFAULT,
                     ),
                 )
             } catch (e: ApiClientException) {
@@ -400,6 +412,46 @@ class PlayerViewModel(application: Application) : AndroidViewModel(application),
         }
     }
 
+    private fun applyMediaSegments(jellyfinMediaSource: JellyfinMediaSource) {
+        viewModelScope.launch {
+            if (jellyfinMediaSource.item != null) {
+                val mediaSegments = runCatching {
+                    mediaSegmentRepository.getSegmentsForItem(jellyfinMediaSource.item)
+                }.getOrNull().orEmpty()
+
+                for (mediaSegment in mediaSegments) {
+                    val action = mediaSegmentRepository.getMediaSegmentAction(mediaSegment)
+
+                    when(action) {
+                        MediaSegmentAction.SKIP -> addSkipAction(mediaSegment)
+                        MediaSegmentAction.NOTHING -> Unit
+                        // Unimplemented
+                        MediaSegmentAction.ASK_TO_SKIP -> Unit
+                    }
+                }
+            }
+        }
+    }
+
+    private fun addSkipAction(mediaSegment: MediaSegmentDto) {
+        val player = playerOrNull ?: return
+
+        player.createMessage { _, _ ->
+                // We can't seek directly on the ExoPlayer instance as not all media is seekable
+                // the seek function in the PlaybackController checks this and optionally starts a transcode
+                // at the requested position
+                // TODO: The above is probably true for jellyfin-android as well.
+                // But I believe there is no such logic here.
+                viewModelScope.launch(Dispatchers.Main) {
+                    player.seekTo(mediaSegment.end.inWholeMilliseconds)
+                }
+            }
+            // Segments at position 0 will never be hit by ExoPlayer so we need to add a minimum value
+            .setPosition(mediaSegment.start.inWholeMilliseconds.coerceAtLeast(1))
+            .setDeleteAfterDelivery(false)
+            .send()
+    }
+
     // Player controls
 
     fun play() {

app/src/main/java/org/jellyfin/mobile/player/audio/MediaService.kt

@@ -44,7 +44,6 @@ import org.jellyfin.mobile.player.cast.ICastPlayerProvider
 import org.jellyfin.mobile.utils.Constants
 import org.jellyfin.mobile.utils.extensions.mediaUri
 import org.jellyfin.mobile.utils.toast
-import org.jellyfin.sdk.api.client.ApiClient
 import org.jellyfin.sdk.api.client.exception.ApiClientException
 import org.koin.android.ext.android.get
 import org.koin.android.ext.android.inject
@@ -53,7 +52,6 @@ import com.google.android.exoplayer2.MediaItem as ExoPlayerMediaItem
 
 class MediaService : MediaBrowserServiceCompat() {
     private val apiClientController: ApiClientController by inject()
-    private val apiClient: ApiClient by inject()
     private val libraryBrowser: LibraryBrowser by inject()
 
     private val serviceScope = MainScope()
@@ -177,12 +175,7 @@ class MediaService : MediaBrowserServiceCompat() {
             loadingJob.join()
 
             val items = try {
-                if (apiClient.userId != null) {
-                    libraryBrowser.loadLibrary(parentId)
-                } else {
-                    Timber.e("Missing userId in ApiClient")
-                    null
-                }
+                libraryBrowser.loadLibrary(parentId)
             } catch (e: ApiClientException) {
                 Timber.e(e)
                 null