Merge pull request #5528 from video-dev/feature/mp4-codec-parsing

MP4 Sample Description CODEC Parsing
video-dev · Jun 2, 2023 · fc112bc · fc112bc
2 parents 548ba93 + d8c6c7a
commit fc112bc
Show file tree

Hide file tree

Showing 5 changed files with 184 additions and 267 deletions.
diff --git a/src/controller/audio-stream-controller.ts b/src/controller/audio-stream-controller.ts
@@ -777,7 +777,12 @@ class AudioStreamController
 
     if (initSegment?.tracks) {
       const mapFragment = frag.initSegment || frag;
-      this._bufferInitSegment(initSegment.tracks, mapFragment, chunkMeta);
+      this._bufferInitSegment(
+        level,
+        initSegment.tracks,
+        mapFragment,
+        chunkMeta
+      );
       hls.trigger(Events.FRAG_PARSING_INIT_SEGMENT, {
         frag: mapFragment,
         id,
@@ -830,6 +835,7 @@ class AudioStreamController
   }
 
   private _bufferInitSegment(
+    currentLevel: Level,
     tracks: TrackSet,
     frag: Fragment,
     chunkMeta: ChunkMetadata
@@ -848,11 +854,16 @@ class AudioStreamController
       return;
     }
 
-    track.levelCodec = track.codec;
     track.id = 'audio';
+
+    const variantAudioCodecs = currentLevel.audioCodec;
     this.log(
-      `Init audio buffer, container:${track.container}, codecs[parsed]=[${track.codec}]`
+      `Init audio buffer, container:${track.container}, codecs[level/parsed]=[${variantAudioCodecs}/${track.codec}]`
     );
+    // SourceBuffer will use track.levelCodec if defined
+    if (variantAudioCodecs && variantAudioCodecs.split(',').length === 1) {
+      track.levelCodec = variantAudioCodecs;
+    }
     this.hls.trigger(Events.BUFFER_CODECS, tracks);
     const initSegment = track.initSegment;
     if (initSegment?.byteLength) {

diff --git a/src/controller/buffer-controller.ts b/src/controller/buffer-controller.ts
@@ -30,7 +30,8 @@ import type Hls from '../hls';
 import type { LevelDetails } from '../loader/level-details';
 
 const MediaSource = getMediaSource();
-const VIDEO_CODEC_PROFILE_REPACE = /([ha]vc.)(?:\.[^.,]+)+/;
+const VIDEO_CODEC_PROFILE_REPLACE =
+  /(avc[1234]|hvc1|hev1|dvh[1e]|vp09|av01)(?:\.[^.,]+)+/;
 
 export default class BufferController implements ComponentAPI {
   // The level details used to determine duration, target-duration and live
@@ -261,11 +262,11 @@ export default class BufferController implements ComponentAPI {
           const { id, codec, levelCodec, container, metadata } =
             data[trackName];
           const currentCodec = (track.levelCodec || track.codec).replace(
-            VIDEO_CODEC_PROFILE_REPACE,
+            VIDEO_CODEC_PROFILE_REPLACE,
             '$1'
           );
           const nextCodec = (levelCodec || codec).replace(
-            VIDEO_CODEC_PROFILE_REPACE,
+            VIDEO_CODEC_PROFILE_REPLACE,
             '$1'
           );
           if (currentCodec !== nextCodec) {

diff --git a/src/controller/error-controller.ts b/src/controller/error-controller.ts
@@ -468,7 +468,7 @@ export default class ErrorController implements NetworkComponentAPI {
     const { hls, penalizedRenditions } = this;
     const levelIndex: number =
       data.parent === PlaylistLevelType.MAIN
-        ? (data.level as number)
+        ? (data.level as number) || 0
         : hls.loadLevel;
     const level = hls.levels[levelIndex];
     const redundantLevels = level.url.length;

diff --git a/src/remux/passthrough-remuxer.ts b/src/remux/passthrough-remuxer.ts
@@ -74,14 +74,14 @@ class PassThroughRemuxer implements Remuxer {
     const initData = (this.initData = parseInitSegment(initSegment));
 
     // Get codec from initSegment or fallback to default
-    if (!audioCodec) {
+    if (initData.audio) {
       audioCodec = getParsedTrackCodec(
         initData.audio,
         ElementaryStreamTypes.AUDIO
       );
     }
 
-    if (!videoCodec) {
+    if (initData.video) {
       videoCodec = getParsedTrackCodec(
         initData.video,
         ElementaryStreamTypes.VIDEO
@@ -263,27 +263,36 @@ function isInvalidInitPts(
 }
 
 function getParsedTrackCodec(
-  track: InitDataTrack | undefined,
+  track: InitDataTrack,
   type: ElementaryStreamTypes.AUDIO | ElementaryStreamTypes.VIDEO
 ): string {
   const parsedCodec = track?.codec;
   if (parsedCodec && parsedCodec.length > 4) {
     return parsedCodec;
   }
+  if (type === ElementaryStreamTypes.AUDIO) {
+    if (
+      parsedCodec === 'ec-3' ||
+      parsedCodec === 'ac-3' ||
+      parsedCodec === 'alac'
+    ) {
+      return parsedCodec;
+    }
+    if (parsedCodec === 'fLaC' || parsedCodec === 'Opus') {
+      return getCodecCompatibleName(parsedCodec);
+    }
+    logger.warn(`Unhandled audio codec "${parsedCodec}" or audio object type`);
+    return 'mp4a.40.5';
+  }
   // Provide defaults based on codec type
   // This allows for some playback of some fmp4 playlists without CODECS defined in manifest
+  logger.warn(`Unhandled video codec "${parsedCodec}"`);
   if (parsedCodec === 'hvc1' || parsedCodec === 'hev1') {
     return 'hvc1.1.6.L120.90';
   }
   if (parsedCodec === 'av01') {
     return 'av01.0.04M.08';
   }
-  if (parsedCodec === 'avc1' || type === ElementaryStreamTypes.VIDEO) {
-    return 'avc1.42e01e';
-  }
-  if (parsedCodec === 'fLaC' || parsedCodec === 'Opus') {
-    return getCodecCompatibleName(parsedCodec);
-  }
-  return 'mp4a.40.5';
+  return 'avc1.42e01e';
 }
 export default PassThroughRemuxer;