From 4db93995e2cdfbf47e4996d0b6928691464ea5c4 Mon Sep 17 00:00:00 2001 From: John Regan Date: Tue, 15 Nov 2022 09:53:50 -0500 Subject: [PATCH] FLAC and Opus support This adds the MP4-registered codec entry "fLaC", as well as the non-registered codec entries "flac" and "FLAC". Browsers added FLAC support before the MP4 Registration Authority listed the codec entry "fLaC", and went with "flac". Bugs have been opened with browsers to address this, but in order to deal with legacy playlist manifests (they may list "flac" or even "FLAC"), this replaces any instance of "flac" and "opus" (any case) with the first compatible codec string. --- src/controller/buffer-controller.ts | 18 ++++++++++++-- src/controller/level-controller.ts | 11 ++++++++- src/controller/stream-controller.ts | 7 +++++- src/remux/passthrough-remuxer.ts | 6 +++++ src/utils/codecs.ts | 38 +++++++++++++++++++++++++++++ 5 files changed, 76 insertions(+), 4 deletions(-) diff --git a/src/controller/buffer-controller.ts b/src/controller/buffer-controller.ts index 93b21be1248..334ce1221c8 100644 --- a/src/controller/buffer-controller.ts +++ b/src/controller/buffer-controller.ts @@ -2,6 +2,7 @@ import { Events } from '../events'; import { logger } from '../utils/logger'; import { ErrorDetails, ErrorTypes } from '../errors'; import { BufferHelper } from '../utils/buffer-helper'; +import { getCodecCompatibleName } from '../utils/codecs'; import { getMediaSource } from '../utils/mediasource-helper'; import { ElementaryStreamTypes } from '../loader/fragment'; import type { TrackSet } from '../types/track'; @@ -29,6 +30,7 @@ import { LevelDetails } from '../loader/level-details'; const MediaSource = getMediaSource(); const VIDEO_CODEC_PROFILE_REPACE = /([ha]vc.)(?:\.[^.,]+)+/; +const AUDIO_CODEC_REGEXP = /flac|opus/gi; export default class BufferController implements ComponentAPI { // The level details used to determine duration, target-duration and live @@ -254,7 +256,14 @@ export default class BufferController implements ComponentAPI { '$1' ); if (currentCodec !== nextCodec) { - const mimeType = `${container};codecs=${levelCodec || codec}`; + let trackCodec = levelCodec || codec; + if (trackName.indexOf('audio') !== -1) { + trackCodec = trackCodec.replace( + AUDIO_CODEC_REGEXP, + getCodecCompatibleName + ); + } + const mimeType = `${container};codecs=${trackCodec}`; this.appendChangeType(trackName, mimeType); logger.log( `[buffer-controller]: switching codec ${currentCodec} to ${nextCodec}` @@ -712,7 +721,12 @@ export default class BufferController implements ComponentAPI { ); } // use levelCodec as first priority - const codec = track.levelCodec || track.codec; + let codec = track.levelCodec || track.codec; + if (codec) { + if (trackName.indexOf('audio') !== -1) { + codec = codec.replace(AUDIO_CODEC_REGEXP, getCodecCompatibleName); + } + } const mimeType = `${track.container};codecs=${codec}`; logger.log(`[buffer-controller]: creating sourceBuffer(${mimeType})`); try { diff --git a/src/controller/level-controller.ts b/src/controller/level-controller.ts index e794edf8b65..581465f7354 100644 --- a/src/controller/level-controller.ts +++ b/src/controller/level-controller.ts @@ -14,7 +14,7 @@ import { import { Level } from '../types/level'; import { Events } from '../events'; import { ErrorTypes, ErrorDetails } from '../errors'; -import { isCodecSupportedInMp4 } from '../utils/codecs'; +import { isCodecSupportedInMp4, getCodecCompatibleName } from '../utils/codecs'; import { addGroupId, assignTrackIdsByGroup } from './level-helper'; import BasePlaylistController from './base-playlist-controller'; import { PlaylistContextType, PlaylistLevelType } from '../types/loader'; @@ -26,6 +26,8 @@ const chromeOrFirefox: boolean = /chrome|firefox/.test( navigator.userAgent.toLowerCase() ); +const AUDIO_CODEC_REGEXP = /flac|opus/gi; + export default class LevelController extends BasePlaylistController { private _levels: Level[] = []; private _firstLevel: number = -1; @@ -109,6 +111,13 @@ export default class LevelController extends BasePlaylistController { levelParsed.audioCodec = undefined; } + if (levelParsed.audioCodec) { + levelParsed.audioCodec = levelParsed.audioCodec.replace( + AUDIO_CODEC_REGEXP, + getCodecCompatibleName + ); + } + const levelKey = `${levelParsed.bitrate}-${levelParsed.attrs.RESOLUTION}-${levelParsed.attrs.CODECS}`; levelFromSet = levelSet[levelKey]; diff --git a/src/controller/stream-controller.ts b/src/controller/stream-controller.ts index ad4f32fcbee..e85279d176b 100644 --- a/src/controller/stream-controller.ts +++ b/src/controller/stream-controller.ts @@ -1220,7 +1220,12 @@ export default class StreamController } } // HE-AAC is broken on Android, always signal audio codec as AAC even if variant manifest states otherwise - if (ua.indexOf('android') !== -1 && audio.container !== 'audio/mpeg') { + if ( + audioCodec && + audioCodec.indexOf('mp4a.40.5') !== -1 && + ua.indexOf('android') !== -1 && + audio.container !== 'audio/mpeg' + ) { // Exclude mpeg audio audioCodec = 'mp4a.40.2'; this.log(`Android: force audio codec to ${audioCodec}`); diff --git a/src/remux/passthrough-remuxer.ts b/src/remux/passthrough-remuxer.ts index ee28c7cb257..cf7c8150395 100644 --- a/src/remux/passthrough-remuxer.ts +++ b/src/remux/passthrough-remuxer.ts @@ -252,6 +252,12 @@ function getParsedTrackCodec( if (parsedCodec === 'avc1' || type === ElementaryStreamTypes.VIDEO) { return 'avc1.42e01e'; } + if (parsedCodec === 'fLaC') { + return 'fLaC'; + } + if (parsedCodec === 'Opus') { + return 'Opus'; + } return 'mp4a.40.5'; } export default PassThroughRemuxer; diff --git a/src/utils/codecs.ts b/src/utils/codecs.ts index 959a364170b..91d84a41a6f 100644 --- a/src/utils/codecs.ts +++ b/src/utils/codecs.ts @@ -14,6 +14,9 @@ const sampleEntryCodesISO = { dtsh: true, 'ec-3': true, enca: true, + fLaC: true, // MP4-RA listed codec entry for FLAC + flac: true, // legacy browser codec name for FLAC + FLAC: true, // some manifests may list "FLAC" with Apple's tools g719: true, g726: true, m4ae: true, @@ -83,3 +86,38 @@ export function isCodecSupportedInMp4(codec: string, type: CodecType): boolean { `${type || 'video'}/mp4;codecs="${codec}"` ); } + +interface CodecNameCache { + flac?: string; + opus?: string; +} + +const CODEC_COMPATIBLE_NAMES: CodecNameCache = {}; + +type LowerCaseCodecType = 'flac' | 'opus'; + +function getCodecCompatibleNameLower( + lowerCaseCodec: LowerCaseCodecType +): string { + if (CODEC_COMPATIBLE_NAMES[lowerCaseCodec]) { + return CODEC_COMPATIBLE_NAMES[lowerCaseCodec]!; + } + + const codecsToCheck = { + flac: ['fLaC', 'flac', 'FLAC'], + opus: ['Opus', 'opus'], + }[lowerCaseCodec]; + + for (let i = 0; i < codecsToCheck.length; i++) { + if (isCodecSupportedInMp4(codecsToCheck[i], 'audio')) { + CODEC_COMPATIBLE_NAMES[lowerCaseCodec] = codecsToCheck[i]; + return codecsToCheck[i]; + } + } + + return lowerCaseCodec; +} + +export function getCodecCompatibleName(codec: string): string { + return getCodecCompatibleNameLower(codec.toLowerCase() as LowerCaseCodecType); +}