feat: Add IAMF support (#1415) (#1416)

CONTRIBUTORS

@@ -32,6 +32,7 @@ Daniel Cantarín <canta@canta.com.ar>
 David Cavar <pal3thorn@gmail.com>
 Dennis E. Mungai (Brainiarc7) <dmngaie@gmail.com>
 Evgeny Zajcev <zevlg@yandex.ru>
+Felicia Lim <flim@google.com>
 Gabe Kopley <gabe@philo.com>
 Geoff Jukes <geoff@jukes.org>
 Haoming Chen <hmchen@google.com>

README.md

@@ -46,6 +46,7 @@ Shaka Packager supports:
   |       FLAC        |    I / O     |      -       |       -      |      -      |       -      |
   |       Opus        |    I / O³    |    I / O     |       -      |      -      |       -      |
   |      Vorbis       |      -       |    I / O     |       -      |      -      |       -      |
+  |       IAMF        |    I / O     |      -       |       -      |      -      |       -      |
 
   NOTES:
   - I for input and O for output.

packager/app/test/packager_test.py

@@ -1104,6 +1104,38 @@ def testAv1WebM(self):
         self._GetFlags(output_dash=True, output_hls=True))
     self._CheckTestResults('av1-webm')
 
+  def testIamfWithBaseProfileAndPcm(self):
+    self.assertPackageSuccess(
+        self._GetStreams(['audio'],
+                         output_format='mp4',
+                         test_files=['bear-iamf-base-pcm.mp4']),
+        self._GetFlags(output_dash=True, output_hls=True))
+    self._CheckTestResults('iamf-base-pcm-mp4')
+
+  def testIamfWithBaseProfileAndOpus(self):
+    self.assertPackageSuccess(
+        self._GetStreams(['audio'],
+                         output_format='mp4',
+                         test_files=['bear-iamf-base-opus.mp4']),
+        self._GetFlags(output_dash=True, output_hls=True))
+    self._CheckTestResults('iamf-base-opus-mp4')
+
+  def testIamfWithSimpleProfileAndAacLc(self):
+    self.assertPackageSuccess(
+        self._GetStreams(['audio'],
+                         output_format='mp4',
+                         test_files=['bear-iamf-simple-aac-lc.mp4']),
+        self._GetFlags(output_dash=True, output_hls=True))
+    self._CheckTestResults('iamf-simple-aac-lc-mp4')
+
+  def testIamfWithSimpleProfileAndFlac(self):
+    self.assertPackageSuccess(
+        self._GetStreams(['audio'],
+                         output_format='mp4',
+                         test_files=['bear-iamf-simple-flac.mp4']),
+        self._GetFlags(output_dash=True, output_hls=True))
+    self._CheckTestResults('iamf-simple-flac-mp4')
+
   def testEncryption(self):
     self.assertPackageSuccess(
         self._GetStreams(['audio', 'video']),

packager/app/test/testdata/iamf-base-opus-mp4/output.m3u8

@@ -0,0 +1,9 @@
+#EXTM3U
+## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>
+
+#EXT-X-INDEPENDENT-SEGMENTS
+
+#EXT-X-MEDIA:TYPE=AUDIO,URI="stream_0.m3u8",GROUP-ID="default-audio-group",NAME="stream_0",DEFAULT=NO,AUTOSELECT=YES,CHANNELS="0"
+
+#EXT-X-STREAM-INF:BANDWIDTH=136796,AVERAGE-BANDWIDTH=134443,CODECS="iamf.001.001.Opus",AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE
+stream_0.m3u8

packager/app/test/testdata/iamf-base-opus-mp4/output.mpd

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
+<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.739958S">
+  <Period id="0">
+    <AdaptationSet id="0" contentType="audio" subsegmentAlignment="true">
+      <Representation id="0" bandwidth="136796" codecs="iamf.001.001.Opus" mimeType="audio/mp4" audioSamplingRate="0">
+        <AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="0"/>
+        <BaseURL>bear-iamf-base-opus-audio.mp4</BaseURL>
+        <SegmentBase indexRange="892-959" timescale="48000">
+          <Initialization range="0-891"/>
+        </SegmentBase>
+      </Representation>
+    </AdaptationSet>
+  </Period>
+</MPD>

packager/app/test/testdata/iamf-base-opus-mp4/stream_0.m3u8

@@ -0,0 +1,16 @@
+#EXTM3U
+#EXT-X-VERSION:6
+## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>
+#EXT-X-TARGETDURATION:2
+#EXT-X-PLAYLIST-TYPE:VOD
+#EXT-X-MAP:URI="bear-iamf-base-opus-audio.mp4",BYTERANGE="892@0"
+#EXTINF:1.014,
+#EXT-X-BYTERANGE:16835@960
+bear-iamf-base-opus-audio.mp4
+#EXTINF:1.000,
+#EXT-X-BYTERANGE:16789
+bear-iamf-base-opus-audio.mp4
+#EXTINF:0.726,
+#EXT-X-BYTERANGE:12422
+bear-iamf-base-opus-audio.mp4
+#EXT-X-ENDLIST

packager/app/test/testdata/iamf-base-pcm-mp4/output.m3u8

@@ -0,0 +1,9 @@
+#EXTM3U
+## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>
+
+#EXT-X-INDEPENDENT-SEGMENTS
+
+#EXT-X-MEDIA:TYPE=AUDIO,URI="stream_0.m3u8",GROUP-ID="default-audio-group",NAME="stream_0",DEFAULT=NO,AUTOSELECT=YES,CHANNELS="0"
+
+#EXT-X-STREAM-INF:BANDWIDTH=2177226,AVERAGE-BANDWIDTH=1803234,CODECS="iamf.001.001.ipcm",AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE
+stream_0.m3u8

packager/app/test/testdata/iamf-base-pcm-mp4/output.mpd

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
+<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.739958S">
+  <Period id="0">
+    <AdaptationSet id="0" contentType="audio" subsegmentAlignment="true">
+      <Representation id="0" bandwidth="2177226" codecs="iamf.001.001.ipcm" mimeType="audio/mp4" audioSamplingRate="0">
+        <AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="0"/>
+        <BaseURL>bear-iamf-base-pcm-audio.mp4</BaseURL>
+        <SegmentBase indexRange="850-917" timescale="48000">
+          <Initialization range="0-849"/>
+        </SegmentBase>
+      </Representation>
+    </AdaptationSet>
+  </Period>
+</MPD>

packager/app/test/testdata/iamf-base-pcm-mp4/stream_0.m3u8

@@ -0,0 +1,16 @@
+#EXTM3U
+#EXT-X-VERSION:6
+## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>
+#EXT-X-TARGETDURATION:1
+#EXT-X-PLAYLIST-TYPE:VOD
+#EXT-X-MAP:URI="bear-iamf-base-pcm-audio.mp4",BYTERANGE="850@0"
+#EXTINF:1.000,
+#EXT-X-BYTERANGE:208108@918
+bear-iamf-base-pcm-audio.mp4
+#EXTINF:1.000,
+#EXT-X-BYTERANGE:208108
+bear-iamf-base-pcm-audio.mp4
+#EXTINF:0.740,
+#EXT-X-BYTERANGE:201382
+bear-iamf-base-pcm-audio.mp4
+#EXT-X-ENDLIST

packager/app/test/testdata/iamf-simple-aac-lc-mp4/output.m3u8

@@ -0,0 +1,9 @@
+#EXTM3U
+## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>
+
+#EXT-X-INDEPENDENT-SEGMENTS
+
+#EXT-X-MEDIA:TYPE=AUDIO,URI="stream_0.m3u8",GROUP-ID="default-audio-group",NAME="stream_0",DEFAULT=NO,AUTOSELECT=YES,CHANNELS="0"
+
+#EXT-X-STREAM-INF:BANDWIDTH=604125,AVERAGE-BANDWIDTH=590976,CODECS="iamf.000.000.mp4a.40.2",AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE
+stream_0.m3u8

packager/app/test/testdata/iamf-simple-aac-lc-mp4/output.mpd

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
+<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.739958S">
+  <Period id="0">
+    <AdaptationSet id="0" contentType="audio" subsegmentAlignment="true">
+      <Representation id="0" bandwidth="604125" codecs="iamf.000.000.mp4a.40.2" mimeType="audio/mp4" audioSamplingRate="0">
+        <AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="0"/>
+        <BaseURL>bear-iamf-simple-aac-lc-audio.mp4</BaseURL>
+        <SegmentBase indexRange="900-967" timescale="48000">
+          <Initialization range="0-899"/>
+        </SegmentBase>
+      </Representation>
+    </AdaptationSet>
+  </Period>
+</MPD>

packager/app/test/testdata/iamf-simple-aac-lc-mp4/stream_0.m3u8

@@ -0,0 +1,16 @@
+#EXTM3U
+#EXT-X-VERSION:6
+## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>
+#EXT-X-TARGETDURATION:2
+#EXT-X-PLAYLIST-TYPE:VOD
+#EXT-X-MAP:URI="bear-iamf-simple-aac-lc-audio.mp4",BYTERANGE="900@0"
+#EXTINF:1.003,
+#EXT-X-BYTERANGE:75717@968
+bear-iamf-simple-aac-lc-audio.mp4
+#EXTINF:1.003,
+#EXT-X-BYTERANGE:72441
+bear-iamf-simple-aac-lc-audio.mp4
+#EXTINF:0.735,
+#EXT-X-BYTERANGE:54248
+bear-iamf-simple-aac-lc-audio.mp4
+#EXT-X-ENDLIST

packager/app/test/testdata/iamf-simple-flac-mp4/output.m3u8

@@ -0,0 +1,9 @@
+#EXTM3U
+## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>
+
+#EXT-X-INDEPENDENT-SEGMENTS
+
+#EXT-X-MEDIA:TYPE=AUDIO,URI="stream_0.m3u8",GROUP-ID="default-audio-group",NAME="stream_0",DEFAULT=NO,AUTOSELECT=YES,CHANNELS="0"
+
+#EXT-X-STREAM-INF:BANDWIDTH=977240,AVERAGE-BANDWIDTH=926327,CODECS="iamf.000.000.fLaC",AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE
+stream_0.m3u8

packager/app/test/testdata/iamf-simple-flac-mp4/output.mpd

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
+<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.739958S">
+  <Period id="0">
+    <AdaptationSet id="0" contentType="audio" subsegmentAlignment="true">
+      <Representation id="0" bandwidth="977240" codecs="iamf.000.000.fLaC" mimeType="audio/mp4" audioSamplingRate="0">
+        <AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="0"/>
+        <BaseURL>bear-iamf-simple-flac-audio.mp4</BaseURL>
+        <SegmentBase indexRange="883-950" timescale="48000">
+          <Initialization range="0-882"/>
+        </SegmentBase>
+      </Representation>
+    </AdaptationSet>
+  </Period>
+</MPD>

packager/app/test/testdata/iamf-simple-flac-mp4/stream_0.m3u8

@@ -0,0 +1,16 @@
+#EXTM3U
+#EXT-X-VERSION:6
+## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>
+#EXT-X-TARGETDURATION:1
+#EXT-X-PLAYLIST-TYPE:VOD
+#EXT-X-MAP:URI="bear-iamf-simple-flac-audio.mp4",BYTERANGE="883@0"
+#EXTINF:1.000,
+#EXT-X-BYTERANGE:116763@951
+bear-iamf-simple-flac-audio.mp4
+#EXTINF:1.000,
+#EXT-X-BYTERANGE:122155
+bear-iamf-simple-flac-audio.mp4
+#EXTINF:0.740,
+#EXT-X-BYTERANGE:78344
+bear-iamf-simple-flac-audio.mp4
+#EXT-X-ENDLIST

packager/media/base/CMakeLists.txt

@@ -80,6 +80,7 @@ target_link_libraries(media_handler_test_base
 add_executable(media_base_unittest
     aes_cryptor_unittest.cc
     aes_pattern_cryptor_unittest.cc
+    audio_stream_info_unittest.cc
     audio_timestamp_helper_unittest.cc
     bit_reader_unittest.cc
     bit_writer_unittest.cc

packager/media/base/audio_stream_info.cc

@@ -45,6 +45,8 @@ std::string AudioCodecToString(Codec codec) {
       return "AC4";
     case kCodecFlac:
       return "FLAC";
+    case kCodecIAMF:
+      return "IAMF";
     case kCodecOpus:
       return "Opus";
     case kCodecVorbis:
@@ -168,6 +170,39 @@ std::string AudioStreamInfo::GetCodecString(Codec codec,
           (audio_object_type & 0x18) >> 3, audio_object_type & 0x7);
     case kCodecFlac:
       return "flac";
+    case kCodecIAMF: {
+      // https://aomediacodec.github.io/iamf/#codecsparameter
+      // The codecs parameter string is composed as
+      //
+      // iamf.xxx.yyy.<standalone_codec_string>
+      //
+      // - xxx is the IAMF primary profile
+      // - yyy is the IAMF additional profile
+      // - <standalone_codec_string> are the elements of the codecs parameter
+      //   string if that stream was carried in its own track
+      //
+      // audio_object_type is composed of primary_profile (2 bits),
+      // additional_profile (2 bits) and (IAMF codec - kCodecAudio) (4 bits).
+      const int iamf_codec = (audio_object_type & 0xF) + kCodecAudio;
+
+      const std::string iamf_codec_string =
+          absl::StrFormat("iamf.%03d.%03d", (audio_object_type & 0xC0) >> 6,
+                          (audio_object_type & 0x30) >> 4);
+
+      switch (iamf_codec) {
+        case kCodecOpus:
+          return absl::StrFormat("%s.%s", iamf_codec_string, "Opus");
+        case kCodecAAC:
+          return absl::StrFormat("%s.%s", iamf_codec_string, "mp4a.40.2");
+        case kCodecFlac:
+          return absl::StrFormat("%s.%s", iamf_codec_string, "fLaC");
+        case kCodecPcm:
+          return absl::StrFormat("%s.%s", iamf_codec_string, "ipcm");
+        default:
+          LOG(WARNING) << "Unknown IAMF codec: " << iamf_codec;
+          return "unknown";
+      }
+    }
     case kCodecOpus:
       return "opus";
     case kCodecMP3:

packager/media/base/audio_stream_info_unittest.cc

@@ -0,0 +1,73 @@
+// Copyright 2024 Google LLC. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file or at
+// https://developers.google.com/open-source/licenses/bsd
+
+#include <packager/media/base/audio_stream_info.h>
+
+#include <gtest/gtest.h>
+
+namespace shaka {
+namespace media {
+
+const int kSimpleProfile = 0;
+const int kBaseProfile = 1;
+
+TEST(AudioStreamInfo, IamfGetCodecStringForSimpleProfilesAndPcm) {
+  const uint8_t audio_object_type =
+      ((kSimpleProfile << 6) |              // primary_profile
+       ((kSimpleProfile << 4) & 0x3F) |     // additional_profile
+       ((kCodecPcm - kCodecAudio) & 0xF));  // IAMF codec
+
+  std::string codec_string =
+      AudioStreamInfo::GetCodecString(kCodecIAMF, audio_object_type);
+  EXPECT_EQ("iamf.000.000.ipcm", codec_string);
+}
+
+TEST(AudioStreamInfo, IamfGetCodecStringForSimpleProfilesAndOpus) {
+  const uint8_t audio_object_type =
+      ((kSimpleProfile << 6) |               // primary_profile
+       ((kSimpleProfile << 4) & 0x3F) |      // additional_profile
+       ((kCodecOpus - kCodecAudio) & 0xF));  // IAMF codec
+
+  std::string codec_string =
+      AudioStreamInfo::GetCodecString(kCodecIAMF, audio_object_type);
+  EXPECT_EQ("iamf.000.000.Opus", codec_string);
+}
+
+TEST(AudioStreamInfo, IamfGetCodecStringForSimpleProfilesAndMp4a) {
+  const uint8_t audio_object_type =
+      ((kSimpleProfile << 6) |              // primary_profile
+       ((kSimpleProfile << 4) & 0x3F) |     // additional_profile
+       ((kCodecAAC - kCodecAudio) & 0xF));  // IAMF codec
+
+  std::string codec_string =
+      AudioStreamInfo::GetCodecString(kCodecIAMF, audio_object_type);
+  EXPECT_EQ("iamf.000.000.mp4a.40.2", codec_string);
+}
+
+TEST(AudioStreamInfo, IamfGetCodecStringForSimpleProfilesAndFlac) {
+  const uint8_t audio_object_type =
+      ((kSimpleProfile << 6) |               // primary_profile
+       ((kSimpleProfile << 4) & 0x3F) |      // additional_profile
+       ((kCodecFlac - kCodecAudio) & 0xF));  // IAMF codec
+
+  std::string codec_string =
+      AudioStreamInfo::GetCodecString(kCodecIAMF, audio_object_type);
+  EXPECT_EQ("iamf.000.000.fLaC", codec_string);
+}
+
+TEST(AudioStreamInfo, IamfGetCodecStringForBaseProfilesAndPcm) {
+  const uint8_t audio_object_type =
+      ((kBaseProfile << 6) |                // primary_profile = 1
+       ((kBaseProfile << 4) & 0x3F) |       // additional_profile = 1
+       ((kCodecPcm - kCodecAudio) & 0xF));  // IAMF codec
+
+  std::string codec_string =
+      AudioStreamInfo::GetCodecString(kCodecIAMF, audio_object_type);
+  EXPECT_EQ("iamf.001.001.ipcm", codec_string);
+}
+
+}  // namespace media
+}  // namespace shaka

packager/media/base/fourccs.h

@@ -86,7 +86,10 @@ enum FourCC : uint32_t {
   FOURCC_hvc1 = 0x68766331,
   FOURCC_hvcC = 0x68766343,
   FOURCC_hvcE = 0x68766345,
+  FOURCC_iamf = 0x69616d66,
+  FOURCC_iacb = 0x69616362,
   FOURCC_iden = 0x6964656e,
+  FOURCC_ipcm = 0x6970636d,
   FOURCC_iso6 = 0x69736f36,
   FOURCC_iso8 = 0x69736f38,
   FOURCC_isom = 0x69736f6d,

packager/media/base/stream_info.h

@@ -53,7 +53,9 @@ enum Codec {
   kCodecDTSX,
   kCodecEAC3,
   kCodecFlac,
+  kCodecIAMF,
   kCodecOpus,
+  kCodecPcm,
   kCodecVorbis,
   kCodecMP3,
   kCodecMha1,

packager/media/codecs/CMakeLists.txt

@@ -24,6 +24,7 @@ add_library(media_codecs STATIC
     h26x_byte_to_unit_stream_converter.cc
     hevc_decoder_configuration_record.cc
     hls_audio_util.cc
+    iamf_audio_util.cc
     nal_unit_to_byte_stream_converter.cc
     nalu_reader.cc
     video_slice_header_parser.cc
@@ -52,6 +53,7 @@ add_executable(media_codecs_unittest
     h26x_bit_reader_unittest.cc
     hevc_decoder_configuration_record_unittest.cc
     hls_audio_util_unittest.cc
+    iamf_audio_util_unittest.cc
     nal_unit_to_byte_stream_converter_unittest.cc
     nalu_reader_unittest.cc
     video_slice_header_parser_unittest.cc