Simulcast support for iOS/Android.

Simulcast support for iOS SDK (#4) Support for simulcast in Android SDK (#3) include simulcast headers for mac also (#10) Fix simulcast using hardware encoder on Android (#48) Add scalabilityMode support for AV1/VP9. (#90) Co-authored-by: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Co-authored-by: Angelika Serwa <angelika.serwa@gmail.com>
webrtc-sdk · May 22, 2024 · b0b9fe9 · b0b9fe9
1 parent 7454824
commit b0b9fe9
Show file tree

Hide file tree

Showing 37 changed files with 523 additions and 41 deletions.
diff --git a/api/video_codecs/video_encoder_factory.h b/api/video_codecs/video_encoder_factory.h
@@ -85,13 +85,8 @@ class VideoEncoderFactory {
   virtual CodecSupport QueryCodecSupport(
       const SdpVideoFormat& format,
       absl::optional<std::string> scalability_mode) const {
-    // Default implementation, query for supported formats and check if the
-    // specified format is supported. Returns false if scalability_mode is
-    // specified.
     CodecSupport codec_support;
-    if (!scalability_mode) {
-      codec_support.is_supported = format.IsCodecInList(GetSupportedFormats());
-    }
+    codec_support.is_supported = format.IsCodecInList(GetSupportedFormats());
     return codec_support;
   }
 

diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn
@@ -723,6 +723,7 @@ if (is_ios || is_mac) {
       ]
 
       deps = [
+        ":simulcast",
         ":base_objc",
         ":native_video",
         ":videocodec_objc",
@@ -822,6 +823,22 @@ if (is_ios || is_mac) {
       ]
     }
 
+    rtc_library("simulcast") {
+      sources = [
+        "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
+        "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm",
+        "objc/api/video_codec/RTCVideoEncoderSimulcast.h",
+        "objc/api/video_codec/RTCVideoEncoderSimulcast.mm",
+      ]
+
+      deps = [
+        ":base_objc",
+        ":wrapped_native_codec_objc",
+        "../media:rtc_media_base",
+        "../media:rtc_simulcast_encoder_adapter",
+      ]
+    }
+
     rtc_library("mediaconstraints_objc") {
       configs += [ "..:no_global_constructors" ]
       sources = [
@@ -1368,6 +1385,9 @@ if (is_ios || is_mac) {
           "objc/api/video_codec/RTCVideoEncoderAV1.h",
           "objc/api/video_frame_buffer/RTCNativeI420Buffer.h",
           "objc/api/video_frame_buffer/RTCNativeMutableI420Buffer.h",
+          # Added for Simulcast support
+          "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
+          "objc/api/video_codec/RTCVideoEncoderSimulcast.h",
         ]
 
         if (!build_with_chromium) {
@@ -1518,6 +1538,9 @@ if (is_ios || is_mac) {
           "objc/components/video_codec/RTCVideoEncoderH264.h",
           "objc/components/video_frame_buffer/RTCCVPixelBuffer.h",
           "objc/helpers/RTCDispatcher.h",
+          # Added for Simulcast support
+          "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
+          "objc/api/video_codec/RTCVideoEncoderSimulcast.h",
         ]
         if (!build_with_chromium) {
           sources += [

diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn
@@ -530,6 +530,8 @@ if (is_android) {
     sources = [
       "api/org/webrtc/SoftwareVideoDecoderFactory.java",
       "api/org/webrtc/SoftwareVideoEncoderFactory.java",
+      "api/org/webrtc/SimulcastVideoEncoder.java",
+      "api/org/webrtc/SimulcastVideoEncoderFactory.java",
     ]
 
     deps = [
@@ -908,6 +910,21 @@ if (current_os == "linux" || is_android) {
     ]
   }
 
+  rtc_library("simulcast_jni") {
+    visibility = [ "*" ]
+    allow_poison = [ "software_video_codecs" ]
+    sources = [
+      "src/jni/simulcast_video_encoder.cc",
+      "src/jni/simulcast_video_encoder.h"
+    ]
+    deps = [
+      ":base_jni",
+      ":video_jni",
+      ":native_api_codecs",
+      "../../media:rtc_simulcast_encoder_adapter"
+    ]
+  }
+
   rtc_library("swcodecs_jni") {
     visibility = [ "*" ]
     allow_poison = [ "software_video_codecs" ]
@@ -921,6 +938,7 @@ if (current_os == "linux" || is_android) {
       ":libvpx_vp8_jni",
       ":libvpx_vp9_jni",
       ":native_api_jni",
+      ":simulcast_jni",
       ":video_jni",
       "../../api/environment",
       "../../api/video_codecs:builtin_video_decoder_factory",

diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java
@@ -143,11 +143,11 @@ public VideoCodecInfo[] getSupportedCodecs() {
         // supported by the decoder.
         if (type == VideoCodecMimeType.H264 && isH264HighProfileSupported(codec)) {
           supportedCodecInfos.add(new VideoCodecInfo(
-              name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true)));
+              name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true), new ArrayList<>()));
         }
 
         supportedCodecInfos.add(new VideoCodecInfo(
-            name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false)));
+            name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false), new ArrayList<>()));
       }
     }
 

diff --git a/sdk/android/api/org/webrtc/LibaomAv1Encoder.java b/sdk/android/api/org/webrtc/LibaomAv1Encoder.java
@@ -9,6 +9,7 @@
  */
 
 package org.webrtc;
+import java.util.List;
 
 public class LibaomAv1Encoder extends WrappedNativeVideoEncoder {
   @Override
@@ -22,4 +23,10 @@ public long createNative(long webrtcEnvRef) {
   public boolean isHardwareEncoder() {
     return false;
   }
+
+  static List<String> scalabilityModes() {
+    return nativeGetSupportedScalabilityModes();
+  }
+
+  static native List<String> nativeGetSupportedScalabilityModes();
 }
diff --git a/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java b/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java
@@ -9,6 +9,7 @@
  */
 
 package org.webrtc;
+import java.util.List;
 
 public class LibvpxVp9Encoder extends WrappedNativeVideoEncoder {
   @Override
@@ -24,4 +25,10 @@ public boolean isHardwareEncoder() {
   }
 
   static native boolean nativeIsSupported();
+
+  static List<String> scalabilityModes() {
+    return nativeGetSupportedScalabilityModes();
+  }
+
+  static native List<String> nativeGetSupportedScalabilityModes();
 }
diff --git a/sdk/android/api/org/webrtc/RtpParameters.java b/sdk/android/api/org/webrtc/RtpParameters.java
@@ -76,6 +76,8 @@ public static class Encoding {
     // If non-null, scale the width and height down by this factor for video. If null,
     // implementation default scaling factor will be used.
     @Nullable public Double scaleResolutionDownBy;
+    // Scalability modes are used to represent simulcast and SVC layers.
+    @Nullable public String scalabilityMode;
     // SSRC to be used by this encoding.
     // Can't be changed between getParameters/setParameters.
     public Long ssrc;
@@ -93,8 +95,8 @@ public Encoding(String rid, boolean active, Double scaleResolutionDownBy) {
     @CalledByNative("Encoding")
     Encoding(String rid, boolean active, double bitratePriority, @Priority int networkPriority,
         Integer maxBitrateBps, Integer minBitrateBps, Integer maxFramerate,
-        Integer numTemporalLayers, Double scaleResolutionDownBy, Long ssrc,
-        boolean adaptiveAudioPacketTime) {
+        Integer numTemporalLayers, Double scaleResolutionDownBy, String scalabilityMode,
+        Long ssrc, boolean adaptiveAudioPacketTime) {
       this.rid = rid;
       this.active = active;
       this.bitratePriority = bitratePriority;
@@ -104,6 +106,7 @@ public Encoding(String rid, boolean active, Double scaleResolutionDownBy) {
       this.maxFramerate = maxFramerate;
       this.numTemporalLayers = numTemporalLayers;
       this.scaleResolutionDownBy = scaleResolutionDownBy;
+      this.scalabilityMode = scalabilityMode;
       this.ssrc = ssrc;
       this.adaptiveAudioPacketTime = adaptiveAudioPacketTime;
     }
@@ -160,6 +163,12 @@ Double getScaleResolutionDownBy() {
       return scaleResolutionDownBy;
     }
 
+    @Nullable
+    @CalledByNative("Encoding")
+    String getScalabilityMode() {
+      return scalabilityMode;
+    }
+
     @CalledByNative("Encoding")
     Long getSsrc() {
       return ssrc;

diff --git a/sdk/android/api/org/webrtc/SimulcastVideoEncoder.java b/sdk/android/api/org/webrtc/SimulcastVideoEncoder.java
@@ -0,0 +1,28 @@
+package org.webrtc;
+
+public class SimulcastVideoEncoder extends WrappedNativeVideoEncoder {
+
+    static native long nativeCreateEncoder(long webrtcEnvRef, VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info);
+
+    VideoEncoderFactory primary;
+    VideoEncoderFactory fallback;
+    VideoCodecInfo info;
+
+    public SimulcastVideoEncoder(VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info) {
+        this.primary = primary;
+        this.fallback = fallback;
+        this.info = info;
+    }
+
+    @Override
+    public long createNative(long webrtcEnvRef) {
+        return nativeCreateEncoder(webrtcEnvRef, primary, fallback, info);
+    }
+
+    @Override
+    public boolean isHardwareEncoder() {
+        return false;
+    }
+
+}
+
diff --git a/sdk/android/api/org/webrtc/SimulcastVideoEncoderFactory.java b/sdk/android/api/org/webrtc/SimulcastVideoEncoderFactory.java
@@ -0,0 +1,43 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+import androidx.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Arrays;
+
+public class SimulcastVideoEncoderFactory implements VideoEncoderFactory {
+
+    VideoEncoderFactory primary;
+    VideoEncoderFactory fallback;
+
+    public SimulcastVideoEncoderFactory(VideoEncoderFactory primary, VideoEncoderFactory fallback) {
+        this.primary = primary;
+        this.fallback = fallback;
+    }
+
+    @Nullable
+    @Override
+    public VideoEncoder createEncoder(VideoCodecInfo info) {
+        return new SimulcastVideoEncoder(primary, fallback, info);
+    }
+
+    @Override
+    public VideoCodecInfo[] getSupportedCodecs() {
+        List<VideoCodecInfo> codecs = new ArrayList<VideoCodecInfo>();
+        codecs.addAll(Arrays.asList(primary.getSupportedCodecs()));
+        codecs.addAll(Arrays.asList(fallback.getSupportedCodecs()));
+        return codecs.toArray(new VideoCodecInfo[codecs.size()]);
+    }
+
+}
diff --git a/sdk/android/api/org/webrtc/VideoCodecInfo.java b/sdk/android/api/org/webrtc/VideoCodecInfo.java
@@ -14,6 +14,8 @@
 import java.util.Arrays;
 import java.util.Locale;
 import java.util.Map;
+import java.util.List;
+import java.util.ArrayList;
 
 /**
  * Represent a video codec as encoded in SDP.
@@ -34,20 +36,24 @@ public class VideoCodecInfo {
 
   public final String name;
   public final Map<String, String> params;
+  public final List<String> scalabilityModes;
+
   @Deprecated public final int payload;
 
   @CalledByNative
-  public VideoCodecInfo(String name, Map<String, String> params) {
+  public VideoCodecInfo(String name, Map<String, String> params, List<String> scalabilityModes) {
     this.payload = 0;
     this.name = name;
     this.params = params;
+    this.scalabilityModes = scalabilityModes;
   }
 
   @Deprecated
   public VideoCodecInfo(int payload, String name, Map<String, String> params) {
     this.payload = payload;
     this.name = name;
     this.params = params;
+    this.scalabilityModes = new ArrayList<>();
   }
 
   @Override
@@ -83,4 +89,9 @@ String getName() {
   Map getParams() {
     return params;
   }
+
+  @CalledByNative
+  List<String> getScalabilityModes() {
+    return scalabilityModes;
+  }
 }
diff --git a/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java b/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java
@@ -48,7 +48,7 @@ public AndroidVideoDecoderInstrumentationTest(String codecName, boolean useEglCo
     if (codecName.equals("H264")) {
       this.codecType = H264Utils.DEFAULT_H264_BASELINE_PROFILE_CODEC;
     } else {
-      this.codecType = new VideoCodecInfo(codecName, new HashMap<>());
+      this.codecType = new VideoCodecInfo(codecName, new HashMap<>(), new ArrayList<>());
     }
     this.useEglContext = useEglContext;
   }

diff --git a/sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java b/sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java
@@ -47,7 +47,7 @@ public void setUp() {
   @SmallTest
   @Test
   public void getSupportedCodecs_hwVp8SameParamsAsSwVp8_oneVp8() {
-    VideoCodecInfo hwVp8Encoder = new VideoCodecInfo("VP8", new HashMap<>());
+    VideoCodecInfo hwVp8Encoder = new VideoCodecInfo("VP8", new HashMap<>(), new ArrayList<>());
     VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(hwVp8Encoder);
     DefaultVideoEncoderFactory defFactory = new DefaultVideoEncoderFactory(hwFactory);
     VideoCodecInfo[] supportedCodecs = defFactory.getSupportedCodecs();
@@ -62,7 +62,7 @@ public void getSupportedCodecs_hwVp8SameParamsAsSwVp8_oneVp8() {
   public void getSupportedCodecs_hwVp8WithDifferentParams_twoVp8() {
     VideoCodecInfo hwVp8Encoder = new VideoCodecInfo("VP8", new HashMap<String, String>() {
       { put("param", "value"); }
-    });
+    }, new ArrayList<>());
     VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(hwVp8Encoder);
     DefaultVideoEncoderFactory defFactory = new DefaultVideoEncoderFactory(hwFactory);
     VideoCodecInfo[] supportedCodecs = defFactory.getSupportedCodecs();

diff --git a/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoDecoderFactoryTest.java b/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoDecoderFactoryTest.java
@@ -14,6 +14,7 @@
 
 import androidx.annotation.Nullable;
 import androidx.test.filters.SmallTest;
+import java.util.ArrayList;
 import java.util.HashMap;
 import org.junit.Before;
 import org.junit.Test;
@@ -55,7 +56,7 @@ public void createDecoder_supportedCodec_returnsNotNull() {
   @Test
   public void createDecoder_unsupportedCodec_returnsNull() {
     VideoDecoderFactory factory = new SoftwareVideoDecoderFactory();
-    VideoCodecInfo codec = new VideoCodecInfo("unsupported", new HashMap<String, String>());
+    VideoCodecInfo codec = new VideoCodecInfo("unsupported", new HashMap<String, String>(), new ArrayList<>());
     VideoDecoder decoder = factory.createDecoder(codec);
     assertThat(decoder).isNull();
   }

diff --git a/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoEncoderFactoryTest.java b/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoEncoderFactoryTest.java
@@ -14,6 +14,7 @@
 
 import androidx.annotation.Nullable;
 import androidx.test.filters.SmallTest;
+import java.util.ArrayList;
 import java.util.HashMap;
 import org.junit.Before;
 import org.junit.Test;
@@ -52,7 +53,7 @@ public void createEncoder_supportedCodec_returnsNotNull() {
   @Test
   public void createEncoder_unsupportedCodec_returnsNull() {
     VideoEncoderFactory factory = new SoftwareVideoEncoderFactory();
-    VideoCodecInfo codec = new VideoCodecInfo("unsupported", new HashMap<String, String>());
+    VideoCodecInfo codec = new VideoCodecInfo("unsupported", new HashMap<String, String>(), new ArrayList<>());
     VideoEncoder encoder = factory.createEncoder(codec);
     assertThat(encoder).isNull();
   }

diff --git a/sdk/android/native_unittests/org/webrtc/CodecsWrapperTestHelper.java b/sdk/android/native_unittests/org/webrtc/CodecsWrapperTestHelper.java
@@ -12,6 +12,7 @@
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.ArrayList;
 
 public class CodecsWrapperTestHelper {
   @CalledByNative
@@ -20,7 +21,7 @@ public static VideoCodecInfo createTestVideoCodecInfo() {
     params.put(
         VideoCodecInfo.H264_FMTP_PROFILE_LEVEL_ID, VideoCodecInfo.H264_CONSTRAINED_BASELINE_3_1);
 
-    VideoCodecInfo codec_info = new VideoCodecInfo("H264", params);
+    VideoCodecInfo codec_info = new VideoCodecInfo("H264", params, new ArrayList<>());
     return codec_info;
   }