feat: Expand binary platforms and hardware encoding (#161)

- Update to the latest Shaka Packager (v3.2.0) and FFmpeg (n7.1)
releases
 - Add support for macOS arm64
 - Add support for hardware encoding on Ubuntu Linux 22.04 and 24.04
 - Adjust AV1 encoding options to match new FFmpeg release

binaries/build_wheels.py

@@ -26,33 +26,43 @@
 
 # Version constants.
 # Change to download different versions.
-FFMPEG_VERSION = 'n4.4-2'
-PACKAGER_VERSION = 'v2.6.1'
+FFMPEG_VERSION = 'n7.1-1'
+PACKAGER_VERSION = 'v3.2.0'
 
 # A map of suffixes that will be combined with the binary download links
 # to achieve a full download link.  Different suffix for each platform.
 # Extend this dictionary to add more platforms.
 PLATFORM_SUFFIXES = {
-    # 64-bit Windows
-    'win_amd64': '-win-x64.exe',
-    # 64-bit Linux
-    'manylinux1_x86_64': '-linux-x64',
-    # Linux on ARM
+    # Linux x64
+    'manylinux2014_x86_64': '-linux-x64',
+    # Linux arm64
     'manylinux2014_aarch64': '-linux-arm64',
-    # 64-bit with 10.9 SDK
+    # macOS x64 with 10.9 SDK
     'macosx_10_9_x86_64': '-osx-x64',
+    # macOS arm64 with 10.9 SDK
+    'macosx_10_9_arm64': '-osx-arm64',
+    # Windows x64
+    'win_amd64': '-win-x64.exe',
 }
 
 FFMPEG_DL_PREFIX = 'https://github.com/shaka-project/static-ffmpeg-binaries/releases/download/' + FFMPEG_VERSION
 PACKAGER_DL_PREFIX = 'https://github.com/shaka-project/shaka-packager/releases/download/' + PACKAGER_VERSION
 
-# The download links to each binary.  These download links
-# aren't complete, they miss the platfrom-specific suffix.
-BINARIES_DL = [
+# The download links to each binary.  These download links aren't complete.
+# They are missing the platfrom-specific suffix and optional distro-specific
+# suffix (Linux only).
+FFMPEG_BINARIES_DL = [
     FFMPEG_DL_PREFIX + '/ffmpeg',
     FFMPEG_DL_PREFIX + '/ffprobe',
+]
+PACKAGER_BINARIES_DL = [
     PACKAGER_DL_PREFIX + '/packager',
 ]
+# Important: wrap map() in list(), because map returns an iterator, and we need
+# a real list.
+UBUNTU_SUFFIXES = list(map(
+    lambda version: '-ubuntu-{}'.format(version),
+    streamer_binaries._ubuntu_versions_with_hw_encoders))
 
 BINARIES_ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
 
@@ -88,34 +98,54 @@ def download_binary(download_url: str, download_dir: str) -> str:
   """Downloads a file and writes it to the file system.
   Returns the file name.
   """
-
   binary_name = download_url.split('/')[-1]
   binary_path = os.path.join(download_dir, binary_name)
+
   print('downloading', binary_name, flush=True, end=' ')
   urllib.request.urlretrieve(download_url, binary_path)
   print('(finished)')
+
   # Set executable permissions for the downloaded binaries.
-  default_permissions = 0o755
-  os.chmod(binary_path, default_permissions)
+  executable_permissions = 0o755
+  os.chmod(binary_path, executable_permissions)
+
   return binary_name
 
 
 def main():
-  # For each platform(OS+CPU), we download the its binaries and
-  # create a binary wheel distribution that contains the executable
-  # binaries specific to this platform.
+  # For each platform(OS+CPU), we download the its binaries and create a binary
+  # wheel distribution that contains the executable binaries specific to this
+  # platform.
   download_dir = os.path.join(BINARIES_ROOT_DIR, streamer_binaries.__name__)
+
   for platform_name, suffix in PLATFORM_SUFFIXES.items():
     binaries_to_include = []
-    # Use the `suffix` specific to this platfrom to achieve
-    # the full download link for each binary.
-    for binary_dl in BINARIES_DL:
+
+    # Use the suffix specific to this platfrom to construct the full download
+    # link for each binary.
+    for binary_dl in PACKAGER_BINARIES_DL:
+      download_link = binary_dl + suffix
+      binary_name = download_binary(download_url=download_link,
+                                    download_dir=download_dir)
+      binaries_to_include.append(binary_name)
+
+    # FFmpeg binaries are like packager binaries, except we have extra variants
+    # for Ubuntu Linux to support hardware encoding.
+    for binary_dl in FFMPEG_BINARIES_DL:
       download_link = binary_dl + suffix
       binary_name = download_binary(download_url=download_link,
                                     download_dir=download_dir)
       binaries_to_include.append(binary_name)
-    # Build a wheel distribution for this platform
-    # and include the binaries we have just downloaded.
+
+      if 'linux' in suffix:
+        for ubuntu_suffix in UBUNTU_SUFFIXES:
+          download_link = binary_dl + suffix + ubuntu_suffix
+          binary_name = download_binary(download_url=download_link,
+                                        download_dir=download_dir)
+          binaries_to_include.append(binary_name)
+
+    # Build a wheel distribution for this platform and include the binaries we
+    # have just downloaded.
     build_bdist_wheel(platform_name, binaries_to_include)
 
 

binaries/setup.py

@@ -41,5 +41,9 @@
   package_data={
       # Only add the corresponding platform specific binaries to the wheel.
       streamer_binaries.__name__: platform_binaries,
-  }
+  },
+  install_requires=[
+      # This is only used for Linux, and only supports Linux.
+      'distro;platform_system=="Linux"',
+  ],
 )

binaries/streamer_binaries/__init__.py

@@ -1,7 +1,8 @@
+import distro
 import os
 import platform
 
-__version__ = '0.5.2'
+__version__ = '0.6.0'
 
 
 # Get the directory path where this __init__.py file resides.
@@ -21,6 +22,12 @@
   'aarch64': 'arm64',
 }[platform.machine()]
 
+# Specific versions of Ubuntu with special builds for hardware-encoding.
+_ubuntu_versions_with_hw_encoders = (
+  '22.04',
+  '24.04',
+)
+
 # Module level variables.
 ffmpeg = os.path.join(_dir_path, 'ffmpeg-{}-{}'.format(_os, _cpu))
 """The path to the installed FFmpeg binary."""
@@ -31,3 +38,11 @@
 packager = os.path.join(_dir_path, 'packager-{}-{}'.format(_os, _cpu))
 """The path to the installed Shaka Packager binary."""
 
+# Special overrides for Ubuntu builds with hardware encoding support.
+# These are not static binaries, and so they must be matched to the distro.
+if _os == 'linux':
+  if distro.id() == 'ubuntu':
+    if distro.version() in _ubuntu_versions_with_hw_encoders:
+      suffix = '-ubuntu-' + distro.version()
+      ffmpeg += suffix
+      ffprobe += suffix

streamer/__init__.py

@@ -1,3 +1,3 @@
-__version__ = '0.5.1'
+__version__ = '0.6.0'
 
 from . import controller_node

streamer/controller_node.py

@@ -30,10 +30,11 @@
 
 from typing import Any, Dict, List, Optional, Tuple, Union
 from streamer import __version__
+from streamer import autodetect
+from streamer import min_versions
 from streamer.cloud_node import CloudNode
 from streamer.bitrate_configuration import BitrateConfig, AudioChannelLayout, VideoResolution
 from streamer.external_command_node import ExternalCommandNode
-from streamer import autodetect
 from streamer.input_configuration import InputConfig, InputType, MediaType, Input
 from streamer.node_base import NodeBase, ProcessStatus
 from streamer.output_stream import AudioOutputStream, OutputStream, TextOutputStream, VideoOutputStream
@@ -134,16 +135,17 @@ def next_short_version(version: str) -> str:
               exact_match=True,
               addendum='Install with: {}'.format(pip_command))
       else:
-        # Check that ffmpeg version is 4.1 or above.
-        _check_command_version('FFmpeg', ['ffmpeg', '-version'], (4, 1))
+        # Check the ffmpeg version.
+        _check_command_version('FFmpeg', ['ffmpeg', '-version'],
+                               min_versions.FFMPEG)
 
-        # Check that ffprobe version (used for autodetect features) is 4.1 or
-        # above.
-        _check_command_version('ffprobe', ['ffprobe', '-version'], (4, 1))
+        # Check the ffprobe version (used for autodetect features).
+        _check_command_version('ffprobe', ['ffprobe', '-version'],
+                               min_versions.FFMPEG)
 
-        # Check that Shaka Packager version is 2.6.0 or above.
+        # Check the Shaka Packager version.
         _check_command_version('Shaka Packager', ['packager', '-version'],
-                               (2, 6, 1))
+                               min_versions.PACKAGER)
 
       if bucket_url:
         # Check that the Google Cloud SDK is at least v212, which introduced

streamer/min_versions.py

@@ -0,0 +1,19 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Minimum versions of tools we depend on."""
+
+# These are minimum semantic versions expressed as tuples of ints.
+FFMPEG = (7, 1)
+PACKAGER = (3, 2, 0)

streamer/transcoder_node.py

@@ -292,16 +292,8 @@ def _encode_video(self, stream: VideoOutputStream, input: Input) -> List[str]:
           '-cpu-used', '8',
           # According to the wiki (https://trac.ffmpeg.org/wiki/Encode/AV1),
           # this allows threaded encoding in AV1, which makes better use of CPU
-          # resources and speeds up encoding.  This will be ignored by libaom
-          # before version 1.0.0-759-g90a15f4f2, and so there may be no benefit
-          # unless libaom and ffmpeg are built from source (as of Oct 2019).
+          # resources and speeds up encoding.
           '-row-mt', '1',
-          # According to the wiki (https://trac.ffmpeg.org/wiki/Encode/AV1),
-          # this allows for threaded _decoding_ in AV1, which will provide a
-          # smoother playback experience for the end user.
-          '-tiles', '2x2',
-          # AV1 is considered "experimental".
-          '-strict', 'experimental',
       ]
 
     keyframe_interval = int(self._pipeline_config.segment_size *