Implement hardware decoding

This implements hardware decoding continuing from the work of @rvillalba-novetta and @mikeboers
with cleanup work by @WyattBlue.

CHANGELOG.rst

@@ -17,7 +17,16 @@ We are operating with `semantic versioning <https://semver.org>`_.
     are merged into the "default" branch.
 
 
+v14.1.0 (Unreleased)
+--------------------
+
+Features
+
+- Add hardware decoding by :gh-user:`matthewlai` and :gh-user:`WyattBlue` in (:pr:`1685`).
+
+
 v14.0.1
+-------
 
 Fixes:
 

av/__init__.py

@@ -17,6 +17,7 @@
 from av.bitstream import BitStreamFilterContext, bitstream_filters_available
 from av.codec.codec import Codec, codecs_available
 from av.codec.context import CodecContext
+from av.codec.hwaccel import HWConfig
 from av.container import open
 from av.format import ContainerFormat, formats_available
 from av.packet import Packet

av/__main__.py

@@ -6,6 +6,8 @@
 def main() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument("--codecs", action="store_true")
+    parser.add_argument("--hwdevices", action="store_true")
+    parser.add_argument("--hwconfigs", action="store_true")
     parser.add_argument("--version", action="store_true")
     args = parser.parse_args()
 
@@ -30,6 +32,18 @@ def main() -> None:
                 version = config["version"]
                 print(f"{libname:<13} {version[0]:3d}.{version[1]:3d}.{version[2]:3d}")
 
+    if args.hwdevices:
+        from av.codec.hwaccel import hwdevices_available
+
+        print("Hardware device types:")
+        for x in hwdevices_available():
+            print("   ", x)
+
+    if args.hwconfigs:
+        from av.codec.codec import dump_hwconfigs
+
+        dump_hwconfigs()
+
     if args.codecs:
         from av.codec.codec import dump_codecs
 

av/about.py

@@ -1 +1 @@
-__version__ = "14.0.1"
+__version__ = "14.1.0"

av/audio/codeccontext.pyx

@@ -3,13 +3,14 @@ cimport libav as lib
 from av.audio.format cimport AudioFormat, get_audio_format
 from av.audio.frame cimport AudioFrame, alloc_audio_frame
 from av.audio.layout cimport AudioLayout, get_audio_layout
+from av.codec.hwaccel cimport HWAccel
 from av.frame cimport Frame
 from av.packet cimport Packet
 
 
 cdef class AudioCodecContext(CodecContext):
-    cdef _init(self, lib.AVCodecContext *ptr, const lib.AVCodec *codec):
-        CodecContext._init(self, ptr, codec)
+    cdef _init(self, lib.AVCodecContext *ptr, const lib.AVCodec *codec, HWAccel hwaccel):
+        CodecContext._init(self, ptr, codec, hwaccel)
 
     cdef _prepare_frames_for_encode(self, Frame input_frame):
 

av/codec/codec.pxd

@@ -7,6 +7,8 @@ cdef class Codec:
     cdef const lib.AVCodecDescriptor *desc
     cdef readonly bint is_encoder
 
+    cdef tuple _hardware_configs
+
     cdef _init(self, name=?)
 
 

av/codec/codec.pyi

@@ -108,3 +108,4 @@ class codec_descriptor:
 codecs_available: set[str]
 
 def dump_codecs() -> None: ...
+def dump_hwconfigs() -> None: ...

av/codec/codec.pyx

@@ -1,4 +1,5 @@
 from av.audio.format cimport get_audio_format
+from av.codec.hwaccel cimport wrap_hwconfig
 from av.descriptor cimport wrap_avclass
 from av.utils cimport avrational_to_fraction
 from av.video.format cimport get_video_format
@@ -117,6 +118,10 @@ cdef class Codec:
         if self.is_encoder and lib.av_codec_is_decoder(self.ptr):
             raise RuntimeError("%s is both encoder and decoder.")
 
+    def __repr__(self):
+        mode = "w" if self.is_encoder else "r"
+        return f"<av.{self.__class__.__name__} {self.name} {mode=}>"
+
     def create(self, kind = None):
         """Create a :class:`.CodecContext` for this codec.
 
@@ -203,6 +208,23 @@ cdef class Codec:
             i += 1
         return ret
 
+    @property
+    def hardware_configs(self):
+        if self._hardware_configs:
+            return self._hardware_configs
+        ret = []
+        cdef int i = 0
+        cdef lib.AVCodecHWConfig *ptr
+        while True:
+            ptr = lib.avcodec_get_hw_config(self.ptr, i)
+            if not ptr:
+                break
+            ret.append(wrap_hwconfig(ptr))
+            i += 1
+        ret = tuple(ret)
+        self._hardware_configs = ret
+        return ret
+
     @property
     def properties(self):
         return self.desc.props
@@ -337,3 +359,19 @@ def dump_codecs():
             )
         except Exception as e:
             print(f"...... {codec.name:<18} ERROR: {e}")
+
+def dump_hwconfigs():
+    print("Hardware configs:")
+    for name in sorted(codecs_available):
+        try:
+            codec = Codec(name, "r")
+        except ValueError:
+            continue
+
+        configs = codec.hardware_configs
+        if not configs:
+            continue
+
+        print("   ", codec.name)
+        for config in configs:
+            print("       ", config)

av/codec/context.pxd

@@ -3,6 +3,7 @@ from libc.stdint cimport int64_t
 
 from av.bytesource cimport ByteSource
 from av.codec.codec cimport Codec
+from av.codec.hwaccel cimport HWAccel
 from av.frame cimport Frame
 from av.packet cimport Packet
 
@@ -18,11 +19,12 @@ cdef class CodecContext:
     cdef int stream_index
 
     cdef lib.AVCodecParserContext *parser
-    cdef _init(self, lib.AVCodecContext *ptr, const lib.AVCodec *codec)
+    cdef _init(self, lib.AVCodecContext *ptr, const lib.AVCodec *codec, HWAccel hwaccel)
 
     # Public API.
     cdef readonly bint is_open
     cdef readonly Codec codec
+    cdef readonly HWAccel hwaccel
     cdef public dict options
     cpdef open(self, bint strict=?)
 
@@ -31,6 +33,9 @@ cdef class CodecContext:
     cpdef decode(self, Packet packet=?)
     cpdef flush_buffers(self)
 
+    # Used by hardware-accelerated decode.
+    cdef HWAccel hwaccel_ctx
+
     # Used by both transcode APIs to setup user-land objects.
     # TODO: Remove the `Packet` from `_setup_decoded_frame` (because flushing packets
     # are bogus). It should take all info it needs from the context and/or stream.
@@ -49,10 +54,11 @@ cdef class CodecContext:
     cdef _send_packet_and_recv(self, Packet packet)
     cdef _recv_frame(self)
 
+    cdef _transfer_hwframe(self, Frame frame)
+
     # Implemented by children for the generic send/recv API, so we have the
     # correct subclass of Frame.
     cdef Frame _next_frame
     cdef Frame _alloc_next_frame(self)
 
-
-cdef CodecContext wrap_codec_context(lib.AVCodecContext*, const lib.AVCodec*)
+cdef CodecContext wrap_codec_context(lib.AVCodecContext*, const lib.AVCodec*, HWAccel hwaccel)

av/codec/context.pyi

@@ -5,6 +5,7 @@ from typing import ClassVar, Literal
 from av.packet import Packet
 
 from .codec import Codec
+from .hwaccel import HWAccel
 
 class ThreadType(Flag):
     NONE: ClassVar[ThreadType]
@@ -83,10 +84,14 @@ class CodecContext:
     def delay(self) -> bool: ...
     @property
     def extradata_size(self) -> int: ...
+    @property
+    def is_hwaccel(self) -> bool: ...
     def open(self, strict: bool = True) -> None: ...
     @staticmethod
     def create(
-        codec: str | Codec, mode: Literal["r", "w"] | None = None
+        codec: str | Codec,
+        mode: Literal["r", "w"] | None = None,
+        hwaccel: HWAccel | None = None,
     ) -> CodecContext: ...
     def parse(
         self, raw_input: bytes | bytearray | memoryview | None = None

av/codec/context.pyx

@@ -18,7 +18,7 @@ from av.dictionary import Dictionary
 cdef object _cinit_sentinel = object()
 
 
-cdef CodecContext wrap_codec_context(lib.AVCodecContext *c_ctx, const lib.AVCodec *c_codec):
+cdef CodecContext wrap_codec_context(lib.AVCodecContext *c_ctx, const lib.AVCodec *c_codec, HWAccel hwaccel):
     """Build an av.CodecContext for an existing AVCodecContext."""
 
     cdef CodecContext py_ctx
@@ -35,7 +35,7 @@ cdef CodecContext wrap_codec_context(lib.AVCodecContext *c_ctx, const lib.AVCode
     else:
         py_ctx = CodecContext(_cinit_sentinel)
 
-    py_ctx._init(c_ctx, c_codec)
+    py_ctx._init(c_ctx, c_codec, hwaccel)
 
     return py_ctx
 
@@ -83,10 +83,10 @@ class Flags2(IntEnum):
 
 cdef class CodecContext:
     @staticmethod
-    def create(codec, mode=None):
+    def create(codec, mode=None, hwaccel=None):
         cdef Codec cy_codec = codec if isinstance(codec, Codec) else Codec(codec, mode)
         cdef lib.AVCodecContext *c_ctx = lib.avcodec_alloc_context3(cy_codec.ptr)
-        return wrap_codec_context(c_ctx, cy_codec.ptr)
+        return wrap_codec_context(c_ctx, cy_codec.ptr, hwaccel)
 
     def __cinit__(self, sentinel=None, *args, **kwargs):
         if sentinel is not _cinit_sentinel:
@@ -96,11 +96,12 @@ cdef class CodecContext:
         self.stream_index = -1  # This is set by the container immediately.
         self.is_open = False
 
-    cdef _init(self, lib.AVCodecContext *ptr, const lib.AVCodec *codec):
+    cdef _init(self, lib.AVCodecContext *ptr, const lib.AVCodec *codec, HWAccel hwaccel):
         self.ptr = ptr
         if self.ptr.codec and codec and self.ptr.codec != codec:
             raise RuntimeError("Wrapping CodecContext with mismatched codec.")
         self.codec = wrap_codec(codec if codec != NULL else self.ptr.codec)
+        self.hwaccel = hwaccel
 
         # Set reasonable threading defaults.
         self.ptr.thread_count = 0  # use as many threads as there are CPUs.
@@ -310,6 +311,13 @@ cdef class CodecContext:
 
         return packets
 
+    @property
+    def is_hwaccel(self):
+        """
+        Returns ``True`` if this codec context is hardware accelerated, ``False`` otherwise.
+        """
+        return self.hwaccel_ctx is not None
+
     def _send_frame_and_recv(self, Frame frame):
         cdef Packet packet
 
@@ -359,10 +367,15 @@ cdef class CodecContext:
             return
         err_check(res)
 
+        frame = self._transfer_hwframe(frame)
+
         if not res:
             self._next_frame = None
             return frame
 
+    cdef _transfer_hwframe(self, Frame frame):
+        return frame
+
     cdef _recv_packet(self):
         cdef Packet packet = Packet()
 

av/codec/hwaccel.pxd

@@ -0,0 +1,20 @@
+cimport libav as lib
+
+from av.codec.codec cimport Codec
+
+
+cdef class HWConfig:
+    cdef object __weakref__
+    cdef lib.AVCodecHWConfig *ptr
+    cdef void _init(self, lib.AVCodecHWConfig *ptr)
+
+cdef HWConfig wrap_hwconfig(lib.AVCodecHWConfig *ptr)
+
+cdef class HWAccel:
+    cdef int _device_type
+    cdef str _device
+    cdef readonly Codec codec
+    cdef readonly HWConfig config
+    cdef lib.AVBufferRef *ptr
+    cdef public bint allow_software_fallback
+    cdef public dict options

av/codec/hwaccel.pyi

@@ -0,0 +1,48 @@
+from enum import IntEnum
+
+from av.codec.codec import Codec
+from av.video.format import VideoFormat
+
+class HWDeviceType(IntEnum):
+    none: int
+    vdpau: int
+    cuda: int
+    vaapi: int
+    dxva2: int
+    qsv: int
+    videotoolbox: int
+    d3d11va: int
+    drm: int
+    opencl: int
+    mediacodec: int
+    vulkan: int
+    d3d12va: int
+
+class HWConfigMethod(IntEnum):
+    none: int
+    hw_device_ctx: int
+    hw_frame_ctx: int
+    internal: int
+    ad_hoc: int
+
+class HWConfig:
+    @property
+    def device_type(self) -> HWDeviceType: ...
+    @property
+    def format(self) -> VideoFormat: ...
+    @property
+    def methods(self) -> HWConfigMethod: ...
+    @property
+    def is_supported(self) -> bool: ...
+
+class HWAccel:
+    def __init__(
+        self,
+        device_type: str | HWDeviceType,
+        device: str | None = None,
+        allow_software_fallback: bool = False,
+        options: dict[str, object] | None = None,
+    ) -> None: ...
+    def create(self, codec: Codec) -> HWAccel: ...
+
+def hwdevices_available() -> list[str]: ...