merge hotfix

Changelog.md

@@ -10,6 +10,12 @@ https://github.com/Etherna/YoutubeExplode/releases
 
 # Changelog
 
+## v6.2.16 (28-Jun-2023)
+
+- Fixed an issue where `ClosedCaptionClient.WriteToAsync(...)` and `ClosedCaptionClient.DownloadAsync(...)` produced invalid SRT timestamps for caption tracks that exceeded 24 hours in length.
+- [Converter] Fixed an issue where processing a video longer than 24 hours failed with an error or resulted in a deadlock.
+- [Converter] Reduced the amount of irrelevant output that is displayed as part of an error message when FFmpeg fails to process a video.
+
 ## v6.2.15 (25-May-2023)
 
 - Fixed an issue where calling `StreamClient.GetManifestAsync(...)` failed on some videos with an error saying `Could not get cipher manifest`.

YoutubeDownloader.Converter.Tests/GeneralSpecs.cs

@@ -186,7 +186,7 @@ await youtube.Videos.DownloadAsync("9bZkp7q19f0", filePath, o => o
     }
 
     [Fact]
-    public async Task I_can_download_a_video_and_track_the_progress()
+    public async Task I_can_download_a_video_while_tracking_progress()
     {
         // Arrange
         var youtube = new YoutubeClient();
@@ -202,8 +202,10 @@ public async Task I_can_download_a_video_and_track_the_progress()
         // Assert
         var progressValues = progress.GetValues();
         progressValues.Should().NotBeEmpty();
+        progressValues.Should().Contain(p => p >= 0.99);
+        progressValues.Should().NotContain(p => p < 0 || p > 1);
 
-        foreach (var value in progress.GetValues())
+        foreach (var value in progressValues)
             _testOutput.WriteLine($"Progress: {value:P2}");
     }
 }

YoutubeDownloader.Converter.Tests/Utils/Extensions/HttpExtensions.cs

@@ -0,0 +1,19 @@
+using System.IO;
+using System.Net.Http;
+using System.Threading.Tasks;
+
+namespace YoutubeExplode.Converter.Tests.Utils.Extensions;
+
+internal static class HttpExtensions
+{
+    public static async Task DownloadAsync(this HttpClient httpClient, string url, string filePath)
+    {
+        using var response = await httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead);
+        response.EnsureSuccessStatusCode();
+
+        await using var source = await response.Content.ReadAsStreamAsync();
+        await using var destination = File.Create(filePath);
+
+        await source.CopyToAsync(destination);
+    }
+}

YoutubeDownloader.Converter.Tests/Utils/FFmpeg.cs

@@ -1,18 +1,23 @@
 using System;
 using System.IO;
 using System.IO.Compression;
+using System.Linq;
 using System.Net.Http;
 using System.Reflection;
 using System.Runtime.InteropServices;
+using System.Security.Cryptography;
 using System.Threading;
 using System.Threading.Tasks;
+using YoutubeExplode.Converter.Tests.Utils.Extensions;
 
 namespace YoutubeExplode.Converter.Tests.Utils;
 
 public static class FFmpeg
 {
     private static readonly SemaphoreSlim Lock = new(1, 1);
 
+    public static Version Version { get; } = new(4, 4, 1);
+
     private static string FileName { get; } =
         RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
             ? "ffmpeg.exe"
@@ -45,7 +50,7 @@ static string GetArchitectureMoniker()
                 return "64";
 
             if (RuntimeInformation.ProcessArchitecture == Architecture.X86)
-                return "86";
+                return "32";
 
             if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
                 return "arm-64";
@@ -56,28 +61,79 @@ static string GetArchitectureMoniker()
             throw new NotSupportedException("Unsupported architecture.");
         }
 
-        const string version = "4.4.1";
         var plat = GetPlatformMoniker();
         var arch = GetArchitectureMoniker();
 
-        return $"https://github.com/vot/ffbinaries-prebuilt/releases/download/v{version}/ffmpeg-{version}-{plat}-{arch}.zip";
+        return $"https://github.com/vot/ffbinaries-prebuilt/releases/download/v{Version}/ffmpeg-{Version}-{plat}-{arch}.zip";
+    }
+
+    private static byte[] GetDownloadHash()
+    {
+        static string GetHashString()
+        {
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            {
+                // Only x64 build is available
+                return "d1124593b7453fc54dd90ca3819dc82c22ffa957937f33dd650082f1a495b10e";
+            }
+
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+            {
+                if (RuntimeInformation.ProcessArchitecture == Architecture.X64)
+                    return "4348301b0d5e18174925e2022da1823aebbdb07282bbe9adb64b2485e1ef2df7";
+
+                if (RuntimeInformation.ProcessArchitecture == Architecture.X86)
+                    return "a292731806fe3733b9e2281edba881d1035e4018599577174a54e275c0afc931";
+
+                if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
+                    return "7d57e730cc34208743cc1a97134541656ecd2c3adcdfad450dedb61d465857da";
+            }
+
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+            {
+                // Only x64 build is available
+                return "e08c670fcbdc2e627aa4c0d0c5ee1ef20e82378af2f14e4e7ae421a148bd49af";
+            }
+
+            throw new NotSupportedException("Unsupported architecture.");
+        }
+
+        var hashString = GetHashString();
+
+        return Enumerable
+            .Range(0, hashString.Length)
+            .Where(x => x % 2 == 0)
+            .Select(x => Convert.ToByte(hashString.Substring(x, 2), 16))
+            .ToArray();
     }
 
     private static async ValueTask DownloadAsync()
     {
+        using var archiveFile = TempFile.Create();
         using var httpClient = new HttpClient();
 
-        // Extract the FFmpeg binary from the downloaded archive
-        await using var zipStream = await httpClient.GetStreamAsync(GetDownloadUrl());
-        using var zip = new ZipArchive(zipStream, ZipArchiveMode.Read);
+        // Download the archive
+        await httpClient.DownloadAsync(GetDownloadUrl(), archiveFile.Path);
 
-        var entry = zip.GetEntry(FileName);
-        if (entry is null)
-            throw new FileNotFoundException("Downloaded archive doesn't contain FFmpeg.");
+        // Verify the hash
+        await using (var archiveStream = File.OpenRead(archiveFile.Path))
+        {
+            var expectedHash = GetDownloadHash();
+            var actualHash = await SHA256.HashDataAsync(archiveStream);
 
-        await using var entryStream = entry.Open();
-        await using var fileStream = File.Create(FilePath);
-        await entryStream.CopyToAsync(fileStream);
+            if (!actualHash.SequenceEqual(expectedHash))
+                throw new Exception("Downloaded archive has invalid hash.");
+        }
+
+        // Extract the executable
+        using (var zip = ZipFile.OpenRead(archiveFile.Path))
+        {
+            var entry =
+                zip.GetEntry(FileName) ??
+                throw new FileNotFoundException("Downloaded archive doesn't contain the FFmpeg executable.");
+
+            entry.ExtractToFile(FilePath, true);
+        }
 
         // Add the execute permission on Unix
         if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))

YoutubeDownloader.Converter.Tests/YoutubeDownloader.Converter.Tests.csproj

@@ -11,7 +11,7 @@
   <ItemGroup>
     <PackageReference Include="FluentAssertions" Version="6.11.0" />
     <PackageReference Include="Gress" Version="2.1.1" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.2" />
     <PackageReference Include="xunit" Version="2.4.2" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5" PrivateAssets="all" />
     <PackageReference Include="coverlet.collector" Version="6.0.0" PrivateAssets="all" />

YoutubeDownloader.Converter/Converter.cs

@@ -44,17 +44,17 @@ private async ValueTask ProcessAsync(
         foreach (var subtitleInput in subtitleInputs)
             arguments.Add("-i").Add(subtitleInput.FilePath);
 
-        // Input mapping
+        // Explicitly specify that all inputs should be used, because by default
+        // FFmpeg only picks one input per stream type (audio, video, subtitle).
         for (var i = 0; i < streamInputs.Count + subtitleInputs.Count; i++)
             arguments.Add("-map").Add(i);
 
-        // Format
-        arguments.Add("-f").Add(container.Name);
-
-        // Preset
-        arguments.Add("-preset").Add(_preset);
+        // Output format and encoding preset
+        arguments
+            .Add("-f").Add(container.Name)
+            .Add("-preset").Add(_preset);
 
-        // Avoid transcoding where possible
+        // Avoid transcoding inputs that have the same container as the output
         {
             var lastAudioStreamIndex = 0;
             var lastVideoStreamIndex = 0;
@@ -91,15 +91,16 @@ private async ValueTask ProcessAsync(
         // MP4
         if (container == Container.Mp4)
         {
-            //specify the codec for subtitles manually, otherwise they may not get injected
+            //explicitly specify the codec for subtitles, otherwise they won't get embedded
             if (subtitleInputs.Any())
                 arguments.Add("-c:s").Add("mov_text");
 
             //move the index (moov atom) to the beginning of the file
             arguments.Add("-movflags").Add("faststart");
         }
 
-        // MP3: set a constant bitrate for audio streams, otherwise the metadata may contain invalid total duration
+        // MP3: explicitly specify the bitrate for audio streams, otherwise their metadata
+        // might contain invalid total duration.
         // https://superuser.com/a/893044
         if (container == Container.Mp3)
         {
@@ -140,17 +141,24 @@ private async ValueTask ProcessAsync(
         }
 
         // Metadata for subtitles
-        for (var i = 0; i < subtitleInputs.Count; i++)
+        foreach (var (subtitleInput, i) in subtitleInputs.WithIndex())
         {
             arguments
                 .Add($"-metadata:s:s:{i}")
-                .Add($"language={subtitleInputs[i].Info.Language.Code}")
+                .Add($"language={subtitleInput.Info.Language.Code}")
                 .Add($"-metadata:s:s:{i}")
-                .Add($"title={subtitleInputs[i].Info.Language.Name}");
+                .Add($"title={subtitleInput.Info.Language.Name}");
         }
 
+        // Enable progress reporting
+        arguments
+            // Info log level is required to extract total stream duration
+            .Add("-loglevel").Add("info")
+            .Add("-stats");
+
         // Misc settings
         arguments
+            .Add("-hide_banner")
             .Add("-threads").Add(Environment.ProcessorCount)
             .Add("-nostdin")
             .Add("-y");

YoutubeDownloader.Converter/FFmpeg.cs

@@ -78,32 +78,48 @@ private static PipeTarget CreateProgressRouter(IProgress<double> progress)
     {
         var totalDuration = default(TimeSpan?);
 
-        return PipeTarget.ToDelegate(l =>
+        return PipeTarget.ToDelegate(line =>
         {
-            totalDuration ??= Regex
-                .Match(l, @"Duration:\s(\d\d:\d\d:\d\d.\d\d)")
-                .Groups[1]
-                .Value
-                .NullIfWhiteSpace()?
-                .Pipe(s => TimeSpan.ParseExact(s, "c", CultureInfo.InvariantCulture));
+            // Extract total stream duration
+            if (totalDuration is null)
+            {
+                // Need to extract all components separately because TimeSpan cannot directly
+                // parse a time string that is greater than 24 hours.
+                var totalDurationMatch = Regex.Match(line, @"Duration:\s(\d+):(\d+):(\d+\.\d+)");
+                if (totalDurationMatch.Success)
+                {
+                    var hours = int.Parse(totalDurationMatch.Groups[1].Value, CultureInfo.InvariantCulture);
+                    var minutes = int.Parse(totalDurationMatch.Groups[2].Value, CultureInfo.InvariantCulture);
+                    var seconds = double.Parse(totalDurationMatch.Groups[3].Value, CultureInfo.InvariantCulture);
+
+                    totalDuration =
+                        TimeSpan.FromHours(hours) +
+                        TimeSpan.FromMinutes(minutes) +
+                        TimeSpan.FromSeconds(seconds);
+                }
+            }
 
             if (totalDuration is null || totalDuration == TimeSpan.Zero)
                 return;
 
-            var processedDuration = Regex
-                .Match(l, @"time=(\d\d:\d\d:\d\d.\d\d)")
-                .Groups[1]
-                .Value
-                .NullIfWhiteSpace()?
-                .Pipe(s => TimeSpan.ParseExact(s, "c", CultureInfo.InvariantCulture));
-
-            if (processedDuration is null)
-                return;
-
-            progress.Report((
-                processedDuration.Value.TotalMilliseconds /
-                totalDuration.Value.TotalMilliseconds
-            ).Clamp(0, 1));
+            // Extract processed stream duration
+            var processedDurationMatch = Regex.Match(line, @"time=(\d+):(\d+):(\d+\.\d+)");
+            if (processedDurationMatch.Success)
+            {
+                var hours = int.Parse(processedDurationMatch.Groups[1].Value, CultureInfo.InvariantCulture);
+                var minutes = int.Parse(processedDurationMatch.Groups[2].Value, CultureInfo.InvariantCulture);
+                var seconds = double.Parse(processedDurationMatch.Groups[3].Value, CultureInfo.InvariantCulture);
+
+                var processedDuration =
+                    TimeSpan.FromHours(hours) +
+                    TimeSpan.FromMinutes(minutes) +
+                    TimeSpan.FromSeconds(seconds);
+
+                progress.Report((
+                    processedDuration.TotalMilliseconds /
+                    totalDuration.Value.TotalMilliseconds
+                ).Clamp(0, 1));
+            }
         });
     }
 }

YoutubeDownloader.Converter/Utils/Extensions/CollectionExtensions.cs

@@ -0,0 +1,13 @@
+using System.Collections.Generic;
+
+namespace YoutubeExplode.Converter.Utils.Extensions;
+
+internal static class CollectionExtensions
+{
+    public static IEnumerable<(T value, int index)> WithIndex<T>(this IEnumerable<T> source)
+    {
+        var i = 0;
+        foreach (var o in source)
+            yield return (o, i++);
+    }
+}

YoutubeDownloader.Converter/YoutubeDownloader.Converter.csproj

@@ -15,14 +15,14 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="CliWrap" Version="3.6.3" />
+    <PackageReference Include="CliWrap" Version="3.6.4" />
     <PackageReference Include="GitVersion.MsBuild" Version="5.12.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.3" PrivateAssets="all" />
     <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1" PrivateAssets="all" />
-    <PackageReference Include="PolyShim" Version="1.4.0" PrivateAssets="all" />
+    <PackageReference Include="PolyShim" Version="1.5.0" PrivateAssets="all" />
   </ItemGroup>
 
   <ItemGroup>

YoutubeDownloader.Tests/StreamSpecs.cs

@@ -1,3 +1,4 @@
+using System.Buffers;
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
@@ -132,7 +133,7 @@ await youtube.Videos.Streams.GetManifestAsync(VideoIds.Deleted)
     public async Task I_can_get_a_specific_stream_from_a_video(string videoId)
     {
         // Arrange
-        var buffer = new byte[1024];
+        using var buffer = MemoryPool<byte>.Shared.Rent(1024);
         var youtube = new YoutubeClient();
 
         // Act
@@ -141,7 +142,7 @@ public async Task I_can_get_a_specific_stream_from_a_video(string videoId)
         foreach (var streamInfo in manifest.Streams)
         {
             await using var stream = await youtube.Videos.Streams.GetAsync(streamInfo);
-            var bytesRead = await stream.ReadAsync(buffer);
+            var bytesRead = await stream.ReadAsync(buffer.Memory);
 
             // Assert
             bytesRead.Should().BeGreaterThan(0);

YoutubeDownloader.Tests/YoutubeDownloader.Tests.csproj

@@ -10,7 +10,7 @@
 
   <ItemGroup>
     <PackageReference Include="FluentAssertions" Version="6.11.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.2" />
     <PackageReference Include="xunit" Version="2.4.2" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5" PrivateAssets="all" />
     <PackageReference Include="coverlet.collector" Version="6.0.0" PrivateAssets="all" />