Dotnet 9

.github/workflows/build.yml

@@ -8,19 +8,18 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
-
-    steps:    
+    runs-on: ubuntu-24.04
+    steps:
       - name: Checkout
         uses: actions/checkout@v2
 
-      - name: Free space
-        run: df -h && rm -rf /opt/hostedtoolcache* && df -h
-
-      - name: Pull Foundation Image
-        uses: addnab/docker-run-action@v3
+      - name: Liberate disk space
+        uses: jlumbroso/free-disk-space@main
         with:
-          image: quantconnect/lean:foundation
+          tool-cache: true
+          large-packages: false
+          docker-images: false
+          swap-storage: false
 
       - name: Checkout Lean Same Branch
         id: lean-same-branch
@@ -41,11 +40,15 @@ jobs:
       - name: Move Lean
         run: mv Lean ../Lean
 
-      - name: BuildDataSource
-        run: dotnet build ./QuantConnect.DataSource.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
-
-      - name: BuildTests
-        run: dotnet build ./tests/Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
-
-      - name: Run Tests
-        run: dotnet test ./tests/bin/Release/net6.0/Tests.dll
+      - uses: addnab/docker-run-action@v3
+        with:
+          image: quantconnect/lean:foundation
+          options: --workdir /__w/Lean.DataSource.SEC/Lean.DataSource.SEC -v /home/runner/work:/__w
+          shell: bash
+          run: |
+            # BuildDataSource
+            dotnet build ./QuantConnect.DataSource.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
+            # BuildTests
+            dotnet build ./tests/Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
+            # Run Tests
+            dotnet test ./tests/bin/Release/net9.0/Tests.dll

DataProcessing/DataProcessing.csproj

@@ -2,7 +2,7 @@
 
     <PropertyGroup>
         <OutputType>Exe</OutputType>
-        <TargetFramework>net6.0</TargetFramework>
+        <TargetFramework>net9.0</TargetFramework>
         <AssemblyName>process</AssemblyName>
     </PropertyGroup>
 

DataProcessing/SECDataConverter.cs

@@ -58,84 +58,6 @@ public class SECDataConverter
         /// </summary>
         public string Destination;
 
-        /// <summary>
-        /// Whitelist of dates to not throw errors on if the file is missing.
-        /// Most of the dates are either Veterans Day or Columbus Day, but
-        /// sometimes there exists random days where the SEC doesn't publish reports
-        /// </summary>
-        public HashSet<DateTime> Holidays = new HashSet<DateTime>
-        {
-            new DateTime(1998, 10, 1),
-            new DateTime(1998, 10, 12),
-            new DateTime(1998, 11, 11),
-            new DateTime(1999, 10, 11),
-            new DateTime(1999, 9, 29),
-            new DateTime(1999, 9, 30),
-            new DateTime(1999, 11, 11),
-            new DateTime(1999, 12, 31),
-            new DateTime(2000, 10, 9),
-            new DateTime(2000, 11, 10),
-            new DateTime(2001, 10, 8),
-            new DateTime(2001, 11, 12),
-            new DateTime(2002, 1, 28),
-            new DateTime(2002, 5, 1),
-            new DateTime(2002, 10, 14),
-            new DateTime(2002, 11, 11),
-            new DateTime(2003, 10, 13),
-            new DateTime(2003, 11, 11),
-            new DateTime(2003, 12, 26),
-            new DateTime(2004, 10, 11),
-            new DateTime(2004, 11, 11),
-            new DateTime(2004, 12, 31),
-            new DateTime(2005, 10, 10),
-            new DateTime(2005, 11, 11),
-            new DateTime(2006, 10, 9),
-            new DateTime(2006, 11, 10),
-            new DateTime(2007, 10, 8),
-            new DateTime(2007, 11, 12),
-            new DateTime(2008, 10, 13),
-            new DateTime(2008, 11, 11),
-            new DateTime(2008, 12, 26),
-            new DateTime(2009, 10, 12),
-            new DateTime(2009, 11, 11),
-            new DateTime(2010, 10, 11),
-            new DateTime(2010, 11, 11),
-            new DateTime(2010, 12, 31),
-            new DateTime(2011, 10, 10),
-            new DateTime(2011, 11, 11),
-            new DateTime(2012, 10, 8),
-            new DateTime(2012, 11, 12),
-            new DateTime(2012, 12, 24),
-            new DateTime(2013, 10, 14),
-            new DateTime(2013, 11, 11),
-            new DateTime(2014, 10, 13),
-            new DateTime(2014, 11, 11),
-            new DateTime(2014, 12, 26),
-            new DateTime(2015, 10, 12),
-            new DateTime(2015, 11, 11),
-            new DateTime(2016, 10, 10),
-            new DateTime(2016, 11, 11),
-            new DateTime(2017, 10, 9),
-            new DateTime(2017, 11, 10),
-            new DateTime(2018, 10, 8),
-            new DateTime(2018, 11, 7),
-            new DateTime(2018, 12, 24),
-            new DateTime(2019, 10, 14),
-            new DateTime(2019, 11, 11),
-            new DateTime(2020, 10, 12),
-            new DateTime(2020, 11, 11),
-            new DateTime(2021, 10, 11),
-            new DateTime(2021, 11, 11),
-            new DateTime(2022, 10, 10),
-            new DateTime(2022, 11, 11),
-            new DateTime(2023, 10, 9),
-            new DateTime(2023, 11, 10),
-            new DateTime(2024, 10, 14),
-            new DateTime(2024, 11, 11),
-            new DateTime(2025, 10, 13),
-            new DateTime(2025, 11, 11)
-        };
-
         /// <summary>
         /// Assets keyed by CIK used to resolve underlying ticker
         /// </summary>
@@ -167,16 +89,6 @@ public SECDataConverter(string rawSource, string destination)
             _mapFileResolver = mapFileProvider.Get(AuxiliaryDataKey.EquityUsa);
 
             _securityDefinitionSymbolResolver = SecurityDefinitionSymbolResolver.GetInstance();
-
-            // Update the hard-coded holidays with dates from the database
-            var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
-            if (marketHoursDatabase.TryGetEntry(Market.USA, "", SecurityType.Equity, out var entry))
-            {
-                foreach (var holiday in entry.ExchangeHours.Holidays)
-                {
-                    Holidays.Add(holiday);
-                }
-            }
         }
 
         /// <summary>
@@ -185,6 +97,7 @@ public SECDataConverter(string rawSource, string destination)
         /// <param name="processingDate">Date to process SEC filings for</param>
         public void Process(DateTime processingDate)
         {
+            var holiday = MarketHoursDatabase.FromDataFolder().GetEntry(Market.USA, (string)null, SecurityType.Equity).ExchangeHours.Holidays;
             // Process data into dictionary of CIK -> List{T} of tickers
             foreach (var line in File.ReadLines(Path.Combine(RawSource, "cik-ticker-mappings.txt")))
             {
@@ -214,7 +127,7 @@ public void Process(DateTime processingDate)
             var remoteRawData = new FileInfo(Path.Combine(RawSource, $"{formattedDate}.nc.tar.gz"));
             if (!remoteRawData.Exists)
             {
-                if (Holidays.Contains(processingDate) || USHoliday.Dates.Contains(processingDate))
+                if (holiday.Contains(processingDate))
                 {
                     Log.Trace("SECDataConverter.Process(): File is missing, but we expected it to be missing. Nothing to do.");
                     return;

DataProcessing/SECDataDownloader.cs

@@ -27,6 +27,7 @@
 using System.Globalization;
 using System.Net.Http;
 using System.Threading;
+using QuantConnect.Securities;
 
 namespace QuantConnect.DataProcessing
 {
@@ -78,6 +79,7 @@ public void Download(string rawDestination, DateTime start, DateTime end)
                 throw new ArgumentException("The SEC requires a company email contact to download data using automation. Please edit `config.json` and add a `sec-user-agent-company-email` entry with your company email address");
             }
 
+            var holiday = MarketHoursDatabase.FromDataFolder().GetEntry(Market.USA, (string)null, SecurityType.Equity).ExchangeHours.Holidays;
             using (var client = new HttpClient())
             {
                 var userAgent = string.Join(" ", companyName, companyEmail);
@@ -88,7 +90,7 @@ public void Download(string rawDestination, DateTime start, DateTime end)
                 for (var currentDate = start; currentDate <= end; currentDate = currentDate.AddDays(1))
                 {
                     // SEC does not publish documents on US federal holidays or weekends
-                    if (!currentDate.IsCommonBusinessDay() || USHoliday.Dates.Contains(currentDate))
+                    if (!currentDate.IsCommonBusinessDay() || holiday.Contains(currentDate))
                     {
                         Log.Trace(
                             $"SECDataDownloader.Download(): Skipping date {currentDate:yyyy-MM-dd} because it was during the weekend or was a holiday");

QuantConnect.DataSource.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net9.0</TargetFramework>
     <RootNamespace>QuantConnect.DataSource</RootNamespace>
     <AssemblyName>QuantConnect.DataSource.SEC</AssemblyName>
     <OutputPath>bin\$(Configuration)</OutputPath>

tests/Tests.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net9.0</TargetFramework>
     <RootNamespace>QuantConnect.DataLibrary.Tests</RootNamespace>
   </PropertyGroup>
   <ItemGroup>
@@ -19,18 +19,18 @@
     </Content>
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="protobuf-net" Version="3.1.33" />
-    <PackageReference Include="NUnit" Version="3.13.3" />
-    <PackageReference Include="NUnit3TestAdapter" Version="4.2.1">
+    <PackageReference Include="NUnit" Version="4.2.2" />
+    <PackageReference Include="NUnit3TestAdapter" Version="4.6.0">
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.9.4" />
     <PackageReference Include="Microsoft.TestPlatform.ObjectModel" Version="16.9.4" />
   </ItemGroup>
+  <ItemGroup>
+    <Using Include="NUnit.Framework.Legacy.ClassicAssert" Alias="Assert" />
+  </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\QuantConnect.DataSource.csproj" />
     <ProjectReference Include="..\DataProcessing\DataProcessing.csproj" />
-    <ProjectReference Include="..\..\Lean\Algorithm\QuantConnect.Algorithm.csproj" />
-    <ProjectReference Include="..\..\Lean\Algorithm.Framework\QuantConnect.Algorithm.Framework.csproj" />
   </ItemGroup>
 </Project>