From 6a13a85862ce44514ac83a2cfea2aa67a8cf654e Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Mon, 23 Dec 2024 18:26:29 -0300 Subject: [PATCH] Dotnet 9 --- .github/workflows/build.yml | 37 +++++------ DataProcessing/DataProcessing.csproj | 2 +- DataProcessing/SECDataConverter.cs | 91 +--------------------------- DataProcessing/SECDataDownloader.cs | 4 +- QuantConnect.DataSource.csproj | 2 +- tests/Tests.csproj | 12 ++-- 6 files changed, 33 insertions(+), 115 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 95e8759..23f274e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,19 +8,18 @@ on: jobs: build: - runs-on: ubuntu-20.04 - - steps: + runs-on: ubuntu-24.04 + steps: - name: Checkout uses: actions/checkout@v2 - - name: Free space - run: df -h && rm -rf /opt/hostedtoolcache* && df -h - - - name: Pull Foundation Image - uses: addnab/docker-run-action@v3 + - name: Liberate disk space + uses: jlumbroso/free-disk-space@main with: - image: quantconnect/lean:foundation + tool-cache: true + large-packages: false + docker-images: false + swap-storage: false - name: Checkout Lean Same Branch id: lean-same-branch @@ -41,11 +40,15 @@ jobs: - name: Move Lean run: mv Lean ../Lean - - name: BuildDataSource - run: dotnet build ./QuantConnect.DataSource.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 - - - name: BuildTests - run: dotnet build ./tests/Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 - - - name: Run Tests - run: dotnet test ./tests/bin/Release/net6.0/Tests.dll + - uses: addnab/docker-run-action@v3 + with: + image: quantconnect/lean:foundation + options: --workdir /__w/Lean.DataSource.SEC/Lean.DataSource.SEC -v /home/runner/work:/__w + shell: bash + run: | + # BuildDataSource + dotnet build ./QuantConnect.DataSource.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 + # BuildTests + dotnet build ./tests/Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 + # Run Tests + dotnet test ./tests/bin/Release/net9.0/Tests.dll diff --git a/DataProcessing/DataProcessing.csproj b/DataProcessing/DataProcessing.csproj index 2fe1f2e..1c22d3b 100644 --- a/DataProcessing/DataProcessing.csproj +++ b/DataProcessing/DataProcessing.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net9.0 process diff --git a/DataProcessing/SECDataConverter.cs b/DataProcessing/SECDataConverter.cs index a9309e8..cda0f0c 100644 --- a/DataProcessing/SECDataConverter.cs +++ b/DataProcessing/SECDataConverter.cs @@ -58,84 +58,6 @@ public class SECDataConverter /// public string Destination; - /// - /// Whitelist of dates to not throw errors on if the file is missing. - /// Most of the dates are either Veterans Day or Columbus Day, but - /// sometimes there exists random days where the SEC doesn't publish reports - /// - public HashSet Holidays = new HashSet - { - new DateTime(1998, 10, 1), - new DateTime(1998, 10, 12), - new DateTime(1998, 11, 11), - new DateTime(1999, 10, 11), - new DateTime(1999, 9, 29), - new DateTime(1999, 9, 30), - new DateTime(1999, 11, 11), - new DateTime(1999, 12, 31), - new DateTime(2000, 10, 9), - new DateTime(2000, 11, 10), - new DateTime(2001, 10, 8), - new DateTime(2001, 11, 12), - new DateTime(2002, 1, 28), - new DateTime(2002, 5, 1), - new DateTime(2002, 10, 14), - new DateTime(2002, 11, 11), - new DateTime(2003, 10, 13), - new DateTime(2003, 11, 11), - new DateTime(2003, 12, 26), - new DateTime(2004, 10, 11), - new DateTime(2004, 11, 11), - new DateTime(2004, 12, 31), - new DateTime(2005, 10, 10), - new DateTime(2005, 11, 11), - new DateTime(2006, 10, 9), - new DateTime(2006, 11, 10), - new DateTime(2007, 10, 8), - new DateTime(2007, 11, 12), - new DateTime(2008, 10, 13), - new DateTime(2008, 11, 11), - new DateTime(2008, 12, 26), - new DateTime(2009, 10, 12), - new DateTime(2009, 11, 11), - new DateTime(2010, 10, 11), - new DateTime(2010, 11, 11), - new DateTime(2010, 12, 31), - new DateTime(2011, 10, 10), - new DateTime(2011, 11, 11), - new DateTime(2012, 10, 8), - new DateTime(2012, 11, 12), - new DateTime(2012, 12, 24), - new DateTime(2013, 10, 14), - new DateTime(2013, 11, 11), - new DateTime(2014, 10, 13), - new DateTime(2014, 11, 11), - new DateTime(2014, 12, 26), - new DateTime(2015, 10, 12), - new DateTime(2015, 11, 11), - new DateTime(2016, 10, 10), - new DateTime(2016, 11, 11), - new DateTime(2017, 10, 9), - new DateTime(2017, 11, 10), - new DateTime(2018, 10, 8), - new DateTime(2018, 11, 7), - new DateTime(2018, 12, 24), - new DateTime(2019, 10, 14), - new DateTime(2019, 11, 11), - new DateTime(2020, 10, 12), - new DateTime(2020, 11, 11), - new DateTime(2021, 10, 11), - new DateTime(2021, 11, 11), - new DateTime(2022, 10, 10), - new DateTime(2022, 11, 11), - new DateTime(2023, 10, 9), - new DateTime(2023, 11, 10), - new DateTime(2024, 10, 14), - new DateTime(2024, 11, 11), - new DateTime(2025, 10, 13), - new DateTime(2025, 11, 11) - }; - /// /// Assets keyed by CIK used to resolve underlying ticker /// @@ -167,16 +89,6 @@ public SECDataConverter(string rawSource, string destination) _mapFileResolver = mapFileProvider.Get(AuxiliaryDataKey.EquityUsa); _securityDefinitionSymbolResolver = SecurityDefinitionSymbolResolver.GetInstance(); - - // Update the hard-coded holidays with dates from the database - var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); - if (marketHoursDatabase.TryGetEntry(Market.USA, "", SecurityType.Equity, out var entry)) - { - foreach (var holiday in entry.ExchangeHours.Holidays) - { - Holidays.Add(holiday); - } - } } /// @@ -185,6 +97,7 @@ public SECDataConverter(string rawSource, string destination) /// Date to process SEC filings for public void Process(DateTime processingDate) { + var holiday = MarketHoursDatabase.FromDataFolder().GetEntry(Market.USA, (string)null, SecurityType.Equity).ExchangeHours.Holidays; // Process data into dictionary of CIK -> List{T} of tickers foreach (var line in File.ReadLines(Path.Combine(RawSource, "cik-ticker-mappings.txt"))) { @@ -214,7 +127,7 @@ public void Process(DateTime processingDate) var remoteRawData = new FileInfo(Path.Combine(RawSource, $"{formattedDate}.nc.tar.gz")); if (!remoteRawData.Exists) { - if (Holidays.Contains(processingDate) || USHoliday.Dates.Contains(processingDate)) + if (holiday.Contains(processingDate)) { Log.Trace("SECDataConverter.Process(): File is missing, but we expected it to be missing. Nothing to do."); return; diff --git a/DataProcessing/SECDataDownloader.cs b/DataProcessing/SECDataDownloader.cs index 1f41340..6b30fab 100644 --- a/DataProcessing/SECDataDownloader.cs +++ b/DataProcessing/SECDataDownloader.cs @@ -27,6 +27,7 @@ using System.Globalization; using System.Net.Http; using System.Threading; +using QuantConnect.Securities; namespace QuantConnect.DataProcessing { @@ -78,6 +79,7 @@ public void Download(string rawDestination, DateTime start, DateTime end) throw new ArgumentException("The SEC requires a company email contact to download data using automation. Please edit `config.json` and add a `sec-user-agent-company-email` entry with your company email address"); } + var holiday = MarketHoursDatabase.FromDataFolder().GetEntry(Market.USA, (string)null, SecurityType.Equity).ExchangeHours.Holidays; using (var client = new HttpClient()) { var userAgent = string.Join(" ", companyName, companyEmail); @@ -88,7 +90,7 @@ public void Download(string rawDestination, DateTime start, DateTime end) for (var currentDate = start; currentDate <= end; currentDate = currentDate.AddDays(1)) { // SEC does not publish documents on US federal holidays or weekends - if (!currentDate.IsCommonBusinessDay() || USHoliday.Dates.Contains(currentDate)) + if (!currentDate.IsCommonBusinessDay() || holiday.Contains(currentDate)) { Log.Trace( $"SECDataDownloader.Download(): Skipping date {currentDate:yyyy-MM-dd} because it was during the weekend or was a holiday"); diff --git a/QuantConnect.DataSource.csproj b/QuantConnect.DataSource.csproj index 50bcf71..7d0fc37 100644 --- a/QuantConnect.DataSource.csproj +++ b/QuantConnect.DataSource.csproj @@ -1,7 +1,7 @@  - net6.0 + net9.0 QuantConnect.DataSource QuantConnect.DataSource.SEC bin\$(Configuration) diff --git a/tests/Tests.csproj b/tests/Tests.csproj index 65863d5..5ac1059 100644 --- a/tests/Tests.csproj +++ b/tests/Tests.csproj @@ -1,6 +1,6 @@ - net6.0 + net9.0 QuantConnect.DataLibrary.Tests @@ -19,18 +19,18 @@ - - - + + all + + + - - \ No newline at end of file