Merge pull request #114 from mukunku/v3.1.0-release

V3.1.0 Release
mukunku · Sep 20, 2024 · 03274e4 · 03274e4
2 parents 50c8ad9 + e8618c5
commit 03274e4
Show file tree

Hide file tree

Showing 28 changed files with 354 additions and 365 deletions.
diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml
@@ -6,6 +6,10 @@ on:
    - master
  pull_request:
 
+permissions:
+  checks: write
+  pull-requests: write
+
 jobs:
  test:
     runs-on: windows-latest
@@ -18,7 +22,7 @@ jobs:
           src
 
     - name: Setup .NET
-      uses: actions/setup-dotnet@v1
+      uses: actions/setup-dotnet@v4.0.1
       with:
         dotnet-version: '8.0.x'
 
@@ -37,3 +41,4 @@ jobs:
         github-token: ${{ secrets.GITHUB_TOKEN }}
         comment-title: 'Unit Test Results'
         results-path: ./src/ParquetViewer.Tests/TestResults/*.trx
+
diff --git a/.gitignore b/.gitignore
@@ -350,5 +350,3 @@ MigrationBackup/
 
 # Ionide (cross platform F# VS Code tools) working folder
 .ionide/
-/src/ParquetFileViewer/FodyWeavers.xml
-/src/ParquetFileViewer/FodyWeavers.xsd
diff --git a/README.md b/README.md
@@ -26,7 +26,7 @@ Checkout the [ParquetViewer Analytics Dashboard](https://app.amplitude.com/analy
 [^1]: Full privacy policy here: https://github.com/mukunku/ParquetViewer/wiki/Privacy-Policy
 
 # Technical Details
-The latest version of this project was written in C# using Microsoft Visual Studio Community 2022 v17.9.6 and .NET 8
+The latest version of this project was written in C# using Microsoft Visual Studio Community 2022 v17.10.3 and .NET 8
 
 # Acknowledgements
 This utility would not be possible without: https://github.com/aloneguid/parquet-dotnet
diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props
@@ -3,14 +3,14 @@
     <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
   </PropertyGroup>
   <ItemGroup>
-    <PackageVersion Include="Apache.Arrow" Version="16.0.0" />
-    <PackageVersion Include="Parquet.Net" Version="4.23.5" />
+    <PackageVersion Include="Apache.Arrow" Version="16.1.0" />
+    <PackageVersion Include="Parquet.Net" Version="4.24.0" />
     <PackageVersion Include="Microsoft.CSharp" Version="4.7.0" />
-    <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.6.0" />
-    <PackageVersion Include="RichardSzalay.MockHttp" Version="6.0.0" />
-    <PackageVersion Include="xunit" Version="2.4.2" />
-    <PackageVersion Include="xunit.runner.visualstudio" Version="2.4.5" />
-    <PackageVersion Include="coverlet.collector" Version="6.0.0" />
+    <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
+    <PackageVersion Include="RichardSzalay.MockHttp" Version="7.0.0" />
+    <PackageVersion Include="xunit" Version="2.8.1" />
+    <PackageVersion Include="xunit.runner.visualstudio" Version="2.8.1" />
+    <PackageVersion Include="coverlet.collector" Version="6.0.2" />
     <PackageVersion Include="System.Data.DataSetExtensions" Version="4.5.0" />
   </ItemGroup>
 </Project>
diff --git a/src/ParquetViewer.Engine/ParquetEngine.cs b/src/ParquetViewer.Engine/ParquetEngine.cs
@@ -70,11 +70,6 @@ private ParquetEngine(string fileOrFolderPath, List<ParquetReader> parquetFiles)
             OpenFileOrFolderPath = fileOrFolderPath;
         }
 
-        public Task<ParquetEngine> CloneAsync(CancellationToken cancellationToken)
-        {
-            return OpenFileOrFolderAsync(this.OpenFileOrFolderPath, cancellationToken);
-        }
-
         public static Task<ParquetEngine> OpenFileOrFolderAsync(string fileOrFolderPath, CancellationToken cancellationToken)
         {
             if (File.Exists(fileOrFolderPath)) //Handles null
@@ -195,7 +190,7 @@ private static IEnumerable<string> ListParquetFiles(string folderPath)
                         file.EndsWith(".parquet.gz")
                 );
 
-            if (parquetFiles.Count() == 0)
+            if (!parquetFiles.Any())
             {
                 //Check for extensionless files
                 parquetFiles = Directory.EnumerateFiles(folderPath, "*", SearchOption.AllDirectories);
@@ -221,19 +216,6 @@ private static void EZDispose(IEnumerable<IDisposable> disposables)
             }
         }
 
-        public void Dispose()
-        {
-            if (_parquetFiles is not null)
-            {
-                foreach (var parquetFile in _parquetFiles)
-                {
-                    try
-                    {
-                        parquetFile?.Dispose();
-                    }
-                    catch { /* Swallow */ }
-                }
-            }
-        }
+        public void Dispose() => EZDispose(_parquetFiles);
     }
 }
diff --git a/src/ParquetViewer.Engine/Types/StructValue.cs b/src/ParquetViewer.Engine/Types/StructValue.cs
@@ -18,11 +18,11 @@ public StructValue(string name, DataRow data)
             Data = data ?? throw new ArgumentNullException(nameof(data));
         }
 
-        public override string ToString() => ToJSONWithJavaScriptSerializer(false);
+        public override string ToString() => ToJSON(false);
 
-        public string ToStringTruncated() => ToJSONWithJavaScriptSerializer(true);
+        public string ToStringTruncated() => ToJSON(true);
 
-        private string ToJSONWithJavaScriptSerializer(bool truncateForDisplay)
+        private string ToJSON(bool truncateForDisplay)
         {
             try
             {
@@ -51,7 +51,7 @@ private string ToJSONWithJavaScriptSerializer(bool truncateForDisplay)
             }
         }
 
-        private static void WriteValue(Utf8JsonWriter jsonWriter, object value, bool truncateForDisplay)
+        public static void WriteValue(Utf8JsonWriter jsonWriter, object value, bool truncateForDisplay)
         {
             if (value is null)
             {

diff --git a/src/ParquetViewer.Tests/Data/COLUMN_NAME_WITH_FORWARD_SLASH1.parquet b/src/ParquetViewer.Tests/Data/COLUMN_NAME_WITH_FORWARD_SLASH1.parquet
diff --git a/src/ParquetViewer.Tests/Data/COLUMN_NAME_WITH_FORWARD_SLASH2.parquet b/src/ParquetViewer.Tests/Data/COLUMN_NAME_WITH_FORWARD_SLASH2.parquet
diff --git a/src/ParquetViewer.Tests/ParquetViewer.Tests.csproj b/src/ParquetViewer.Tests/ParquetViewer.Tests.csproj
@@ -4,32 +4,23 @@
     <TargetFramework>net8.0-windows</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
-
     <IsPackable>false</IsPackable>
-
     <PlatformTarget>x64</PlatformTarget>
-
     <Configurations>Debug;Release;Release_SelfContained</Configurations>
-
     <ProduceReferenceAssembly>False</ProduceReferenceAssembly>
-
     <EnforceCodeStyleInBuild>True</EnforceCodeStyleInBuild>
   </PropertyGroup>
-
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
     <LangVersion>default</LangVersion>
     <Optimize>True</Optimize>
   </PropertyGroup>
-
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release_SelfContained|AnyCPU'">
     <LangVersion>default</LangVersion>
     <Optimize>True</Optimize>
   </PropertyGroup>
-
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
     <LangVersion>default</LangVersion>
   </PropertyGroup>
-
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
     <PackageReference Include="RichardSzalay.MockHttp" />
@@ -43,82 +34,13 @@
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
   </ItemGroup>
-
   <ItemGroup>
     <ProjectReference Include="..\ParquetViewer.Engine\ParquetViewer.Engine.csproj" />
     <ProjectReference Include="..\ParquetViewer\ParquetViewer.csproj" />
   </ItemGroup>
-
   <ItemGroup>
-    <None Update="Data\COLUMN_ENDING_IN_PERIOD_TEST1.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\COLUMN_NAME_WITH_FORWARD_SLASH1.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\DATETIME_TEST1.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\DATETIME_TEST2.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\DECIMALS_AND_BOOLS_TEST.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\DECIMALS_AND_BOOLS_TEST1.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\DECIMALS_TEST.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\EMPTY_LIST_OF_STRUCTS1.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\LIST_OF_STRUCTS1.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\LIST_TYPE_TEST1.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\MALFORMED_DATETIME_TEST1.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\MAP_TYPE_TEST1.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\MAP_TYPE_TEST2.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\NULLABLE_GUID_TEST1.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\ORACLE_MALFORMED_INT64_TEST1.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\PARTITIONED_PARQUET_FILE_TEST1\bldgtype=B\bd8c129da60e412db4b21800b9e0b983.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\PARTITIONED_PARQUET_FILE_TEST1\bldgtype=M\7405c310a7484964a45695e6b7bf507f.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\PARTITIONED_PARQUET_FILE_TEST1\bldgtype=R\25f774cc6eaf48189d6995e99edfb594.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\PARTITIONED_PARQUET_FILE_TEST1\bldgtype=X\4527cef6cc044c939656ec2a4cfcc992.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\RANDOM_TEST_FILE1.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\SAME_COLUMN_NAME_DIFFERENT_CASING.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\SAME_COLUMN_NAME_DIFFERENT_CASING1.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Data\STRUCT_TEST1.parquet">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
+    <Content Include="Data\**\*.parquet">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
   </ItemGroup>
-
 </Project>
diff --git a/src/ParquetViewer.Tests/SanityTests.cs b/src/ParquetViewer.Tests/SanityTests.cs
@@ -323,10 +323,9 @@ public async Task MALFORMED_DATETIME_TEST1()
         [Fact]
         public async Task COLUMN_NAME_WITH_FORWARD_SLASH_TEST1()
         {
-            //TODO: need to make this file smaller
-            using var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/COLUMN_NAME_WITH_FORWARD_SLASH1.parquet", default);
+            using var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/COLUMN_NAME_WITH_FORWARD_SLASH2.parquet", default);
 
-            Assert.Equal(181966, parquetEngine.RecordCount);
+            Assert.Equal(1, parquetEngine.RecordCount);
             Assert.Equal(320, parquetEngine.Fields.Count);
 
             var dataTable = (await parquetEngine.ReadRowsAsync(parquetEngine.Fields, 0, 1, default))(false);

diff --git a/src/ParquetViewer/App.config b/src/ParquetViewer/App.config
@@ -1,21 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <configuration>
-    <startup> 
-    </startup>
+  <startup>
+  </startup>
   <runtime>
-    <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
-      <dependentAssembly>
-        <assemblyIdentity name="System.Buffers" publicKeyToken="cc7b13ffcd2ddd51" culture="neutral"/>
-        <bindingRedirect oldVersion="0.0.0.0-4.0.3.0" newVersion="4.0.3.0"/>
-      </dependentAssembly>
-      <dependentAssembly>
-        <assemblyIdentity name="IronSnappy" publicKeyToken="b1d4b1dc83bdcf31" culture="neutral"/>
-        <bindingRedirect oldVersion="0.0.0.0-1.3.0.0" newVersion="1.3.0.0"/>
-      </dependentAssembly>
-      <dependentAssembly>
-        <assemblyIdentity name="System.Runtime.CompilerServices.Unsafe" publicKeyToken="b03f5f7f11d50a3a" culture="neutral"/>
-        <bindingRedirect oldVersion="0.0.0.0-5.0.0.0" newVersion="5.0.0.0"/>
-      </dependentAssembly>
-    </assemblyBinding>
   </runtime>
 </configuration>