Fix buffer read for windows style newlines (#67)

Implement Joel's test for complex large files. Correct the counting of positions when a chunk ends with a `\r` and the next chunk begins with a `\n`. Update to version 3.2.0.
tspence · Aug 6, 2024 · 4355ed8 · 4355ed8
1 parent 5af4695
commit 4355ed8
Show file tree

Hide file tree

Showing 11 changed files with 1,787 additions and 8 deletions.
diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml
@@ -27,19 +27,22 @@ jobs:
     - name: Build (Framework 2.0 Tests)
       run: msbuild ./tests/net20/tests.net20.csproj
     - name: Test (net20)
-      run: ./NUnit.ConsoleRunner.3.4.0/tools/nunit3-console.exe ./tests/net20/bin/Debug/tests.net20.dll
+      working-directory: ./tests/net20/bin/Debug/
+      run: ../../../../NUnit.ConsoleRunner.3.4.0/tools/nunit3-console.exe ./tests.net20.dll
     - name: Build (Framework 4.0)
       run: msbuild ./src/net40/src.net40.csproj
     - name: Build (Framework 4.0 Tests)
       run: msbuild ./tests/net40/tests.net40.csproj
     - name: Test (net40)
-      run: ./NUnit.ConsoleRunner.3.4.0/tools/nunit3-console.exe ./tests/net40/bin/Debug/tests.net40.dll
+      working-directory: ./tests/net40/bin/Debug
+      run: ../../../../NUnit.ConsoleRunner.3.4.0/tools/nunit3-console.exe ./tests.net40.dll
     - name: Build (Framework 4.5)
       run: msbuild ./src/net45/src.net45.csproj
     - name: Build (Framework 4.5 Tests)
       run: msbuild ./tests/net45/tests.net45.csproj
     - name: Test (net45)
-      run: ./NUnit.ConsoleRunner.3.4.0/tools/nunit3-console.exe ./tests/net45/bin/Debug/tests.net45.dll
+      working-directory: ./tests/net45/bin/Debug/
+      run: ../../../../NUnit.ConsoleRunner.3.4.0/tools/nunit3-console.exe ./tests.net45.dll
     - name: Build (DotNet Core 5.0 and NetStandard 2.0)
       run: dotnet build ./csharp-csv-reader.sln
     - name: Test (net50)

diff --git a/CSVFile.nuspec b/CSVFile.nuspec
@@ -2,7 +2,7 @@
 <package >
 	<metadata>
 		<id>CSVFile</id>
-		<version>3.1.4</version>
+		<version>3.2.0</version>
 		<title>CSVFile</title>
 		<authors>Ted Spence</authors>
 		<owners>Ted Spence</owners>
@@ -15,8 +15,8 @@
 		<releaseNotes>
 			August 5, 2024
 
-			* Add serialization options for arrays and objects
-			* Fix bad deploy of 3.1.3
+			* Fix issue with Windows-style newlines crossing chunks found by @joelverhagen
+			* Fix issue with endless loops reported by @wvvegt
 		</releaseNotes>
 		<readme>docs/README.md</readme>
 		<copyright>Copyright 2006 - 2024</copyright>

diff --git a/src/CSV.cs b/src/CSV.cs
@@ -73,6 +73,10 @@ public static IEnumerable<string[]> ParseStream(StreamReader inStream, CSVSettin
                 {
                     yield return row;
                 } 
+                else if (inStream.EndOfStream)
+                {
+                    break;
+                }
             }
         }
 
@@ -101,6 +105,10 @@ public static async IAsyncEnumerable<string[]> ParseStreamAsync(StreamReader inS
                 {
                     yield return row;
                 }
+                else if (inStream.EndOfStream)
+                {
+                    break;
+                }
             }
         }
 #endif

diff --git a/src/CSVStateMachine.cs b/src/CSVStateMachine.cs
@@ -65,7 +65,7 @@ public class CSVStateMachine
         /// <returns></returns>
         public bool NeedsMoreText()
         {
-            return String.IsNullOrEmpty(_line) || _position >= _line.Length;
+            return String.IsNullOrEmpty(_line) || _position + _settings.LineSeparator.Length >= _line.Length;
         }
 
         /// <summary>
@@ -202,11 +202,13 @@ public string[] ParseChunk(string chunk, bool reachedEnd)
                     var notEnoughChars = _position + _settings.LineSeparator.Length > _line.Length;
                     if (notEnoughChars && !reachedEnd)
                     {
+                        // Backtrack one character so we can pick up the line separator completely next time
+                        _position--;
                         return null;
                     }
 
                     // If we have reached the end, but this isn't a complete line separator, it's just text
-                    if (notEnoughChars && reachedEnd)
+                    if (notEnoughChars)
                     {
                         _work.Append(c);
                     }

diff --git a/tests/PackageAssets.csv b/tests/PackageAssets.csv
diff --git a/tests/ReaderTest.cs b/tests/ReaderTest.cs
@@ -4,6 +4,10 @@
  * Home page: https://github.com/tspence/csharp-csv-reader
  */
 using System;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Text;
 using NUnit.Framework;
 using CSVFile;
 #if HAS_ASYNC
@@ -336,6 +340,41 @@ public void TestMultipleNewlines()
             }
         }
 
+        [Test]
+        public void TestIssue62()
+        {
+            var inputLines = File.ReadAllLines("PackageAssets.csv");
+            var desiredLines = 53_543;
+            var linesToRead = Enumerable
+                .Repeat(inputLines, desiredLines / inputLines.Length + 1)
+                .SelectMany(x => x)
+                .Take(desiredLines)
+                .ToArray();
+
+            var config = new CSVSettings
+            {
+                HeaderRowIncluded = false,
+            };
+
+            var outputLines = 0;
+            var rawText = string.Join(Environment.NewLine, linesToRead);
+            var rawBytes = Encoding.UTF8.GetBytes(rawText);
+            using (var memoryStream = new MemoryStream(rawBytes))
+            {
+                using (var streamReader = new StreamReader(memoryStream))
+                {
+                    using (var csvReader = new CSVReader(streamReader, config))
+                    {
+                        foreach (var row in csvReader)
+                        {
+                            outputLines++;
+                        }
+                    }
+                }
+            }
+            Assert.AreEqual(desiredLines, outputLines);
+        }
+
 #if HAS_ASYNC_IENUM
         [Test]
         public async Task TestAsyncReader()

diff --git a/tests/net20/tests.net20.csproj b/tests/net20/tests.net20.csproj
@@ -74,6 +74,12 @@
       <Name>src.net20</Name>
     </ProjectReference>
   </ItemGroup>
+  <ItemGroup>
+    <Content Include="..\PackageAssets.csv">
+      <Link>PackageAssets.csv</Link>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
   <ItemGroup>
     <None Include="packages.config" />
   </ItemGroup>

diff --git a/tests/net40/tests.net40.csproj b/tests/net40/tests.net40.csproj
@@ -71,6 +71,12 @@
       <Name>src.net40</Name>
     </ProjectReference>
   </ItemGroup>
+  <ItemGroup>
+    <Content Include="..\PackageAssets.csv">
+      <Link>PackageAssets.csv</Link>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
   <ItemGroup>
     <None Include="packages.config" />
   </ItemGroup>

diff --git a/tests/net45/tests.net45.csproj b/tests/net45/tests.net45.csproj
@@ -71,6 +71,12 @@
       <Name>src.net45</Name>
     </ProjectReference>
   </ItemGroup>
+  <ItemGroup>
+    <Content Include="..\PackageAssets.csv">
+      <Link>PackageAssets.csv</Link>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
   <ItemGroup>
     <None Include="packages.config" />
   </ItemGroup>

diff --git a/tests/net50/tests.net50.csproj b/tests/net50/tests.net50.csproj
@@ -30,4 +30,11 @@
     <ProjectReference Include="..\..\src\net50\src.net50.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Content Include="..\PackageAssets.csv">
+      <Link>PackageAssets.csv</Link>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+
 </Project>
diff --git a/tests/net60/tests.net60.csproj b/tests/net60/tests.net60.csproj
@@ -32,5 +32,12 @@
     <ProjectReference Include="..\..\src\netstandard20\src.netstandard20.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Content Include="..\PackageAssets.csv">
+      <Link>PackageAssets.csv</Link>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+
 </Project>
-Original file line number
+Diff line change
@@ Expand Up @@
                     {
                         yield return row;
                     }
+                    else if (inStream.EndOfStream)
+                    {
+                        break;
+                    }
                 }
             }
@@ Expand Down Expand Up @@
                     {
                         yield return row;
                     }
+                    else if (inStream.EndOfStream)
+                    {
+                        break;
+                    }
                 }
             }
     #endif
@@ Expand Down @@