Add support for +4GB files

SAPTeamDEV · Aug 17, 2024 · ad5063f · ad5063f
1 parent fd1f3f6
commit ad5063f
Show file tree

Hide file tree

Showing 4 changed files with 53 additions and 15 deletions.
diff --git a/cli/Kryptor.Cli.Shared/Wordlist/CompileSession.cs b/cli/Kryptor.Cli.Shared/Wordlist/CompileSession.cs
@@ -27,6 +27,7 @@ public class CompileSession : Session
         public WordlistIndexEntryV2 IndexEntry;
         private readonly bool Indexing;
         private readonly bool Importing;
+        private bool LowMemory;
 
         public CompileSession(string path, string destination, WordlistIndexEntryV2 entry, bool indexing, bool importing)
         {
@@ -42,6 +43,8 @@ public CompileSession(string path, string destination, WordlistIndexEntryV2 entr
 
             Indexing = indexing;
             Importing = importing;
+
+            LowMemory = false;
         }
 
         protected override async Task<bool> RunAsync(ISessionHost sessionHost, CancellationToken cancellationToken)
@@ -65,6 +68,10 @@ protected override async Task<bool> RunAsync(ISessionHost sessionHost, Cancellat
                 installer.FinalizeInstallation(IndexEntry);
 
                 Description = Indexing ? $"{IndexEntry.Id} Indexed" : $"{IndexEntry.Id} Installed";
+                if (LowMemory)
+                {
+                    Description += " in low-memory mode";
+                }
             }
             catch
             {
@@ -79,6 +86,11 @@ protected override async Task<bool> RunAsync(ISessionHost sessionHost, Cancellat
             return true;
         }
 
+        private bool TryAddLine(string line)
+        {
+            return LowMemory || uniqueLines.Add(line);
+        }
+
         private void Cleanup(bool deleteInstallation)
         {
             if (cleaned) return;
@@ -107,17 +119,19 @@ private async Task Compile(CancellationToken cancellationToken)
             {
                 double steps = 1.0 / streamReader.BaseStream.Length * 100;
 
-                int readChars = 0;
+                long readChars = 0;
                 string line;
                 while ((line = await streamReader.ReadLineAsync()) != null)
                 {
                     cancellationToken.ThrowIfCancellationRequested();
                     readChars += line.Length;
                     lines++;
 
+                    Progress = readChars * steps;
+
                     line = line.Trim();
 
-                    if (string.IsNullOrEmpty(line) || line.Length < 4 && uniqueLines.Add(line) && !regex.IsMatch(line)) continue;
+                    if (string.IsNullOrEmpty(line) || line.Length < 4 || !TryAddLine(line) || regex.IsMatch(line)) continue;
 
                     byte[] data = Encoding.UTF8.GetBytes(line + "\n");
                     words++;
@@ -129,8 +143,6 @@ private async Task Compile(CancellationToken cancellationToken)
                     }
 
                     fileStreams[c].Write(data, 0, data.Length);
-
-                    Progress = readChars * steps;
                 }
             }
 

diff --git a/cli/Kryptor.Cli.Shared/Wordlist/DownloadSession.cs b/cli/Kryptor.Cli.Shared/Wordlist/DownloadSession.cs
@@ -97,7 +97,7 @@ private void SetEndStatus(object sender, System.ComponentModel.AsyncCompletedEve
 
                 Description = $"{IndexEntry.Id}: Verifying file";
 
-                VerifyHash(File.OpenRead(Downloader.Package.FileName), CancellationToken).Wait();
+                VerifyHash(File.OpenRead(Downloader.Package.FileName), CancellationToken);
 
                 if (IndexEntry.Compressed)
                 {
@@ -190,13 +190,11 @@ public void DeleteCache()
             }
         }
 
-        private async Task VerifyHash(Stream stream, CancellationToken cancellationToken)
+        private void VerifyHash(Stream stream, CancellationToken cancellationToken)
         {
             try
             {
-                byte[] buffer = new byte[stream.Length];
-                await stream.ReadAsync(buffer, 0, buffer.Length, cancellationToken);
-                byte[] hash = buffer.Sha256();
+                byte[] hash = stream.Sha256();
 
                 if (IndexEntry.Hash == null)
                 {

diff --git a/cli/Kryptor.Cli.Shared/Wordlist/ImportSessionHost.cs b/cli/Kryptor.Cli.Shared/Wordlist/ImportSessionHost.cs
@@ -30,10 +30,7 @@ public override void Start(ClientContext context)
             {
                 IndexEntry.Size = f.Length;
 
-                byte[] buffer = new byte[f.Length];
-                f.Read(buffer, 0, buffer.Length);
-
-                IndexEntry.Hash = buffer.Sha256();
+                IndexEntry.Hash = f.Sha256();
             }
 
             if (GetInstallationPermission(IndexEntry))

diff --git a/src/Kryptor/Extensions/CryptoExtensions.cs b/src/Kryptor/Extensions/CryptoExtensions.cs
@@ -1,4 +1,5 @@
-using System.Security.Cryptography;
+using System.IO;
+using System.Security.Cryptography;
 
 namespace SAPTeam.Kryptor.Extensions
 {
@@ -10,7 +11,7 @@ public static class CryptoExtensions
         /// <summary>
         /// Computes the SHA256 hash value for the specified byte array.
         /// </summary>
-        /// <param name="buffer">The input to compute the hash code for.</param>
+        /// <param name="buffer">The input to compute the hash code for it.</param>
         /// <returns>The computed hash array.</returns>
         public static byte[] Sha256(this byte[] buffer)
         {
@@ -20,6 +21,36 @@ public static byte[] Sha256(this byte[] buffer)
             }
         }
 
+        /// <summary>
+        /// Computes the SHA256 hash value for the specified stream.
+        /// </summary>
+        /// <param name="stream">The input to compute the hash code for it.</param>
+        /// <param name="startFromOrigin">If set to <see langword="true"/>, the stream will be moved to the beginning and at the end will be moved to the previous position.</param>
+        /// <returns>The computed hash array.</returns>
+        public static byte[] Sha256(this Stream stream, bool startFromOrigin = true)
+        {
+            var curPos = stream.Position;
+
+            if (startFromOrigin && stream.CanSeek)
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+            }
+
+            byte[] hash;
+
+            using (SHA256 sha256 = SHA256.Create())
+            {
+                hash = sha256.ComputeHash(stream);
+            }
+
+            if (startFromOrigin && stream.CanSeek)
+            {
+                stream.Seek(curPos, SeekOrigin.Begin);
+            }
+
+            return hash;
+        }
+
         /// <summary>
         /// Computes the SHA384 hash value for the specified byte array.
         /// </summary>