(#102) FileCacheTests: preliminary test API

Emulsion.ContentProxy/FileCache.fs

@@ -15,18 +15,20 @@ type DownloadRequest = {
     Size: uint64
 }
 
+module FileCache =
+    let FileName(sha256: SHA256, cacheKey: string): string =
+        cacheKey
+        |> Encoding.UTF8.GetBytes
+        |> sha256.ComputeHash
+        |> Convert.ToBase64String
+
 // TODO: Total cache limit
 type FileCache(logger: ILogger,
                settings: FileCacheSettings,
                sha256: SHA256) =
 
-    let getFilePath (cacheKey: string) =
-        let hash =
-            cacheKey
-            |> Encoding.UTF8.GetBytes
-            |> sha256.ComputeHash
-            |> Convert.ToBase64String
-        Path.Combine(settings.Directory, hash)
+    let getFilePath(cacheKey: string) =
+        Path.Combine(settings.Directory, FileCache.FileName(sha256, cacheKey))
 
     let getFromCache(cacheKey: string) = async {
         let path = getFilePath cacheKey
@@ -57,6 +59,7 @@ type FileCache(logger: ILogger,
 
         use cachedFile = new FileStream(path, FileMode.Open, FileAccess.Write, FileShare.None)
         do! Async.AwaitTask(stream.CopyToAsync(cachedFile, ct))
+        logger.Information("Download successful: \"{Uri}\" to \"{Path}\".")
 
         let! file = getFromCache cacheKey
         return upcast Option.get file

Emulsion.Settings/Settings.fs

@@ -37,6 +37,7 @@ type HostingSettings = {
 type FileCacheSettings = {
     Directory: string
     FileSizeLimitBytes: uint64
+    TotalCacheSizeLimitBytes: uint64
 }
 
 type EmulsionSettings = {
@@ -107,7 +108,8 @@ let read (config : IConfiguration) : EmulsionSettings =
         Option.ofObj section["directory"]
         |> Option.map(fun directory -> {
             Directory = directory
-            FileSizeLimitBytes = uint64OrDefault section["fileSizeLimitBytes"] 1048576UL
+            FileSizeLimitBytes = uint64OrDefault section["fileSizeLimitBytes"] 1024UL * 1024UL
+            TotalCacheSizeLimitBytes = uint64OrDefault section["fileSizeLimitBytes"] 20UL * 1024UL * 1024UL
         })
 
     { Xmpp = readXmpp <| config.GetSection("xmpp")

Emulsion.Tests/ContentProxy/FileCacheTests.fs

@@ -1,20 +1,96 @@
 namespace Emulsion.Tests.ContentProxy
 
+open System
+open System.Collections.Generic
+open System.IO
+open System.Security.Cryptography
+open System.Threading.Tasks
+
 open Xunit
 open Xunit.Abstractions
 
+open Emulsion.ContentProxy
+open Emulsion.Settings
+open Emulsion.Tests.TestUtils
+open Emulsion.Tests.TestUtils.Logging
+
 type FileCacheTests(outputHelper: ITestOutputHelper) =
+
+    let sha256 = SHA256.Create()
+
+    let cacheDirectory = lazy (
+        let path = Path.GetTempFileName()
+        File.Delete path
+        Directory.CreateDirectory path |> ignore
+        path
+    )
+
+    let setUpFileCache(totalLimitBytes: uint64) =
+        let settings = {
+            Directory = cacheDirectory.Value
+            FileSizeLimitBytes = 1048576UL
+            TotalCacheSizeLimitBytes = totalLimitBytes
+        }
+
+        new FileCache(xunitLogger outputHelper, settings, sha256)
+
+    let assertCacheState(entries: (string * byte[]) seq) =
+        let files =
+            Directory.EnumerateFileSystemEntries(cacheDirectory.Value)
+            |> Seq.map(fun file ->
+                let name = Path.GetFileName file
+                let content = File.ReadAllBytes file
+                name, content
+            )
+            |> Map.ofSeq
+
+        let entries =
+            entries
+            |> Seq.map(fun (k, v) -> FileCache.FileName(sha256, k), v)
+            |> Map.ofSeq
+
+        Assert.Equal<IEnumerable<_>>(entries, files)
+
+    [<Fact>]
     member _.``File should be cached``(): unit =
         Assert.False true
 
+    [<Fact>]
     member _.``Too big file should be proxied``(): unit =
         Assert.False true
 
-    member _.``Cleanup should be triggered``(): unit =
-        Assert.False true
+    [<Fact>]
+    member _.``Cleanup should be triggered``(): Task = task {
+        use fileCache = setUpFileCache 129UL
+        use fileStorage = new WebFileStorage(Map.ofArray [|
+            "a", [| for _ in 1 .. 128 do yield 1uy |]
+            "b", [| for _ in 1 .. 128 do yield 2uy |]
+            "c", [| 3uy |]
+        |])
+
+        let! file = fileCache.Download(fileStorage.Link("a"), "a", 128UL)
+        Assert.True file.IsSome
+        assertCacheState [| "a", fileStorage.Content("a") |]
 
+        let! file = fileCache.Download(fileStorage.Link("b"), "b", 128UL)
+        Assert.True file.IsSome
+        assertCacheState [| "b", fileStorage.Content("b") |]
+
+        let! file = fileCache.Download(fileStorage.Link("c"), "c", 1UL)
+        Assert.True file.IsSome
+        assertCacheState [|
+            "b", fileStorage.Content("b")
+            "c", fileStorage.Content("c")
+        |]
+    }
+
+    [<Fact>]
     member _.``File should be read even after cleanup``(): unit =
         Assert.False true
 
+    [<Fact>]
     member _.``File should be re-downloaded after cleanup even if there's a outdated read session in progress``(): unit =
         Assert.False true
+
+    interface IDisposable with
+        member _.Dispose() = sha256.Dispose()

Emulsion.Tests/Emulsion.Tests.fsproj

@@ -10,6 +10,7 @@
     <Compile Include="TestUtils\TestDataStorage.fs" />
     <Compile Include="TestUtils\Exceptions.fs" />
     <Compile Include="TestUtils\TelegramClientMock.fs" />
+    <Compile Include="TestUtils\WebFileStorage.fs" />
     <Compile Include="ExceptionUtilsTests.fs" />
     <Compile Include="LifetimesTests.fs" />
     <Compile Include="MessageSenderTests.fs" />

Emulsion.Tests/TestUtils/WebFileStorage.fs

@@ -0,0 +1,14 @@
+namespace Emulsion.Tests.TestUtils
+
+open System
+
+type WebFileStorage(data: Map<string, byte[]>) =
+    member _.Link(entry: string): Uri =
+        failwith "todo"
+
+    member _.Content(entry: string): byte[] =
+        failwith "todo"
+
+    interface IDisposable with
+        member this.Dispose(): unit = failwith "todo"
+

README.md

@@ -27,7 +27,8 @@ Copy `emulsion.example.json` to `emulsion.json` and set the settings. For some s
         "pingTimeout": "00:00:30"
     },
     "fileCache": {
-        "fileSizeLimitBytes": 1048576
+        "fileSizeLimitBytes": 1048576,
+        "totalCacheSizeLimitBytes": 20971520
     }
 }
 ```
@@ -54,7 +55,8 @@ To enable it, configure the `database`, `hosting` and `fileCache` configuration
     },
     "fileCache": {
         "directory": "/tmp/emulsion/cache",
-        "fileSizeLimitBytes": 1048576
+        "fileSizeLimitBytes": 1048576,
+        "totalCacheSizeLimitBytes": 20971520
     }
 }
 ```