Merge pull request #2489 from JKamsker/ReadTransform

Add support for on-the-fly document upgrades

LiteDB.Tests/Database/DocumentUpgrade_Tests.cs

@@ -0,0 +1,133 @@
+using FluentAssertions;
+
+using LiteDB.Engine;
+
+using System.IO;
+
+using Xunit;
+
+namespace LiteDB.Tests.Database;
+
+public class DocumentUpgrade_Tests
+{
+    [Fact]
+    public void DocumentUpgrade_Test()
+    {
+        var ms = new MemoryStream();
+        using (var db = new LiteDatabase(ms))
+        {
+            var col = db.GetCollection("col");
+
+            col.Insert(new BsonDocument { ["version"] = 1, ["_id"] = 1, ["name"] = "John" });
+        }
+
+        ms.Position = 0;
+
+        using (var db = new LiteDatabase(ms))
+        {
+            var col = db.GetCollection("col");
+
+            col.Count().Should().Be(1);
+
+            var doc = col.FindById(1);
+
+            doc["version"].AsInt32.Should().Be(1);
+            doc["name"].AsString.Should().Be("John");
+            doc["age"].AsInt32.Should().Be(0);
+        }
+
+        ms.Position = 0;
+
+        using var engine = new LiteEngine(new EngineSettings
+        {
+            DataStream = ms,
+            ReadTransform = (collectionName, val) =>
+            {
+                if (val is not BsonDocument doc)
+                {
+                    return val;
+                }
+
+                if (doc.TryGetValue("version", out var version) && version.AsInt32 == 1)
+                {
+                    doc["version"] = 2;
+                    doc["age"] = 30;
+                }
+
+                return val;
+            }
+        });
+
+        using (var db = new LiteDatabase(engine))
+        {
+            var col = db.GetCollection("col");
+
+            col.Count().Should().Be(1);
+
+            var doc = col.FindById(1);
+
+            doc["version"].AsInt32.Should().Be(2);
+            doc["name"].AsString.Should().Be("John");
+            doc["age"].AsInt32.Should().Be(30);
+        }
+    }
+
+    [Fact]
+    public void DocumentUpgrade_BsonMapper_Test()
+    {
+        var ms = new MemoryStream();
+        using (var db = new LiteDatabase(ms))
+        {
+            var col = db.GetCollection("col");
+
+            col.Insert(new BsonDocument { ["version"] = 1, ["_id"] = 1, ["name"] = "John" });
+        }
+
+        ms.Position = 0;
+
+        using (var db = new LiteDatabase(ms))
+        {
+            var col = db.GetCollection("col");
+
+            col.Count().Should().Be(1);
+
+            var doc = col.FindById(1);
+
+            doc["version"].AsInt32.Should().Be(1);
+            doc["name"].AsString.Should().Be("John");
+            doc["age"].AsInt32.Should().Be(0);
+        }
+
+        ms.Position = 0;
+
+        var mapper = new BsonMapper();
+        mapper.OnDeserialization = (sender, type, val) =>
+        {
+            if (val is not BsonDocument doc)
+            {
+                return val;
+            }
+
+            if (doc.TryGetValue("version", out var version) && version.AsInt32 == 1)
+            {
+                doc["version"] = 2;
+                doc["age"] = 30;
+            }
+
+            return doc;
+        };
+
+        using (var db = new LiteDatabase(ms, mapper))
+        {
+            var col = db.GetCollection("col");
+
+            col.Count().Should().Be(1);
+
+            var doc = col.FindById(1);
+
+            doc["version"].AsInt32.Should().Be(2);
+            doc["name"].AsString.Should().Be("John");
+            doc["age"].AsInt32.Should().Be(30);
+        }
+    }
+}

LiteDB/Client/Mapper/BsonMapper.Deserialize.cs

@@ -9,6 +9,24 @@ namespace LiteDB
 {
     public partial class BsonMapper
     {
+        #region Deserialization Hooks
+
+        /// <summary>
+        /// Delegate for deserialization callback.
+        /// </summary>
+        /// <param name="sender">The BsonMapper instance that triggered the deserialization.</param>
+        /// <param name="target">The target type for deserialization.</param>
+        /// <param name="value">The BsonValue to be deserialized.</param>
+        /// <returns>The deserialized BsonValue.</returns>
+        public delegate BsonValue DeserializationCallback(BsonMapper sender, Type target, BsonValue value);
+
+        /// <summary>
+        /// Gets called before deserialization of a value
+        /// </summary>
+        public DeserializationCallback? OnDeserialization { get; set; }
+
+        #endregion Deserialization Hooks
+
         #region Basic direct .NET convert types
 
         // direct bson types
@@ -78,6 +96,15 @@ public T Deserialize<T>(BsonValue value)
         /// </summary>
         public object Deserialize(Type type, BsonValue value)
         {
+            if (OnDeserialization is not null)
+            {
+                var result = OnDeserialization(this, type, value);
+                if (result is not null)
+                {
+                    value = result;
+                }
+            }
+
             // null value - null returns
             if (value.IsNull) return null;
 

LiteDB/Client/Structures/ConnectionString.cs

@@ -1,4 +1,4 @@
-using LiteDB.Engine;
+using LiteDB.Engine;
 using System;
 using System.Collections.Generic;
 using System.Globalization;
@@ -107,7 +107,7 @@ public ConnectionString(string connectionString)
         /// <summary>
         /// Create ILiteEngine instance according string connection parameters. For now, only Local/Shared are supported
         /// </summary>
-        internal ILiteEngine CreateEngine()
+        internal ILiteEngine CreateEngine(Action<EngineSettings> engineSettingsAction = null)
         {
             var settings = new EngineSettings
             {
@@ -120,6 +120,8 @@ internal ILiteEngine CreateEngine()
                 AutoRebuild = this.AutoRebuild,
             };
 
+            engineSettingsAction?.Invoke(settings);
+
             // create engine implementation as Connection Type
             if (this.Connection == ConnectionType.Direct)
             {

LiteDB/Document/DataReader/BsonDataReader.cs

@@ -1,4 +1,4 @@
-using LiteDB.Engine;
+using LiteDB.Engine;
 using System;
 using System.Collections;
 using System.Collections.Generic;
@@ -56,10 +56,10 @@ internal BsonDataReader(IEnumerable<BsonValue> values, string collection, Engine
                 if (_source.MoveNext())
                 {
                     _hasValues = _isFirst = true;
-                    _current = _source.Current;
+                    _current = _state.ReadTransform(_collection, _source.Current);
                 }
             }
-            catch(Exception ex)
+            catch (Exception ex)
             {
                 _state.Handle(ex);
                 throw;
@@ -102,10 +102,10 @@ public bool Read()
                     try
                     {
                         var read = _source.MoveNext(); // can throw any error here
-                        _current = _source.Current;
+                        _current = _state.ReadTransform(_collection, _source.Current);
                         return read;
                     }
-                    catch(Exception ex)
+                    catch (Exception ex)
                     {
                         _state.Handle(ex);
                         throw ex;
@@ -117,7 +117,7 @@ public bool Read()
                 }
             }
         }
-        
+
         public BsonValue this[string field]
         {
             get

LiteDB/Engine/EngineSettings.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.ComponentModel;
@@ -7,6 +7,7 @@
 using System.Reflection;
 using System.Text;
 using System.Text.RegularExpressions;
+
 using static LiteDB.Constants;
 
 namespace LiteDB.Engine
@@ -66,6 +67,11 @@ public class EngineSettings
         /// </summary>
         public bool Upgrade { get; set; } = false;
 
+        /// <summary>
+        /// Is used to transform a <see cref="BsonValue"/> from the database on read. This can be used to upgrade data from older versions.
+        /// </summary>
+        public Func<string, BsonValue, BsonValue> ReadTransform { get; set; }
+
         /// <summary>
         /// Create new IStreamFactory for datafile
         /// </summary>

LiteDB/Engine/EngineState.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
@@ -9,7 +9,6 @@
 
 using static LiteDB.Constants;
 
-
 namespace LiteDB.Engine
 {
     internal class EngineState
@@ -25,7 +24,7 @@ internal class EngineState
 #endif
 
         public EngineState(LiteEngine engine, EngineSettings settings)
-        { 
+        {
             _engine = engine;
             _settings = settings;
         }
@@ -39,7 +38,7 @@ public bool Handle(Exception ex)
         {
             LOG(ex.Message, "ERROR");
 
-            if (ex is IOException || 
+            if (ex is IOException ||
                 (ex is LiteException lex && lex.ErrorCode == LiteException.INVALID_DATAFILE_STATE))
             {
                 _exception = ex;
@@ -51,5 +50,12 @@ public bool Handle(Exception ex)
 
             return true;
         }
+
+        public BsonValue ReadTransform(string collection, BsonValue value)
+        {
+            if (_settings?.ReadTransform is null) return value;
+
+            return _settings.ReadTransform(collection, value);
+        }
     }
-}
+}

LiteDB/LiteDB.csproj

@@ -28,7 +28,7 @@
     <SignAssembly Condition="'$(OS)'=='Windows_NT'">true</SignAssembly>
     <AssemblyOriginatorKeyFile Condition="'$(Configuration)' == 'Release'">LiteDB.snk</AssemblyOriginatorKeyFile>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <LangVersion>8.0</LangVersion>
+    <LangVersion>latest</LangVersion>
   </PropertyGroup>
 
   <!--