From 25b0940984f5246a775fe4f3766a7aacf3842a54 Mon Sep 17 00:00:00 2001 From: JKamsker <11245306+JKamsker@users.noreply.github.com> Date: Fri, 7 Jun 2024 00:30:41 +0200 Subject: [PATCH 1/4] Add support for on-the-fly document upgrades --- .../Database/DocumentUpgrade_Tests.cs | 76 +++++++++++++++++++ LiteDB/Client/Structures/ConnectionString.cs | 6 +- LiteDB/Document/DataReader/BsonDataReader.cs | 12 +-- LiteDB/Engine/EngineSettings.cs | 8 +- LiteDB/Engine/EngineState.cs | 16 ++-- 5 files changed, 104 insertions(+), 14 deletions(-) create mode 100644 LiteDB.Tests/Database/DocumentUpgrade_Tests.cs diff --git a/LiteDB.Tests/Database/DocumentUpgrade_Tests.cs b/LiteDB.Tests/Database/DocumentUpgrade_Tests.cs new file mode 100644 index 000000000..b265326b0 --- /dev/null +++ b/LiteDB.Tests/Database/DocumentUpgrade_Tests.cs @@ -0,0 +1,76 @@ +using FluentAssertions; + +using LiteDB.Engine; + +using System.IO; + +using Xunit; + +namespace LiteDB.Tests.Database; + +public class DocumentUpgrade_Tests +{ + [Fact] + public void DocumentUpgrade_Test() + { + var ms = new MemoryStream(); + using (var db = new LiteDatabase(ms)) + { + var col = db.GetCollection("col"); + + col.Insert(new BsonDocument { ["version"] = 1, ["_id"] = 1, ["name"] = "John" }); + } + + ms.Position = 0; + + using (var db = new LiteDatabase(ms)) + { + var col = db.GetCollection("col"); + + col.Count().Should().Be(1); + + var doc = col.FindById(1); + + doc["version"].AsInt32.Should().Be(1); + doc["name"].AsString.Should().Be("John"); + doc["age"].AsInt32.Should().Be(0); + } + + ms.Position = 0; + + var engine = new LiteEngine(new EngineSettings + { + DataStream = ms, + ReadTransform = ReadTransform + }); + + using (var db = new LiteDatabase(engine)) + { + var col = db.GetCollection("col"); + + col.Count().Should().Be(1); + + var doc = col.FindById(1); + + doc["version"].AsInt32.Should().Be(2); + doc["name"].AsString.Should().Be("John"); + doc["age"].AsInt32.Should().Be(30); + } + } + + private BsonValue ReadTransform(string arg1, BsonValue val) + { + if (!(val is BsonDocument bdoc)) + { + return val; + } + + if (bdoc.TryGetValue("version", out var version) && version.AsInt32 == 1) + { + bdoc["version"] = 2; + bdoc["age"] = 30; + } + + return val; + } +} \ No newline at end of file diff --git a/LiteDB/Client/Structures/ConnectionString.cs b/LiteDB/Client/Structures/ConnectionString.cs index 3f44acd76..945649764 100644 --- a/LiteDB/Client/Structures/ConnectionString.cs +++ b/LiteDB/Client/Structures/ConnectionString.cs @@ -1,4 +1,4 @@ -using LiteDB.Engine; +using LiteDB.Engine; using System; using System.Collections.Generic; using System.Globalization; @@ -107,7 +107,7 @@ public ConnectionString(string connectionString) /// /// Create ILiteEngine instance according string connection parameters. For now, only Local/Shared are supported /// - internal ILiteEngine CreateEngine() + internal ILiteEngine CreateEngine(Action engineSettingsAction = null) { var settings = new EngineSettings { @@ -120,6 +120,8 @@ internal ILiteEngine CreateEngine() AutoRebuild = this.AutoRebuild, }; + engineSettingsAction?.Invoke(settings); + // create engine implementation as Connection Type if (this.Connection == ConnectionType.Direct) { diff --git a/LiteDB/Document/DataReader/BsonDataReader.cs b/LiteDB/Document/DataReader/BsonDataReader.cs index c76125085..33131362a 100644 --- a/LiteDB/Document/DataReader/BsonDataReader.cs +++ b/LiteDB/Document/DataReader/BsonDataReader.cs @@ -1,4 +1,4 @@ -using LiteDB.Engine; +using LiteDB.Engine; using System; using System.Collections; using System.Collections.Generic; @@ -56,10 +56,10 @@ internal BsonDataReader(IEnumerable values, string collection, Engine if (_source.MoveNext()) { _hasValues = _isFirst = true; - _current = _source.Current; + _current = _state.ReadTransform(_collection, _source.Current); } } - catch(Exception ex) + catch (Exception ex) { _state.Handle(ex); throw; @@ -102,10 +102,10 @@ public bool Read() try { var read = _source.MoveNext(); // can throw any error here - _current = _source.Current; + _current = _state.ReadTransform(_collection, _source.Current); return read; } - catch(Exception ex) + catch (Exception ex) { _state.Handle(ex); throw ex; @@ -117,7 +117,7 @@ public bool Read() } } } - + public BsonValue this[string field] { get diff --git a/LiteDB/Engine/EngineSettings.cs b/LiteDB/Engine/EngineSettings.cs index d2bb96372..609dedfad 100644 --- a/LiteDB/Engine/EngineSettings.cs +++ b/LiteDB/Engine/EngineSettings.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.ComponentModel; @@ -7,6 +7,7 @@ using System.Reflection; using System.Text; using System.Text.RegularExpressions; + using static LiteDB.Constants; namespace LiteDB.Engine @@ -66,6 +67,11 @@ public class EngineSettings /// public bool Upgrade { get; set; } = false; + /// + /// Is used to transform a from the database on read. This can be used to upgrade data from older versions. + /// + public Func ReadTransform { get; set; } + /// /// Create new IStreamFactory for datafile /// diff --git a/LiteDB/Engine/EngineState.cs b/LiteDB/Engine/EngineState.cs index cfdad8a25..bd3dafac7 100644 --- a/LiteDB/Engine/EngineState.cs +++ b/LiteDB/Engine/EngineState.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -9,7 +9,6 @@ using static LiteDB.Constants; - namespace LiteDB.Engine { internal class EngineState @@ -25,7 +24,7 @@ internal class EngineState #endif public EngineState(LiteEngine engine, EngineSettings settings) - { + { _engine = engine; _settings = settings; } @@ -39,7 +38,7 @@ public bool Handle(Exception ex) { LOG(ex.Message, "ERROR"); - if (ex is IOException || + if (ex is IOException || (ex is LiteException lex && lex.ErrorCode == LiteException.INVALID_DATAFILE_STATE)) { _exception = ex; @@ -51,5 +50,12 @@ public bool Handle(Exception ex) return true; } + + public BsonValue ReadTransform(string collection, BsonValue value) + { + if (_settings?.ReadTransform is null) return value; + + return _settings.ReadTransform(collection, value); + } } -} +} \ No newline at end of file From 02e23d92361a0eb1817f1f11c03a096d5cda5679 Mon Sep 17 00:00:00 2001 From: JKamsker Date: Fri, 7 Jun 2024 00:52:58 +0200 Subject: [PATCH 2/4] Add deserialization hook to bsonmapper --- .../Database/DocumentUpgrade_Tests.cs | 77 ++++++++++++++++--- .../Client/Mapper/BsonMapper.Deserialize.cs | 27 +++++++ LiteDB/LiteDB.csproj | 1 + 3 files changed, 95 insertions(+), 10 deletions(-) diff --git a/LiteDB.Tests/Database/DocumentUpgrade_Tests.cs b/LiteDB.Tests/Database/DocumentUpgrade_Tests.cs index b265326b0..5bf3ad3a2 100644 --- a/LiteDB.Tests/Database/DocumentUpgrade_Tests.cs +++ b/LiteDB.Tests/Database/DocumentUpgrade_Tests.cs @@ -1,4 +1,4 @@ -using FluentAssertions; +using FluentAssertions; using LiteDB.Engine; @@ -38,10 +38,24 @@ public void DocumentUpgrade_Test() ms.Position = 0; - var engine = new LiteEngine(new EngineSettings + using var engine = new LiteEngine(new EngineSettings { DataStream = ms, - ReadTransform = ReadTransform + ReadTransform = (collectionName, val) => + { + if (val is not BsonDocument doc) + { + return val; + } + + if (doc.TryGetValue("version", out var version) && version.AsInt32 == 1) + { + doc["version"] = 2; + doc["age"] = 30; + } + + return val; + } }); using (var db = new LiteDatabase(engine)) @@ -58,19 +72,62 @@ public void DocumentUpgrade_Test() } } - private BsonValue ReadTransform(string arg1, BsonValue val) + [Fact] + public void DocumentUpgrade_BsonMapper_Test() { - if (!(val is BsonDocument bdoc)) + var ms = new MemoryStream(); + using (var db = new LiteDatabase(ms)) { - return val; + var col = db.GetCollection("col"); + + col.Insert(new BsonDocument { ["version"] = 1, ["_id"] = 1, ["name"] = "John" }); } - if (bdoc.TryGetValue("version", out var version) && version.AsInt32 == 1) + ms.Position = 0; + + using (var db = new LiteDatabase(ms)) { - bdoc["version"] = 2; - bdoc["age"] = 30; + var col = db.GetCollection("col"); + + col.Count().Should().Be(1); + + var doc = col.FindById(1); + + doc["version"].AsInt32.Should().Be(1); + doc["name"].AsString.Should().Be("John"); + doc["age"].AsInt32.Should().Be(0); } - return val; + ms.Position = 0; + + var mapper = new BsonMapper(); + mapper.OnDeserialization = (sender, type, val) => + { + if (val is not BsonDocument doc) + { + return val; + } + + if (doc.TryGetValue("version", out var version) && version.AsInt32 == 1) + { + doc["version"] = 2; + doc["age"] = 30; + } + + return doc; + }; + + using (var db = new LiteDatabase(ms, mapper)) + { + var col = db.GetCollection("col"); + + col.Count().Should().Be(1); + + var doc = col.FindById(1); + + doc["version"].AsInt32.Should().Be(2); + doc["name"].AsString.Should().Be("John"); + doc["age"].AsInt32.Should().Be(30); + } } } \ No newline at end of file diff --git a/LiteDB/Client/Mapper/BsonMapper.Deserialize.cs b/LiteDB/Client/Mapper/BsonMapper.Deserialize.cs index 3b24d4b2c..ef7f1b10d 100644 --- a/LiteDB/Client/Mapper/BsonMapper.Deserialize.cs +++ b/LiteDB/Client/Mapper/BsonMapper.Deserialize.cs @@ -9,6 +9,24 @@ namespace LiteDB { public partial class BsonMapper { + #region Deserialization Hooks + + /// + /// Delegate for deserialization callback. + /// + /// The BsonMapper instance that triggered the deserialization. + /// The target type for deserialization. + /// The BsonValue to be deserialized. + /// The deserialized BsonValue. + public delegate BsonValue DeserializationCallback(BsonMapper sender, Type target, BsonValue value); + + /// + /// Gets called before deserialization of a value + /// + public DeserializationCallback? OnDeserialization { get; set; } + + #endregion Deserialization Hooks + #region Basic direct .NET convert types // direct bson types @@ -78,6 +96,15 @@ public T Deserialize(BsonValue value) /// public object Deserialize(Type type, BsonValue value) { + if (OnDeserialization is not null) + { + var result = OnDeserialization(this, type, value); + if (result is not null) + { + value = result; + } + } + // null value - null returns if (value.IsNull) return null; diff --git a/LiteDB/LiteDB.csproj b/LiteDB/LiteDB.csproj index 721395e91..5d26fbd8e 100644 --- a/LiteDB/LiteDB.csproj +++ b/LiteDB/LiteDB.csproj @@ -28,6 +28,7 @@ true LiteDB.snk true + 9.0