Merge pull request #759 from dolittle/migrations

Added support for painless upgrades from V6 & older runtimes.

Runtime.sln

@@ -168,6 +168,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Events.Store.MongoDB", "Int
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Events.Processing.Tests", "Specifications\Events.Processing.Tests\Events.Processing.Tests.csproj", "{F83E289C-91DC-42B9-BD8C-ED15E0D044E0}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Events.Store.MongoDB.Tests", "Specifications\Events.Store.MongoDB.Tests\Events.Store.MongoDB.Tests.csproj", "{044D8D0D-BD6D-4B13-AEA1-03C851497C99}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -901,6 +903,18 @@ Global
 		{F83E289C-91DC-42B9-BD8C-ED15E0D044E0}.Release|x64.Build.0 = Release|Any CPU
 		{F83E289C-91DC-42B9-BD8C-ED15E0D044E0}.Release|x86.ActiveCfg = Release|Any CPU
 		{F83E289C-91DC-42B9-BD8C-ED15E0D044E0}.Release|x86.Build.0 = Release|Any CPU
+		{044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Debug|x64.Build.0 = Debug|Any CPU
+		{044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Debug|x86.Build.0 = Debug|Any CPU
+		{044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Release|Any CPU.Build.0 = Release|Any CPU
+		{044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Release|x64.ActiveCfg = Release|Any CPU
+		{044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Release|x64.Build.0 = Release|Any CPU
+		{044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Release|x86.ActiveCfg = Release|Any CPU
+		{044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{79AB0F5F-4DA5-40C5-9A0B-BE392F16B7C0} = {562DC4C5-91AB-4280-B40C-A2E0C43AA21C}
@@ -967,5 +981,6 @@ Global
 		{AC1D9095-A820-4B4B-BD98-A88E08790C16} = {C4446054-773D-4B63-A88B-3D35B71F6F66}
 		{02CE4263-12B0-4DDC-AB20-452D2257D4D5} = {37343684-7BA7-40C1-9904-B8C434B8BD19}
 		{F83E289C-91DC-42B9-BD8C-ED15E0D044E0} = {95D3ED76-BB5F-442F-A99A-F48FF34FEDA6}
+		{044D8D0D-BD6D-4B13-AEA1-03C851497C99} = {95D3ED76-BB5F-442F-A99A-F48FF34FEDA6}
 	EndGlobalSection
 EndGlobal

Source/Bootstrap/BoostrapProcedures.cs

@@ -6,6 +6,7 @@
 using System.Threading.Tasks;
 using Dolittle.Runtime.Bootstrap;
 using Dolittle.Runtime.DependencyInversion.Lifecycle;
+using Dolittle.Runtime.Diagnostics.OpenTelemetry;
 using Dolittle.Runtime.Domain.Tenancy;
 using Dolittle.Runtime.Tenancy;
 
@@ -30,6 +31,9 @@ public BoostrapProcedures(IEnumerable<ICanPerformBoostrapProcedure> procedures,
     /// <inheritdoc />
     public async Task PerformAll()
     {
+        // ReSharper disable once ExplicitCallerInfoArgument
+        using var activity = RuntimeActivity.Source.StartActivity("PerformBootstrapProcedures");
+
         if (_performedBootstrap)
         {
             throw new BootstrapProceduresAlreadyPerformed();

Source/Bootstrap/Bootstrap.csproj

@@ -9,6 +9,7 @@
     <ItemGroup>
       <ProjectReference Include="../DependencyInversion/DependencyInversion.csproj" />
       <ProjectReference Include="../Tenancy/Tenancy.csproj" />
+      <ProjectReference Include="..\Diagnostics\Diagnostics.csproj" />
     </ItemGroup>
 
 </Project>

Source/Bootstrap/ICanPerformBoostrapProcedure.cs

@@ -15,13 +15,15 @@ public interface ICanPerformBoostrapProcedure
     /// Performs a bootstrap procedure.
     /// </summary>
     /// <returns>The <see cref="Task"/> representing the asynchronous action.</returns>
-    Task Perform();
+    Task Perform() => Task.CompletedTask;
 
     /// <summary>
     /// Performs a bootstrap procedure for a specific tenant. 
     /// </summary>
     /// <returns>The <see cref="Task"/> representing the asynchronous action.</returns>
-    Task PerformForTenant(TenantId tenant);
+    Task PerformForTenant(TenantId tenant) => Task.CompletedTask;
 
     int Priority => 0;
+
+
 }

Source/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensions.cs

@@ -77,6 +77,7 @@ static void AddOpenTelemetryTracing(this IHostBuilder builder, ResourceBuilder r
                 .WithTracing(_ =>
                 {
                     _.SetResourceBuilder(resourceBuilder)
+                        .AddSource(RuntimeActivity.SourceName)
                         .AddHttpClientInstrumentation()
                         .AddAspNetCoreInstrumentation()
                         .AddMongoDBInstrumentation()

Source/Diagnostics/OpenTelemetry/Tracing/RuntimeActivity.cs

@@ -0,0 +1,10 @@
+// Copyright (c) Dolittle. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Dolittle.Runtime.Diagnostics.OpenTelemetry;
+
+public static class RuntimeActivity
+{
+    public const string SourceName = "Dolittle.Runtime";
+    public static readonly System.Diagnostics.ActivitySource Source = new(SourceName);
+}

Source/Diagnostics/OpenTelemetry/Tracing/Tags.cs

@@ -0,0 +1,9 @@
+// Copyright (c) Dolittle. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Dolittle.Runtime.Diagnostics.OpenTelemetry;
+
+public static class Tags
+{
+    public const string TenantId = "tenant_id";
+}

Source/Events.Store.MongoDB/Events.Store.MongoDB.csproj

@@ -17,5 +17,6 @@
         <ProjectReference Include="../EventHorizon/EventHorizon.csproj" />
         <ProjectReference Include="../Events/Events.csproj" />
         <ProjectReference Include="../MongoDB/MongoDB.csproj" />
+        <ProjectReference Include="..\Diagnostics\Diagnostics.csproj" />
     </ItemGroup>
 </Project>

Source/Events.Store.MongoDB/Legacy/BackwardsCompatibility.cs

@@ -22,9 +22,9 @@ public class BackwardsCompatibility : IConfigureBackwardsCompatibility
     /// Initializes a new instance of the <see cref="BackwardsCompatibility"/> class.
     /// </summary>
     /// <param name="configuration">The <see cref="IOptions{TOptions}"/> for <see cref="EventStoreBackwardsCompatibilityConfiguration"/>.</param>
-    public BackwardsCompatibility(IOptions<EventStoreBackwardsCompatibilityConfiguration> configuration)
+    public BackwardsCompatibility()
     {
-        var serializer = new EventSourceAndPartitionSerializer(configuration.Value.Version);
+        var serializer = new EventSourceAndPartitionSerializer();
 
         _eventSourceAndPartitionConventions = new ConventionPack();
         _eventSourceAndPartitionConventions.AddClassMapConvention("EventSource and Partition backwards compatibility", cm =>

Source/Events.Store.MongoDB/Legacy/EventSourceAndPartitionSerializer.cs

@@ -1,7 +1,6 @@
 // Copyright (c) Dolittle. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
-using System;
 using MongoDB.Bson;
 using MongoDB.Bson.Serialization;
 using MongoDB.Bson.Serialization.Serializers;
@@ -16,16 +15,13 @@ namespace Dolittle.Runtime.Events.Store.MongoDB.Legacy;
 /// </remarks>
 public class EventSourceAndPartitionSerializer : SerializerBase<string>
 {
-    readonly EventStoreBackwardsCompatibleVersion _backwardsCompatibleVersion;
 
     /// <summary>
     /// Initializes a new instance of the <see cref="EventSourceAndPartitionSerializer"/> class.
     /// </summary>
     /// <param name="backwardsCompatibleVersion">The version to be backwards compatible with.</param>
-    public EventSourceAndPartitionSerializer(EventStoreBackwardsCompatibleVersion backwardsCompatibleVersion)
+    public EventSourceAndPartitionSerializer()
     {
-        ThrowIfBackwardsCompatibleVersionNotSet(backwardsCompatibleVersion);
-        _backwardsCompatibleVersion = backwardsCompatibleVersion;
     }
 
     /// <inheritdoc />
@@ -43,21 +39,6 @@ public override string Deserialize(BsonDeserializationContext context, BsonDeser
     /// <inheritdoc />
     public override void Serialize(BsonSerializationContext context, BsonSerializationArgs args, string value)
     {
-        if (_backwardsCompatibleVersion == EventStoreBackwardsCompatibleVersion.V6 && Guid.TryParse(value, out var guid))
-        {
-            context.Writer.WriteBinaryData(new BsonBinaryData(guid, GuidRepresentation.Standard));
-        }
-        else
-        {
-            context.Writer.WriteString(value);
-        }
-    }
-
-    static void ThrowIfBackwardsCompatibleVersionNotSet(EventStoreBackwardsCompatibleVersion backwardsCompatibleVersion)
-    {
-        if (backwardsCompatibleVersion == EventStoreBackwardsCompatibleVersion.NotSet)
-        {
-            throw new EventSourceBackwardsCompatibilityMustBeConfigured();
-        }
+        context.Writer.WriteString(value);
     }
 }

Source/Events.Store.MongoDB/Migrations/DatabaseMetadata.cs

@@ -0,0 +1,22 @@
+// Copyright (c) Dolittle. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections.Generic;
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace Dolittle.Runtime.Events.Store.MongoDB.Migrations;
+
+public class DatabaseMetadata
+{
+    [BsonId] public string Id { get; set; } = "database";
+    public List<Migration> Migrations { get; set; } = new();
+    public required string CurrentVersion { get; set; }
+    public required DateTimeOffset UpdatedAt { get; set; }
+
+    public class Migration
+    {
+        public required string Version { get; set; }
+        public required DateTimeOffset Timestamp { get; set; }
+    }
+}

Source/Events.Store.MongoDB/Migrations/DatabaseMigrator.cs

@@ -0,0 +1,28 @@
+// Copyright (c) Dolittle. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Threading.Tasks;
+using Dolittle.Runtime.Domain.Tenancy;
+using Dolittle.Runtime.Events.Store.MongoDB.Migrations;
+using Dolittle.Runtime.Server.Bootstrap;
+
+namespace Dolittle.Runtime.Events.Store.MongoDB;
+
+public class DatabaseMigrator : ICanPerformBoostrapProcedure
+{
+    readonly Func<TenantId, IDbMigration> _getMigrator;
+
+    public DatabaseMigrator(Func<TenantId, IDbMigration> getMigrator)
+    {
+        _getMigrator = getMigrator;
+    }
+
+    public async Task PerformForTenant(TenantId tenant)
+    {
+        await _getMigrator(tenant).MigrateTenant();
+    }
+
+
+    public int Priority => 1000;
+}

Source/Events.Store.MongoDB/Migrations/Extensions.cs

@@ -0,0 +1,9 @@
+// Copyright (c) Dolittle. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Dolittle.Runtime.Events.Store.MongoDB.Migrations;
+
+static class Extensions
+{
+
+}

Source/Events.Store.MongoDB/Migrations/MetadataRepository.cs

@@ -0,0 +1,35 @@
+// Copyright (c) Dolittle. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Threading.Tasks;
+using Dolittle.Runtime.DependencyInversion.Scoping;
+using MongoDB.Driver;
+
+namespace Dolittle.Runtime.Events.Store.MongoDB.Migrations;
+
+public interface IManageDatabaseMetadata
+{
+    public Task<DatabaseMetadata?> Get();
+    public Task Set(DatabaseMetadata metadata);
+}
+
+[PerTenant]
+public class MetadataRepository : IManageDatabaseMetadata
+{
+    const string CollectionName = "metadata";
+    const string MetadataId = "database";
+
+
+    IMongoCollection<DatabaseMetadata> _collection;
+
+    public MetadataRepository(IDatabaseConnection db)
+    {
+        _collection = db.Database.GetCollection<DatabaseMetadata>(CollectionName);
+    }
+
+    public async Task<DatabaseMetadata?> Get() => await _collection.Find(Builders<DatabaseMetadata>.Filter.Eq(it => it.Id, MetadataId))
+        .FirstOrDefaultAsync();
+
+    public Task Set(DatabaseMetadata metadata) => _collection.ReplaceOneAsync(Builders<DatabaseMetadata>.Filter.Eq(it => it.Id, MetadataId), metadata,
+        new ReplaceOptions { IsUpsert = true });
+}