From 4d6926d3270a1496edefac35f7dbb03e9a8680a1 Mon Sep 17 00:00:00 2001 From: Magne Helleborg Date: Fri, 10 Nov 2023 12:34:03 +0100 Subject: [PATCH] Improved structure of version migrations, and added a migration for V6 eventsource IDs (which were guids). This eliminates the need for the V6 compatibility mode --- Runtime.sln | 15 ++ Source/Bootstrap/BoostrapProcedures.cs | 4 + Source/Bootstrap/Bootstrap.csproj | 1 + .../Bootstrap/ICanPerformBoostrapProcedure.cs | 6 +- .../OpenTelemetryConfigurationExtensions.cs | 1 + .../OpenTelemetry/Tracing/RuntimeActivity.cs | 10 ++ .../Diagnostics/OpenTelemetry/Tracing/Tags.cs | 9 + .../Events.Store.MongoDB/DatabaseMigrator.cs | 75 -------- .../Events.Store.MongoDB.csproj | 1 + .../Legacy/BackwardsCompatibility.cs | 4 +- .../EventSourceAndPartitionSerializer.cs | 23 +-- ...toreBackwardsCompatibilityConfiguration.cs | 14 -- .../EventStoreBackwardsCompatibleVersion.cs | 30 ---- .../Migrations/DatabaseMetadata.cs | 22 +++ .../Migrations/DatabaseMigrator.cs | 28 +++ .../Migrations/Extensions.cs | 9 + .../Migrations/MetadataRepository.cs | 35 ++++ .../Migrations/StreamIdMatcher.cs | 38 ++++ .../Migrations/V6EventSourceMigrator.cs | 169 ++++++++++++++++++ .../Migrations/V9Migrations.cs | 124 +++++++++++++ .../Events.Store.MongoDB/StreamIdMatcher.cs | 17 -- .../Events.Store.MongoDB/Streams/Streams.cs | 26 --- Source/Server/Program.cs | 14 -- .../Events.Store.MongoDB.Tests.csproj | 13 ++ .../StreamIdMatcherTests.cs | 48 +++++ 25 files changed, 535 insertions(+), 201 deletions(-) create mode 100644 Source/Diagnostics/OpenTelemetry/Tracing/RuntimeActivity.cs create mode 100644 Source/Diagnostics/OpenTelemetry/Tracing/Tags.cs delete mode 100644 Source/Events.Store.MongoDB/DatabaseMigrator.cs delete mode 100644 Source/Events.Store.MongoDB/Legacy/EventStoreBackwardsCompatibilityConfiguration.cs delete mode 100644 Source/Events.Store.MongoDB/Legacy/EventStoreBackwardsCompatibleVersion.cs create mode 100644 Source/Events.Store.MongoDB/Migrations/DatabaseMetadata.cs create mode 100644 Source/Events.Store.MongoDB/Migrations/DatabaseMigrator.cs create mode 100644 Source/Events.Store.MongoDB/Migrations/Extensions.cs create mode 100644 Source/Events.Store.MongoDB/Migrations/MetadataRepository.cs create mode 100644 Source/Events.Store.MongoDB/Migrations/StreamIdMatcher.cs create mode 100644 Source/Events.Store.MongoDB/Migrations/V6EventSourceMigrator.cs create mode 100644 Source/Events.Store.MongoDB/Migrations/V9Migrations.cs delete mode 100644 Source/Events.Store.MongoDB/StreamIdMatcher.cs create mode 100644 Specifications/Events.Store.MongoDB.Tests/Events.Store.MongoDB.Tests.csproj create mode 100644 Specifications/Events.Store.MongoDB.Tests/StreamIdMatcherTests.cs diff --git a/Runtime.sln b/Runtime.sln index f16dc5773..dc7320f13 100644 --- a/Runtime.sln +++ b/Runtime.sln @@ -168,6 +168,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Events.Store.MongoDB", "Int EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Events.Processing.Tests", "Specifications\Events.Processing.Tests\Events.Processing.Tests.csproj", "{F83E289C-91DC-42B9-BD8C-ED15E0D044E0}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Events.Store.MongoDB.Tests", "Specifications\Events.Store.MongoDB.Tests\Events.Store.MongoDB.Tests.csproj", "{044D8D0D-BD6D-4B13-AEA1-03C851497C99}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -901,6 +903,18 @@ Global {F83E289C-91DC-42B9-BD8C-ED15E0D044E0}.Release|x64.Build.0 = Release|Any CPU {F83E289C-91DC-42B9-BD8C-ED15E0D044E0}.Release|x86.ActiveCfg = Release|Any CPU {F83E289C-91DC-42B9-BD8C-ED15E0D044E0}.Release|x86.Build.0 = Release|Any CPU + {044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Debug|Any CPU.Build.0 = Debug|Any CPU + {044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Debug|x64.ActiveCfg = Debug|Any CPU + {044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Debug|x64.Build.0 = Debug|Any CPU + {044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Debug|x86.ActiveCfg = Debug|Any CPU + {044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Debug|x86.Build.0 = Debug|Any CPU + {044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Release|Any CPU.ActiveCfg = Release|Any CPU + {044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Release|Any CPU.Build.0 = Release|Any CPU + {044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Release|x64.ActiveCfg = Release|Any CPU + {044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Release|x64.Build.0 = Release|Any CPU + {044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Release|x86.ActiveCfg = Release|Any CPU + {044D8D0D-BD6D-4B13-AEA1-03C851497C99}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {79AB0F5F-4DA5-40C5-9A0B-BE392F16B7C0} = {562DC4C5-91AB-4280-B40C-A2E0C43AA21C} @@ -967,5 +981,6 @@ Global {AC1D9095-A820-4B4B-BD98-A88E08790C16} = {C4446054-773D-4B63-A88B-3D35B71F6F66} {02CE4263-12B0-4DDC-AB20-452D2257D4D5} = {37343684-7BA7-40C1-9904-B8C434B8BD19} {F83E289C-91DC-42B9-BD8C-ED15E0D044E0} = {95D3ED76-BB5F-442F-A99A-F48FF34FEDA6} + {044D8D0D-BD6D-4B13-AEA1-03C851497C99} = {95D3ED76-BB5F-442F-A99A-F48FF34FEDA6} EndGlobalSection EndGlobal diff --git a/Source/Bootstrap/BoostrapProcedures.cs b/Source/Bootstrap/BoostrapProcedures.cs index fda6a9139..106cf7b29 100644 --- a/Source/Bootstrap/BoostrapProcedures.cs +++ b/Source/Bootstrap/BoostrapProcedures.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Dolittle.Runtime.Bootstrap; using Dolittle.Runtime.DependencyInversion.Lifecycle; +using Dolittle.Runtime.Diagnostics.OpenTelemetry; using Dolittle.Runtime.Domain.Tenancy; using Dolittle.Runtime.Tenancy; @@ -30,6 +31,9 @@ public BoostrapProcedures(IEnumerable procedures, /// public async Task PerformAll() { + // ReSharper disable once ExplicitCallerInfoArgument + using var activity = RuntimeActivity.Source.StartActivity("PerformBootstrapProcedures"); + if (_performedBootstrap) { throw new BootstrapProceduresAlreadyPerformed(); diff --git a/Source/Bootstrap/Bootstrap.csproj b/Source/Bootstrap/Bootstrap.csproj index 9cb639675..89b6b2982 100644 --- a/Source/Bootstrap/Bootstrap.csproj +++ b/Source/Bootstrap/Bootstrap.csproj @@ -9,6 +9,7 @@ + diff --git a/Source/Bootstrap/ICanPerformBoostrapProcedure.cs b/Source/Bootstrap/ICanPerformBoostrapProcedure.cs index e33510b20..46031911c 100644 --- a/Source/Bootstrap/ICanPerformBoostrapProcedure.cs +++ b/Source/Bootstrap/ICanPerformBoostrapProcedure.cs @@ -15,13 +15,15 @@ public interface ICanPerformBoostrapProcedure /// Performs a bootstrap procedure. /// /// The representing the asynchronous action. - Task Perform(); + Task Perform() => Task.CompletedTask; /// /// Performs a bootstrap procedure for a specific tenant. /// /// The representing the asynchronous action. - Task PerformForTenant(TenantId tenant); + Task PerformForTenant(TenantId tenant) => Task.CompletedTask; int Priority => 0; + + } diff --git a/Source/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensions.cs b/Source/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensions.cs index b0ae8293c..d3f8d343f 100644 --- a/Source/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensions.cs +++ b/Source/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensions.cs @@ -77,6 +77,7 @@ static void AddOpenTelemetryTracing(this IHostBuilder builder, ResourceBuilder r .WithTracing(_ => { _.SetResourceBuilder(resourceBuilder) + .AddSource(RuntimeActivity.SourceName) .AddHttpClientInstrumentation() .AddAspNetCoreInstrumentation() .AddMongoDBInstrumentation() diff --git a/Source/Diagnostics/OpenTelemetry/Tracing/RuntimeActivity.cs b/Source/Diagnostics/OpenTelemetry/Tracing/RuntimeActivity.cs new file mode 100644 index 000000000..eee512fd9 --- /dev/null +++ b/Source/Diagnostics/OpenTelemetry/Tracing/RuntimeActivity.cs @@ -0,0 +1,10 @@ +// Copyright (c) Dolittle. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Dolittle.Runtime.Diagnostics.OpenTelemetry; + +public static class RuntimeActivity +{ + public const string SourceName = "Dolittle.Runtime"; + public static readonly System.Diagnostics.ActivitySource Source = new(SourceName); +} diff --git a/Source/Diagnostics/OpenTelemetry/Tracing/Tags.cs b/Source/Diagnostics/OpenTelemetry/Tracing/Tags.cs new file mode 100644 index 000000000..66c85898d --- /dev/null +++ b/Source/Diagnostics/OpenTelemetry/Tracing/Tags.cs @@ -0,0 +1,9 @@ +// Copyright (c) Dolittle. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Dolittle.Runtime.Diagnostics.OpenTelemetry; + +public static class Tags +{ + public const string TenantId = "tenant_id"; +} diff --git a/Source/Events.Store.MongoDB/DatabaseMigrator.cs b/Source/Events.Store.MongoDB/DatabaseMigrator.cs deleted file mode 100644 index c0382ab65..000000000 --- a/Source/Events.Store.MongoDB/DatabaseMigrator.cs +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) Dolittle. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Dolittle.Runtime.Domain.Tenancy; -using Dolittle.Runtime.Server.Bootstrap; -using MongoDB.Bson; -using MongoDB.Driver; - -namespace Dolittle.Runtime.Events.Store.MongoDB; - -public class DatabaseMigrator : ICanPerformBoostrapProcedure -{ - readonly Func _getDatabase; - - public DatabaseMigrator(Func getDatabase) - { - _getDatabase = getDatabase; - } - - public Task Perform() => Task.CompletedTask; - - public async Task PerformForTenant(TenantId tenant) - { - var db = _getDatabase(tenant).Database; - var streams = await GetStreams(db); - await CleanupDefaultMetadata(db, streams); - } - - async Task CleanupDefaultMetadata(IMongoDatabase db, IList streams) - { - foreach (var stream in streams) - { - var collection = db.GetCollection(stream); - await RemoveEventHorizonDefaultMetadata(collection); - await RemoveAggregateDefaultDefaultMetadata(collection); - } - } - - static async Task> GetStreams(IMongoDatabase db) - { - var collections = await (await db.ListCollectionNamesAsync()).ToListAsync(); - var streams = collections.Where(StreamIdMatcher.IsMatch).ToList(); - return streams; - } - - /// - /// Removes the event log default values for the EventHorizon metadata property for events not sent via the Event Horizon. - /// - /// - Task RemoveEventHorizonDefaultMetadata(IMongoCollection collection) - { - var filter = Builders.Filter.Eq("EventHorizon.FromEventHorizon", false); - var update = Builders.Update.Unset("EventHorizon"); - - return collection.UpdateManyAsync(filter, update); - } - - /// - /// Removes the event log default values for the Aggregate metadata property for events not applied by an Aggregate. - /// - /// - Task RemoveAggregateDefaultDefaultMetadata(IMongoCollection collection) - { - var filter = Builders.Filter.Eq("Aggregate.WasAppliedByAggregate", false); - var update = Builders.Update.Unset("Aggregate"); - - return collection.UpdateManyAsync(filter, update); - } - - public int Priority => 1000; -} diff --git a/Source/Events.Store.MongoDB/Events.Store.MongoDB.csproj b/Source/Events.Store.MongoDB/Events.Store.MongoDB.csproj index a35534947..d81fc6668 100644 --- a/Source/Events.Store.MongoDB/Events.Store.MongoDB.csproj +++ b/Source/Events.Store.MongoDB/Events.Store.MongoDB.csproj @@ -17,5 +17,6 @@ + diff --git a/Source/Events.Store.MongoDB/Legacy/BackwardsCompatibility.cs b/Source/Events.Store.MongoDB/Legacy/BackwardsCompatibility.cs index f3fd80b39..df5572b67 100644 --- a/Source/Events.Store.MongoDB/Legacy/BackwardsCompatibility.cs +++ b/Source/Events.Store.MongoDB/Legacy/BackwardsCompatibility.cs @@ -22,9 +22,9 @@ public class BackwardsCompatibility : IConfigureBackwardsCompatibility /// Initializes a new instance of the class. /// /// The for . - public BackwardsCompatibility(IOptions configuration) + public BackwardsCompatibility() { - var serializer = new EventSourceAndPartitionSerializer(configuration.Value.Version); + var serializer = new EventSourceAndPartitionSerializer(); _eventSourceAndPartitionConventions = new ConventionPack(); _eventSourceAndPartitionConventions.AddClassMapConvention("EventSource and Partition backwards compatibility", cm => diff --git a/Source/Events.Store.MongoDB/Legacy/EventSourceAndPartitionSerializer.cs b/Source/Events.Store.MongoDB/Legacy/EventSourceAndPartitionSerializer.cs index d6e959535..ca2965df3 100644 --- a/Source/Events.Store.MongoDB/Legacy/EventSourceAndPartitionSerializer.cs +++ b/Source/Events.Store.MongoDB/Legacy/EventSourceAndPartitionSerializer.cs @@ -1,7 +1,6 @@ // Copyright (c) Dolittle. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System; using MongoDB.Bson; using MongoDB.Bson.Serialization; using MongoDB.Bson.Serialization.Serializers; @@ -16,16 +15,13 @@ namespace Dolittle.Runtime.Events.Store.MongoDB.Legacy; /// public class EventSourceAndPartitionSerializer : SerializerBase { - readonly EventStoreBackwardsCompatibleVersion _backwardsCompatibleVersion; /// /// Initializes a new instance of the class. /// /// The version to be backwards compatible with. - public EventSourceAndPartitionSerializer(EventStoreBackwardsCompatibleVersion backwardsCompatibleVersion) + public EventSourceAndPartitionSerializer() { - ThrowIfBackwardsCompatibleVersionNotSet(backwardsCompatibleVersion); - _backwardsCompatibleVersion = backwardsCompatibleVersion; } /// @@ -43,21 +39,6 @@ public override string Deserialize(BsonDeserializationContext context, BsonDeser /// public override void Serialize(BsonSerializationContext context, BsonSerializationArgs args, string value) { - if (_backwardsCompatibleVersion == EventStoreBackwardsCompatibleVersion.V6 && Guid.TryParse(value, out var guid)) - { - context.Writer.WriteBinaryData(new BsonBinaryData(guid, GuidRepresentation.Standard)); - } - else - { - context.Writer.WriteString(value); - } - } - - static void ThrowIfBackwardsCompatibleVersionNotSet(EventStoreBackwardsCompatibleVersion backwardsCompatibleVersion) - { - if (backwardsCompatibleVersion == EventStoreBackwardsCompatibleVersion.NotSet) - { - throw new EventSourceBackwardsCompatibilityMustBeConfigured(); - } + context.Writer.WriteString(value); } } diff --git a/Source/Events.Store.MongoDB/Legacy/EventStoreBackwardsCompatibilityConfiguration.cs b/Source/Events.Store.MongoDB/Legacy/EventStoreBackwardsCompatibilityConfiguration.cs deleted file mode 100644 index 0460793f9..000000000 --- a/Source/Events.Store.MongoDB/Legacy/EventStoreBackwardsCompatibilityConfiguration.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) Dolittle. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using Dolittle.Runtime.Configuration; - -namespace Dolittle.Runtime.Events.Store.MongoDB.Legacy; - -/// -/// Represents the configuration of Event Store backwards compatibility. -/// -/// The previous version of the Runtime to be backwards compatible with. -[Configuration("eventStore:backwardsCompatibility")] -public record EventStoreBackwardsCompatibilityConfiguration( - EventStoreBackwardsCompatibleVersion Version = EventStoreBackwardsCompatibleVersion.NotSet); diff --git a/Source/Events.Store.MongoDB/Legacy/EventStoreBackwardsCompatibleVersion.cs b/Source/Events.Store.MongoDB/Legacy/EventStoreBackwardsCompatibleVersion.cs deleted file mode 100644 index 55dc17a1f..000000000 --- a/Source/Events.Store.MongoDB/Legacy/EventStoreBackwardsCompatibleVersion.cs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Dolittle. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -namespace Dolittle.Runtime.Events.Store.MongoDB.Legacy; - -/// -/// Represents the version of the Event Store schema to be compatible with. -/// -public enum EventStoreBackwardsCompatibleVersion -{ - /// - /// The default configured value, where compatibility has not been selected. - /// - /// - /// If this value is used, all use of the Event Store will crash at Runtime - as we cannot decide which version to be backwards compatible with dynamically. - /// - NotSet, - - /// - /// Configures the Event Store persistence to be backwards compatible with the v6 Runtime as far as possible. - /// Meaning that EventSourceIds and PartitionIds will be persisted as Guids when the string is convertible to a Guid. - /// - V6, - - /// - /// Configures the Event Store persistence to be backwards compatible with the v7 Runtime. - /// EventSourceIds and PartitionIds will always be persisted as strings. - /// - V7, -} diff --git a/Source/Events.Store.MongoDB/Migrations/DatabaseMetadata.cs b/Source/Events.Store.MongoDB/Migrations/DatabaseMetadata.cs new file mode 100644 index 000000000..ac10e73c1 --- /dev/null +++ b/Source/Events.Store.MongoDB/Migrations/DatabaseMetadata.cs @@ -0,0 +1,22 @@ +// Copyright (c) Dolittle. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using MongoDB.Bson.Serialization.Attributes; + +namespace Dolittle.Runtime.Events.Store.MongoDB.Migrations; + +public class DatabaseMetadata +{ + [BsonId] public string Id { get; set; } = "database"; + public List Migrations { get; set; } = new(); + public required string CurrentVersion { get; set; } + public required DateTimeOffset UpdatedAt { get; set; } + + public class Migration + { + public required string Version { get; set; } + public required DateTimeOffset Timestamp { get; set; } + } +} diff --git a/Source/Events.Store.MongoDB/Migrations/DatabaseMigrator.cs b/Source/Events.Store.MongoDB/Migrations/DatabaseMigrator.cs new file mode 100644 index 000000000..5f1b588ba --- /dev/null +++ b/Source/Events.Store.MongoDB/Migrations/DatabaseMigrator.cs @@ -0,0 +1,28 @@ +// Copyright (c) Dolittle. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Threading.Tasks; +using Dolittle.Runtime.Domain.Tenancy; +using Dolittle.Runtime.Events.Store.MongoDB.Migrations; +using Dolittle.Runtime.Server.Bootstrap; + +namespace Dolittle.Runtime.Events.Store.MongoDB; + +public class DatabaseMigrator : ICanPerformBoostrapProcedure +{ + readonly Func _getMigrator; + + public DatabaseMigrator(Func getMigrator) + { + _getMigrator = getMigrator; + } + + public async Task PerformForTenant(TenantId tenant) + { + await _getMigrator(tenant).MigrateTenant(); + } + + + public int Priority => 1000; +} diff --git a/Source/Events.Store.MongoDB/Migrations/Extensions.cs b/Source/Events.Store.MongoDB/Migrations/Extensions.cs new file mode 100644 index 000000000..6808e6205 --- /dev/null +++ b/Source/Events.Store.MongoDB/Migrations/Extensions.cs @@ -0,0 +1,9 @@ +// Copyright (c) Dolittle. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Dolittle.Runtime.Events.Store.MongoDB.Migrations; + +static class Extensions +{ + +} diff --git a/Source/Events.Store.MongoDB/Migrations/MetadataRepository.cs b/Source/Events.Store.MongoDB/Migrations/MetadataRepository.cs new file mode 100644 index 000000000..5c10a9770 --- /dev/null +++ b/Source/Events.Store.MongoDB/Migrations/MetadataRepository.cs @@ -0,0 +1,35 @@ +// Copyright (c) Dolittle. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Threading.Tasks; +using Dolittle.Runtime.DependencyInversion.Scoping; +using MongoDB.Driver; + +namespace Dolittle.Runtime.Events.Store.MongoDB.Migrations; + +public interface IManageDatabaseMetadata +{ + public Task Get(); + public Task Set(DatabaseMetadata metadata); +} + +[PerTenant] +public class MetadataRepository : IManageDatabaseMetadata +{ + const string CollectionName = "metadata"; + const string MetadataId = "database"; + + + IMongoCollection _collection; + + public MetadataRepository(IDatabaseConnection db) + { + _collection = db.Database.GetCollection(CollectionName); + } + + public async Task Get() => await _collection.Find(Builders.Filter.Eq(it => it.Id, MetadataId)) + .FirstOrDefaultAsync(); + + public Task Set(DatabaseMetadata metadata) => _collection.ReplaceOneAsync(Builders.Filter.Eq(it => it.Id, MetadataId), metadata, + new ReplaceOptions { IsUpsert = true }); +} diff --git a/Source/Events.Store.MongoDB/Migrations/StreamIdMatcher.cs b/Source/Events.Store.MongoDB/Migrations/StreamIdMatcher.cs new file mode 100644 index 000000000..ba7f0047a --- /dev/null +++ b/Source/Events.Store.MongoDB/Migrations/StreamIdMatcher.cs @@ -0,0 +1,38 @@ +// Copyright (c) Dolittle. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Text.RegularExpressions; + +namespace Dolittle.Runtime.Events.Store.MongoDB.Migrations; + +public static partial class StreamIdMatcher +{ + public static bool IsStreamOrEventLog(string input) => input.Equals("event-log") || IsStream(input) || IsScopedEventLog(input) || IsScopedStream(input); + + public static bool IsStream(string input) => IsNormalStream(input) || IsScopedStream(input); + + public static bool IsNormalStream(string input) + { + return StreamIdRegex().IsMatch(input); + } + + public static bool IsScopedStream(string input) + { + return ScopedStreamRegex().IsMatch(input); + } + + public static bool IsScopedEventLog(string input) + { + return ScopedEventLogRegex().IsMatch(input); + } + + [GeneratedRegex("^stream-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", RegexOptions.IgnoreCase | RegexOptions.Compiled)] + private static partial Regex StreamIdRegex(); + + [GeneratedRegex("^x-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-stream-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + RegexOptions.IgnoreCase | RegexOptions.Compiled)] + private static partial Regex ScopedStreamRegex(); + + [GeneratedRegex("^x-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-event-log$", RegexOptions.IgnoreCase | RegexOptions.Compiled)] + private static partial Regex ScopedEventLogRegex(); +} diff --git a/Source/Events.Store.MongoDB/Migrations/V6EventSourceMigrator.cs b/Source/Events.Store.MongoDB/Migrations/V6EventSourceMigrator.cs new file mode 100644 index 000000000..1cc5cbe3d --- /dev/null +++ b/Source/Events.Store.MongoDB/Migrations/V6EventSourceMigrator.cs @@ -0,0 +1,169 @@ +// Copyright (c) Dolittle. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Threading.Tasks; +using Dolittle.Runtime.Diagnostics.OpenTelemetry; +using MongoDB.Bson; +using MongoDB.Driver; + +namespace Dolittle.Runtime.Events.Store.MongoDB.Migrations; + +public static class V6EventSourceMigrator +{ + static readonly string[] _migratedFields = { "Metadata.EventSource" }; + + public static async Task MigrateEventsourceId(IMongoCollection collection) + { + using var activity = RuntimeActivity.Source.StartActivity(); + + var eventLogHasUuidFields = await ContainsUuidFieldsAsync(collection, _migratedFields); + if (eventLogHasUuidFields) + { + await ConvertUuidFieldsToStringAsync(collection, _migratedFields); + } + } + + public static async Task ContainsUuidFieldsAsync(IMongoCollection collection, params string[] uuidFieldNames) + { + // Build the filter to check for any UUID fields + var queryFilter = AnyIsBinaryFilter(uuidFieldNames); + + // Check if any document matches the criteria + var isUuidPresent = await collection.Find(queryFilter).AnyAsync(); + + return isUuidPresent; + } + + static FilterDefinition AnyIsBinaryFilter(string[] uuidFieldNames) + { + var filters = new List>(); + foreach (var fieldName in uuidFieldNames) + { + // Check for UUID type + var binaryTypeFilter = Builders.Filter.Type(fieldName, BsonType.Binary); + + filters.Add(binaryTypeFilter); + } + + // Combine all filters using $or to check if any of the fields are of UUID type + var queryFilter = Builders.Filter.Or(filters); + return queryFilter; + } + + /// + /// Creates a copy of the collection with the given UUID fields converted to strings + /// + /// Source collection + /// Which fields to convert + public static async Task ConvertUuidFieldsToStringAsync(IMongoCollection collection, params string[] uuidFieldNames) + { + var collectionName = collection.CollectionNamespace.CollectionName; + var tempCollectionName = $"{collectionName}_temp"; + // Create a temporary collection + var tempCollection = collection.Database.GetCollection(tempCollectionName); + // Backup the original collection to the temporary collection + + + // Create a projection to include only the fields that need to be converted + var projection = Builders.Projection.Include("_id"); + foreach (var fieldName in uuidFieldNames) + { + projection = projection.Include(fieldName); + } + + // Find all documents that contain the specified fields + using var cursor = await collection.Find(AnyIsBinaryFilter(uuidFieldNames)).Project(projection).ToCursorAsync(); + var bulkOps = new List>(); + while (await cursor.MoveNextAsync()) + { + foreach (var doc in cursor.Current) + { + var updateDoc = new BsonDocument(); + foreach (var fieldName in uuidFieldNames) + { + if (!TryGetValue(doc,fieldName, out var element)) + { + Console.WriteLine($"Field {fieldName} not found for id {doc["_id"]}"); + } + else if (element!.IsBsonBinaryData) + { + // Console.WriteLine($"Converting {fieldName} to string for id {doc["_id"]}"); + var binaryData = element.AsBsonBinaryData; + if (binaryData.SubType is BsonBinarySubType.UuidLegacy or BsonBinarySubType.UuidStandard) + { + var guid = binaryData.ToGuid(); + updateDoc.Add(fieldName, guid.ToString()); + } + } + else + { + Console.WriteLine($"Field {fieldName} is not a UUID for id {doc["_id"]}"); + } + } + + if (updateDoc.ElementCount > 0) + { + // Create the updated document with UUID fields converted to strings + var filter = Builders.Filter.Eq("_id", doc["_id"]); + var update = Builders.Update.Combine(updateDoc.Select(field => Builders.Update.Set(field.Name, field.Value))); + bulkOps.Add(new UpdateOneModel(filter, update)); + } + + // Execute in batches of 1000 for efficiency + if (bulkOps.Count >= 1000) + { + await collection.BulkWriteAsync(bulkOps); + bulkOps.Clear(); + } + } + } + + // Write any remaining operations + if (bulkOps.Count > 0) + { + await collection.BulkWriteAsync(bulkOps); + } + } + + + static bool TryGetValue(BsonDocument doc, string path, [NotNullWhen(true)] out BsonValue? value) + { + // Initialize the output value + value = default; + + // Split the path into parts for nested documents + var parts = path.Split('.'); + var currentDoc = doc; + + for (var i = 0; i < parts.Length; i++) + { + if (currentDoc.TryGetValue(parts[i], out var bsonValue)) + { + if (i == parts.Length - 1) + { + value = bsonValue; + return true; + } + else if (bsonValue.IsBsonDocument) + { + // Move deeper into the nested documents + currentDoc = bsonValue.AsBsonDocument; + } + else + { + return false; // The path is not valid + } + } + else + { + return false; // The key does not exist + } + } + + return false; // The path was not valid + } +} diff --git a/Source/Events.Store.MongoDB/Migrations/V9Migrations.cs b/Source/Events.Store.MongoDB/Migrations/V9Migrations.cs new file mode 100644 index 000000000..d63dc02be --- /dev/null +++ b/Source/Events.Store.MongoDB/Migrations/V9Migrations.cs @@ -0,0 +1,124 @@ +// Copyright (c) Dolittle. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using Dolittle.Runtime.DependencyInversion.Scoping; +using Dolittle.Runtime.Domain.Tenancy; +using Microsoft.Extensions.Logging; +using MongoDB.Bson; +using MongoDB.Driver; + +namespace Dolittle.Runtime.Events.Store.MongoDB.Migrations; + +public interface IDbMigration +{ + public Task MigrateTenant(); +} + +[PerTenant] +public class V9Migrations : IDbMigration +{ + readonly IManageDatabaseMetadata _metadataManager; + readonly IMongoDatabase _db; + readonly ILogger _logger; + readonly TenantId _tenantId; + + const string MigrationVersion = "V9.0.0"; + + + public V9Migrations(IManageDatabaseMetadata metadataManager, IDatabaseConnection db, ILogger logger, TenantId tenantId) + { + _metadataManager = metadataManager; + _db = db.Database; + _logger = logger; + _tenantId = tenantId; + } + + public async Task MigrateTenant() + { + var metadata = await _metadataManager.Get(); + + if (metadata?.Migrations.Any(it => it.Version.Equals(MigrationVersion)) == true) return; // Already migrated + + _logger.LogInformation("Migrating tenant {TenantId} to {Version}", _tenantId.Value, MigrationVersion); + + var before = Stopwatch.GetTimestamp(); + + await MigrateEventCollections(await GetEventCollections(_db)); + + if (metadata == null) + { + metadata = new DatabaseMetadata + { + CurrentVersion = "V9.0.0", + UpdatedAt = DateTimeOffset.UtcNow + }; + } + + metadata.Migrations.Add(new DatabaseMetadata.Migration() + { + Version = MigrationVersion, + Timestamp = DateTimeOffset.UtcNow + }); + metadata.CurrentVersion = MigrationVersion; + metadata.UpdatedAt = DateTimeOffset.UtcNow; + + await _metadataManager.Set(metadata); + + _logger.LogInformation("Completed migration of tenant {TenantId} to {Version} in {Elapsed}", _tenantId.Value, MigrationVersion, + Stopwatch.GetElapsedTime(before)); + } + + async Task MigrateEventCollection(string collectionName) + { + var eventCollection = _db.GetCollection(collectionName); + + await RemoveEventHorizonDefaultMetadata(eventCollection); + await RemoveAggregateDefaultDefaultMetadata(eventCollection); + await V6EventSourceMigrator.MigrateEventsourceId(eventCollection); + } + + async Task MigrateEventCollections(IList streams) + { + foreach (var collectionNames in streams) + { + _logger.LogInformation("Migrating {CollectionName}", collectionNames); + await MigrateEventCollection(collectionNames); + } + } + + static async Task> GetEventCollections(IMongoDatabase db) + { + var collections = await (await db.ListCollectionNamesAsync()).ToListAsync(); + var streams = collections.Where(StreamIdMatcher.IsStreamOrEventLog).ToList(); + return streams; + } + + /// + /// Removes the event log default values for the EventHorizon metadata property for events not sent via the Event Horizon. + /// + /// + Task RemoveEventHorizonDefaultMetadata(IMongoCollection collection) + { + var filter = Builders.Filter.Eq("EventHorizon.FromEventHorizon", false); + var update = Builders.Update.Unset("EventHorizon"); + + return collection.UpdateManyAsync(filter, update); + } + + /// + /// Removes the event log default values for the Aggregate metadata property for events not applied by an Aggregate. + /// + /// + Task RemoveAggregateDefaultDefaultMetadata(IMongoCollection collection) + { + var filter = Builders.Filter.Eq("Aggregate.WasAppliedByAggregate", false); + var update = Builders.Update.Unset("Aggregate"); + + return collection.UpdateManyAsync(filter, update); + } +} diff --git a/Source/Events.Store.MongoDB/StreamIdMatcher.cs b/Source/Events.Store.MongoDB/StreamIdMatcher.cs deleted file mode 100644 index 6bde7771b..000000000 --- a/Source/Events.Store.MongoDB/StreamIdMatcher.cs +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) Dolittle. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System.Text.RegularExpressions; - -namespace Dolittle.Runtime.Events.Store.MongoDB; - -public static partial class StreamIdMatcher -{ - public static bool IsMatch(string input) - { - return StreamIdRegex().IsMatch(input); - } - - [GeneratedRegex("^stream-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", RegexOptions.IgnoreCase | RegexOptions.Compiled)] - private static partial Regex StreamIdRegex(); -} diff --git a/Source/Events.Store.MongoDB/Streams/Streams.cs b/Source/Events.Store.MongoDB/Streams/Streams.cs index 2e781652e..085d455f8 100644 --- a/Source/Events.Store.MongoDB/Streams/Streams.cs +++ b/Source/Events.Store.MongoDB/Streams/Streams.cs @@ -38,8 +38,6 @@ public Streams(IDatabaseConnection connection, ILogger logger) CreateCollectionsAndIndexesForEventLog(); CreateCollectionsAndIndexesForStreamDefinitions(); - RemoveAggregateDefaultDefaultMetadata(); - RemoveEventHorizonDefaultMetadata(); } /// @@ -239,28 +237,4 @@ await streamDefinitions.Indexes.CreateOneAsync( .Ascending(_ => _.StreamId)), cancellationToken: cancellationToken).ConfigureAwait(false); } - - /// - /// Removes the event log default values for the EventHorizon property for events not sent via the Event Horizon. - /// - /// - void RemoveEventHorizonDefaultMetadata() - { - var filter = Builders.Filter.Eq("EventHorizon.FromEventHorizon", false); - var update = Builders.Update.Unset("EventHorizon"); - - DefaultEventLog.UpdateMany(filter, update); - } - - /// - /// Removes the event log default values for the Aggregate property for events not applied by an Aggregate. - /// - /// - void RemoveAggregateDefaultDefaultMetadata() - { - var filter = Builders.Filter.Eq("Aggregate.WasAppliedByAggregate", false); - var update = Builders.Update.Unset("Aggregate"); - - DefaultEventLog.UpdateMany(filter, update); - } } diff --git a/Source/Server/Program.cs b/Source/Server/Program.cs index 8742ca77b..c290411d8 100644 --- a/Source/Server/Program.cs +++ b/Source/Server/Program.cs @@ -61,18 +61,4 @@ static void VerifyConfiguration(IServiceProvider provider) logger.LogError(e, "It seems like the Runtime is missing its 'tenants' configuration. Without any tenants the Runtime will no function properly."); throw; } - try - { - var config = provider.GetRequiredService>(); - if (config.Value.Version != EventStoreBackwardsCompatibleVersion.V6 && config.Value.Version != EventStoreBackwardsCompatibleVersion.V7) - { - throw new Exception("Event Store Backwards Compatability Version needs to be set to either 'V6' or 'V7'"); - } - } - catch (Exception e) - { - logger.LogCritical(e, @"Cannot start Runtime because it is missing the event store backwards compatability configuration. -Make sure that the dolittle:runtime:eventStore:backwardsCompatibility configuration is provided by setting the 'DOLITTLE__RUNTIME__EVENTSTORE__BACKWARDSCOMPATIBILITY__VERSION' environment variable to either V6 (store PartitionId and EventSourceId as Guid) or V7 (store PartitionId and EventSourceId as string)"); - throw; - } } diff --git a/Specifications/Events.Store.MongoDB.Tests/Events.Store.MongoDB.Tests.csproj b/Specifications/Events.Store.MongoDB.Tests/Events.Store.MongoDB.Tests.csproj new file mode 100644 index 000000000..35dde849f --- /dev/null +++ b/Specifications/Events.Store.MongoDB.Tests/Events.Store.MongoDB.Tests.csproj @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/Specifications/Events.Store.MongoDB.Tests/StreamIdMatcherTests.cs b/Specifications/Events.Store.MongoDB.Tests/StreamIdMatcherTests.cs new file mode 100644 index 000000000..4626d7e14 --- /dev/null +++ b/Specifications/Events.Store.MongoDB.Tests/StreamIdMatcherTests.cs @@ -0,0 +1,48 @@ +using Dolittle.Runtime.Events.Store.MongoDB.Migrations; +using FluentAssertions; +using Xunit; + +namespace Events.Store.MongoDB.Tests; + +public class StreamIdMatcherTests +{ + [Theory] + [InlineData("x-06fd3dcf-a457-4e76-917e-5049ef49bfd3-stream-6a080414-d493-4ce1-a11b-bd60208b9d7a")] + [InlineData("x-16fd3dcf-a457-4e76-917e-5049ef49bfd3-stream-6a080414-d493-4ce1-a11b-bd60208b9d7b")] + + public void VerifyMatchesScopedStream(string input) + { + StreamIdMatcher.IsScopedStream(input).Should().BeTrue(); + } + + [Theory] + [InlineData("x-06fd3dcf-a457-4e76-917e-5049ef49bfd3-event-log")] + [InlineData("x-06fd3dcf-a457-4e76-917e-5049ef49bfd4-event-log")] + + public void VerifyMatchesScopedEventLog(string input) + { + StreamIdMatcher.IsScopedEventLog(input).Should().BeTrue(); + } + + + [Theory] + [InlineData("event-log")] + [InlineData("stream-6a080414-d493-4ce1-a11b-bd60208b9d7a")] + [InlineData("x-06fd3dcf-a457-4e76-917e-5049ef49bfd3-event-log")] + [InlineData("x-16fd3dcf-a457-4e76-917e-5049ef49bfd3-stream-6a080414-d493-4ce1-a11b-bd60208b9d7b")] + public void VerifyMatches(string input) + { + StreamIdMatcher.IsStreamOrEventLog(input).Should().BeTrue(); + } + + [Theory] + [InlineData("stream-processor-states")] + [InlineData("stream-definitions")] + [InlineData("x-06fd3dcf-a457-4e76-917e-5049ef49bfd3-stream-definitions")] + [InlineData("x-06fd3dcf-a457-4e76-917e-5049ef49bfd3-stream-processor-states")] + [InlineData("x-06fd3dcf-a457-4e76-917e-5049ef49bfd3-subscription-states")] + public void DoesNotMatchOtherCollections(string input) + { + StreamIdMatcher.IsStreamOrEventLog(input).Should().BeFalse(); + } +} \ No newline at end of file