Skip to content

Commit

Permalink
GDPR redaction support (#247)
Browse files Browse the repository at this point in the history
Added support for retroactively redacting personal data details from previously committed events. Redactions are scoped to a single EventSourceId, and a single event type, and allow you to overwrite or remove specific fields.

Rewritten to central package management for nuget.
  • Loading branch information
mhelleborg authored Oct 23, 2024
1 parent 5283a7f commit 6368abb
Show file tree
Hide file tree
Showing 82 changed files with 1,738 additions and 367 deletions.
4 changes: 2 additions & 2 deletions Benchmarks/Harness/Harness.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="BenchmarkDotNet" Version="0.13.12" />
<PackageReference Include="Docker.DotNet" Version="3.125.15" />
<PackageReference Include="BenchmarkDotNet"/>
<PackageReference Include="Docker.DotNet"/>
</ItemGroup>

</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,17 @@ namespace Dolittle.Benchmarks.SDK.AggregateRoots.with_1_tenant;

public class applying_events_with_1000_events_to_replay : SingleRuntimeSetup
{
IAggregates _aggregates;
EventSourceId aggregate_root_event_source = "aggregate root source";
IAggregates _aggregates = null!;
readonly EventSourceId _aggregateRootEventSource = "aggregate root source";
public override void IterationSetup()
{
base.IterationSetup();
var client = GetConnectedClient(_ => _
.WithEventTypes(_ => _.Register<AnEvent>().Register<LastEvent>())
.WithEventTypes(types => types.Register<AnEvent>().Register<LastEvent>())
.WithAggregateRoots(_ => _
.Register<AnAggregateRoot>()));
_aggregates = client.Aggregates.ForTenant(TenantId.Development);
_aggregates.Get<AnAggregateRoot>(aggregate_root_event_source).Perform(_ =>
_aggregates.Get<AnAggregateRoot>(_aggregateRootEventSource).Perform(_ =>
{
for (var i = 0; i < 1000; i++)
{
Expand All @@ -33,13 +33,13 @@ public override void IterationSetup()
[Benchmark]
public async Task Applying1EventWith1000EventsToReplay()
{
await _aggregates.Get<AnAggregateRoot>(aggregate_root_event_source).Perform(_ => _.Finish()).ConfigureAwait(false);
await _aggregates.Get<AnAggregateRoot>(_aggregateRootEventSource).Perform(_ => _.Finish()).ConfigureAwait(false);
}

[Benchmark]
public async Task Applying100EventsWith1000EventsToReplay()
{
await _aggregates.Get<AnAggregateRoot>(aggregate_root_event_source).Perform(_ =>
await _aggregates.Get<AnAggregateRoot>(_aggregateRootEventSource).Perform(_ =>
{
for (var i = 0; i < 99; i++)
{
Expand All @@ -54,8 +54,8 @@ public async Task Applying1Event100TimesWith1000EventsToReplay()
{
for (var i = 0; i < 99; i++)
{
await _aggregates.Get<AnAggregateRoot>(aggregate_root_event_source).Perform(_ => _.DoSomething()).ConfigureAwait(false);
await _aggregates.Get<AnAggregateRoot>(_aggregateRootEventSource).Perform(_ => _.DoSomething()).ConfigureAwait(false);
}
await _aggregates.Get<AnAggregateRoot>(aggregate_root_event_source).Perform(_ => _.Finish()).ConfigureAwait(false);
await _aggregates.Get<AnAggregateRoot>(_aggregateRootEventSource).Perform(_ => _.Finish()).ConfigureAwait(false);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,17 @@ namespace Dolittle.Benchmarks.SDK.AggregateRoots.with_1_tenant;

public class applying_events_with_100_events_to_replay : SingleRuntimeSetup
{
IAggregates _aggregates;
EventSourceId aggregate_root_event_source = "aggregate root source";
IAggregates _aggregates = null!;
readonly EventSourceId _aggregateRootEventSource = "aggregate root source";
public override void IterationSetup()
{
base.IterationSetup();
var client = GetConnectedClient(_ => _
.WithEventTypes(_ => _.Register<AnEvent>().Register<LastEvent>())
.WithEventTypes(types => types.Register<AnEvent>().Register<LastEvent>())
.WithAggregateRoots(_ => _
.Register<AnAggregateRoot>()));
_aggregates = client.Aggregates.ForTenant(TenantId.Development);
_aggregates.Get<AnAggregateRoot>(aggregate_root_event_source).Perform(_ =>
_aggregates.Get<AnAggregateRoot>(_aggregateRootEventSource).Perform(_ =>
{
for (var i = 0; i < 100; i++)
{
Expand All @@ -33,13 +33,13 @@ public override void IterationSetup()
[Benchmark]
public async Task Applying1EventWith100EventsToReplay()
{
await _aggregates.Get<AnAggregateRoot>(aggregate_root_event_source).Perform(_ => _.Finish()).ConfigureAwait(false);
await _aggregates.Get<AnAggregateRoot>(_aggregateRootEventSource).Perform(_ => _.Finish()).ConfigureAwait(false);
}

[Benchmark]
public async Task Applying100EventsWith100EventsToReplay()
{
await _aggregates.Get<AnAggregateRoot>(aggregate_root_event_source).Perform(_ =>
await _aggregates.Get<AnAggregateRoot>(_aggregateRootEventSource).Perform(_ =>
{
for (var i = 0; i < 99; i++)
{
Expand All @@ -54,8 +54,8 @@ public async Task Applying1Event100TimesWith100EventsToReplay()
{
for (var i = 0; i < 99; i++)
{
await _aggregates.Get<AnAggregateRoot>(aggregate_root_event_source).Perform(_ => _.DoSomething()).ConfigureAwait(false);
await _aggregates.Get<AnAggregateRoot>(_aggregateRootEventSource).Perform(_ => _.DoSomething()).ConfigureAwait(false);
}
await _aggregates.Get<AnAggregateRoot>(aggregate_root_event_source).Perform(_ => _.Finish()).ConfigureAwait(false);
await _aggregates.Get<AnAggregateRoot>(_aggregateRootEventSource).Perform(_ => _.Finish()).ConfigureAwait(false);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,13 @@ namespace Dolittle.Benchmarks.SDK.AggregateRoots.with_1_tenant;

public class applying_events_with_nothing_to_replay : SingleRuntimeSetup
{
IAggregates _aggregates;
IAggregates _aggregates = null!;
public override void IterationSetup()
{
base.IterationSetup();
var client = GetConnectedClient(_ => _
.WithEventTypes(_ => _.Register<AnEvent>().Register<LastEvent>())
.WithAggregateRoots(_ => _
.Register<AnAggregateRoot>()));
var client = GetConnectedClient(builder => builder
.WithEventTypes(types => types.Register<AnEvent>().Register<LastEvent>())
.WithAggregateRoots(roots => roots.Register<AnAggregateRoot>()));
_aggregates = client.Aggregates.ForTenant(TenantId.Development);
client.EventStore.ForTenant(TenantId.Development).CommitEvent(new AnEvent(), "source").Wait();
}
Expand All @@ -31,7 +30,7 @@ public async Task Applying1EventWithNothingToReplay()
[Benchmark]
public async Task Applying100EventsWithNothingToReplay()
{
await _aggregates.Get<AnAggregateRoot>("an aggregate").Perform(_ =>
await _aggregates!.Get<AnAggregateRoot>("an aggregate").Perform(_ =>
{
for (var i = 0; i < 99; i++)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@ namespace Dolittle.Benchmarks.SDK.AggregateRoots.with_1_tenant;

public class applying_events_with_nothing_to_replay_for_four_aggregate_roots : SingleRuntimeSetup
{
IAggregates _aggregates;
IAggregates _aggregates = null!;
public override void IterationSetup()
{
base.IterationSetup();
var client = GetConnectedClient(_ => _
.WithEventTypes(_ => _.Register<AnEvent>().Register<LastEvent>())
.WithEventTypes(types => types.Register<AnEvent>().Register<LastEvent>())
.WithAggregateRoots(_ => _
.Register<AnAggregateRoot>()));
_aggregates = client.Aggregates.ForTenant(TenantId.Development);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,13 @@ namespace Dolittle.Benchmarks.SDK.AggregateRoots.with_1_tenant;

public class applying_events_with_nothing_to_replay_for_ten_aggregate_roots : SingleRuntimeSetup
{
IAggregates _aggregates;
IAggregates _aggregates = null!;
public override void IterationSetup()
{
base.IterationSetup();
var client = GetConnectedClient(_ => _
.WithEventTypes(_ => _.Register<AnEvent>().Register<LastEvent>())
.WithAggregateRoots(_ => _
.Register<AnAggregateRoot>()));
var client = GetConnectedClient(builder => builder
.WithEventTypes(types => types.Register<AnEvent>().Register<LastEvent>())
.WithAggregateRoots(roots => roots.Register<AnAggregateRoot>()));
_aggregates = client.Aggregates.ForTenant(TenantId.Development);
client.EventStore.ForTenant(TenantId.Development).CommitEvent(new AnEvent(), "source").Wait();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,16 +22,16 @@ public class commiting_and_processing_events : SingleRuntimeSetup
public override void IterationSetup()
{
base.IterationSetup();
var client = GetConnectedClient(_ => _
.WithEventTypes(_ => _.Register<AnEvent>().Register<LastEvent>())
.WithEventHandlers(_ => _
_finishedProcessing = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
var client = GetConnectedClient(builder => builder
.WithEventTypes(types => types.Register<AnEvent>().Register<LastEvent>())
.WithEventHandlers(handlers => handlers
.Create("63c974e5-1381-4757-a5de-04ef9d729d16")
.WithConcurrency(Concurrency)
.Partitioned()
.Handle<AnEvent>((evt, ctx) => { })
.Handle<LastEvent>((evt, ctx) => { _finishedProcessing.SetResult(true); })));
_eventStore = client.EventStore.ForTenant(TenantId.Development);
_finishedProcessing = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
_eventStore.CommitEvent(_anEvent, "source").Wait();
_eventSources = new EventSourceId[EventSourceIds];
for (var i = 0; i < EventSourceIds; i++)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ namespace Dolittle.Benchmarks.SDK.EventStore.with_1_tenant;

public class committing_100_events : SingleRuntimeSetup
{
IEventStore _eventStore;
IEventStore _eventStore = null!;

public override void IterationSetup()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ public async Task Commit100Loop()
{
for (var i = 0; i < 100; i++)
{
await _eventStore.Commit(_ => _.CreateEvent(new some_event())
await EventStore!.Commit(builder => builder.CreateEvent(new some_event())
.FromEventSource("source")
.WithEventType("3065740c-f2e3-4ef4-9738-fe2b1a104399"))
.ConfigureAwait(false);
Expand All @@ -29,7 +29,7 @@ await _eventStore.Commit(_ => _.CreateEvent(new some_event())
public async Task Commit100()
{
var @event = new some_event();
await _eventStore.Commit(_ =>
await EventStore!.Commit(_ =>
{
for (var i = 0; i < 100; i++)
{
Expand All @@ -43,6 +43,6 @@ await _eventStore.Commit(_ =>
public async Task Commit1()
{
var @event = new some_event();
await _eventStore.Commit(_ => _.CreateEvent(@event).FromEventSource("source").WithEventType("3065740c-f2e3-4ef4-9738-fe2b1a104399")).ConfigureAwait(false);
await EventStore!.Commit(_ => _.CreateEvent(@event).FromEventSource("source").WithEventType("3065740c-f2e3-4ef4-9738-fe2b1a104399")).ConfigureAwait(false);
}
}
4 changes: 2 additions & 2 deletions Benchmarks/SDK/SDK.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="BenchmarkDotNet" Version="0.13.12" />
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.10.0" />
<PackageReference Include="BenchmarkDotNet"/>
<PackageReference Include="Microsoft.CodeAnalysis.CSharp"/>
</ItemGroup>

<ItemGroup>
Expand Down
14 changes: 7 additions & 7 deletions Benchmarks/SDK/SingleRuntimeSetup.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@ namespace Dolittle.Benchmarks.SDK;

public class SingleRuntimeSetup
{
Harness.Harness _harness;
Harness.IRuntimeWithMongo _singleRuntime;
protected IEventStore _eventStore;
Harness.Harness? _harness;
Harness.IRuntimeWithMongo? _singleRuntime;
protected IEventStore? EventStore;

[GlobalSetup]
public void GlobalSetup()
Expand All @@ -31,8 +31,8 @@ public void GlobalSetup()

_singleRuntime.Start().Wait();
var client = GetConnectedClient();
_eventStore = client.EventStore.ForTenant(TenantId.Development);
_eventStore.Commit(_ => _.CreateEvent(new some_event())
EventStore = client.EventStore.ForTenant(TenantId.Development);
EventStore.Commit(it => it.CreateEvent(new some_event())
.FromEventSource("source")
.WithEventType("3065740c-f2e3-4ef4-9738-fe2b1a104399")).Wait();

Expand All @@ -53,7 +53,7 @@ public virtual void IterationCleanup()
[GlobalCleanup]
public async Task GlobalCleanup()
{
_harness.Dispose();
_harness?.Dispose();
if (_singleRuntime is not null)
{
await _singleRuntime.DisposeAsync().ConfigureAwait(false);
Expand All @@ -64,7 +64,7 @@ public IDolittleClient GetConnectedClient(bool withDiscovery = false)
=> GetConnectedClient(_ => _.WithoutDiscovery());
public IDolittleClient GetConnectedClient(SetupDolittleClient setup)
=> Host.CreateDefaultBuilder().UseDolittle(setup).Build().GetDolittleClient(_ => _
.WithRuntimeOn("localhost", (ushort)_singleRuntime.Endpoints.Private)
.WithRuntimeOn("localhost", (ushort)_singleRuntime!.Endpoints.Private)
.WithServiceProvider(new ServiceCollection().BuildServiceProvider())
.WithLogging(new NullLoggerFactory())).GetAwaiter().GetResult();
}
71 changes: 71 additions & 0 deletions Directory.Packages.props
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
<Project>
<PropertyGroup>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<PropertyGroup>
<ContractsVersion>7.8.0</ContractsVersion>
</PropertyGroup>
<ItemGroup>
<PackageVersion Include="Dolittle.Contracts" Version="$(ContractsVersion)" />
<PackageVersion Include="Autofac" Version="8.1.1" />
<PackageVersion Include="Autofac.Extensions.DependencyInjection" Version="10.0.0" />
<PackageVersion Include="BenchmarkDotNet" Version="0.14.0" />
<PackageVersion Include="ConsoleTables" Version="2.6.1" />
<PackageVersion Include="coverlet.collector" Version="6.0.2" />
<PackageVersion Include="Docker.DotNet" Version="3.125.15" />
<PackageVersion Include="Grpc.AspNetCore" Version="2.65.0" />
<PackageVersion Include="Grpc.Core.Testing" Version="2.46.6" />
<PackageVersion Include="Grpc.Tools" Version="2.66.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageVersion>
<PackageVersion Include="Grpc.Net.Client" Version="2.66.0" />
<PackageVersion Include="Lamar" Version="13.1.0" />
<PackageVersion Include="Lamar.Microsoft.DependencyInjection" Version="13.1.0" />
<PackageVersion Include="Machine.Specifications" Version="1.1.2" />
<PackageVersion Include="Machine.Specifications.Should" Version="1.0.0" />
<PackageVersion Include="Machine.Specifications.Runner.VisualStudio" Version="2.10.2" />
<PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="4.11.0" />
<PackageVersion Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="4.11.0" />
<PackageVersion Include="Microsoft.CodeAnalysis.CSharp.Analyzer.Testing.XUnit" Version="1.1.2" />
<PackageVersion Include="Microsoft.CodeAnalysis.CSharp.CodeFix.Testing.XUnit" Version="1.1.2" />
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.2" />
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.Logging" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.2" />
<PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.Options" Version="8.0.2" />
<PackageVersion Include="Microsoft.Reactive.Testing" Version="6.0.1" />
<PackageVersion Include="MongoDB.Driver" Version="2.30.0" />
<PackageVersion Include="MongoDB.Driver.Core.Extensions.DiagnosticSources" Version="1.5.0" />
<PackageVersion Include="Moq" Version="4.18.4" />
<PackageVersion Include="FluentAssertions" Version="6.12.1" />
<PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
<PackageVersion Include="Polly" Version="8.4.1" />
<PackageVersion Include="Proto.Actor" Version="1.7.1-alpha.0.1" />
<PackageVersion Include="Proto.Remote" Version="1.7.1-alpha.0.1" />
<PackageVersion Include="Proto.OpenTelemetry" Version="1.7.1-alpha.0.1" />
<PackageVersion Include="Proto.Cluster" Version="1.7.1-alpha.0.1" />
<PackageVersion Include="Google.Protobuf" Version="3.28.3" />
<PackageVersion Include="Swashbuckle.AspNetCore" Version="6.9.0" />
<PackageVersion Include="System.Linq.Async" Version="6.0.1" />
<PackageVersion Include="System.Collections.Immutable" Version="8.0.0" />
<PackageVersion Include="System.Net.Http" Version="4.3.4" />
<PackageVersion Include="System.Reactive" Version="6.0.1" />
<PackageVersion Include="System.Security.Claims" Version="4.3.0" />
<PackageVersion Include="System.Text.Json" Version="8.0.5" />
<PackageVersion Include="OpenTelemetry.Api" Version="1.9.0" />
<PackageVersion Include="OpenTelemetry.Instrumentation.Runtime" Version="1.9.0" />
<PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.9.0" />
<PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.9.0" />
<PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.9.0" />
<PackageVersion Include="OpenTelemetry.Instrumentation.GrpcNetClient" Version="1.9.0-beta.1" />
<PackageVersion Include="OpenTelemetry.Instrumentation.Http" Version="1.9.0" />
<PackageVersion Include="System.Text.RegularExpressions" Version="4.3.1" />
<PackageVersion Include="xunit" Version="2.9.2" />
<PackageVersion Include="xunit.runner.visualstudio" Version="2.8.2" />
</ItemGroup>
</Project>
2 changes: 1 addition & 1 deletion DotNET.SDK.sln
Original file line number Diff line number Diff line change
Expand Up @@ -115,11 +115,11 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Meta", "Meta", "{8E6E3E7B-E
ProjectSection(SolutionItems) = preProject
README.md = README.md
specs.props = specs.props
versions.props = versions.props
default.props = default.props
tests.props = tests.props
base.props = base.props
netstandard.props = netstandard.props
Directory.Packages.props = Directory.Packages.props
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".github", ".github", "{9A48B407-5649-44A1-9433-04CCD42F48E5}"
Expand Down
3 changes: 2 additions & 1 deletion Samples/ASP.NET.Specs/ASP.NET.Specs.csproj
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
<Project Sdk="Microsoft.NET.Sdk">

<Import Project="../../specs.props" />

<PropertyGroup>
<AssemblyName>Dolittle.SDK.ASP.NET.Specs</AssemblyName>
<NoWarn>CS8981</NoWarn>
</PropertyGroup>

<ItemGroup>
Expand Down
10 changes: 5 additions & 5 deletions Samples/ASP.NET.Tests/ASP.NET.Tests.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="FluentAssertions" Version="6.12.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.10.0"/>
<PackageReference Include="xunit" Version="2.9.0"/>
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
<PackageReference Include="FluentAssertions"/>
<PackageReference Include="Microsoft.NET.Test.Sdk"/>
<PackageReference Include="xunit"/>
<PackageReference Include="xunit.runner.visualstudio">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="coverlet.collector" Version="6.0.2">
<PackageReference Include="coverlet.collector">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
Expand Down
Loading

0 comments on commit 6368abb

Please sign in to comment.