Skip to content

Commit

Permalink
feat(webAPI): Require UUIDv7 (#1032)
Browse files Browse the repository at this point in the history
<!--- Provide a general summary of your changes in the Title above -->

## Description

This adds a requirement for user-supplied IDs to be UUIDv7, and that the
timestamp on these are in the past.
While implementing this I stumbled over a bug where the old way of
creating IDs meant they were sorted in the wrong order because GUIDs in
.NET are little endian.
(Sort by ID ascending would have the newest at the top, and oldest at
the bottom)

The idea with UUIDv7 is that the first 6 bytes is the timestamp of
creation, and this helps us avoid a fragmented index
There will only be appends on the index b-tree when creating new
entities, and no inserts further up the index.
If we were to allow old UUIDv4 for the legacy archive imports, the index
would forever be fragmented and have a performance hit.

We now require that all incoming IDs are V7 and big endian (this is the
standard in most other languages)

Sorting by ID ascending on f.ex. DialogID should now list out oldest at
the top, and newest at the bottom.

## Related Issue(s)
- #969 

## Verification

- [ ] **Your** code builds clean without any errors or warnings
- [ ] Manual testing done (required)
- [ ] Relevant automated test added (if you find this hard, leave it and
we'll help out)

## Documentation

- [ ] Documentation is updated (either in `docs`-directory, Altinnpedia
or a separate linked PR in
[altinn-studio-docs.](https://github.com/Altinn/altinn-studio-docs), if
applicable)
  • Loading branch information
oskogstad authored Aug 30, 2024
1 parent b82d4d7 commit e9b844f
Show file tree
Hide file tree
Showing 15 changed files with 229 additions and 99 deletions.
26 changes: 6 additions & 20 deletions docs/schema/V1/swagger.verified.json
Original file line number Diff line number Diff line change
Expand Up @@ -3494,7 +3494,7 @@
"type": "array"
},
"id": {
"description": "The UUDIv7 of the action may be provided to support idempotent additions to the list of API actions.\nIf not supplied, a new UUIDv7 will be generated.",
"description": "A UUIDv7 used for merging existing data, unknown IDs will be ignored as this entity does not support user-defined IDs.",
"example": "01913cd5-784f-7d3b-abef-4c77b1f0972d",
"format": "guid",
"nullable": true,
Expand Down Expand Up @@ -3525,7 +3525,7 @@
]
},
"id": {
"description": "The UUDIv7 of the action may be provided to support idempotent additions to the list of API action endpoints.\nIf not supplied, a new UUIDv7 will be generated.",
"description": "A UUIDv7 used for merging existing data, unknown IDs will be ignored as this entity does not support user-defined IDs.",
"example": "01913cd5-784f-7d3b-abef-4c77b1f0972d",
"format": "guid",
"nullable": true,
Expand Down Expand Up @@ -3573,7 +3573,7 @@
"type": "array"
},
"id": {
"description": "A self-defined UUIDv7 may be provided in order to support idempotent updates of attachments. If not supplied,\na new UUIDv7 will be generated.",
"description": "A UUIDv7 used for merging existing data, unknown IDs will be ignored as this entity does not support user-defined IDs.",
"example": "01913cd5-784f-7d3b-abef-4c77b1f0972d",
"format": "guid",
"nullable": true,
Expand Down Expand Up @@ -3601,7 +3601,7 @@
]
},
"id": {
"description": "A self-defined UUIDv7 may be provided in order to support idempotent updates of attachment URLs. If not supplied,\na new UUIDv7 will be generated.",
"description": "A UUIDv7 used for merging existing data, unknown IDs will be ignored as this entity does not support user-defined IDs.",
"example": "01913cd5-784f-7d3b-abef-4c77b1f0972d",
"format": "guid",
"nullable": true,
Expand Down Expand Up @@ -3644,7 +3644,7 @@
]
},
"id": {
"description": "The UUDIv7 of the action may be provided to support idempotent additions to the list of actions. If not supplied,\na new UUIDv7 will be generated.",
"description": "A UUIDv7 used for merging existing data, unknown IDs will be ignored as this entity does not support user-defined IDs.",
"example": "01913cd5-784f-7d3b-abef-4c77b1f0972d",
"format": "guid",
"nullable": true,
Expand Down Expand Up @@ -3931,13 +3931,6 @@
},
"type": "array"
},
"id": {
"description": "A self-defined UUIDv7 may be provided in order to support idempotent updates of attachments. If not supplied,\na new UUIDv7 will be generated.",
"example": "01913cd5-784f-7d3b-abef-4c77b1f0972d",
"format": "guid",
"nullable": true,
"type": "string"
},
"urls": {
"description": "The URLs associated with the attachment, each referring to a different representation of the attachment.",
"items": {
Expand All @@ -3959,13 +3952,6 @@
}
]
},
"id": {
"description": "A self-defined UUIDv7 may be provided in order to support idempotent updates of attachment URLs. If not supplied,\na new UUIDv7 will be generated.",
"example": "01913cd5-784f-7d3b-abef-4c77b1f0972d",
"format": "guid",
"nullable": true,
"type": "string"
},
"mediaType": {
"description": "The media type of the attachment.",
"example": "application/pdf\napplication/zip",
Expand Down Expand Up @@ -6121,4 +6107,4 @@
"url": "https://altinn-dev-api.azure-api.net/dialogporten"
}
]
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
using Digdir.Domain.Dialogporten.Domain.Common;
using FluentValidation;
using FluentValidation.Validators;

namespace Digdir.Domain.Dialogporten.Application.Common.Extensions.FluentValidation;

public static class FluentValidationUuiDv7Extensions
{
public static IRuleBuilderOptions<T, Guid?> IsValidUuidV7<T>(this IRuleBuilder<T, Guid?> ruleBuilder)
=> ruleBuilder.SetValidator(new UuidV7TimestampIsInPastValidator<T>());

public static IRuleBuilderOptions<T, Guid?> UuidV7TimestampIsInPast<T>(this IRuleBuilder<T, Guid?> ruleBuilder)
=> ruleBuilder.SetValidator(new IsValidUuidV7TimestampValidator<T>());
}

internal sealed class UuidV7TimestampIsInPastValidator<T> : PropertyValidator<T, Guid?>
{
public override bool IsValid(ValidationContext<T> context, Guid? value)
=> value is null || UuidV7.IsValid(value.Value);

public override string Name { get; } = "Uuid7Validator";

protected override string GetDefaultMessageTemplate(string errorCode) =>
"Invalid {PropertyName}. Expected big endian UUIDv7 format. Got '{PropertyValue}'. See [link TBD].";
}

internal sealed class IsValidUuidV7TimestampValidator<T> : PropertyValidator<T, Guid?>
{
public override bool IsValid(ValidationContext<T> context, Guid? value)
{
if (value is null)
{
return true;
}

if (!UuidV7.TryToDateTimeOffset(value.Value, out var date))
{
context.MessageFormatter.AppendArgument("date", "invalid date");
return false;
}

context.MessageFormatter.AppendArgument("date", date.ToString("o"));
return date < DateTimeOffset.UtcNow;
}

public override string Name { get; } = "Uuid7TimestampValidator";

protected override string GetDefaultMessageTemplate(string errorCode)
=> "Invalid {PropertyName}. Expected the unix timestamp portion of the UUIDv7 to be in the past. Extrapolated '{date}' from '{PropertyValue}'. See [link TBD].";
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System.Reflection;
using Digdir.Domain.Dialogporten.Application.Common.Extensions.Enumerables;
using Digdir.Domain.Dialogporten.Application.Common.Extensions.FluentValidation;
using Digdir.Domain.Dialogporten.Application.Features.V1.Common;
using Digdir.Domain.Dialogporten.Application.Features.V1.Common.Content;
using Digdir.Domain.Dialogporten.Application.Features.V1.Common.Localizations;
using Digdir.Domain.Dialogporten.Domain.Actors;
Expand All @@ -26,7 +27,8 @@ public CreateDialogCommandValidator(
IValidator<CreateDialogContentDto> contentValidator)
{
RuleFor(x => x.Id)
.NotEqual(default(Guid));
.IsValidUuidV7()
.UuidV7TimestampIsInPast();

RuleFor(x => x.ServiceResource)
.NotNull()
Expand Down Expand Up @@ -126,7 +128,8 @@ public CreateDialogDialogTransmissionDtoValidator(
IValidator<CreateDialogTransmissionAttachmentDto> attachmentValidator)
{
RuleFor(x => x.Id)
.NotEqual(default(Guid));
.IsValidUuidV7()
.UuidV7TimestampIsInPast();
RuleFor(x => x.CreatedAt)
.IsInPast();
RuleFor(x => x.ExtendedType)
Expand Down Expand Up @@ -354,7 +357,8 @@ public CreateDialogDialogActivityDtoValidator(
IValidator<CreateDialogDialogActivityPerformedByActorDto> actorValidator)
{
RuleFor(x => x.Id)
.NotEqual(default(Guid));
.IsValidUuidV7()
.UuidV7TimestampIsInPast();
RuleFor(x => x.CreatedAt)
.IsInPast();
RuleFor(x => x.ExtendedType)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System.Reflection;
using Digdir.Domain.Dialogporten.Application.Common.Extensions.Enumerables;
using Digdir.Domain.Dialogporten.Application.Common.Extensions.FluentValidation;
using Digdir.Domain.Dialogporten.Application.Features.V1.Common;
using Digdir.Domain.Dialogporten.Application.Features.V1.Common.Content;
using Digdir.Domain.Dialogporten.Application.Features.V1.Common.Localizations;
using Digdir.Domain.Dialogporten.Domain.Actors;
Expand Down Expand Up @@ -112,12 +113,8 @@ public UpdateDialogTransmissionAttachmentDtoValidator(
IValidator<IEnumerable<LocalizationDto>> localizationsValidator,
IValidator<UpdateDialogTransmissionAttachmentUrlDto> urlValidator)
{
RuleFor(x => x.Id)
.NotEqual(default(Guid));
RuleFor(x => x.DisplayName)
.SetValidator(localizationsValidator);
RuleFor(x => x.Urls)
.UniqueBy(x => x.Id);
RuleFor(x => x.Urls)
.NotEmpty()
.ForEach(x => x.SetValidator(urlValidator));
Expand Down Expand Up @@ -185,7 +182,8 @@ public UpdateDialogDialogTransmissionDtoValidator(
IValidator<UpdateDialogTransmissionAttachmentDto> attachmentValidator)
{
RuleFor(x => x.Id)
.NotEqual(default(Guid));
.IsValidUuidV7()
.UuidV7TimestampIsInPast();
RuleFor(x => x.CreatedAt)
.IsInPast();
RuleFor(x => x.ExtendedType)
Expand All @@ -203,8 +201,6 @@ public UpdateDialogDialogTransmissionDtoValidator(
.SetValidator(actorValidator);
RuleFor(x => x.AuthorizationAttribute)
.MaximumLength(Constants.DefaultMaxStringLength);
RuleFor(x => x.Attachments)
.UniqueBy(x => x.Id);
RuleForEach(x => x.Attachments)
.SetValidator(attachmentValidator);
RuleFor(x => x.Content)
Expand Down Expand Up @@ -261,7 +257,8 @@ public UpdateDialogDialogAttachmentDtoValidator(
IValidator<UpdateDialogDialogAttachmentUrlDto> urlValidator)
{
RuleFor(x => x.Id)
.NotEqual(default(Guid));
.IsValidUuidV7()
.UuidV7TimestampIsInPast();
RuleFor(x => x.DisplayName)
.SetValidator(localizationsValidator);
RuleFor(x => x.Urls)
Expand Down Expand Up @@ -376,7 +373,8 @@ public UpdateDialogDialogActivityDtoValidator(
IValidator<UpdateDialogDialogActivityPerformedByActorDto> actorValidator)
{
RuleFor(x => x.Id)
.NotEqual(default(Guid));
.IsValidUuidV7()
.UuidV7TimestampIsInPast();
RuleFor(x => x.CreatedAt)
.IsInPast();
RuleFor(x => x.ExtendedType)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -306,8 +306,7 @@ public sealed class UpdateDialogDialogActivityPerformedByActorDto
public sealed class UpdateDialogDialogApiActionDto
{
/// <summary>
/// The UUDIv7 of the action may be provided to support idempotent additions to the list of API actions.
/// If not supplied, a new UUIDv7 will be generated.
/// A UUIDv7 used for merging existing data, unknown IDs will be ignored as this entity does not support user-defined IDs.
/// </summary>
/// <example>01913cd5-784f-7d3b-abef-4c77b1f0972d</example>
public Guid? Id { get; set; }
Expand Down Expand Up @@ -344,8 +343,7 @@ public sealed class UpdateDialogDialogApiActionDto
public sealed class UpdateDialogDialogApiActionEndpointDto
{
/// <summary>
/// The UUDIv7 of the action may be provided to support idempotent additions to the list of API action endpoints.
/// If not supplied, a new UUIDv7 will be generated.
/// A UUIDv7 used for merging existing data, unknown IDs will be ignored as this entity does not support user-defined IDs.
/// </summary>
/// <example>01913cd5-784f-7d3b-abef-4c77b1f0972d</example>
public Guid? Id { get; set; }
Expand Down Expand Up @@ -398,8 +396,7 @@ public sealed class UpdateDialogDialogApiActionEndpointDto
public sealed class UpdateDialogDialogGuiActionDto
{
/// <summary>
/// The UUDIv7 of the action may be provided to support idempotent additions to the list of actions. If not supplied,
/// a new UUIDv7 will be generated.
/// A UUIDv7 used for merging existing data, unknown IDs will be ignored as this entity does not support user-defined IDs.
/// </summary>
/// <example>01913cd5-784f-7d3b-abef-4c77b1f0972d</example>
public Guid? Id { get; set; }
Expand Down Expand Up @@ -467,8 +464,7 @@ public sealed class UpdateDialogDialogGuiActionDto
public class UpdateDialogDialogAttachmentDto
{
/// <summary>
/// A self-defined UUIDv7 may be provided in order to support idempotent updates of attachments. If not supplied,
/// a new UUIDv7 will be generated.
/// A UUIDv7 used for merging existing data, unknown IDs will be ignored as this entity does not support user-defined IDs.
/// </summary>
/// <example>01913cd5-784f-7d3b-abef-4c77b1f0972d</example>
public Guid? Id { get; set; }
Expand All @@ -487,8 +483,7 @@ public class UpdateDialogDialogAttachmentDto
public sealed class UpdateDialogDialogAttachmentUrlDto
{
/// <summary>
/// A self-defined UUIDv7 may be provided in order to support idempotent updates of attachment URLs. If not supplied,
/// a new UUIDv7 will be generated.
/// A UUIDv7 used for merging existing data, unknown IDs will be ignored as this entity does not support user-defined IDs.
/// </summary>
/// <example>01913cd5-784f-7d3b-abef-4c77b1f0972d</example>
public Guid? Id { get; set; }
Expand All @@ -515,13 +510,6 @@ public sealed class UpdateDialogDialogAttachmentUrlDto

public class UpdateDialogTransmissionAttachmentDto
{
/// <summary>
/// A self-defined UUIDv7 may be provided in order to support idempotent updates of attachments. If not supplied,
/// a new UUIDv7 will be generated.
/// </summary>
/// <example>01913cd5-784f-7d3b-abef-4c77b1f0972d</example>
public Guid? Id { get; set; }

/// <summary>
/// The display name of the attachment that should be used in GUIs.
/// </summary>
Expand All @@ -535,13 +523,6 @@ public class UpdateDialogTransmissionAttachmentDto

public sealed class UpdateDialogTransmissionAttachmentUrlDto
{
/// <summary>
/// A self-defined UUIDv7 may be provided in order to support idempotent updates of attachment URLs. If not supplied,
/// a new UUIDv7 will be generated.
/// </summary>
/// <example>01913cd5-784f-7d3b-abef-4c77b1f0972d</example>
public Guid? Id { get; set; }

/// <summary>
/// The fully qualified URL of the attachment.
/// </summary>
Expand Down
50 changes: 34 additions & 16 deletions src/Digdir.Domain.Dialogporten.Domain/Common/UuidV7.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,31 +2,49 @@

public static class UuidV7
{
public static bool IsValid(string uuid)
private const int Version = 7;
private const int Variant = 2;
private const long UnixEpochMilliseconds = 62_135_596_800_000;
private const long TicksPerMillisecond = 10_000;

public static bool TryParse(ReadOnlySpan<char> value, out Guid result)
=> Guid.TryParse(value, out result) && IsValid(result);

public static bool IsValid(Guid value) => IsValid(value.ToByteArray());

public static bool TryToDateTimeOffset(Guid value, out DateTimeOffset result)
=> TryToDateTimeOffset(value.ToByteArray(), out result);

private static bool TryToDateTimeOffset(ReadOnlySpan<byte> bytes, out DateTimeOffset result)
{
if (!Guid.TryParse(uuid, out var guid))
// The timestamp is stored in the first 6 bytes, in little endian order.
var unixMs = ((long)bytes[3] << 40)
| ((long)bytes[2] << 32)
| ((long)bytes[1] << 24)
| ((long)bytes[0] << 16)
| ((long)bytes[5] << 8)
| bytes[4];

var ticks = (UnixEpochMilliseconds + unixMs) * TicksPerMillisecond;

if (ticks < DateTimeOffset.MinValue.Ticks || ticks > DateTimeOffset.MaxValue.Ticks)
{
result = default;
return false;
}

return IsValid(guid);
result = new DateTimeOffset(ticks, TimeSpan.Zero);
return true;
}

public static bool IsValid(Guid value)
private static bool IsValid(ReadOnlySpan<byte> bytes)
{
var bytes = value.ToByteArray();
// Version is stored in the 7th byte, but the nibbles are reversed so version is actually in the higher 4 bits
// The version is stored in the 7th byte, but the nibbles are reversed, so the version is actually in the higher 4 bits.
var version = bytes[7] >> 4;
if (version != 7) // Ensure version is 7
{
return false;
}
// Variant is stored in the 8th byte, in the higher bits

// The variant is stored in the 9th byte, in the higher bits.
var variant = bytes[8] >> 6;
if (variant != 2) // The variant for UUIDv7 should be '10' in binary
{
return false;
}
return true;

return version == Version && variant == Variant; // The variant for UUIDv7 should be '10' in binary
}
}
Loading

0 comments on commit e9b844f

Please sign in to comment.