feat: Synchronization of resource policy metadata (#1411)

## Description

This adds a command akin to sync-subject-resources which fetches
information about resource policies and stores the information to a
table in postgres. For now, this only parses and stores XACML
obligations for required authentication level, but can be fairly easily
extended to implement eg. #40

There are however some real limitations to the Resource Registry as of
now for Altinn Apps, as they are not really stored there, but merely
have a representation that does not include the policy. ~~For these, we
store a default authentication level.~~ Edit: We don't do this, as we
cannot safely assume anything here. So we do not store anything unless
we have an explicit obligation for authentication level set in a policy
we can reach. For all other resources, no authentication level handling
will be performed.

## Related Issue(s)

- #1214 
 
## Verification

- [x] **Your** code builds clean without any errors or warnings
- [x] Manual testing done (required)
- [ ] Relevant automated test added (if you find this hard, leave it and
we'll help out)

## Documentation

- [ ] Documentation is updated (either in `docs`-directory, Altinnpedia
or a separate linked PR in
[altinn-studio-docs.](https://github.com/Altinn/altinn-studio-docs), if
applicable)


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

## Release Notes

- **New Features**
	- Introduced a new command for synchronizing resource policies.
- Added support for managing resource policy information in various
environments (production, staging, test).
- Enhanced deployment capabilities with a new scheduled job for resource
policy synchronization.
  
- **Bug Fixes**
	- Improved error handling and logging during synchronization processes.

- **Documentation**
- Updated command documentation to include new synchronization options
and parameters.

- **Chores**
- Added new settings to local development configurations to manage
synchronization behavior at startup.
- Introduced additional configuration settings for local development to
control synchronization processes.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: Magnus Sandgren <5285192+MagnusSandgren@users.noreply.github.com>
Co-authored-by: Ole Jørgen Skogstad <skogstad@softis.net>
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

.azure/applications/sync-resource-policy-information-job/main.bicep

@@ -0,0 +1,109 @@
+targetScope = 'resourceGroup'
+
+@description('The tag of the image to be used')
+@minLength(3)
+param imageTag string
+
+@description('The environment for the deployment')
+@minLength(3)
+param environment string
+
+@description('The location where the resources will be deployed')
+@minLength(3)
+param location string
+
+@description('The name of the container app environment')
+@minLength(3)
+@secure()
+param containerAppEnvironmentName string
+
+@description('The name of the Key Vault for the environment')
+@minLength(3)
+@secure()
+param environmentKeyVaultName string
+
+@description('The cron expression for the job schedule')
+@minLength(9)
+param jobSchedule string
+
+@description('The connection string for Application Insights')
+@minLength(3)
+@secure()
+param appInsightConnectionString string
+
+var namePrefix = 'dp-be-${environment}'
+var baseImageUrl = 'ghcr.io/digdir/dialogporten-'
+var tags = {
+  FullName: '${namePrefix}-sync-resource-policy-information'
+  Environment: environment
+  Product: 'Dialogporten'
+  Description: 'Synchronizes resource policy information'
+  JobType: 'Scheduled'
+}
+var name = '${namePrefix}-sync-rp-info'
+
+resource containerAppEnvironment 'Microsoft.App/managedEnvironments@2024-03-01' existing = {
+  name: containerAppEnvironmentName
+}
+
+var containerAppEnvVars = [
+  {
+    name: 'Infrastructure__DialogDbConnectionString'
+    secretRef: 'dbconnectionstring'
+  }
+  {
+    name: 'Infrastructure__Redis__ConnectionString'
+    secretRef: 'redisconnectionstring'
+  }
+  {
+    name: 'DOTNET_ENVIRONMENT'
+    value: environment
+  }
+  {
+    name: 'APPLICATIONINSIGHTS_CONNECTION_STRING'
+    value: appInsightConnectionString
+  }
+]
+
+// Base URL for accessing secrets in the Key Vault
+// https://learn.microsoft.com/en-us/azure/azure-resource-manager/bicep/bicep-functions-deployment#example-1
+var keyVaultBaseUrl = 'https://${environmentKeyVaultName}${az.environment().suffixes.keyvaultDns}/secrets'
+
+var secrets = [
+  {
+    name: 'dbconnectionstring'
+    keyVaultUrl: '${keyVaultBaseUrl}/dialogportenAdoConnectionString'
+    identity: 'System'
+  }
+  {
+    name: 'redisconnectionstring'
+    keyVaultUrl: '${keyVaultBaseUrl}/dialogportenRedisConnectionString'
+    identity: 'System'
+  }
+]
+
+module migrationJob '../../modules/containerAppJob/main.bicep' = {
+  name: name
+  params: {
+    name: name
+    location: location
+    image: '${baseImageUrl}janitor:${imageTag}'
+    containerAppEnvId: containerAppEnvironment.id
+    environmentVariables: containerAppEnvVars
+    secrets: secrets
+    tags: tags
+    cronExpression: jobSchedule
+    args: 'sync-resource-policy-information'
+  }
+}
+
+module keyVaultReaderAccessPolicy '../../modules/keyvault/addReaderRoles.bicep' = {
+  name: 'keyVaultReaderAccessPolicy-${name}'
+  params: {
+    keyvaultName: environmentKeyVaultName
+    principalIds: [migrationJob.outputs.identityPrincipalId]
+  }
+}
+
+output identityPrincipalId string = migrationJob.outputs.identityPrincipalId
+output name string = migrationJob.outputs.name

.azure/applications/sync-resource-policy-information-job/prod.bicepparam

@@ -0,0 +1,11 @@
+using './main.bicep'
+
+param environment = 'prod'
+param location = 'norwayeast'
+param imageTag = readEnvironmentVariable('IMAGE_TAG')
+param jobSchedule = '10 3 * * *' // 3:10AM every night
+
+//secrets
+param containerAppEnvironmentName = readEnvironmentVariable('AZURE_CONTAINER_APP_ENVIRONMENT_NAME')
+param environmentKeyVaultName = readEnvironmentVariable('AZURE_ENVIRONMENT_KEY_VAULT_NAME')
+param appInsightConnectionString = readEnvironmentVariable('AZURE_APP_INSIGHTS_CONNECTION_STRING')

.azure/applications/sync-resource-policy-information-job/staging.bicepparam

@@ -0,0 +1,11 @@
+using './main.bicep'
+
+param environment = 'staging'
+param location = 'norwayeast'
+param imageTag = readEnvironmentVariable('IMAGE_TAG')
+param jobSchedule = '15 3 * * *' // 3:15AM every night
+
+//secrets
+param containerAppEnvironmentName = readEnvironmentVariable('AZURE_CONTAINER_APP_ENVIRONMENT_NAME')
+param environmentKeyVaultName = readEnvironmentVariable('AZURE_ENVIRONMENT_KEY_VAULT_NAME')
+param appInsightConnectionString = readEnvironmentVariable('AZURE_APP_INSIGHTS_CONNECTION_STRING')

.azure/applications/sync-resource-policy-information-job/test.bicepparam

@@ -0,0 +1,11 @@
+using './main.bicep'
+
+param environment = 'test'
+param location = 'norwayeast'
+param imageTag = readEnvironmentVariable('IMAGE_TAG')
+param jobSchedule = '20 3 * * *' // 3:20AM every night
+
+//secrets
+param containerAppEnvironmentName = readEnvironmentVariable('AZURE_CONTAINER_APP_ENVIRONMENT_NAME')
+param environmentKeyVaultName = readEnvironmentVariable('AZURE_ENVIRONMENT_KEY_VAULT_NAME')
+param appInsightConnectionString = readEnvironmentVariable('AZURE_APP_INSIGHTS_CONNECTION_STRING')

.azure/applications/sync-resource-policy-information-job/yt01.bicepparam

@@ -0,0 +1,11 @@
+using './main.bicep'
+
+param environment = 'yt01'
+param location = 'norwayeast'
+param imageTag = readEnvironmentVariable('IMAGE_TAG')
+param jobSchedule = '25 3 * * *' // 3:25AM every night
+
+//secrets
+param containerAppEnvironmentName = readEnvironmentVariable('AZURE_CONTAINER_APP_ENVIRONMENT_NAME')
+param environmentKeyVaultName = readEnvironmentVariable('AZURE_ENVIRONMENT_KEY_VAULT_NAME')
+param appInsightConnectionString = readEnvironmentVariable('AZURE_APP_INSIGHTS_CONNECTION_STRING')

.github/workflows/workflow-deploy-apps.yml

@@ -243,6 +243,7 @@ jobs:
       matrix:
         include:
           - name: sync-subject-resource-mappings-job
+          - name: sync-resource-policy-information-job
     steps:
       - name: "Checkout GitHub Action"
         uses: actions/checkout@v4

src/Digdir.Domain.Dialogporten.Application/Externals/IDialogDbContext.cs

@@ -8,6 +8,7 @@
 using Digdir.Domain.Dialogporten.Domain.Dialogs.Entities.Contents;
 using Digdir.Domain.Dialogporten.Domain.Dialogs.Entities.Transmissions;
 using Digdir.Domain.Dialogporten.Domain.Dialogs.Entities.Transmissions.Contents;
+using Digdir.Domain.Dialogporten.Domain.ResourcePolicyInformation;
 using Digdir.Domain.Dialogporten.Domain.SubjectResources;
 
 namespace Digdir.Domain.Dialogporten.Application.Externals;
@@ -42,6 +43,7 @@ public interface IDialogDbContext
     DbSet<SubjectResource> SubjectResources { get; }
     DbSet<DialogEndUserContext> DialogEndUserContexts { get; }
     DbSet<LabelAssignmentLog> LabelAssignmentLogs { get; }
+    DbSet<ResourcePolicyInformation> ResourcePolicyInformation { get; }
 
     /// <summary>
     /// Validate a property on the <typeparamref name="TEntity"/> using a lambda

src/Digdir.Domain.Dialogporten.Application/Externals/IResourcePolicyInformationRepository.cs

@@ -0,0 +1,25 @@
+using Digdir.Domain.Dialogporten.Domain.ResourcePolicyInformation;
+using Digdir.Library.Entity.Abstractions.Features.Identifiable;
+
+namespace Digdir.Domain.Dialogporten.Application.Externals;
+
+public interface IResourcePolicyInformationRepository
+{
+    Task<int> Merge(IReadOnlyCollection<ResourcePolicyInformation> resourceMetadata, CancellationToken cancellationToken = default);
+    Task<DateTimeOffset> GetLastUpdatedAt(TimeSpan? timeSkew = null, CancellationToken cancellationToken = default);
+}
+
+public static class ResourcePolicyInformationExtensions
+{
+    public static ResourcePolicyInformation ToResourcePolicyInformation(this UpdatedResourcePolicyInformation updatedResourcePolicyInformation, DateTimeOffset createdAt)
+    {
+        return new ResourcePolicyInformation
+        {
+            Id = IdentifiableExtensions.CreateVersion7(),
+            Resource = updatedResourcePolicyInformation.ResourceUrn.ToString()!,
+            MinimumAuthenticationLevel = updatedResourcePolicyInformation.MinimumAuthenticationLevel,
+            CreatedAt = createdAt.ToUniversalTime(),
+            UpdatedAt = updatedResourcePolicyInformation.UpdatedAt.ToUniversalTime()
+        };
+    }
+}

src/Digdir.Domain.Dialogporten.Application/Externals/IResourceRegistry.cs

@@ -1,11 +1,13 @@
-namespace Digdir.Domain.Dialogporten.Application.Externals;
+namespace Digdir.Domain.Dialogporten.Application.Externals;
 
 public interface IResourceRegistry
 {
     Task<IReadOnlyCollection<ServiceResourceInformation>> GetResourceInformationForOrg(string orgNumber, CancellationToken cancellationToken);
     Task<ServiceResourceInformation?> GetResourceInformation(string serviceResourceId, CancellationToken cancellationToken);
     IAsyncEnumerable<List<UpdatedSubjectResource>> GetUpdatedSubjectResources(DateTimeOffset since, int batchSize,
         CancellationToken cancellationToken);
+    Task<IReadOnlyCollection<UpdatedResourcePolicyInformation>> GetUpdatedResourcePolicyInformation(DateTimeOffset since, int numberOfConcurrentRequests,
+        CancellationToken cancellationToken);
 }
 
 public sealed record ServiceResourceInformation
@@ -24,3 +26,4 @@ public ServiceResourceInformation(string resourceId, string resourceType, string
 
 
 public sealed record UpdatedSubjectResource(Uri SubjectUrn, Uri ResourceUrn, DateTimeOffset UpdatedAt, bool Deleted);
+public sealed record UpdatedResourcePolicyInformation(Uri ResourceUrn, int MinimumAuthenticationLevel, DateTimeOffset UpdatedAt);

src/Digdir.Domain.Dialogporten.Application/Features/V1/ResourceRegistry/Commands/SyncPolicy/SyncPolicyCommand.cs

@@ -0,0 +1,76 @@
+using Digdir.Domain.Dialogporten.Application.Common.ReturnTypes;
+using Digdir.Domain.Dialogporten.Application.Externals;
+using MediatR;
+using Microsoft.Extensions.Logging;
+using OneOf;
+using OneOf.Types;
+
+namespace Digdir.Domain.Dialogporten.Application.Features.V1.ResourceRegistry.Commands.SyncPolicy;
+
+public sealed class SyncPolicyCommand : IRequest<SyncPolicyResult>
+{
+    public DateTimeOffset? Since { get; set; }
+    public int? NumberOfConcurrentRequests { get; set; }
+}
+
+[GenerateOneOf]
+public sealed partial class SyncPolicyResult : OneOfBase<Success, ValidationError>;
+
+internal sealed class SyncPolicyCommandHandler : IRequestHandler<SyncPolicyCommand, SyncPolicyResult>
+{
+    private const int DefaultNumberOfConcurrentRequests = 15;
+    private readonly IResourceRegistry _resourceRegistry;
+    private readonly IResourcePolicyInformationRepository _resourcePolicyMetadataRepository;
+    private readonly ILogger<SyncPolicyCommandHandler> _logger;
+
+    public SyncPolicyCommandHandler(
+        IResourceRegistry resourceRegistry,
+        IResourcePolicyInformationRepository resourcePolicyMetadataRepository,
+        ILogger<SyncPolicyCommandHandler> logger)
+    {
+        _resourceRegistry = resourceRegistry ?? throw new ArgumentNullException(nameof(resourceRegistry));
+        _resourcePolicyMetadataRepository = resourcePolicyMetadataRepository ?? throw new ArgumentNullException(nameof(resourcePolicyMetadataRepository));
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+    }
+
+    public async Task<SyncPolicyResult> Handle(SyncPolicyCommand request,
+        CancellationToken cancellationToken)
+    {
+        // Get the last updated timestamp from parameter, or the database (with a time skew), or use a default
+        var lastUpdated = request.Since
+                          ?? await _resourcePolicyMetadataRepository.GetLastUpdatedAt(
+                              timeSkew: TimeSpan.FromMicroseconds(1),
+                              cancellationToken: cancellationToken);
+
+        _logger.LogInformation("Fetching updated resource policy information since {LastUpdated:O}.", lastUpdated);
+
+        try
+        {
+            var syncTime = DateTimeOffset.Now;
+            var updatedResourcePolicyInformation = await _resourceRegistry
+                               .GetUpdatedResourcePolicyInformation(lastUpdated, request.NumberOfConcurrentRequests ?? DefaultNumberOfConcurrentRequests, cancellationToken);
+
+            var mergeableResourcePolicyInformation = updatedResourcePolicyInformation
+                .Select(x => x.ToResourcePolicyInformation(syncTime))
+                .ToList();
+            var mergeCount = await _resourcePolicyMetadataRepository.Merge(mergeableResourcePolicyInformation, cancellationToken);
+            _logger.LogInformation("{MergeCount} copies of resource policy information updated.", mergeCount);
+
+            if (mergeCount > 0)
+            {
+                _logger.LogInformation("Successfully synced information from {MergeCount} policies", mergeCount);
+            }
+            else
+            {
+                _logger.LogInformation("Resource policy information are already up-to-date.");
+            }
+
+            return new Success();
+        }
+        catch (Exception e)
+        {
+            _logger.LogError(e, "Failed to sync resource policy information.");
+            throw;
+        }
+    }
+}

src/Digdir.Domain.Dialogporten.Application/Features/V1/ResourceRegistry/Commands/SyncPolicy/SyncPolicyCommandValidator.cs

@@ -0,0 +1,11 @@
+using FluentValidation;
+
+namespace Digdir.Domain.Dialogporten.Application.Features.V1.ResourceRegistry.Commands.SyncPolicy;
+
+internal sealed class SyncPolicyCommandValidator : AbstractValidator<SyncPolicyCommand>
+{
+    public SyncPolicyCommandValidator()
+    {
+        RuleFor(x => x.NumberOfConcurrentRequests).InclusiveBetween(1, 50);
+    }
+}

src/Digdir.Domain.Dialogporten.Application/Features/V1/ResourceRegistry/Commands/Synchronize/SynchronizeSubjectResourceMappingsCommand.cs → src/Digdir.Domain.Dialogporten.Application/Features/V1/ResourceRegistry/Commands/SyncSubjectMap/SyncSubjectMapCommand.cs

@@ -5,38 +5,38 @@
 using OneOf;
 using OneOf.Types;
 
-namespace Digdir.Domain.Dialogporten.Application.Features.V1.ResourceRegistry.Commands.Synchronize;
+namespace Digdir.Domain.Dialogporten.Application.Features.V1.ResourceRegistry.Commands.SyncSubjectMap;
 
-public sealed class SynchronizeSubjectResourceMappingsCommand : IRequest<SynchronizeResourceRegistryResult>
+public sealed class SyncSubjectMapCommand : IRequest<SyncSubjectMapResult>
 {
     public DateTimeOffset? Since { get; set; }
     public int? BatchSize { get; set; }
 }
 
 [GenerateOneOf]
-public sealed partial class SynchronizeResourceRegistryResult : OneOfBase<Success, ValidationError>;
+public sealed partial class SyncSubjectMapResult : OneOfBase<Success, ValidationError>;
 
-internal sealed class SynchronizeResourceRegistryCommandHandler : IRequestHandler<SynchronizeSubjectResourceMappingsCommand, SynchronizeResourceRegistryResult>
+internal sealed class SyncSubjectMapCommandHandler : IRequestHandler<SyncSubjectMapCommand, SyncSubjectMapResult>
 {
     private const int DefaultBatchSize = 1000;
     private readonly IResourceRegistry _resourceRegistry;
     private readonly ISubjectResourceRepository _subjectResourceRepository;
     private readonly IUnitOfWork _unitOfWork;
-    private readonly ILogger<SynchronizeResourceRegistryCommandHandler> _logger;
+    private readonly ILogger<SyncSubjectMapCommandHandler> _logger;
 
-    public SynchronizeResourceRegistryCommandHandler(
+    public SyncSubjectMapCommandHandler(
         IResourceRegistry resourceRegistry,
         ISubjectResourceRepository subjectResourceRepository,
         IUnitOfWork unitOfWork,
-        ILogger<SynchronizeResourceRegistryCommandHandler> logger)
+        ILogger<SyncSubjectMapCommandHandler> logger)
     {
         _resourceRegistry = resourceRegistry ?? throw new ArgumentNullException(nameof(resourceRegistry));
         _subjectResourceRepository = subjectResourceRepository ?? throw new ArgumentNullException(nameof(subjectResourceRepository));
         _unitOfWork = unitOfWork;
         _logger = logger ?? throw new ArgumentNullException(nameof(logger));
     }
 
-    public async Task<SynchronizeResourceRegistryResult> Handle(SynchronizeSubjectResourceMappingsCommand request, CancellationToken cancellationToken)
+    public async Task<SyncSubjectMapResult> Handle(SyncSubjectMapCommand request, CancellationToken cancellationToken)
     {
         // Get the last updated timestamp from parameter, or the database (with a time skew), or use a default
         var lastUpdated = request.Since

src/Digdir.Domain.Dialogporten.Application/Features/V1/ResourceRegistry/Commands/SyncSubjectMap/SyncSubjectMapCommandValidator.cs

@@ -0,0 +1,11 @@
+using FluentValidation;
+
+namespace Digdir.Domain.Dialogporten.Application.Features.V1.ResourceRegistry.Commands.SyncSubjectMap;
+
+internal sealed class SyncSubjectMapCommandValidator : AbstractValidator<SyncSubjectMapCommand>
+{
+    public SyncSubjectMapCommandValidator()
+    {
+        RuleFor(x => x.BatchSize).InclusiveBetween(1, 1000);
+    }
+}

src/Digdir.Domain.Dialogporten.Application/LocalDevelopmentSettings.cs

@@ -15,6 +15,8 @@ public sealed class LocalDevelopmentSettings
     public bool UseLocalDevelopmentOrganizationRegister { get; set; } = true;
     public bool UseLocalDevelopmentCompactJwsGenerator { get; set; } = true;
     public bool UseInMemoryServiceBusTransport { get; set; } = true;
+    public bool DisableSubjectResourceSyncOnStartup { get; set; } = true;
+    public bool DisablePolicyInformationSyncOnStartup { get; set; } = true;
 }
 
 public static class LocalDevelopmentSettingsExtensions