fix: Refactor probes and add more health checks

.azure/applications/web-api-eu/main.bicep

@@ -79,6 +79,38 @@ resource environmentKeyVaultResource 'Microsoft.KeyVault/vaults@2023-07-01' exis
 
 var containerAppName = '${namePrefix}-webapi-eu-ca'
 
+var port = 8080
+
+var probes = [
+  {
+    periodSeconds: 5
+    initialDelaySeconds: 2
+    type: 'Liveness'
+    httpGet: {
+      path: '/liveness'
+      port: port
+    }
+  }
+  {
+    periodSeconds: 5
+    initialDelaySeconds: 2
+    type: 'Readiness'
+    httpGet: {
+      path: '/readiness'
+      port: port
+    }
+  }
+  {
+    periodSeconds: 5
+    initialDelaySeconds: 2
+    type: 'Startup'
+    httpGet: {
+      path: '/startup'
+      port: port
+    }
+  }
+]
+
 module containerApp '../../modules/containerApp/main.bicep' = {
   name: containerAppName
   params: {
@@ -91,6 +123,7 @@ module containerApp '../../modules/containerApp/main.bicep' = {
     tags: tags
     resources: resources
     revisionSuffix: imageTag
+    probes: probes
   }
 }
 

.azure/applications/web-api-so/main.bicep

@@ -83,6 +83,38 @@ resource environmentKeyVaultResource 'Microsoft.KeyVault/vaults@2023-07-01' exis
 
 var containerAppName = '${namePrefix}-webapi-so-ca'
 
+var port = 8080
+
+var probes = [
+  {
+    periodSeconds: 5
+    initialDelaySeconds: 2
+    type: 'Liveness'
+    httpGet: {
+      path: '/liveness'
+      port: port
+    }
+  }
+  {
+    periodSeconds: 5
+    initialDelaySeconds: 2
+    type: 'Readiness'
+    httpGet: {
+      path: '/readiness'
+      port: port
+    }
+  }
+  {
+    periodSeconds: 5
+    initialDelaySeconds: 2
+    type: 'Startup'
+    httpGet: {
+      path: '/startup'
+      port: port
+    }
+  }
+]
+
 module containerApp '../../modules/containerApp/main.bicep' = {
   name: containerAppName
   params: {
@@ -95,6 +127,8 @@ module containerApp '../../modules/containerApp/main.bicep' = {
     tags: tags
     resources: resources
     revisionSuffix: imageTag
+    probes: probes
+    port: port
   }
 }
 

.azure/modules/containerApp/main.bicep

@@ -28,30 +28,12 @@ param resources object?
 @description('The suffix for the revision of the container app')
 param revisionSuffix string
 
+@description('The probes for the container app')
+param probes array = []
+
 // Container app revision name does not allow '.' character
 var cleanedRevisionSuffix = replace(revisionSuffix, '.', '-')
 
-var probes = [
-  {
-    periodSeconds: 5
-    initialDelaySeconds: 2
-    type: 'Liveness'
-    httpGet: {
-      path: '/healthz'
-      port: port
-    }
-  }
-  {
-    periodSeconds: 5
-    initialDelaySeconds: 2
-    type: 'Readiness'
-    httpGet: {
-      path: '/healthz'
-      port: port
-    }
-  }
-]
-
 var ipSecurityRestrictions = empty(apimIp)
   ? []
   : [
@@ -74,7 +56,6 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = {
   identity: {
     type: 'SystemAssigned'
   }
-
   properties: {
     configuration: {
       ingress: ingress

src/Digdir.Domain.Dialogporten.Infrastructure/Digdir.Domain.Dialogporten.Infrastructure.csproj

@@ -1,12 +1,16 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <ItemGroup>
+        <PackageReference Include="AspNetCore.HealthChecks.NpgSql" Version="8.0.2" />
+        <PackageReference Include="AspNetCore.HealthChecks.Redis" Version="8.0.1" />
         <PackageReference Include="Altinn.ApiClients.Maskinporten" Version="9.1.0"/>
+        <PackageReference Include="AspNetCore.HealthChecks.UI.Client" Version="8.0.1" />
         <PackageReference Include="HotChocolate.Subscriptions.Redis" Version="13.9.12" />
         <PackageReference Include="Microsoft.Extensions.Caching.StackExchangeRedis" Version="8.0.8" />
         <PackageReference Include="Altinn.Authorization.ABAC" Version="0.0.8"/>
         <PackageReference Include="Bogus" Version="35.6.1"/>
         <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="8.0.0"/>
+        <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
         <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="8.0.8" />
         <PackageReference Include="Npgsql" Version="8.0.4"/>
         <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.4"/>

src/Digdir.Domain.Dialogporten.Infrastructure/HealthChecks/WellKnownHealthChecks.cs

@@ -0,0 +1,77 @@
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace Digdir.Domain.Dialogporten.Infrastructure.HealthChecks;
+
+internal sealed class WellKnownEndpointsHealthCheck : IHealthCheck
+{
+    private readonly IHttpClientFactory _httpClientFactory;
+    private readonly ILogger<WellKnownEndpointsHealthCheck> _logger;
+    private readonly IConfiguration _configuration;
+
+    public WellKnownEndpointsHealthCheck(
+        IHttpClientFactory httpClientFactory,
+        ILogger<WellKnownEndpointsHealthCheck> logger,
+        IConfiguration configuration)
+    {
+        _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
+    }
+
+    public async Task<HealthCheckResult> CheckHealthAsync(
+        HealthCheckContext context,
+        CancellationToken cancellationToken = default)
+    {
+        var wellKnownEndpoints = _configuration.GetSection("WebApi:Authentication:JwtBearerTokenSchemas")
+            .GetChildren()
+            .Select(section => section.GetValue<string>("WellKnown"))
+            .ToList();
+
+        if (wellKnownEndpoints.Count == 0)
+        {
+            _logger.LogWarning("No Well-Known endpoints found in configuration.");
+            return HealthCheckResult.Unhealthy("No Well-Known endpoints are configured.");
+        }
+
+        var client = _httpClientFactory.CreateClient("HealthCheckClient");
+
+        var unhealthyEndpoints = new List<string>();
+
+        foreach (var url in wellKnownEndpoints)
+        {
+            try
+            {
+                var response = await client.GetAsync(url, cancellationToken);
+                if (!response.IsSuccessStatusCode)
+                {
+                    _logger.LogWarning("Health check failed for Well-Known endpoint: {Url}. Status Code: {StatusCode}", url, response.StatusCode);
+                    unhealthyEndpoints.Add($"{url} (Status Code: {response.StatusCode})");
+                }
+            }
+            catch (HttpRequestException ex)
+            {
+                _logger.LogError(ex, "Exception occurred while checking Well-Known endpoint: {Url}", url);
+                return HealthCheckResult.Unhealthy($"Exception occurred while checking Well-Known endpoint {url}.");
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "An unexpected error occurred while checking Well-Known endpoint: {Url}", url);
+                return HealthCheckResult.Unhealthy($"An unexpected error occurred while checking Well-Known endpoint {url}.");
+            }
+        }
+
+        if (unhealthyEndpoints.Count > 0)
+        {
+            var description = $"The following endpoints are unhealthy: {string.Join(", ", unhealthyEndpoints)}";
+            return HealthCheckResult.Unhealthy(description);
+        }
+
+        return HealthCheckResult.Healthy("All Well-Known endpoints are healthy.");
+    }
+}

src/Digdir.Domain.Dialogporten.Infrastructure/InfrastructureExtensions.cs

@@ -30,9 +30,11 @@
 using Digdir.Domain.Dialogporten.Infrastructure.Persistence.Configurations.Actors;
 using Digdir.Domain.Dialogporten.Infrastructure.Persistence.Repositories;
 using HotChocolate.Subscriptions;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
 using StackExchange.Redis;
 using ZiggyCreatures.Caching.Fusion;
 using ZiggyCreatures.Caching.Fusion.NullObjects;
+using Digdir.Domain.Dialogporten.Infrastructure.HealthChecks;
 
 namespace Digdir.Domain.Dialogporten.Infrastructure;
 
@@ -199,6 +201,31 @@ public static IServiceCollection AddInfrastructure(this IServiceCollection servi
             })
             .AddPolicyHandlerFromRegistry(PollyPolicy.DefaultHttpRetryPolicy);
 
+        services.AddHttpClient("HealthCheckClient")
+            .ConfigureHttpClient((services, client) =>
+            {
+                client.Timeout = TimeSpan.FromSeconds(5);
+            })
+            .SetHandlerLifetime(TimeSpan.FromMinutes(2));
+
+        services.AddHealthChecks()
+            .AddRedis(
+                redisConnectionString: infrastructureSettings.Redis.ConnectionString,
+                name: "redis",
+                failureStatus: HealthStatus.Unhealthy,
+                tags: ["dependencies"])
+            .AddNpgSql(
+                connectionString: infrastructureSettings.DialogDbConnectionString,
+                name: "postgres",
+                healthQuery: "SELECT 1",
+                failureStatus: HealthStatus.Unhealthy,
+                tags: ["dependencies", "critical"])
+            .AddCheck<WellKnownEndpointsHealthCheck>(
+                "Well-Known Endpoints",
+                failureStatus: HealthStatus.Unhealthy,
+                tags: ["dependencies"]);
+
+
         if (environment.IsDevelopment())
         {
             var localDeveloperSettings = configuration.GetLocalDevelopmentSettings();

src/Digdir.Domain.Dialogporten.WebApi/Program.cs

@@ -19,8 +19,11 @@
 using FastEndpoints;
 using FastEndpoints.Swagger;
 using FluentValidation;
+using HealthChecks.UI.Client;
 using Microsoft.ApplicationInsights.Extensibility;
 using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
 using NSwag;
 using Serilog;
 
@@ -123,14 +126,14 @@ static void BuildAndRun(string[] args)
             };
         })
         .AddControllers(options => options.InputFormatters.Insert(0, JsonPatchInputFormatter.Get()))
-            .AddNewtonsoftJson()
-            .Services
-
+        .AddNewtonsoftJson()
+        .Services
 
         // Auth
         .AddDialogportenAuthentication(builder.Configuration)
         .AddAuthorization()
-        .AddHealthChecks();
+        .AddHealthChecks()
+        .AddCheck("self", () => HealthCheckResult.Healthy(), tags: ["self"]);
 
     if (builder.Environment.IsDevelopment())
     {
@@ -200,7 +203,27 @@ static void BuildAndRun(string[] args)
             uiConfig.DocumentPath = dialogPrefix + "/swagger/{documentName}/swagger.json";
         });
     app.MapControllers();
-    app.MapHealthChecks("/healthz");
+    app.MapHealthChecks("/startup", new HealthCheckOptions
+    {
+        Predicate = check => check.Tags.Contains("dependencies"),
+        ResponseWriter = UIResponseWriter.WriteHealthCheckUIResponse
+    });
+    app.MapHealthChecks("/liveness", new HealthCheckOptions
+    {
+        Predicate = check => check.Tags.Contains("self"),
+        ResponseWriter = UIResponseWriter.WriteHealthCheckUIResponse
+    });
+    app.MapHealthChecks("/readiness", new HealthCheckOptions
+    {
+        Predicate = check => check.Tags.Contains("critical"),
+        ResponseWriter = UIResponseWriter.WriteHealthCheckUIResponse
+    });
+    app.MapHealthChecks("/health", new HealthCheckOptions
+    {
+        Predicate = check => check.Tags.Contains("dependencies"),
+        ResponseWriter = UIResponseWriter.WriteHealthCheckUIResponse
+    });
+
     app.Run();
 }