Skip to content

Commit

Permalink
fix: Refactor probes and add more health checks (#1159)
Browse files Browse the repository at this point in the history
- Adds health check for Redis, PosgreSQL and the wellknown-endpoints. 
- Ensures that we have different endpoints for
readiness/liveness/startup/health

Related to #292 

<img width="542" alt="image"
src="https://github.com/user-attachments/assets/5b71bfbc-1e83-427c-8042-e363ffbf8faa">


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

## Summary by CodeRabbit

- **New Features**
- Added health check capabilities for Redis, PostgreSQL, and well-known
endpoints.
- Introduced multiple health check endpoints: `/startup`, `/liveness`,
`/readiness`, and `/health`.
- Integrated health checks into the service collection for better
monitoring.
  - Added a new project for utility functions related to health checks.

- **Enhancements**
- Improved health monitoring with a new HTTP client and health check
configurations, including a self-check feature.
- Added support for dynamic configuration of health check probes in
deployment templates.
- Updated API specifications to reflect new health check schemas and
structures.

- **Bug Fixes**
- Enhanced error handling for health checks to provide clearer feedback
on endpoint status.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: Are Almaas <arealmaas@gmail.com>
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Co-authored-by: Dialogporten Automation Bot <164321870+dialogporten-bot@users.noreply.github.com>
Co-authored-by: Magnus Sandgren <5285192+MagnusSandgren@users.noreply.github.com>
  • Loading branch information
6 people authored Oct 8, 2024
1 parent 4c43e2f commit 6889a96
Show file tree
Hide file tree
Showing 14 changed files with 280 additions and 30 deletions.
33 changes: 33 additions & 0 deletions .azure/applications/web-api-eu/main.bicep
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,38 @@ resource environmentKeyVaultResource 'Microsoft.KeyVault/vaults@2023-07-01' exis

var containerAppName = '${namePrefix}-webapi-eu-ca'

var port = 8080

var probes = [
{
periodSeconds: 5
initialDelaySeconds: 2
type: 'Liveness'
httpGet: {
path: '/liveness'
port: port
}
}
{
periodSeconds: 5
initialDelaySeconds: 2
type: 'Readiness'
httpGet: {
path: '/readiness'
port: port
}
}
{
periodSeconds: 5
initialDelaySeconds: 2
type: 'Startup'
httpGet: {
path: '/startup'
port: port
}
}
]

module containerApp '../../modules/containerApp/main.bicep' = {
name: containerAppName
params: {
Expand All @@ -94,6 +126,7 @@ module containerApp '../../modules/containerApp/main.bicep' = {
apimIp: apimIp
tags: tags
resources: resources
probes: probes
revisionSuffix: revisionSuffix
}
}
Expand Down
34 changes: 34 additions & 0 deletions .azure/applications/web-api-so/main.bicep
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,38 @@ resource environmentKeyVaultResource 'Microsoft.KeyVault/vaults@2023-07-01' exis

var containerAppName = '${namePrefix}-webapi-so-ca'

var port = 8080

var probes = [
{
periodSeconds: 5
initialDelaySeconds: 2
type: 'Liveness'
httpGet: {
path: '/liveness'
port: port
}
}
{
periodSeconds: 5
initialDelaySeconds: 2
type: 'Readiness'
httpGet: {
path: '/readiness'
port: port
}
}
{
periodSeconds: 5
initialDelaySeconds: 2
type: 'Startup'
httpGet: {
path: '/startup'
port: port
}
}
]

module containerApp '../../modules/containerApp/main.bicep' = {
name: containerAppName
params: {
Expand All @@ -98,6 +130,8 @@ module containerApp '../../modules/containerApp/main.bicep' = {
apimIp: apimIp
tags: tags
resources: resources
probes: probes
port: port
revisionSuffix: revisionSuffix
}
}
Expand Down
25 changes: 3 additions & 22 deletions .azure/modules/containerApp/main.bicep
Original file line number Diff line number Diff line change
Expand Up @@ -28,30 +28,12 @@ param resources object?
@description('The suffix for the revision of the container app')
param revisionSuffix string

@description('The probes for the container app')
param probes array = []

// Container app revision name does not allow '.' character
var cleanedRevisionSuffix = replace(revisionSuffix, '.', '-')

var probes = [
{
periodSeconds: 5
initialDelaySeconds: 2
type: 'Liveness'
httpGet: {
path: '/healthz'
port: port
}
}
{
periodSeconds: 5
initialDelaySeconds: 2
type: 'Readiness'
httpGet: {
path: '/healthz'
port: port
}
}
]

var ipSecurityRestrictions = empty(apimIp)
? []
: [
Expand All @@ -74,7 +56,6 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = {
identity: {
type: 'SystemAssigned'
}

properties: {
configuration: {
ingress: ingress
Expand Down
7 changes: 7 additions & 0 deletions Digdir.Domain.Dialogporten.sln
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Digdir.Domain.Dialogporten.
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Digdir.Domain.Dialogporten.Architecture.Tests", "tests\Digdir.Domain.Dialogporten.Architecture.Tests\Digdir.Domain.Dialogporten.Architecture.Tests.csproj", "{E389C7C8-9610-40AC-86DC-769B1B7DC78E}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Digdir.Library.Utils.AspNet", "src\Digdir.Library.Utils.AspNet\Digdir.Library.Utils.AspNet.csproj", "{6A485C65-3613-4A49-A16F-2789119F6F38}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Digdir.Tool.Dialogporten.SlackNotifier.Tests", "tests\Digdir.Tool.Dialogporten.SlackNotifier.Tests\Digdir.Tool.Dialogporten.SlackNotifier.Tests.csproj", "{F7DF2792-9C83-49F7-B7DD-556E8EC577DB}"
EndProject
Global
Expand Down Expand Up @@ -155,6 +157,10 @@ Global
{E389C7C8-9610-40AC-86DC-769B1B7DC78E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E389C7C8-9610-40AC-86DC-769B1B7DC78E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E389C7C8-9610-40AC-86DC-769B1B7DC78E}.Release|Any CPU.Build.0 = Release|Any CPU
{6A485C65-3613-4A49-A16F-2789119F6F38}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6A485C65-3613-4A49-A16F-2789119F6F38}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6A485C65-3613-4A49-A16F-2789119F6F38}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6A485C65-3613-4A49-A16F-2789119F6F38}.Release|Any CPU.Build.0 = Release|Any CPU
{F7DF2792-9C83-49F7-B7DD-556E8EC577DB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F7DF2792-9C83-49F7-B7DD-556E8EC577DB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F7DF2792-9C83-49F7-B7DD-556E8EC577DB}.Release|Any CPU.ActiveCfg = Release|Any CPU
Expand Down Expand Up @@ -188,6 +194,7 @@ Global
{AF35FFCA-1206-4C08-A003-DA4A1344CCD5} = {CADB8189-4AA1-4732-844A-C41DBF3EC8B7}
{0900E3CF-F9D8-4B29-957F-484B3B028D6D} = {320B47A0-5EB8-4B6E-8C84-90633A1849CA}
{E389C7C8-9610-40AC-86DC-769B1B7DC78E} = {CADB8189-4AA1-4732-844A-C41DBF3EC8B7}
{6A485C65-3613-4A49-A16F-2789119F6F38} = {096E9B69-6783-4446-A895-0B6D7729A0D9}
{F7DF2792-9C83-49F7-B7DD-556E8EC577DB} = {CADB8189-4AA1-4732-844A-C41DBF3EC8B7}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
Expand Down
2 changes: 1 addition & 1 deletion docs/schema/V1/swagger.verified.json
Original file line number Diff line number Diff line change
Expand Up @@ -6474,4 +6474,4 @@
"url": "https://altinn-dev-api.azure-api.net/dialogporten"
}
]
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<TreatAsUsed>true</TreatAsUsed>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="8.0.0"/>
<PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks.EntityFrameworkCore" Version="8.0.0"/>
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0"/>
<PackageReference Include="Polly.Contrib.WaitAndRetry" Version="1.1.1"/>
<PackageReference Include="ZiggyCreatures.FusionCache" Version="1.4.0" />
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using StackExchange.Redis;
using Microsoft.Extensions.Options;

namespace Digdir.Domain.Dialogporten.Infrastructure.HealthChecks;

internal sealed class RedisHealthCheck : IHealthCheck
{
private readonly InfrastructureSettings _settings;

public RedisHealthCheck(IOptions<InfrastructureSettings> options)
{
_settings = options?.Value ?? throw new ArgumentNullException(nameof(options));
}

public async Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
{
try
{
using var redis = await ConnectionMultiplexer.ConnectAsync(_settings.Redis.ConnectionString);
var db = redis.GetDatabase();
await db.PingAsync();
return HealthCheckResult.Healthy("Redis connection is healthy.");
}
catch (RedisConnectionException ex)
{
return HealthCheckResult.Unhealthy("Unable to connect to Redis.", exception: ex);
}
catch (Exception ex)
{
return HealthCheckResult.Unhealthy("An unexpected error occurred while checking Redis health.", exception: ex);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@
using StackExchange.Redis;
using ZiggyCreatures.Caching.Fusion;
using ZiggyCreatures.Caching.Fusion.NullObjects;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Digdir.Domain.Dialogporten.Infrastructure.HealthChecks;

namespace Digdir.Domain.Dialogporten.Infrastructure;

Expand Down Expand Up @@ -199,6 +201,8 @@ public static IServiceCollection AddInfrastructure(this IServiceCollection servi
})
.AddPolicyHandlerFromRegistry(PollyPolicy.DefaultHttpRetryPolicy);

services.AddCustomHealthChecks();

if (environment.IsDevelopment())
{
var localDeveloperSettings = configuration.GetLocalDevelopmentSettings();
Expand All @@ -212,6 +216,17 @@ public static IServiceCollection AddInfrastructure(this IServiceCollection servi
return services;
}

private static IServiceCollection AddCustomHealthChecks(this IServiceCollection services)
{
services.AddHealthChecks()
.AddCheck<RedisHealthCheck>("redis", tags: ["dependencies", "redis"])
.AddDbContextCheck<DialogDbContext>("postgres", tags: ["dependencies", "critical"]);

services.AddSingleton<RedisHealthCheck>();

return services;
}

private static IServiceCollection AddGraphQlRedisSubscriptions(this IServiceCollection services,
string redisConnectionString)
{
Expand Down Expand Up @@ -295,4 +310,4 @@ private static IServiceCollection ConfigureFusionCache(this IServiceCollection s

return services;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@
<ItemGroup>
<ProjectReference Include="..\Digdir.Domain.Dialogporten.Application\Digdir.Domain.Dialogporten.Application.csproj"/>
<ProjectReference Include="..\Digdir.Domain.Dialogporten.Infrastructure\Digdir.Domain.Dialogporten.Infrastructure.csproj"/>
<ProjectReference Include="..\Digdir.Library.Utils.AspNet\Digdir.Library.Utils.AspNet.csproj"/>
<ProjectReference Include="..\Digdir.Tool.Dialogporten.GenerateFakeData\Digdir.Tool.Dialogporten.GenerateFakeData.csproj"/>
</ItemGroup>

</Project>
</Project>
19 changes: 14 additions & 5 deletions src/Digdir.Domain.Dialogporten.WebApi/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
using Microsoft.AspNetCore.Authorization;
using NSwag;
using Serilog;
using Digdir.Library.Utils.AspNet;
using Microsoft.Extensions.Options;

// Using two-stage initialization to catch startup errors.
Log.Logger = new LoggerConfiguration()
Expand Down Expand Up @@ -125,12 +127,17 @@ static void BuildAndRun(string[] args)
.AddControllers(options => options.InputFormatters.Insert(0, JsonPatchInputFormatter.Get()))
.AddNewtonsoftJson()
.Services

// Add health checks with the retrieved URLs
.AddAspNetHealthChecks((x, y) => x.HealthCheckSettings.HttpGetEndpointsToCheck = y
.GetRequiredService<IOptions<WebApiSettings>>().Value?
.Authentication?
.JwtBearerTokenSchemas?
.Select(z => z.WellKnown)
.ToList() ?? [])

// Auth
.AddDialogportenAuthentication(builder.Configuration)
.AddAuthorization()
.AddHealthChecks();
.AddAuthorization();

if (builder.Environment.IsDevelopment())
{
Expand All @@ -147,6 +154,9 @@ static void BuildAndRun(string[] args)

var app = builder.Build();

app.MapAspNetHealthChecks()
.MapControllers();

app.UseHttpsRedirection()
.UseSerilogRequestLogging()
.UseDefaultExceptionHandler()
Expand Down Expand Up @@ -199,8 +209,7 @@ static void BuildAndRun(string[] args)
var dialogPrefix = builder.Environment.IsDevelopment() ? "" : "/dialogporten";
uiConfig.DocumentPath = dialogPrefix + "/swagger/{documentName}/swagger.json";
});
app.MapControllers();
app.MapHealthChecks("/healthz");

app.Run();
}

Expand Down
48 changes: 48 additions & 0 deletions src/Digdir.Library.Utils.AspNet/AspNetUtilitiesExtensions.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
using Digdir.Library.Utils.AspNet.HealthChecks;
using HealthChecks.UI.Client;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Diagnostics.HealthChecks;

namespace Digdir.Library.Utils.AspNet;

public static class AspNetUtilitiesExtensions
{
public static IServiceCollection AddAspNetHealthChecks(
this IServiceCollection services,
Action<AspNetUtilitiesSettings>? configure = null)
=> services.AddAspNetHealthChecks((x, _) => configure?.Invoke(x));

public static IServiceCollection AddAspNetHealthChecks(this IServiceCollection services, Action<AspNetUtilitiesSettings, IServiceProvider>? configure = null)
{
var optionsBuilder = services.AddOptions<AspNetUtilitiesSettings>();

if (configure is not null)
{
optionsBuilder.Configure(configure);
}

return services
.AddHealthChecks()
.AddCheck("self", () => HealthCheckResult.Healthy(), tags: ["self"])
.AddCheck<EndpointsHealthCheck>(
"Endpoints",
failureStatus: HealthStatus.Unhealthy,
tags: ["external"])
.Services;
}

public static WebApplication MapAspNetHealthChecks(this WebApplication app) =>
app.MapHealthCheckEndpoint("/health/startup", check => check.Tags.Contains("dependencies"))
.MapHealthCheckEndpoint("/health/liveness", check => check.Tags.Contains("self"))
.MapHealthCheckEndpoint("/health/readiness", check => check.Tags.Contains("critical"))
.MapHealthCheckEndpoint("/health", check => check.Tags.Contains("dependencies"))
.MapHealthCheckEndpoint("/health/deep", check => check.Tags.Contains("dependencies") || check.Tags.Contains("external"));

private static WebApplication MapHealthCheckEndpoint(this WebApplication app, string path, Func<HealthCheckRegistration, bool> predicate)
{
app.MapHealthChecks(path, new HealthCheckOptions { Predicate = predicate, ResponseWriter = UIResponseWriter.WriteHealthCheckUIResponse });
return app;
}
}
11 changes: 11 additions & 0 deletions src/Digdir.Library.Utils.AspNet/AspNetUtilitiesSettings.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
namespace Digdir.Library.Utils.AspNet;

public sealed class AspNetUtilitiesSettings
{
public HealthCheckSettings HealthCheckSettings { get; set; } = new();
}

public sealed class HealthCheckSettings
{
public List<string> HttpGetEndpointsToCheck { get; set; } = [];
}
13 changes: 13 additions & 0 deletions src/Digdir.Library.Utils.AspNet/Digdir.Library.Utils.AspNet.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
<NoWarn>1591</NoWarn> <!-- Disable warnings for missing XML comments -->
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
<PackageReference Include="AspNetCore.HealthChecks.UI.Client" Version="8.0.1" />
</ItemGroup>

</Project>
Loading

0 comments on commit 6889a96

Please sign in to comment.