From ef268bbc4a115ea823f9ecee2ae3f22a71e016b2 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Sat, 7 Sep 2024 14:20:09 -0700 Subject: [PATCH] Report both CPU and Memory issues --- .../ResourceUtilizationHealthCheck.cs | 58 +++++++--- .../ResourceHealthCheckTests.cs | 105 +++++++++++------- 2 files changed, 107 insertions(+), 56 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.Diagnostics.HealthChecks.ResourceUtilization/ResourceUtilizationHealthCheck.cs b/src/Libraries/Microsoft.Extensions.Diagnostics.HealthChecks.ResourceUtilization/ResourceUtilizationHealthCheck.cs index b60a6068b50..8965393487c 100644 --- a/src/Libraries/Microsoft.Extensions.Diagnostics.HealthChecks.ResourceUtilization/ResourceUtilizationHealthCheck.cs +++ b/src/Libraries/Microsoft.Extensions.Diagnostics.HealthChecks.ResourceUtilization/ResourceUtilizationHealthCheck.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Diagnostics.ResourceMonitoring; @@ -14,7 +15,6 @@ namespace Microsoft.Extensions.Diagnostics.HealthChecks; /// internal sealed class ResourceUtilizationHealthCheck : IHealthCheck { - private static readonly Task _healthy = Task.FromResult(HealthCheckResult.Healthy()); private readonly ResourceUtilizationHealthCheckOptions _options; private readonly IResourceMonitor _dataTracker; @@ -39,26 +39,58 @@ public ResourceUtilizationHealthCheck(IOptions CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) { var utilization = _dataTracker.GetUtilization(_options.SamplingWindow); - if (utilization.CpuUsedPercentage > _options.CpuThresholds.UnhealthyUtilizationPercentage) + IReadOnlyDictionary data = new Dictionary { - return Task.FromResult(HealthCheckResult.Unhealthy("CPU usage is above the limit")); - } + { nameof(utilization.CpuUsedPercentage), utilization.CpuUsedPercentage }, + { nameof(utilization.MemoryUsedPercentage), utilization.MemoryUsedPercentage }, + }; - if (utilization.MemoryUsedPercentage > _options.MemoryThresholds.UnhealthyUtilizationPercentage) - { - return Task.FromResult(HealthCheckResult.Unhealthy("Memory usage is above the limit")); - } + bool cpuUnhealthy = utilization.CpuUsedPercentage > _options.CpuThresholds.UnhealthyUtilizationPercentage; + bool memoryUnhealthy = utilization.MemoryUsedPercentage > _options.MemoryThresholds.UnhealthyUtilizationPercentage; - if (utilization.CpuUsedPercentage > _options.CpuThresholds.DegradedUtilizationPercentage) + if (cpuUnhealthy || memoryUnhealthy) { - return Task.FromResult(HealthCheckResult.Degraded("CPU usage is close to the limit")); + string message = string.Empty; + if (cpuUnhealthy && memoryUnhealthy) + { + message = "CPU and Memory"; + } + else if (cpuUnhealthy) + { + message = "CPU"; + } + else + { + message = "Memory"; + } + + message += " usage is above the limit"; + return Task.FromResult(HealthCheckResult.Unhealthy(message, default, data)); } - if (utilization.MemoryUsedPercentage > _options.MemoryThresholds.DegradedUtilizationPercentage) + bool cpuDegraded = utilization.CpuUsedPercentage > _options.CpuThresholds.DegradedUtilizationPercentage; + bool memoryDegraded = utilization.MemoryUsedPercentage > _options.MemoryThresholds.DegradedUtilizationPercentage; + + if (cpuDegraded || memoryDegraded) { - return Task.FromResult(HealthCheckResult.Degraded("Memory usage is close to the limit")); + string message = string.Empty; + if (cpuDegraded && memoryDegraded) + { + message = "CPU and Memory"; + } + else if (cpuDegraded) + { + message = "CPU"; + } + else + { + message = "Memory"; + } + + message += " usage is close to the limit"; + return Task.FromResult(HealthCheckResult.Degraded(message, default, data)); } - return _healthy; + return Task.FromResult(HealthCheckResult.Healthy(default, data)); } } diff --git a/test/Libraries/Microsoft.Extensions.Diagnostics.HealthChecks.ResourceUtilization.Tests/ResourceHealthCheckTests.cs b/test/Libraries/Microsoft.Extensions.Diagnostics.HealthChecks.ResourceUtilization.Tests/ResourceHealthCheckTests.cs index d6129017d79..0ab298ec3f4 100644 --- a/test/Libraries/Microsoft.Extensions.Diagnostics.HealthChecks.ResourceUtilization.Tests/ResourceHealthCheckTests.cs +++ b/test/Libraries/Microsoft.Extensions.Diagnostics.HealthChecks.ResourceUtilization.Tests/ResourceHealthCheckTests.cs @@ -23,6 +23,7 @@ public class ResourceHealthCheckTests 0UL, 1000UL, new ResourceUsageThresholds(), + new ResourceUsageThresholds(), "", }, new object[] @@ -32,6 +33,7 @@ public class ResourceHealthCheckTests 0UL, 1000UL, new ResourceUsageThresholds { DegradedUtilizationPercentage = 0.2, UnhealthyUtilizationPercentage = 0.2 }, + new ResourceUsageThresholds { DegradedUtilizationPercentage = 0.2, UnhealthyUtilizationPercentage = 0.2 }, "" }, new object[] @@ -41,6 +43,7 @@ public class ResourceHealthCheckTests 2UL, 1000UL, new ResourceUsageThresholds { DegradedUtilizationPercentage = 0.2, UnhealthyUtilizationPercentage = 0.2 }, + new ResourceUsageThresholds { DegradedUtilizationPercentage = 0.2, UnhealthyUtilizationPercentage = 0.2 }, "" }, new object[] @@ -50,7 +53,8 @@ public class ResourceHealthCheckTests 3UL, 1000UL, new ResourceUsageThresholds { DegradedUtilizationPercentage = 0.2, UnhealthyUtilizationPercentage = 0.4 }, - " usage is close to the limit" + new ResourceUsageThresholds { DegradedUtilizationPercentage = 0.2, UnhealthyUtilizationPercentage = 0.4 }, + "CPU and Memory usage is close to the limit" }, new object[] { @@ -59,7 +63,8 @@ public class ResourceHealthCheckTests 5UL, 1000UL, new ResourceUsageThresholds { DegradedUtilizationPercentage = 0.2, UnhealthyUtilizationPercentage = 0.4 }, - " usage is above the limit" + new ResourceUsageThresholds { DegradedUtilizationPercentage = 0.2, UnhealthyUtilizationPercentage = 0.4 }, + "CPU and Memory usage is above the limit" }, new object[] { @@ -68,7 +73,8 @@ public class ResourceHealthCheckTests 5UL, 1000UL, new ResourceUsageThresholds { DegradedUtilizationPercentage = 0.4, UnhealthyUtilizationPercentage = 0.2 }, - " usage is above the limit" + new ResourceUsageThresholds { DegradedUtilizationPercentage = 0.4, UnhealthyUtilizationPercentage = 0.2 }, + "CPU and Memory usage is above the limit" }, new object[] { @@ -77,7 +83,8 @@ public class ResourceHealthCheckTests 3UL, 1000UL, new ResourceUsageThresholds { DegradedUtilizationPercentage = 0.2 }, - " usage is close to the limit" + new ResourceUsageThresholds { DegradedUtilizationPercentage = 0.2 }, + "CPU and Memory usage is close to the limit" }, new object[] { @@ -86,67 +93,79 @@ public class ResourceHealthCheckTests 5UL, 1000UL, new ResourceUsageThresholds { UnhealthyUtilizationPercentage = 0.4 }, - " usage is above the limit" + new ResourceUsageThresholds { UnhealthyUtilizationPercentage = 0.4 }, + "CPU and Memory usage is above the limit" + }, + new object[] + { + HealthStatus.Degraded, + 0.3, + 3UL, + 1000UL, + new ResourceUsageThresholds { DegradedUtilizationPercentage = 0.2, UnhealthyUtilizationPercentage = 0.4 }, + new ResourceUsageThresholds { DegradedUtilizationPercentage = 0.9, UnhealthyUtilizationPercentage = 0.9 }, + "CPU usage is close to the limit" + }, + new object[] + { + HealthStatus.Degraded, + 0.1, + 3UL, + 1000UL, + new ResourceUsageThresholds { DegradedUtilizationPercentage = 0.9, UnhealthyUtilizationPercentage = 0.9 }, + new ResourceUsageThresholds { DegradedUtilizationPercentage = 0.2, UnhealthyUtilizationPercentage = 0.4 }, + "Memory usage is close to the limit" + }, + new object[] + { + HealthStatus.Unhealthy, + 0.5, + 5UL, + 1000UL, + new ResourceUsageThresholds { DegradedUtilizationPercentage = 0.2, UnhealthyUtilizationPercentage = 0.4 }, + new ResourceUsageThresholds { DegradedUtilizationPercentage = 0.9, UnhealthyUtilizationPercentage = 0.9 }, + "CPU usage is above the limit" + }, + new object[] + { + HealthStatus.Unhealthy, + 0.1, + 5UL, + 1000UL, + new ResourceUsageThresholds { DegradedUtilizationPercentage = 0.9, UnhealthyUtilizationPercentage = 0.9 }, + new ResourceUsageThresholds { DegradedUtilizationPercentage = 0.2, UnhealthyUtilizationPercentage = 0.4 }, + "Memory usage is above the limit" }, }; [Theory] [MemberData(nameof(Data))] -#pragma warning disable xUnit1026 // Theory methods should use all of their parameters - public async Task TestCpuChecks(HealthStatus expected, double utilization, ulong _, ulong totalMemory, ResourceUsageThresholds thresholds, string expectedDescription) -#pragma warning restore xUnit1026 // Theory methods should use all of their parameters - { - var systemResources = new SystemResources(1.0, 1.0, totalMemory, totalMemory); - var dataTracker = new Mock(); - var samplingWindow = TimeSpan.FromSeconds(1); - dataTracker - .Setup(tracker => tracker.GetUtilization(samplingWindow)) - .Returns(new ResourceUtilization(cpuUsedPercentage: utilization, memoryUsedInBytes: 0, systemResources)); - - var checkContext = new HealthCheckContext(); - var cpuCheckOptions = new ResourceUtilizationHealthCheckOptions - { - CpuThresholds = thresholds, - SamplingWindow = samplingWindow - }; - - var options = Microsoft.Extensions.Options.Options.Create(cpuCheckOptions); - var healthCheck = new ResourceUtilizationHealthCheck(options, dataTracker.Object); - var healthCheckResult = await healthCheck.CheckHealthAsync(checkContext); - Assert.Equal(expected, healthCheckResult.Status); - if (healthCheckResult.Status != HealthStatus.Healthy) - { - Assert.Equal("CPU" + expectedDescription, healthCheckResult.Description); - } - } - - [Theory] - [MemberData(nameof(Data))] -#pragma warning disable xUnit1026 // Theory methods should use all of their parameters - public async Task TestMemoryChecks(HealthStatus expected, double _, ulong memoryUsed, ulong totalMemory, ResourceUsageThresholds thresholds, string expectedDescription) -#pragma warning restore xUnit1026 // Theory methods should use all of their parameters + public async Task TestCpuAndMemoryChecks(HealthStatus expected, double utilization, ulong memoryUsed, ulong totalMemory, + ResourceUsageThresholds cpuThresholds, ResourceUsageThresholds memoryThresholds, string expectedDescription) { var systemResources = new SystemResources(1.0, 1.0, totalMemory, totalMemory); var dataTracker = new Mock(); var samplingWindow = TimeSpan.FromSeconds(1); dataTracker .Setup(tracker => tracker.GetUtilization(samplingWindow)) - .Returns(new ResourceUtilization(cpuUsedPercentage: 0, memoryUsedInBytes: memoryUsed, systemResources)); + .Returns(new ResourceUtilization(cpuUsedPercentage: utilization, memoryUsedInBytes: memoryUsed, systemResources)); var checkContext = new HealthCheckContext(); - var memCheckOptions = new ResourceUtilizationHealthCheckOptions + var checkOptions = new ResourceUtilizationHealthCheckOptions { - MemoryThresholds = thresholds, + CpuThresholds = cpuThresholds, + MemoryThresholds = memoryThresholds, SamplingWindow = samplingWindow }; - var options = Microsoft.Extensions.Options.Options.Create(memCheckOptions); + var options = Microsoft.Extensions.Options.Options.Create(checkOptions); var healthCheck = new ResourceUtilizationHealthCheck(options, dataTracker.Object); var healthCheckResult = await healthCheck.CheckHealthAsync(checkContext); Assert.Equal(expected, healthCheckResult.Status); + Assert.NotEmpty(healthCheckResult.Data); if (healthCheckResult.Status != HealthStatus.Healthy) { - Assert.Equal("Memory" + expectedDescription, healthCheckResult.Description); + Assert.Equal(expectedDescription, healthCheckResult.Description); } }