-
Notifications
You must be signed in to change notification settings - Fork 648
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add API for configuring rate limiting (#6196)
Add basic system outage rate limiter Co-authored-by: Szymon Pobiega <szymon.pobiega@gmail.com> Co-authored-by: Michał Wójcik <michal.wojcik@particular.net>
- Loading branch information
1 parent
0fae8bb
commit 7d94fcf
Showing
17 changed files
with
810 additions
and
26 deletions.
There are no files selected for viewing
124 changes: 124 additions & 0 deletions
124
src/NServiceBus.AcceptanceTests/Core/Recoverability/When_messages_never_succeed.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,124 @@ | ||
namespace NServiceBus.AcceptanceTests.Core.Recoverability | ||
{ | ||
using System; | ||
using System.Threading; | ||
using System.Threading.Tasks; | ||
using AcceptanceTesting; | ||
using EndpointTemplates; | ||
using NUnit.Framework; | ||
|
||
public class When_messages_never_succeed : NServiceBusAcceptanceTest | ||
{ | ||
public static int NumberOfConsecutiveFailuresBeforeThrottling = 1; | ||
public static TimeSpan TimeToWaitBetweenThrottledAttempts = TimeSpan.FromSeconds(0); | ||
|
||
[Test] | ||
public async Task Should_throttle_pipeline_after_configured_number_of_consecutive_failures() | ||
{ | ||
NumberOfConsecutiveFailuresBeforeThrottling = 5; | ||
|
||
var context = await Scenario.Define<Context>() | ||
.WithEndpoint<EndpointWithFailingHandler>(b => b | ||
.DoNotFailOnErrorMessages() | ||
.When(async (session, ctx) => | ||
{ | ||
for (var x = 0; x < 10; x++) | ||
{ | ||
await session.SendLocal(new InitiatingMessage | ||
{ | ||
Id = ctx.TestRunId | ||
}); | ||
} | ||
}) | ||
) | ||
.Done(c => c.ThrottleModeEntered && c.FailuresBeforeThrottling >= NumberOfConsecutiveFailuresBeforeThrottling) | ||
.Run(); | ||
} | ||
|
||
[Test] | ||
public async Task Should_not_throttle_pipeline_if_number_of_consecutive_failures_is_below_threshold() | ||
{ | ||
NumberOfConsecutiveFailuresBeforeThrottling = 100; | ||
|
||
var context = await Scenario.Define<Context>() | ||
.WithEndpoint<EndpointWithFailingHandler>(b => b | ||
.DoNotFailOnErrorMessages() | ||
.When(async (session, ctx) => | ||
{ | ||
for (var x = 0; x < 10; x++) | ||
{ | ||
await session.SendLocal(new InitiatingMessage | ||
{ | ||
Id = ctx.TestRunId | ||
}); | ||
} | ||
}) | ||
) | ||
.Done(c => c.FailuresBeforeThrottling == 10 && !c.ThrottleModeEntered) | ||
.Run(); | ||
} | ||
|
||
class Context : ScenarioContext | ||
{ | ||
public bool ThrottleModeEntered { get; set; } | ||
public static int failuresBeforeThrottling; | ||
public DateTime LastProcessedTimeStamp { get; set; } | ||
public TimeSpan TimeBetweenProcessingAttempts { get; set; } | ||
|
||
public int FailuresBeforeThrottling => failuresBeforeThrottling; | ||
} | ||
|
||
class EndpointWithFailingHandler : EndpointConfigurationBuilder | ||
{ | ||
public EndpointWithFailingHandler() | ||
{ | ||
EndpointSetup<DefaultServer>((config, context) => | ||
{ | ||
config.LimitMessageProcessingConcurrencyTo(3); | ||
var scenarioContext = (Context)context.ScenarioContext; | ||
var recoverability = config.Recoverability(); | ||
recoverability.Immediate(i => i.NumberOfRetries(0)); | ||
recoverability.Delayed(d => d.NumberOfRetries(0)); | ||
var rateLimitingSettings = new RateLimitSettings(TimeToWaitBetweenThrottledAttempts, () => | ||
{ | ||
scenarioContext.ThrottleModeEntered = true; | ||
return Task.FromResult(0); | ||
}); | ||
recoverability.OnConsecutiveFailures(NumberOfConsecutiveFailuresBeforeThrottling, rateLimitingSettings); | ||
}); | ||
} | ||
|
||
class InitiatingHandler : IHandleMessages<InitiatingMessage> | ||
{ | ||
public InitiatingHandler(Context testContext) | ||
{ | ||
this.testContext = testContext; | ||
} | ||
public Task Handle(InitiatingMessage initiatingMessage, IMessageHandlerContext context) | ||
{ | ||
if (testContext.ThrottleModeEntered) | ||
{ | ||
testContext.TimeBetweenProcessingAttempts = DateTime.Now - testContext.LastProcessedTimeStamp; | ||
} | ||
|
||
testContext.LastProcessedTimeStamp = DateTime.Now; | ||
|
||
Interlocked.Increment(ref Context.failuresBeforeThrottling); | ||
|
||
throw new SimulatedException("THIS IS A MESSAGE THAT WILL NEVER SUCCEED"); | ||
} | ||
Context testContext; | ||
} | ||
} | ||
|
||
public class InitiatingMessage : IMessage | ||
{ | ||
public Guid Id { get; set; } | ||
} | ||
} | ||
} |
94 changes: 94 additions & 0 deletions
94
....AcceptanceTests/Core/Recoverability/When_messages_never_succeed_with_delays_specified.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,94 @@ | ||
namespace NServiceBus.AcceptanceTests.Core.Recoverability | ||
{ | ||
using System; | ||
using System.Threading; | ||
using System.Threading.Tasks; | ||
using AcceptanceTesting; | ||
using EndpointTemplates; | ||
using NUnit.Framework; | ||
|
||
public class When_messages_never_succeed_with_delays_specified : NServiceBusAcceptanceTest | ||
{ | ||
[Test] | ||
public async Task Should_wait_the_configured_delay_between_processing_attempts_in_throttled_mode() | ||
{ | ||
var context = await Scenario.Define<Context>() | ||
.WithEndpoint<EndpointWithFailingHandler>(b => b | ||
.DoNotFailOnErrorMessages() | ||
.When(async (session, ctx) => | ||
{ | ||
for (var x = 0; x < 5; x++) | ||
{ | ||
await session.SendLocal(new InitiatingMessage | ||
{ | ||
Id = ctx.TestRunId | ||
}); | ||
} | ||
}) | ||
) | ||
.Done(c => c.ThrottleModeEntered && c.TimeBetweenProcessingAttempts >= TimeSpan.FromSeconds(2)) | ||
.Run(); | ||
} | ||
|
||
class Context : ScenarioContext | ||
{ | ||
public bool ThrottleModeEntered { get; set; } | ||
public static int failuresBeforeThrottling; | ||
public DateTime LastProcessedTimeStamp { get; set; } | ||
public TimeSpan TimeBetweenProcessingAttempts { get; set; } | ||
} | ||
|
||
class EndpointWithFailingHandler : EndpointConfigurationBuilder | ||
{ | ||
public EndpointWithFailingHandler() | ||
{ | ||
EndpointSetup<DefaultServer>((config, context) => | ||
{ | ||
var scenarioContext = (Context)context.ScenarioContext; | ||
config.LimitMessageProcessingConcurrencyTo(1); | ||
var recoverability = config.Recoverability(); | ||
recoverability.Immediate(i => i.NumberOfRetries(0)); | ||
recoverability.Delayed(d => d.NumberOfRetries(0)); | ||
var rateLimitingSettings = new RateLimitSettings(TimeSpan.FromSeconds(2), () => | ||
{ | ||
scenarioContext.ThrottleModeEntered = true; | ||
return Task.FromResult(0); | ||
}); | ||
recoverability.OnConsecutiveFailures(1, rateLimitingSettings); | ||
}); | ||
} | ||
|
||
class InitiatingHandler : IHandleMessages<InitiatingMessage> | ||
{ | ||
public InitiatingHandler(Context context) | ||
{ | ||
testContext = context; | ||
} | ||
public Task Handle(InitiatingMessage initiatingMessage, IMessageHandlerContext context) | ||
{ | ||
if (testContext.ThrottleModeEntered) | ||
{ | ||
testContext.TimeBetweenProcessingAttempts = DateTime.Now - testContext.LastProcessedTimeStamp; | ||
} | ||
|
||
testContext.LastProcessedTimeStamp = DateTime.Now; | ||
|
||
Interlocked.Increment(ref Context.failuresBeforeThrottling); | ||
|
||
throw new SimulatedException("THIS IS A MESSAGE THAT WILL NEVER SUCCEED"); | ||
} | ||
Context testContext; | ||
} | ||
} | ||
|
||
public class InitiatingMessage : IMessage | ||
{ | ||
public Guid Id { get; set; } | ||
} | ||
} | ||
} |
96 changes: 96 additions & 0 deletions
96
...NServiceBus.AcceptanceTests/Core/Recoverability/When_messages_succeed_after_throttling.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,96 @@ | ||
namespace NServiceBus.AcceptanceTests.Core.Recoverability | ||
{ | ||
using System; | ||
using System.Threading.Tasks; | ||
using AcceptanceTesting; | ||
using EndpointTemplates; | ||
using NUnit.Framework; | ||
|
||
public class When_messages_succeed_after_throttling : NServiceBusAcceptanceTest | ||
{ | ||
[Test] | ||
public async Task Should_end_throttling_mode_after_a_single_successful_message() | ||
{ | ||
var context = await Scenario.Define<Context>() | ||
.WithEndpoint<EndpointWithFailingHandler>(b => b | ||
.DoNotFailOnErrorMessages() | ||
.When(async (session, ctx) => | ||
{ | ||
for (var x = 0; x < 10; x++) | ||
{ | ||
await session.SendLocal(new InitiatingMessage | ||
{ | ||
Id = ctx.TestRunId | ||
}); | ||
} | ||
}) | ||
) | ||
.Done(c => c.MessageProcessedNormally && c.ThrottleModeEnded) | ||
.Run(); | ||
} | ||
|
||
class Context : ScenarioContext | ||
{ | ||
public bool ThrottleModeEntered { get; set; } | ||
public bool ThrottleModeEnded { get; set; } | ||
public bool MessageProcessedNormally { get; set; } | ||
} | ||
|
||
class EndpointWithFailingHandler : EndpointConfigurationBuilder | ||
{ | ||
public EndpointWithFailingHandler() | ||
{ | ||
EndpointSetup<DefaultServer>((config, context) => | ||
{ | ||
var scenarioContext = (Context)context.ScenarioContext; | ||
config.LimitMessageProcessingConcurrencyTo(3); | ||
var recoverability = config.Recoverability(); | ||
recoverability.Immediate(i => i.NumberOfRetries(0)); | ||
recoverability.Delayed(d => d.NumberOfRetries(0)); | ||
var rateLimitingSettings = new RateLimitSettings(TimeSpan.FromSeconds(5), () => | ||
{ | ||
scenarioContext.ThrottleModeEntered = true; | ||
return Task.FromResult(0); | ||
}, | ||
() => | ||
{ | ||
scenarioContext.ThrottleModeEnded = true; | ||
return Task.FromResult(0); | ||
}); | ||
recoverability.OnConsecutiveFailures(2, rateLimitingSettings); | ||
}); | ||
} | ||
|
||
class InitiatingHandler : IHandleMessages<InitiatingMessage> | ||
{ | ||
public InitiatingHandler(Context testContext) | ||
{ | ||
this.testContext = testContext; | ||
} | ||
public Task Handle(InitiatingMessage initiatingMessage, IMessageHandlerContext context) | ||
{ | ||
if (!testContext.ThrottleModeEntered) | ||
{ | ||
throw new SimulatedException("THIS IS A MESSAGE THAT WILL NEVER SUCCEED"); | ||
} | ||
|
||
testContext.MessageProcessedNormally = true; | ||
|
||
return Task.FromResult(0); | ||
} | ||
Context testContext; | ||
} | ||
} | ||
|
||
public class InitiatingMessage : IMessage | ||
{ | ||
public Guid Id { get; set; } | ||
} | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.