Skip to content

Commit

Permalink
Implement backoff algorithm for configuration worker (#324) (#337)
Browse files Browse the repository at this point in the history
  • Loading branch information
maennchen committed Feb 21, 2024
1 parent 302092f commit ab661fd
Show file tree
Hide file tree
Showing 5 changed files with 313 additions and 24 deletions.
52 changes: 52 additions & 0 deletions src/oidcc_backoff.erl
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
%%%-------------------------------------------------------------------
%% @doc Backoff Handling
%%
%% Based on `db_connection':
%% [https://github.com/elixir-ecto/db_connection/blob/8ef1f2ea54922873590b8939f2dad6b031c5b49c/lib/db_connection/backoff.ex#L24]
%% @end
%% @since 3.2.0
%%%-------------------------------------------------------------------
-module(oidcc_backoff).

-export_type([type/0]).
-export_type([min/0]).
-export_type([max/0]).
-export_type([state/0]).

-export([handle_retry/4]).

-type type() :: stop | exponential | random | random_exponential.

-type min() :: pos_integer().

-type max() :: pos_integer().

-opaque state() :: pos_integer() | {pos_integer(), pos_integer()}.

%% @private
-spec handle_retry(Type, Min, Max, State) -> stop | {Wait, State} when
Type :: type(), Min :: min(), Max :: max(), State :: undefined | state(), Wait :: pos_integer().
handle_retry(Type, Min, Max, State) when Min > 0, Max > 0, Max >= Min ->
priv_handle_retry(Type, Min, Max, State).

-spec priv_handle_retry(Type, Min, Max, State) -> stop | {Wait, State} when
Type :: type(), Min :: min(), Max :: max(), State :: undefined | state(), Wait :: pos_integer().
priv_handle_retry(stop, _Min, _Max, undefined) ->
stop;
priv_handle_retry(random, Min, Max, State) ->
{rand(Min, Max), State};
priv_handle_retry(exponential, Min, _Max, undefined) ->
{Min, Min};
priv_handle_retry(exponential, _Min, Max, State) ->
Wait = min(State * 2, Max),
{Wait, Wait};
priv_handle_retry(random_exponential, Min, Max, undefined) ->
Lower = max(Min, Max div 3),
priv_handle_retry(random_exponential, Min, Max, {Lower, Lower});
priv_handle_retry(random_exponential, _Min, Max, {Prev, Lower}) ->
NextMin = min(Prev, Lower),
NextMax = min(Prev * 3, Max),
Next = rand(NextMin, NextMax),
priv_handle_retry(random, NextMin, NextMax, {Next, Lower}).

rand(Min, Max) -> rand:uniform(Max - Min + 1) + Min - 1.
26 changes: 18 additions & 8 deletions src/oidcc_client_context.erl
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
%%%-------------------------------------------------------------------
-module(oidcc_client_context).

-feature(maybe_expr, enable).

-include("oidcc_client_context.hrl").
-include("oidcc_provider_configuration.hrl").

Expand Down Expand Up @@ -129,14 +131,22 @@ from_configuration_worker(ProviderName, ClientId, ClientSecret) ->
ClientSecret :: unauthenticated,
Opts :: unauthenticated_opts().
from_configuration_worker(ProviderName, ClientId, ClientSecret, Opts) when is_pid(ProviderName) ->
{ok,
from_manual(
oidcc_provider_configuration_worker:get_provider_configuration(ProviderName),
oidcc_provider_configuration_worker:get_jwks(ProviderName),
ClientId,
ClientSecret,
Opts
)};
maybe
#oidcc_provider_configuration{} =
ProviderConfiguration ?=
oidcc_provider_configuration_worker:get_provider_configuration(ProviderName),
#jose_jwk{} = Jwks ?= oidcc_provider_configuration_worker:get_jwks(ProviderName),
{ok,
from_manual(
ProviderConfiguration,
Jwks,
ClientId,
ClientSecret,
Opts
)}
else
undefined -> {error, provider_not_ready}
end;
from_configuration_worker(ProviderName, ClientId, ClientSecret, Opts) ->
case erlang:whereis(ProviderName) of
undefined ->
Expand Down
90 changes: 74 additions & 16 deletions src/oidcc_provider_configuration_worker.erl
Original file line number Diff line number Diff line change
Expand Up @@ -34,18 +34,29 @@

-export_type([opts/0]).

-type opts() :: #{
name => gen_server:server_name(),
issuer := uri_string:uri_string(),
provider_configuration_opts => oidcc_provider_configuration:opts(),
backoff_min => oidcc_backoff:min(),
backoff_max => oidcc_backoff:max(),
backoff_type => oidcc_backoff:type()
}.
%% Configuration Options
%%
%% <ul>
%% <li>`name' - The gen_server name of the provider.</li>
%% <li>`issuer' - The issuer URI.</li>
%% <li>`provider_configuration_opts' - Options for the provider configuration fetching.</li>
%% <li>`provider_configuration_opts' - Options for the provider configuration
%% fetching.</li>
%% <li>`backoff_min' - The minimum backoff interval in ms
%% (default: `1_000`)</li>
%% <li>`backoff_max' - The maximum backoff interval in ms
%% (default: `30_000`)</li>
%% <li>`backoff_type' - The backoff strategy, `stop' for no backoff and
%% to stop, `exponential' for exponential, `random' for random and
%% `random_exponential' for random exponential (default: `stop')</li>
%% </ul>
-type opts() :: #{
name => gen_server:server_name(),
issuer := uri_string:uri_string(),
provider_configuration_opts => oidcc_provider_configuration:opts()
}.

-record(state, {
provider_configuration = undefined :: #oidcc_provider_configuration{} | undefined,
Expand All @@ -54,9 +65,15 @@
provider_configuration_opts :: oidcc_provider_configuration:opts(),
configuration_refresh_timer = undefined :: timer:tref() | undefined,
jwks_refresh_timer = undefined :: timer:tref() | undefined,
ets_table = undefined :: ets:table() | undefined
ets_table = undefined :: ets:table() | undefined,
backoff_min = 1000 :: oidcc_backoff:min(),
backoff_max = 30000 :: oidcc_backoff:max(),
backoff_type = stop :: oidcc_backoff:type(),
backoff_state = undefined :: oidcc_backoff:state() | undefined
}).

-type state() :: #state{}.

%% @doc Start Configuration Provider
%%
%% <h2>Examples</h2>
Expand Down Expand Up @@ -110,7 +127,10 @@ init(Opts) ->
#state{
issuer = Issuer,
provider_configuration_opts = ProviderConfigurationOpts,
ets_table = EtsTable
ets_table = EtsTable,
backoff_min = maps:get(backoff_min, Opts, 1000),
backoff_max = maps:get(backoff_max, Opts, 30000),
backoff_type = maps:get(backoff_type, Opts, stop)
},
{continue, load_configuration}}
end.
Expand Down Expand Up @@ -161,12 +181,12 @@ handle_continue(
ok = store_in_ets(EtsTable, provider_configuration, Configuration),
{noreply,
State#state{
provider_configuration = Configuration, configuration_refresh_timer = NewTimer
provider_configuration = Configuration,
configuration_refresh_timer = NewTimer
},
{continue, load_jwks}}
else
{error, Reason} ->
{stop, {configuration_load_failed, Reason}, State}
{error, Reason} -> handle_backoff_retry(configuration_load_failed, Reason, State)
end;
handle_continue(
load_jwks,
Expand All @@ -187,13 +207,18 @@ handle_continue(
oidcc_provider_configuration:load_jwks(JwksUri, ProviderConfigurationOpts),
{ok, NewTimer} = timer:send_after(Expiry, jwks_expired),
ok = store_in_ets(EtsTable, jwks, Jwks),
{noreply, State#state{jwks = Jwks, jwks_refresh_timer = NewTimer}}
{noreply, State#state{
jwks = Jwks,
jwks_refresh_timer = NewTimer,
backoff_state = undefined
}}
else
{error, Reason} ->
{stop, {jwks_load_failed, Reason}, State}
{error, Reason} -> handle_backoff_retry(jwks_load_failed, Reason, State)
end.

%% @private
handle_info(backoff_retry, State) ->
{noreply, State, {continue, load_configuration}};
handle_info(configuration_expired, State) ->
{noreply, State#state{configuration_refresh_timer = undefined, jwks_refresh_timer = undefined},
{continue, load_configuration}};
Expand All @@ -202,12 +227,12 @@ handle_info(jwks_expired, State) ->

%% @doc Get Configuration
-spec get_provider_configuration(Name :: gen_server:server_ref()) ->
oidcc_provider_configuration:t().
oidcc_provider_configuration:t() | undefined.
get_provider_configuration(Name) ->
lookup_in_ets_or_call(Name, provider_configuration, get_provider_configuration).

%% @doc Get Parsed Jwks
-spec get_jwks(Name :: gen_server:server_ref()) -> jose_jwk:key().
-spec get_jwks(Name :: gen_server:server_ref()) -> jose_jwk:key() | undefined.
get_jwks(Name) ->
lookup_in_ets_or_call(Name, jwks, get_jwks).

Expand Down Expand Up @@ -366,3 +391,36 @@ register_ets_table(Opts) ->
_OtherName ->
undefined
end.

-spec handle_backoff_retry(ErrorType, Reason, State) ->
{stop, {ErrorType, Reason}, State} | {noreply, State}
when
ErrorType :: jwks_load_failed | configuration_load_failed,
Reason :: term(),
State :: state().
handle_backoff_retry(
ErrorType,
Reason,
#state{
issuer = Issuer,
backoff_min = BackoffMin,
backoff_max = BackoffMax,
backoff_type = BackoffType,
backoff_state = BackoffState
} = State
) ->
ErrorDetails = {ErrorType, Reason},
case oidcc_backoff:handle_retry(BackoffType, BackoffMin, BackoffMax, BackoffState) of
stop ->
{stop, ErrorDetails, State};
{Wait, NewBackoffState} ->
logger:error(
"Metadata load failed for issuer ~s. Retrying in ~w ms. Error Details: ~w",
[Issuer, Wait, ErrorDetails],
#{error => ErrorDetails}
),
timer:send_after(Wait, backoff_retry),
{noreply, State#state{
backoff_state = NewBackoffState
}}
end.
114 changes: 114 additions & 0 deletions test/oidcc_backoff_test.erl
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
%% Based on https://github.com/elixir-ecto/db_connection/blob/8ef1f2ea54922873590b8939f2dad6b031c5b49c/test/db_connection/backoff_test.exs

-module(oidcc_backoff_test).

-include_lib("eunit/include/eunit.hrl").

exp_backoff_in_min_max_test() ->
Min = 1000,
Max = 30000,
lists:map(
fun(Retry) ->
?assertMatch({wait, _}, Retry),

{wait, Wait} = Retry,

?assert(Wait >= Min),
?assert(Wait =< Max)
end,
calculate_backoffs(20, exponential, Min, Max)
).

exp_backoff_double_until_max_test() ->
Min = 1000,
Max = 30000,
lists:foldl(
fun
({wait, Wait}, undefined) ->
Wait;
(Retry, Prev) ->
?assertMatch({wait, _}, Retry),

{wait, Wait} = Retry,

?assert(((Wait div 2) =:= Prev) or (Wait =:= Max)),

Wait
end,
undefined,
calculate_backoffs(20, exponential, Min, Max)
).

rand_backoff_in_min_max_test() ->
Min = 1000,
Max = 30000,
lists:map(
fun(Retry) ->
?assertMatch({wait, _}, Retry),

{wait, Wait} = Retry,

?assert(Wait >= Min),
?assert(Wait =< Max)
end,
calculate_backoffs(20, random, Min, Max)
).

rand_backoff_different_every_time_test() ->
Min = 1000,
Max = 30000,
Comparison = calculate_backoffs(20, random, Min, Max),
lists:map(
fun(_) ->
?assertNotEqual(Comparison, calculate_backoffs(20, random, Min, Max))
end,
lists:seq(1, 100)
).

rand_exp_backoff_in_min_max_test() ->
Min = 1000,
Max = 30000,
lists:map(
fun(Retry) ->
?assertMatch({wait, _}, Retry),

{wait, Wait} = Retry,

?assert(Wait >= Min),
?assert(Wait =< Max)
end,
calculate_backoffs(20, random_exponential, Min, Max)
).

rand_exp_backoff_increase_until_third_max_test() ->
Min = 1000,
Max = 30000,
lists:foldl(
fun
({wait, Wait}, undefined) ->
Wait;
(Retry, Prev) ->
?assertMatch({wait, _}, Retry),

{wait, Wait} = Retry,

?assert((Wait >= Prev) or (Wait >= (Max div 3))),

Wait
end,
undefined,
calculate_backoffs(20, random_exponential, Min, Max)
).

calculate_backoffs(N, Type, Min, Max) ->
lists:reverse(calculate_backoffs(N, Type, Min, Max, undefined, [])).

calculate_backoffs(0, _Type, _Min, _Max, _State, Acc) ->
Acc;
calculate_backoffs(N, Type, Min, Max, State, Acc) ->
case oidcc_backoff:handle_retry(Type, Min, Max, State) of
stop ->
[stop | Acc];
{Wait, NewState} ->
calculate_backoffs(N - 1, Type, Min, Max, NewState, [{wait, Wait} | Acc])
end.
Loading

0 comments on commit ab661fd

Please sign in to comment.