diff --git a/src/weatherreport/src/weatherreport.erl b/src/weatherreport/src/weatherreport.erl index d15ba568086..67283178b36 100644 --- a/src/weatherreport/src/weatherreport.erl +++ b/src/weatherreport/src/weatherreport.erl @@ -61,7 +61,8 @@ {expert, $e, "expert", undefined, "Perform more detailed diagnostics" }, {usage, $h, "help", undefined, "Display help/usage" }, {list, $l, "list", undefined, "Describe available diagnostic tasks" }, - {all_nodes, $a, "all-nodes", undefined, "Run weatherreport on all cluster nodes" } + {all_nodes, $a, "all-nodes", undefined, "Run weatherreport on all cluster nodes" }, + {timeout, $t, "timeout", integer, "Timeout value (in ms) for each diagnostic check" } ]). -define(USAGE_OPTS, [ O || O <- ?OPTS, @@ -115,7 +116,6 @@ run(InputChecks) -> weatherreport_runner:run(Checks, all); _ -> weatherreport_runner:run(Checks) - end, case Messages of [] -> @@ -163,6 +163,9 @@ process_option({etc,Path}, Result) -> process_option({level, Level}, Result) -> application:set_env(weatherreport, log_level, Level), Result; +process_option({timeout, Timeout}, Result) -> + application:set_env(weatherreport, timeout, Timeout), + Result; process_option(expert, Result) -> application:set_env(weatherreport, expert, true), Result; diff --git a/src/weatherreport/src/weatherreport_config.erl b/src/weatherreport/src/weatherreport_config.erl index e93da83595c..cee6c9a75e9 100644 --- a/src/weatherreport/src/weatherreport_config.erl +++ b/src/weatherreport/src/weatherreport_config.erl @@ -39,6 +39,7 @@ data_directories/0, get_vm_env/1, etc_dir/0, + timeout/0, node_name/0, cookie/0, user/0]). @@ -87,6 +88,16 @@ user() -> lists:reverse(Resp1) end. +%% @doc The specified timeout value for diagnostic checks run via RPC +-spec timeout() -> integer(). +timeout() -> + case application:get_env(weatherreport, timeout) of + {ok, Timeout} -> + Timeout; + _ -> + 300000 + end. + %% @doc The CouchDB configuration directory. -spec etc_dir() -> file:filename(). etc_dir() -> diff --git a/src/weatherreport/src/weatherreport_node.erl b/src/weatherreport/src/weatherreport_node.erl index 1df6cc8497f..e353bf3bfa5 100644 --- a/src/weatherreport/src/weatherreport_node.erl +++ b/src/weatherreport/src/weatherreport_node.erl @@ -55,7 +55,7 @@ local_command(Module, Function) -> %% @see can_connect/0 -spec local_command(Module::atom(), Function::atom(), Args::[term()]) -> term(). local_command(Module, Function, Args) -> - local_command(Module, Function, Args, 5000). + local_command(Module, Function, Args, weatherreport_config:timeout()). %% @doc Calls the given module and function with the given arguments %% on the local node and returns the result of that call, @@ -88,7 +88,12 @@ local_command(Module, Function, Args, Timeout) -> %% escript. -spec multicall([node()], Module::atom(), Function::atom(), Args::[term()], Timeout::integer()) -> term(). multicall(Nodes, Module, Function, Args, Timeout) -> - local_command(rpc, multicall, [Nodes, Module, Function, Args, Timeout]). + case local_command(rpc, multicall, [Nodes, Module, Function, Args, Timeout]) of + {badrpc, Reason} -> + {[{badrpc, Reason}], []}; + Resp -> + Resp + end. %% @doc Retrieves the operating system's process ID of the local %% node. diff --git a/src/weatherreport/src/weatherreport_runner.erl b/src/weatherreport/src/weatherreport_runner.erl index 95da6911e00..1beb02e0e98 100644 --- a/src/weatherreport/src/weatherreport_runner.erl +++ b/src/weatherreport/src/weatherreport_runner.erl @@ -57,21 +57,35 @@ run(Checks, Nodes) -> erlang, apply, [fun() -> {node(), weatherreport_check:check(Mod, CheckOpts)} end, []], - 5000 + weatherreport_config:timeout() ), lists:map(fun(Node) -> weatherreport_util:log( node(), error, - io_lib:format("Could not run checks on cluster node ~w", [Node]) + io_lib:format( + "Could not run check ~w on cluster node ~w", + [Mod, Node] + ) ) end, BadNodes), + LogBadRpc = fun({badrpc, Error}) -> + weatherreport_util:log( + node(), + error, + io_lib:format( + "Bad rpc call executing check ~w: ~w", + [Mod, Error] + ) + ) + end, + [LogBadRpc(Resp) || {badrpc, _Error}=Resp <- Resps], TransformResponse = fun({Node, Messages}) -> [{Node, Lvl, Module, Msg} || {Lvl, Module, Msg} <- Messages] end, ResponsesWithNode = [ TransformResponse({Node, Messages}) || {Node, Messages} <- Resps, - Node =/= bad_rpc + Node =/= badrpc ], [lists:concat(ResponsesWithNode) | Acc] end, [], Checks)).