Merge pull request #8 from cloudant/33697-make-timeout-configurable-a…

…nd-raise-default

33697 make timeout configurable and raise default

src/weatherreport/src/weatherreport.erl

@@ -61,7 +61,8 @@
                {expert, $e,       "expert", undefined,     "Perform more detailed diagnostics"               },
                {usage, $h,        "help",  undefined,      "Display help/usage"                              },
                {list,  $l,        "list",  undefined,      "Describe available diagnostic tasks"             },
-               {all_nodes, $a,    "all-nodes", undefined,  "Run weatherreport on all cluster nodes"          }
+               {all_nodes, $a,    "all-nodes", undefined,  "Run weatherreport on all cluster nodes"          },
+               {timeout, $t,      "timeout", integer,      "Timeout value (in ms) for each diagnostic check"              }
               ]).
 
 -define(USAGE_OPTS, [ O || O <- ?OPTS,
@@ -115,7 +116,6 @@ run(InputChecks) ->
             weatherreport_runner:run(Checks, all);
         _ ->
             weatherreport_runner:run(Checks)
-
     end,
     case Messages of
     [] ->
@@ -163,6 +163,9 @@ process_option({etc,Path}, Result) ->
 process_option({level, Level}, Result) ->
     application:set_env(weatherreport, log_level, Level),
     Result;
+process_option({timeout, Timeout}, Result) ->
+    application:set_env(weatherreport, timeout, Timeout),
+    Result;
 process_option(expert, Result) ->
     application:set_env(weatherreport, expert, true),
     Result;

src/weatherreport/src/weatherreport_config.erl

@@ -39,6 +39,7 @@
          data_directories/0,
          get_vm_env/1,
          etc_dir/0,
+         timeout/0,
          node_name/0,
          cookie/0,
          user/0]).
@@ -87,6 +88,16 @@ user() ->
             lists:reverse(Resp1)
     end.
 
+%% @doc The specified timeout value for diagnostic checks run via RPC
+-spec timeout() -> integer().
+timeout() ->
+    case application:get_env(weatherreport, timeout) of
+        {ok, Timeout} ->
+            Timeout;
+        _ ->
+            300000
+    end.
+
 %% @doc The CouchDB configuration directory.
 -spec etc_dir() -> file:filename().
 etc_dir() ->

src/weatherreport/src/weatherreport_node.erl

@@ -55,7 +55,7 @@ local_command(Module, Function) ->
 %% @see can_connect/0
 -spec local_command(Module::atom(), Function::atom(), Args::[term()]) -> term().
 local_command(Module, Function, Args) ->
-    local_command(Module, Function, Args, 5000).
+    local_command(Module, Function, Args, weatherreport_config:timeout()).
 
 %% @doc Calls the given module and function with the given arguments
 %% on the local node and returns the result of that call,
@@ -88,7 +88,12 @@ local_command(Module, Function, Args, Timeout) ->
 %% escript.
 -spec multicall([node()], Module::atom(), Function::atom(), Args::[term()], Timeout::integer()) -> term().
 multicall(Nodes, Module, Function, Args, Timeout) ->
-    local_command(rpc, multicall, [Nodes, Module, Function, Args, Timeout]).
+    case local_command(rpc, multicall, [Nodes, Module, Function, Args, Timeout]) of
+        {badrpc, Reason} ->
+            {[{badrpc, Reason}], []};
+        Resp ->
+            Resp
+    end.
 
 %% @doc Retrieves the operating system's process ID of the local
 %% node.

src/weatherreport/src/weatherreport_runner.erl

@@ -57,21 +57,35 @@ run(Checks, Nodes) ->
             erlang,
             apply,
             [fun() -> {node(), weatherreport_check:check(Mod, CheckOpts)} end, []],
-            5000
+            weatherreport_config:timeout()
         ),
         lists:map(fun(Node) ->
             weatherreport_util:log(
                 node(),
                 error,
-                io_lib:format("Could not run checks on cluster node ~w", [Node])
+                io_lib:format(
+                    "Could not run check ~w on cluster node ~w",
+                    [Mod, Node]
+                )
             )
         end, BadNodes),
+        LogBadRpc = fun({badrpc, Error}) ->
+            weatherreport_util:log(
+                node(),
+                error,
+                io_lib:format(
+                    "Bad rpc call executing check ~w: ~w",
+                    [Mod, Error]
+                )
+            )
+        end,
+        [LogBadRpc(Resp) || {badrpc, _Error}=Resp <- Resps],
         TransformResponse = fun({Node, Messages}) ->
             [{Node, Lvl, Module, Msg} || {Lvl, Module, Msg} <- Messages]
         end,
         ResponsesWithNode = [
             TransformResponse({Node, Messages}) || {Node, Messages} <- Resps,
-            Node =/= bad_rpc
+            Node =/= badrpc
         ],
         [lists:concat(ResponsesWithNode) | Acc]
     end, [], Checks)).