sync routes (#303)

* Sync Routes can add a new route from config service

If a route does not exist locally, we will insert it, and fetch all of its
parts. DevaddrRanges, Skfs, Euis.

* handle removing routes that are left over after syncing

* move syncing routes to cli module

- add cli config test suite
- pretty print the added and removed ids or routes

The diffing for the routes is done in the cli module, this way the route
stream worker doesn't start to become like the xor filter worker where
it started to hold the kitchen sink.

Routes are only sent to the route stream worker after they have been
inserted and need their parts updated.

* add modules for org list request and response

config/ct.config

@@ -72,6 +72,7 @@
                 grpc_opts => #{
                     service_protos => [iot_config_pb],
                     services => #{
+                        'helium.iot_config.org' => hpr_test_ics_org_service,
                         'helium.iot_config.gateway' => hpr_test_ics_gateway_service,
                         'helium.iot_config.route' => hpr_test_ics_route_service
                     }

src/cli/hpr_cli_config.erl

@@ -6,8 +6,23 @@
 
 -behavior(clique_handler).
 
+-include("hpr.hrl").
+
 -export([register_cli/0]).
 
+-export([
+    get_api_routes/0,
+    get_api_routes_for_oui/1
+]).
+
+-ifdef(TEST).
+
+-export([
+    config_route_sync/3
+]).
+
+-endif.
+
 register_cli() ->
     register_all_usage(),
     register_all_cmds().
@@ -42,9 +57,11 @@ config_usage() ->
             "    [--display_skfs] default: false (SKFs not included)\n",
             "config route refresh_all                    - Refresh all routes\n",
             "    [--minimum] default: 1 (need a minimum of 1 SKFs ro be updated)\n",
-            "config route refresh <route_id>             - Refresh route\n",
+            "config route refresh <route_id>             - Refresh route's EUIs, SKFs, DevAddrRanges\n",
             "config route activate <route_id>            - Activate route\n",
             "config route deactivate <route_id>          - Deactivate route\n",
+            "config route remove <route_id>              - Delete all remnants of a route\n",
+            "config route sync [--oui=<oui>]             - Fetch all Routes from Config Service, creating new, removing old\n"
             "config skf <DevAddr/Session Key>            - List all Session Key Filters for Devaddr or Session Key\n",
             "config eui --app <app_eui> --dev <dev_eui>  - List all Routes with EUI pair\n"
             "\n\n",
@@ -104,6 +121,18 @@ config_cmd() ->
             [],
             fun config_route_deactivate/3
         ],
+        [
+            ["config", "route", "remove", '*'],
+            [],
+            [],
+            fun config_route_remove/3
+        ],
+        [
+            ["config", "route", "sync"],
+            [],
+            [{oui, [{shortname, "o"}, {longname, "oui"}]}],
+            fun config_route_sync/3
+        ],
         [["config", "skf", '*'], [], [], fun config_skf/3],
         [
             ["config", "eui"],
@@ -398,6 +427,45 @@ config_route_deactivate(["config", "route", "deactivate", RouteID], [], _Flags)
 config_route_deactivate(_, _, _) ->
     usage.
 
+config_route_remove(["config", "route", "remove", RouteID], [], []) ->
+    case hpr_route_storage:lookup(RouteID) of
+        {ok, RouteETS} ->
+            Route = hpr_route_ets:route(RouteETS),
+            hpr_route_storage:delete(Route),
+            c_text("Deleted Route: ~p", [RouteID]);
+        {error, not_found} ->
+            c_text("Could not find ~p", [RouteID])
+    end;
+config_route_remove(_, _, _) ->
+    usage.
+
+config_route_sync(["config", "route", "sync"], [], Flags) ->
+    Updates =
+        case maps:from_list(Flags) of
+            #{oui := OUI0} ->
+                OUI = erlang:list_to_integer(OUI0, 10),
+                APIRoutes = get_api_routes_for_oui(OUI),
+                ExistingRoutes = hpr_route_storage:oui_routes(OUI),
+                sync_routes(APIRoutes, ExistingRoutes);
+            #{} ->
+                APIRoutes = get_api_routes(),
+                ExistingRoutes = hpr_route_storage:all_routes(),
+                sync_routes(APIRoutes, ExistingRoutes)
+        end,
+
+    FormatRoute = fun(Route) ->
+        io_lib:format(" - ~s OUI=~p~n", [hpr_route:id(Route), hpr_route:oui(Route)])
+    end,
+    Added = lists:map(FormatRoute, maps:get(added, Updates, [])),
+    Removed = lists:map(FormatRoute, maps:get(removed, Updates, [])),
+
+    c_list(
+        [io_lib:format("=== Added (~p) ===~n", [length(Added)])] ++ Added ++
+            [io_lib:format("=== Removed (~p) ===~n", [length(Removed)])] ++ Removed
+    );
+config_route_sync(_, _, _) ->
+    usage.
+
 config_skf(["config", "skf", DevAddrOrSKF], [], []) ->
     SKFS =
         case hpr_utils:hex_to_bin(erlang:list_to_binary(DevAddrOrSKF)) of
@@ -728,3 +796,81 @@ format_skf({{SKF, DevAddr}, MaxCopies}) ->
         hpr_utils:bin_to_hex_string(SKF),
         MaxCopies
     ]).
+
+-spec get_api_routes() -> list(hpr_route:route()).
+get_api_routes() ->
+    {ok, OrgList, _Meta} = helium_iot_config_org_client:list(
+        hpr_org_list_req:new(),
+        #{channel => ?IOT_CONFIG_CHANNEL}
+    ),
+
+    lists:flatmap(
+        fun(OUI) -> get_api_routes_for_oui(OUI) end,
+        hpr_org_list_res:org_ouis(OrgList)
+    ).
+
+-spec get_api_routes_for_oui(OUI :: non_neg_integer()) -> list(hpr_route:route()).
+get_api_routes_for_oui(OUI) ->
+    PubKeyBin = hpr_utils:pubkey_bin(),
+    SigFun = hpr_utils:sig_fun(),
+
+    ListReq = hpr_route_list_req:new(PubKeyBin, OUI),
+    SignedReq = hpr_route_list_req:sign(ListReq, SigFun),
+    {ok, RouteListRes, _Meta} = helium_iot_config_route_client:list(
+        SignedReq,
+        #{channel => ?IOT_CONFIG_CHANNEL}
+    ),
+    hpr_route_list_res:routes(RouteListRes).
+
+-spec sync_routes(
+    APIRoutes :: list(hpr_route:route()),
+    ExistingRoutes :: list(hpr_route:route())
+) -> #{added => list(hpr_route:route()), removed => list(hpr_route:route())}.
+sync_routes(APIRoutes, ExistingRoutes) ->
+    sync_routes(APIRoutes, ExistingRoutes, #{added => [], removed => []}).
+
+-spec sync_routes(
+    APIRoutes :: list(hpr_route:route()),
+    ExistingRoutes :: list(hpr_route:route()),
+    Updates :: map()
+) -> #{added => list(hpr_route:route()), removed => list(hpr_route:route())}.
+sync_routes([], [], Updates) ->
+    Updates;
+sync_routes([], [Route | LeftoverRoutes], #{removed := RemovedRoutes} = Updates) ->
+    RouteID = hpr_route:id(Route),
+    lager:info([{route_id, RouteID}], "removing leftover route: ~p"),
+    ok = hpr_route_storage:delete(Route),
+    sync_routes(
+        [],
+        LeftoverRoutes,
+        Updates#{removed => [Route | RemovedRoutes]}
+    );
+sync_routes([Route | Routes], ExistingRoutes, #{added := AddedRoutes} = Updates) ->
+    RouteID = hpr_route:id(Route),
+    case hpr_route_storage:lookup(RouteID) of
+        {ok, _Route} ->
+            lager:info([{route_id, RouteID}], "doing nothing, route already exists"),
+            sync_routes(
+                Routes,
+                remove_route(Route, ExistingRoutes),
+                Updates
+            );
+        {error, not_found} ->
+            lager:info([{route_id, RouteID}], "syncing new route"),
+            ok = hpr_route_storage:insert(Route),
+            hpr_route_stream_worker:refresh_route(hpr_route:id(Route)),
+
+            sync_routes(
+                Routes,
+                remove_route(Route, ExistingRoutes),
+                Updates#{added => [Route | AddedRoutes]}
+            )
+    end.
+
+-spec remove_route(
+    Target :: hpr_route:route(),
+    Coll :: list(hpr_route:route())
+) -> list(hpr_route:route()).
+remove_route(Target, RouteList) ->
+    ID = hpr_route:id(Target),
+    lists:filter(fun(R) -> hpr_route:id(R) =/= ID end, RouteList).

src/grpc/iot_config/hpr_org.erl

@@ -0,0 +1,56 @@
+-module(hpr_org).
+
+-include("../autogen/iot_config_pb.hrl").
+
+-export([
+    oui/1,
+    owner/1,
+    payer/1,
+    delegate_keys/1,
+    locked/1
+]).
+
+-type org() :: #iot_config_org_v1_pb{}.
+
+-ifdef(TEST).
+
+-export([test_new/1]).
+
+-endif.
+
+-spec oui(Org :: org()) -> non_neg_integer().
+oui(Org) ->
+    Org#iot_config_org_v1_pb.oui.
+
+-spec owner(Org :: org()) -> binary().
+owner(Org) ->
+    Org#iot_config_org_v1_pb.owner.
+
+-spec payer(Org :: org()) -> binary().
+payer(Org) ->
+    Org#iot_config_org_v1_pb.payer.
+
+-spec delegate_keys(Org :: org()) -> list(binary()).
+delegate_keys(Org) ->
+    Org#iot_config_org_v1_pb.delegate_keys.
+
+-spec locked(Org :: org()) -> boolean().
+locked(Org) ->
+    Org#iot_config_org_v1_pb.locked.
+
+%% ------------------------------------------------------------------
+%% Tests Functions
+%% ------------------------------------------------------------------
+-ifdef(TEST).
+
+-spec test_new(RouteMap :: map()) -> org().
+test_new(RouteMap) ->
+    #iot_config_org_v1_pb{
+        oui = maps:get(oui, RouteMap),
+        owner = maps:get(owner, RouteMap, <<"owner-test-value">>),
+        payer = maps:get(payer, RouteMap, <<"payer-test-value">>),
+        delegate_keys = maps:get(delegate_keys, RouteMap, []),
+        locked = maps:get(locked, RouteMap, false)
+    }.
+
+-endif.

src/grpc/iot_config/hpr_org_list_req.erl

@@ -0,0 +1,11 @@
+-module(hpr_org_list_req).
+
+-include("../autogen/iot_config_pb.hrl").
+
+-export([new/0]).
+
+-type req() :: #iot_config_org_list_req_v1_pb{}.
+
+-spec new() -> req().
+new() ->
+     #iot_config_org_list_req_v1_pb{}.

src/grpc/iot_config/hpr_org_list_res.erl

@@ -0,0 +1,36 @@
+-module(hpr_org_list_res).
+
+-include("../autogen/iot_config_pb.hrl").
+
+-export([
+    orgs/1,
+    timestamp/1,
+    signer/1,
+    signature/1
+]).
+
+-export([
+    org_ouis/1
+]).
+
+-type res() :: #iot_config_org_list_res_v1_pb{}.
+
+-spec orgs(Res :: res()) -> list(hpr_org:org()).
+orgs(Res) ->
+    Res#iot_config_org_list_res_v1_pb.orgs.
+
+-spec timestamp(Res :: res()) -> non_neg_integer().
+timestamp(Res) ->
+    Res#iot_config_org_list_res_v1_pb.timestamp.
+
+-spec signer(Res :: res()) -> binary().
+signer(Res) ->
+    Res#iot_config_org_list_res_v1_pb.signer.
+
+-spec signature(Res :: res()) -> binary().
+signature(Res) ->
+    Res#iot_config_org_list_res_v1_pb.signature.
+
+-spec org_ouis(Res :: res()) -> list(non_neg_integer()).
+org_ouis(Res) ->
+    [hpr_org:oui(Org) || Org <- ?MODULE:orgs(Res)].

src/grpc/iot_config/hpr_route_list_req.erl

@@ -0,0 +1,58 @@
+-module(hpr_route_list_req).
+
+-include("../autogen/iot_config_pb.hrl").
+
+-export([
+    new/2,
+    timestamp/1,
+    signer/1,
+    signature/1
+]).
+
+-export([
+    sign/2,
+    verify/1
+]).
+
+-type req() :: #iot_config_route_list_req_v1_pb{}.
+
+-spec new(Signer :: libp2p_crypto:pubkey_bin(), Oui :: non_neg_integer()) -> req().
+new(Signer, Oui) ->
+    #iot_config_route_list_req_v1_pb{
+        oui = Oui,
+        timestamp = erlang:system_time(millisecond),
+        signer = Signer
+    }.
+
+-spec timestamp(Req :: req()) -> non_neg_integer().
+timestamp(Req) ->
+     Req#iot_config_route_list_req_v1_pb.timestamp.
+
+-spec signer(Req:: req()) -> binary().
+signer(Req) ->
+     Req#iot_config_route_list_req_v1_pb.signer.
+
+-spec signature(Req:: req()) -> binary().
+signature(Req) ->
+     Req#iot_config_route_list_req_v1_pb.signature.
+
+-spec sign(RouteListReq :: req(), SigFun :: fun()) -> req().
+sign(RouteListReq, SigFun) ->
+    EncodedRouteListReq = iot_config_pb:encode_msg(
+        RouteListReq, iot_config_route_list_req_v1_pb
+    ),
+    RouteListReq#iot_config_route_list_req_v1_pb{signature = SigFun(EncodedRouteListReq)}.
+
+-spec verify(RouteListReq :: req()) -> boolean().
+verify(RouteListReq) ->
+    EncodedRouteListReq = iot_config_pb:encode_msg(
+        RouteListReq#iot_config_route_list_req_v1_pb{
+            signature = <<>>
+        },
+        iot_config_route_list_req_v1_pb
+    ),
+    libp2p_crypto:verify(
+        EncodedRouteListReq,
+        ?MODULE:signature(RouteListReq),
+        libp2p_crypto:bin_to_pubkey(?MODULE:signer(RouteListReq))
+    ).

src/grpc/iot_config/hpr_route_list_res.erl

@@ -0,0 +1,13 @@
+-module(hpr_route_list_res).
+
+-include("../autogen/iot_config_pb.hrl").
+
+-export([
+    routes/1
+]).
+
+-type res() :: #iot_config_route_list_res_v1_pb{}.
+
+-spec routes(Res :: res()) -> list(hpr_route:route()).
+routes(Res) ->
+    Res#iot_config_route_list_res_v1_pb.routes.