sync routes

config/ct.config

@@ -72,6 +72,7 @@
                 grpc_opts => #{
                     service_protos => [iot_config_pb],
                     services => #{
+                        'helium.iot_config.org' => hpr_test_ics_org_service,
                         'helium.iot_config.gateway' => hpr_test_ics_gateway_service,
                         'helium.iot_config.route' => hpr_test_ics_route_service
                     }

src/cli/hpr_cli_config.erl

@@ -6,8 +6,23 @@
 
 -behavior(clique_handler).
 
+-include("hpr.hrl").
+
 -export([register_cli/0]).
 
+-export([
+    get_api_routes/0,
+    get_api_routes_for_oui/1
+]).
+
+-ifdef(TEST).
+
+-export([
+    config_route_sync/3
+]).
+
+-endif.
+
 register_cli() ->
     register_all_usage(),
     register_all_cmds().
@@ -42,9 +57,11 @@ config_usage() ->
             "    [--display_skfs] default: false (SKFs not included)\n",
             "config route refresh_all                    - Refresh all routes\n",
             "    [--minimum] default: 1 (need a minimum of 1 SKFs ro be updated)\n",
-            "config route refresh <route_id>             - Refresh route\n",
+            "config route refresh <route_id>             - Refresh route's EUIs, SKFs, DevAddrRanges\n",
             "config route activate <route_id>            - Activate route\n",
             "config route deactivate <route_id>          - Deactivate route\n",
+            "config route remove <route_id>              - Delete all remnants of a route\n",
+            "config route sync [--oui=<oui>]             - Fetch all Routes from Config Service, creating new, removing old\n"
             "config skf <DevAddr/Session Key>            - List all Session Key Filters for Devaddr or Session Key\n",
             "config eui --app <app_eui> --dev <dev_eui>  - List all Routes with EUI pair\n"
             "\n\n",
@@ -104,6 +121,18 @@ config_cmd() ->
             [],
             fun config_route_deactivate/3
         ],
+        [
+            ["config", "route", "remove", '*'],
+            [],
+            [],
+            fun config_route_remove/3
+        ],
+        [
+            ["config", "route", "sync"],
+            [],
+            [{oui, [{shortname, "o"}, {longname, "oui"}]}],
+            fun config_route_sync/3
+        ],
         [["config", "skf", '*'], [], [], fun config_skf/3],
         [
             ["config", "eui"],
@@ -398,6 +427,45 @@ config_route_deactivate(["config", "route", "deactivate", RouteID], [], _Flags)
 config_route_deactivate(_, _, _) ->
     usage.
 
+config_route_remove(["config", "route", "remove", RouteID], [], []) ->
+    case hpr_route_storage:lookup(RouteID) of
+        {ok, RouteETS} ->
+            Route = hpr_route_ets:route(RouteETS),
+            hpr_route_storage:delete(Route),
+            c_text("Deleted Route: ~p", [RouteID]);
+        {error, not_found} ->
+            c_text("Could not find ~p", [RouteID])
+    end;
+config_route_remove(_, _, _) ->
+    usage.
+
+config_route_sync(["config", "route", "sync"], [], Flags) ->
+    Updates =
+        case maps:from_list(Flags) of
+            #{oui := OUI0} ->
+                OUI = erlang:list_to_integer(OUI0, 10),
+                APIRoutes = get_api_routes_for_oui(OUI),
+                ExistingRoutes = hpr_route_storage:oui_routes(OUI),
+                sync_routes(APIRoutes, ExistingRoutes);
+            #{} ->
+                APIRoutes = get_api_routes(),
+                ExistingRoutes = hpr_route_storage:all_routes(),
+                sync_routes(APIRoutes, ExistingRoutes)
+        end,
+
+    FormatRoute = fun(Route) ->
+        io_lib:format(" - ~s OUI=~p~n", [hpr_route:id(Route), hpr_route:oui(Route)])
+    end,
+    Added = lists:map(FormatRoute, maps:get(added, Updates, [])),
+    Removed = lists:map(FormatRoute, maps:get(removed, Updates, [])),
+
+    c_list(
+        [io_lib:format("=== Added (~p) ===~n", [length(Added)])] ++ Added ++
+            [io_lib:format("=== Removed (~p) ===~n", [length(Removed)])] ++ Removed
+    );
+config_route_sync(_, _, _) ->
+    usage.
+
 config_skf(["config", "skf", DevAddrOrSKF], [], []) ->
     SKFS =
         case hpr_utils:hex_to_bin(erlang:list_to_binary(DevAddrOrSKF)) of
@@ -728,3 +796,81 @@ format_skf({{SKF, DevAddr}, MaxCopies}) ->
         hpr_utils:bin_to_hex_string(SKF),
         MaxCopies
     ]).
+
+-spec get_api_routes() -> list(hpr_route:route()).
+get_api_routes() ->
+    {ok, OrgList, _Meta} = helium_iot_config_org_client:list(
+        hpr_org_list_req:new(),
+        #{channel => ?IOT_CONFIG_CHANNEL}
+    ),
+
+    lists:flatmap(
+        fun(OUI) -> get_api_routes_for_oui(OUI) end,
+        hpr_org_list_res:org_ouis(OrgList)
+    ).
+
+-spec get_api_routes_for_oui(OUI :: non_neg_integer()) -> list(hpr_route:route()).
+get_api_routes_for_oui(OUI) ->
+    PubKeyBin = hpr_utils:pubkey_bin(),
+    SigFun = hpr_utils:sig_fun(),
+
+    ListReq = hpr_route_list_req:new(PubKeyBin, OUI),
+    SignedReq = hpr_route_list_req:sign(ListReq, SigFun),
+    {ok, RouteListRes, _Meta} = helium_iot_config_route_client:list(
+        SignedReq,
+        #{channel => ?IOT_CONFIG_CHANNEL}
+    ),
+    hpr_route_list_res:routes(RouteListRes).
+
+-spec sync_routes(
+    APIRoutes :: list(hpr_route:route()),
+    ExistingRoutes :: list(hpr_route:route())
+) -> #{added => list(hpr_route:route()), removed => list(hpr_route:route())}.
+sync_routes(APIRoutes, ExistingRoutes) ->
+    sync_routes(APIRoutes, ExistingRoutes, #{added => [], removed => []}).
+
+-spec sync_routes(
+    APIRoutes :: list(hpr_route:route()),
+    ExistingRoutes :: list(hpr_route:route()),
+    Updates :: map()
+) -> #{added => list(hpr_route:route()), removed => list(hpr_route:route())}.
+sync_routes([], [], Updates) ->
+    Updates;
+sync_routes([], [Route | LeftoverRoutes], #{removed := RemovedRoutes} = Updates) ->
+    RouteID = hpr_route:id(Route),
+    lager:info([{route_id, RouteID}], "removing leftover route: ~p"),
+    ok = hpr_route_storage:delete(Route),
+    sync_routes(
+        [],
+        LeftoverRoutes,
+        Updates#{removed => [Route | RemovedRoutes]}
+    );
+sync_routes([Route | Routes], ExistingRoutes, #{added := AddedRoutes} = Updates) ->
+    RouteID = hpr_route:id(Route),
+    case hpr_route_storage:lookup(RouteID) of
+        {ok, _Route} ->
+            lager:info([{route_id, RouteID}], "doing nothing, route already exists"),
+            sync_routes(
+                Routes,
+                remove_route(Route, ExistingRoutes),
+                Updates
+            );
+        {error, not_found} ->
+            lager:info([{route_id, RouteID}], "syncing new route"),
+            ok = hpr_route_storage:insert(Route),
+            hpr_route_stream_worker:refresh_route(hpr_route:id(Route)),
+
+            sync_routes(
+                Routes,
+                remove_route(Route, ExistingRoutes),
+                Updates#{added => [Route | AddedRoutes]}
+            )
+    end.
+
+-spec remove_route(
+    Target :: hpr_route:route(),
+    Coll :: list(hpr_route:route())
+) -> list(hpr_route:route()).
+remove_route(Target, RouteList) ->
+    ID = hpr_route:id(Target),
+    lists:filter(fun(R) -> hpr_route:id(R) =/= ID end, RouteList).

src/grpc/iot_config/hpr_org.erl

@@ -0,0 +1,56 @@
+-module(hpr_org).
+
+-include("../autogen/iot_config_pb.hrl").
+
+-export([
+    oui/1,
+    owner/1,
+    payer/1,
+    delegate_keys/1,
+    locked/1
+]).
+
+-type org() :: #iot_config_org_v1_pb{}.
+
+-ifdef(TEST).
+
+-export([test_new/1]).
+
+-endif.
+
+-spec oui(Org :: org()) -> non_neg_integer().
+oui(Org) ->
+    Org#iot_config_org_v1_pb.oui.
+
+-spec owner(Org :: org()) -> binary().
+owner(Org) ->
+    Org#iot_config_org_v1_pb.owner.
+
+-spec payer(Org :: org()) -> binary().
+payer(Org) ->
+    Org#iot_config_org_v1_pb.payer.
+
+-spec delegate_keys(Org :: org()) -> list(binary()).
+delegate_keys(Org) ->
+    Org#iot_config_org_v1_pb.delegate_keys.
+
+-spec locked(Org :: org()) -> boolean().
+locked(Org) ->
+    Org#iot_config_org_v1_pb.locked.
+
+%% ------------------------------------------------------------------
+%% Tests Functions
+%% ------------------------------------------------------------------
+-ifdef(TEST).
+
+-spec test_new(RouteMap :: map()) -> org().
+test_new(RouteMap) ->
+    #iot_config_org_v1_pb{
+        oui = maps:get(oui, RouteMap),
+        owner = maps:get(owner, RouteMap, <<"owner-test-value">>),
+        payer = maps:get(payer, RouteMap, <<"payer-test-value">>),
+        delegate_keys = maps:get(delegate_keys, RouteMap, []),
+        locked = maps:get(locked, RouteMap, false)
+    }.
+
+-endif.

src/grpc/iot_config/hpr_org_list_req.erl

@@ -0,0 +1,11 @@
+-module(hpr_org_list_req).
+
+-include("../autogen/iot_config_pb.hrl").
+
+-export([new/0]).
+
+-type req() :: #iot_config_org_list_req_v1_pb{}.
+
+-spec new() -> req().
+new() ->
+     #iot_config_org_list_req_v1_pb{}.

src/grpc/iot_config/hpr_org_list_res.erl

@@ -0,0 +1,36 @@
+-module(hpr_org_list_res).
+
+-include("../autogen/iot_config_pb.hrl").
+
+-export([
+    orgs/1,
+    timestamp/1,
+    signer/1,
+    signature/1
+]).
+
+-export([
+    org_ouis/1
+]).
+
+-type res() :: #iot_config_org_list_res_v1_pb{}.
+
+-spec orgs(Res :: res()) -> list(hpr_org:org()).
+orgs(Res) ->
+    Res#iot_config_org_list_res_v1_pb.orgs.
+
+-spec timestamp(Res :: res()) -> non_neg_integer().
+timestamp(Res) ->
+    Res#iot_config_org_list_res_v1_pb.timestamp.
+
+-spec signer(Res :: res()) -> binary().
+signer(Res) ->
+    Res#iot_config_org_list_res_v1_pb.signer.
+
+-spec signature(Res :: res()) -> binary().
+signature(Res) ->
+    Res#iot_config_org_list_res_v1_pb.signature.
+
+-spec org_ouis(Res :: res()) -> list(non_neg_integer()).
+org_ouis(Res) ->
+    [hpr_org:oui(Org) || Org <- ?MODULE:orgs(Res)].

src/grpc/iot_config/hpr_route_list_req.erl

@@ -0,0 +1,58 @@
+-module(hpr_route_list_req).
+
+-include("../autogen/iot_config_pb.hrl").
+
+-export([
+    new/2,
+    timestamp/1,
+    signer/1,
+    signature/1
+]).
+
+-export([
+    sign/2,
+    verify/1
+]).
+
+-type req() :: #iot_config_route_list_req_v1_pb{}.
+
+-spec new(Signer :: libp2p_crypto:pubkey_bin(), Oui :: non_neg_integer()) -> req().
+new(Signer, Oui) ->
+    #iot_config_route_list_req_v1_pb{
+        oui = Oui,
+        timestamp = erlang:system_time(millisecond),
+        signer = Signer
+    }.
+
+-spec timestamp(Req :: req()) -> non_neg_integer().
+timestamp(Req) ->
+     Req#iot_config_route_list_req_v1_pb.timestamp.
+
+-spec signer(Req:: req()) -> binary().
+signer(Req) ->
+     Req#iot_config_route_list_req_v1_pb.signer.
+
+-spec signature(Req:: req()) -> binary().
+signature(Req) ->
+     Req#iot_config_route_list_req_v1_pb.signature.
+
+-spec sign(RouteListReq :: req(), SigFun :: fun()) -> req().
+sign(RouteListReq, SigFun) ->
+    EncodedRouteListReq = iot_config_pb:encode_msg(
+        RouteListReq, iot_config_route_list_req_v1_pb
+    ),
+    RouteListReq#iot_config_route_list_req_v1_pb{signature = SigFun(EncodedRouteListReq)}.
+
+-spec verify(RouteListReq :: req()) -> boolean().
+verify(RouteListReq) ->
+    EncodedRouteListReq = iot_config_pb:encode_msg(
+        RouteListReq#iot_config_route_list_req_v1_pb{
+            signature = <<>>
+        },
+        iot_config_route_list_req_v1_pb
+    ),
+    libp2p_crypto:verify(
+        EncodedRouteListReq,
+        ?MODULE:signature(RouteListReq),
+        libp2p_crypto:bin_to_pubkey(?MODULE:signer(RouteListReq))
+    ).

src/grpc/iot_config/hpr_route_list_res.erl

@@ -0,0 +1,13 @@
+-module(hpr_route_list_res).
+
+-include("../autogen/iot_config_pb.hrl").
+
+-export([
+    routes/1
+]).
+
+-type res() :: #iot_config_route_list_res_v1_pb{}.
+
+-spec routes(Res :: res()) -> list(hpr_route:route()).
+routes(Res) ->
+    Res#iot_config_route_list_res_v1_pb.routes.