From 8e3e7bf16010a61dee37671e162c1df92b2108d2 Mon Sep 17 00:00:00 2001 From: shuai Date: Thu, 25 May 2023 16:24:13 +0800 Subject: [PATCH 1/5] srv6 vpn --- orchagent/nexthopgroupkey.h | 31 ++- orchagent/nexthopkey.h | 64 ++++- orchagent/routeorch.cpp | 190 +++++++++---- orchagent/routeorch.h | 1 + orchagent/srv6orch.cpp | 536 +++++++++++++++++++++++++++++++++++- orchagent/srv6orch.h | 97 ++++++- 6 files changed, 827 insertions(+), 92 deletions(-) diff --git a/orchagent/nexthopgroupkey.h b/orchagent/nexthopgroupkey.h index d012cbe41a..9c176b5819 100644 --- a/orchagent/nexthopgroupkey.h +++ b/orchagent/nexthopgroupkey.h @@ -13,6 +13,7 @@ class NextHopGroupKey { m_overlay_nexthops = false; m_srv6_nexthops = false; + m_srv6_vpn = false; auto nhv = tokenize(nexthops, NHG_DELIMITER); for (const auto &nh : nhv) { @@ -21,12 +22,13 @@ class NextHopGroupKey } /* ip_string|if_alias|vni|router_mac separated by ',' */ - NextHopGroupKey(const std::string &nexthops, bool overlay_nh, bool srv6_nh = false) + NextHopGroupKey(const std::string &nexthops, bool overlay_nh, bool srv6_nh = false, bool srv6_vpn = false) { if (overlay_nh) { m_overlay_nexthops = true; m_srv6_nexthops = false; + m_srv6_vpn = false; auto nhv = tokenize(nexthops, NHG_DELIMITER); for (const auto &nh_str : nhv) { @@ -38,6 +40,9 @@ class NextHopGroupKey { m_overlay_nexthops = false; m_srv6_nexthops = true; + m_srv6_vpn = srv6_vpn; + m_srv6_str = nexthops; + auto nhv = tokenize(nexthops, NHG_DELIMITER); for (const auto &nh_str : nhv) { @@ -47,10 +52,12 @@ class NextHopGroupKey } } + NextHopGroupKey(const std::string &nexthops, const std::string &weights) { m_overlay_nexthops = false; m_srv6_nexthops = false; + m_srv6_vpn = false; std::vector nhv = tokenize(nexthops, NHG_DELIMITER); std::vector wtv = tokenize(weights, NHG_DELIMITER); bool set_weight = wtv.size() == nhv.size(); @@ -74,6 +81,11 @@ class NextHopGroupKey inline bool operator<(const NextHopGroupKey &o) const { + if (m_srv6_nexthops) + { + return m_srv6_str < o.m_srv6_str; + } + if (m_nexthops < o.m_nexthops) { return true; @@ -193,6 +205,9 @@ class NextHopGroupKey const std::string to_string() const { + if (m_srv6_nexthops) { + return m_srv6_str; + } string nhs_str; for (auto it = m_nexthops.begin(); it != m_nexthops.end(); ++it) @@ -221,6 +236,17 @@ class NextHopGroupKey return m_srv6_nexthops; } + inline bool is_srv6_vpn() const + { + return m_srv6_vpn; + } + + inline string get_srv6_vpn_key() const + { + // use nexthopgroupkey as vpn key + return m_srv6_str; + } + void clear() { m_nexthops.clear(); @@ -230,6 +256,9 @@ class NextHopGroupKey std::set m_nexthops; bool m_overlay_nexthops; bool m_srv6_nexthops; + + string m_srv6_str; + bool m_srv6_vpn; }; #endif /* SWSS_NEXTHOPGROUPKEY_H */ diff --git a/orchagent/nexthopkey.h b/orchagent/nexthopkey.h index 2f03e9fd49..49e36fb825 100644 --- a/orchagent/nexthopkey.h +++ b/orchagent/nexthopkey.h @@ -22,18 +22,33 @@ struct NextHopKey uint32_t weight; // NH weight for NHGs string srv6_segment; // SRV6 segment string string srv6_source; // SRV6 source address + string srv6_vpn_sid; // SRV6 vpn sid - NextHopKey() : weight(0) {} + NextHopKey() : weight(0), + srv6_vpn_sid(""), + srv6_source(""), + srv6_segment("") + {} NextHopKey(const std::string &str, const std::string &alias) : - alias(alias), vni(0), mac_address(), weight(0) + alias(alias), vni(0), mac_address(), weight(0), + srv6_vpn_sid(""), + srv6_source(""), + srv6_segment("") { std::string ip_str = parseMplsNextHop(str); ip_address = ip_str; } NextHopKey(const IpAddress &ip, const std::string &alias) : - ip_address(ip), alias(alias), vni(0), mac_address(), weight(0) {} + ip_address(ip), alias(alias), vni(0), mac_address(), weight(0), + srv6_vpn_sid(""), + srv6_source(""), + srv6_segment("") + {} NextHopKey(const std::string &str) : - vni(0), mac_address() + vni(0), mac_address(), + srv6_vpn_sid(""), + srv6_source(""), + srv6_segment("") { if (str.find(NHG_DELIMITER) != string::npos) { @@ -74,16 +89,16 @@ struct NextHopKey { weight = 0; vni = 0; - weight = 0; auto keys = tokenize(str, NH_DELIMITER); - if (keys.size() != 3) + if (keys.size() != 4) { std::string err = "Error converting " + str + " to Nexthop"; throw std::invalid_argument(err); } ip_address = keys[0]; - srv6_segment = keys[1]; + srv6_vpn_sid = keys[1]; srv6_source = keys[2]; + srv6_segment = keys[3]; } else { @@ -99,10 +114,22 @@ struct NextHopKey vni = static_cast(std::stoul(keys[2])); mac_address = keys[3]; weight = 0; + srv6_vpn_sid = ""; + srv6_source = ""; + srv6_segment = ""; } } - NextHopKey(const IpAddress &ip, const MacAddress &mac, const uint32_t &vni, bool overlay_nh) : ip_address(ip), alias(""), vni(vni), mac_address(mac), weight(0){} + NextHopKey(const IpAddress &ip, const MacAddress &mac, const uint32_t &vni, bool overlay_nh) : + ip_address(ip), + alias(""), + vni(vni), + mac_address(mac), + weight(0), + srv6_vpn_sid(""), + srv6_source(""), + srv6_segment("") + {} const std::string to_string() const { @@ -111,11 +138,14 @@ struct NextHopKey return str; } - const std::string to_string(bool overlay_nh, bool srv6_nh) const + const std::string to_string(bool overlay_nh, bool srv6_nh = false) const { if (srv6_nh) { - return ip_address.to_string() + NH_DELIMITER + srv6_segment + NH_DELIMITER + srv6_source; + return ip_address.to_string() + NH_DELIMITER + + srv6_vpn_sid + NH_DELIMITER + + srv6_source + NH_DELIMITER + + srv6_segment + NH_DELIMITER; } std::string str = formatMplsNextHop(); str += (ip_address.to_string() + NH_DELIMITER + alias + NH_DELIMITER + @@ -125,8 +155,8 @@ struct NextHopKey bool operator<(const NextHopKey &o) const { - return tie(ip_address, alias, label_stack, vni, mac_address, srv6_segment, srv6_source) < - tie(o.ip_address, o.alias, o.label_stack, o.vni, o.mac_address, o.srv6_segment, o.srv6_source); + return tie(ip_address, alias, label_stack, vni, mac_address, srv6_segment, srv6_source, srv6_vpn_sid) < + tie(o.ip_address, o.alias, o.label_stack, o.vni, o.mac_address, o.srv6_segment, o.srv6_source, o.srv6_vpn_sid); } bool operator==(const NextHopKey &o) const @@ -134,7 +164,8 @@ struct NextHopKey return (ip_address == o.ip_address) && (alias == o.alias) && (label_stack == o.label_stack) && (vni == o.vni) && (mac_address == o.mac_address) && - (srv6_segment == o.srv6_segment) && (srv6_source == o.srv6_source); + (srv6_segment == o.srv6_segment) && (srv6_source == o.srv6_source) && + (srv6_vpn_sid == o.srv6_vpn_sid); } bool operator!=(const NextHopKey &o) const @@ -154,7 +185,12 @@ struct NextHopKey bool isSrv6NextHop() const { - return (srv6_segment != ""); + return ((srv6_segment != "") || (srv6_vpn_sid != "")); + } + + bool isSrv6Vpn() const + { + return (srv6_vpn_sid != ""); } std::string parseMplsNextHop(const std::string& str) diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index b8b9056439..4e18725651 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -597,7 +597,8 @@ void RouteOrch::doTask(Consumer& consumer) string srv6_segments; string srv6_source; bool srv6_nh = false; - + string srv6_vpn_sids; + bool srv6_vpn = false; for (auto i : kfvFieldsValues(t)) { if (fvField(i) == "nexthop") @@ -634,6 +635,13 @@ void RouteOrch::doTask(Consumer& consumer) if (fvField(i) == "seg_src") srv6_source = fvValue(i); + if (fvField(i) == "vpn_sid" && fvValue(i) != "") + { + srv6_vpn_sids = fvValue(i); + srv6_nh = true; + srv6_vpn = true; + } + if (fvField(i) == "protocol") { ctx.protocol = fvValue(i); @@ -666,6 +674,7 @@ void RouteOrch::doTask(Consumer& consumer) NextHopGroupKey& nhg = ctx.nhg; vector srv6_segv; vector srv6_src; + vector srv6_vpn_sidv; bool l3Vni = true; uint32_t vni = 0; @@ -679,6 +688,7 @@ void RouteOrch::doTask(Consumer& consumer) rmacv = tokenize(remote_macs, ','); srv6_segv = tokenize(srv6_segments, ','); srv6_src = tokenize(srv6_source, ','); + srv6_vpn_sidv = tokenize(srv6_vpn_sids, ','); /* * For backward compatibility, adjust ip string from old format to @@ -762,26 +772,52 @@ void RouteOrch::doTask(Consumer& consumer) } else if (srv6_nh == true) { - string ip; - if (ipv.empty()) + if (srv6_vpn == true) { - ip = "0.0.0.0"; + if (ipv.size() != srv6_src.size()) + { + SWSS_LOG_ERROR("inconsistent number of endpoints and srv6_srcs."); + it = consumer.m_toSync.erase(it); + continue; + } + + if (ipv.size() != srv6_vpn_sidv.size()) + { + SWSS_LOG_ERROR("inconsistent number of endpoints and srv6 vpn sids."); + it = consumer.m_toSync.erase(it); + continue; + } + + for (uint32_t i = 0; i < ipv.size(); i++) + { + if (i) nhg_str += NHG_DELIMITER; + nhg_str += ipv[i] + NH_DELIMITER; // ip address + nhg_str += srv6_vpn_sidv[i] + NH_DELIMITER; // srv6 vpn sid + nhg_str += srv6_src[i] + NH_DELIMITER; // srv6 source + nhg_str += NH_DELIMITER; // srv6 segment (empty) + } } else { - SWSS_LOG_ERROR("For SRV6 nexthop ipv should be empty"); - it = consumer.m_toSync.erase(it); - continue; - } - nhg_str = ip + NH_DELIMITER + srv6_segv[0] + NH_DELIMITER + srv6_src[0]; + if (srv6_segv.size() != srv6_src.size()) + { + SWSS_LOG_ERROR("inconsistent number of srv6_segv and srv6_srcs."); + it = consumer.m_toSync.erase(it); + continue; + } - for (uint32_t i = 1; i < srv6_segv.size(); i++) - { - nhg_str += NHG_DELIMITER + ip; - nhg_str += NH_DELIMITER + srv6_segv[i]; - nhg_str += NH_DELIMITER + srv6_src[i]; + string ip = "0.0.0.0"; + for (uint32_t i = 0; i < srv6_segv.size(); i++) + { + if (i) nhg_str += NHG_DELIMITER; + nhg_str += ip + NH_DELIMITER; // ip address + nhg_str += NH_DELIMITER; // srv6 vpn sid (empty) + nhg_str += srv6_src[i] + NH_DELIMITER; // srv6 source + nhg_str += srv6_segv[i] + NH_DELIMITER; // srv6 segment + } } - nhg = NextHopGroupKey(nhg_str, overlay_nh, srv6_nh); + + nhg = NextHopGroupKey(nhg_str, overlay_nh, srv6_nh, srv6_vpn); SWSS_LOG_INFO("SRV6 route with nhg %s", nhg.to_string().c_str()); } else if (overlay_nh == false) @@ -925,6 +961,7 @@ void RouteOrch::doTask(Consumer& consumer) // Go through the bulker results auto it_prev = consumer.m_toSync.begin(); m_bulkNhgReducedRefCnt.clear(); + m_bulkSrv6NhgReducedVec.clear(); while (it_prev != it) { KeyOpFieldsValuesTuple t = it_prev->second; @@ -1001,25 +1038,18 @@ void RouteOrch::doTask(Consumer& consumer) { removeOverlayNextHops(it_nhg.second, it_nhg.first); } - else if (it_nhg.first.is_srv6_nexthop()) - { - if(it_nhg.first.getSize() > 1) - { - if(m_syncdNextHopGroups[it_nhg.first].ref_count == 0) - { - removeNextHopGroup(it_nhg.first); - } - else - { - SWSS_LOG_ERROR("SRV6 ECMP %s REF count is not zero", it_nhg.first.to_string().c_str()); - } - } - } else if (m_syncdNextHopGroups[it_nhg.first].ref_count == 0) { removeNextHopGroup(it_nhg.first); } } + + /* Reduce reference for srv6 next hop group */ + /* Later delete for increase refcnt early */ + if (!m_bulkSrv6NhgReducedVec.empty()) + { + m_srv6Orch->removeSrv6Nexthops(m_bulkSrv6NhgReducedVec); + } } } @@ -1123,6 +1153,11 @@ void RouteOrch::increaseNextHopRefCount(const NextHopGroupKey &nexthops) { m_syncdNextHopGroups[nexthops].ref_count ++; } + + if (nexthops.is_srv6_vpn()) + { + m_srv6Orch->increasePrefixAggIdRefCount(nexthops); + } } void RouteOrch::decreaseNextHopRefCount(const NextHopGroupKey &nexthops) @@ -1143,6 +1178,12 @@ void RouteOrch::decreaseNextHopRefCount(const NextHopGroupKey &nexthops) else { m_syncdNextHopGroups[nexthops].ref_count --; + + } + + if (nexthops.is_srv6_vpn()) + { + m_srv6Orch->decreasePrefixAggIdRefCount(nexthops); } } @@ -1503,18 +1544,6 @@ bool RouteOrch::removeNextHopGroup(const NextHopGroupKey &nexthops) } } - if (srv6_nh) - { - if (!m_srv6Orch->removeSrv6Nexthops(nexthops)) - { - SWSS_LOG_ERROR("Failed to remove Srv6 Nexthop %s", nexthops.to_string().c_str()); - } - else - { - SWSS_LOG_INFO("Remove ECMP Srv6 nexthops %s", nexthops.to_string().c_str()); - } - } - m_syncdNextHopGroups.erase(nexthops); return true; @@ -1787,6 +1816,15 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) if (m_neighOrch->hasNextHop(nexthop)) { next_hop_id = m_neighOrch->getNextHopId(nexthop); + if (srv6_nh) + { + SWSS_LOG_INFO("Single NH: create srv6 vpn %s", nextHops.to_string().c_str()); + if (!m_srv6Orch->srv6Nexthops(nextHops, next_hop_id)) + { + SWSS_LOG_ERROR("Failed to create SRV6 vpn %s", nextHops.to_string().c_str()); + return false; + } + } } /* For non-existent MPLS NH, check if IP neighbor NH exists */ else if (nexthop.isMplsNextHop() && @@ -1837,22 +1875,31 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) /* The route is pointing to a next hop group */ else { - /* Check if there is already an existing next hop group */ - if (!hasNextHopGroup(nextHops)) + /* Need to call srv6nexthops() always for srv6 route, */ + /* regardless of whether there is already an existing next hop group */ + /* because vpn refcount need to be add if need */ + if (srv6_nh) { - if(srv6_nh) + sai_object_id_t temp_nh_id; + SWSS_LOG_INFO("ECMP SRV6 NH: handle srv6 nexthops %s", nextHops.to_string().c_str()); + if(!m_srv6Orch->srv6Nexthops(nextHops, temp_nh_id)) { - sai_object_id_t temp_nh_id; - SWSS_LOG_INFO("ECMP SRV6 NH: create srv6 nexthops %s", nextHops.to_string().c_str()); - if(!m_srv6Orch->srv6Nexthops(nextHops, temp_nh_id)) - { - SWSS_LOG_ERROR("Failed to create SRV6 nexthops for %s", nextHops.to_string().c_str()); - return false; - } + SWSS_LOG_ERROR("Failed to handle SRV6 nexthops for %s", nextHops.to_string().c_str()); + return false; } + } + + /* Check if there is already an existing next hop group */ + if (!hasNextHopGroup(nextHops)) + { /* Try to create a new next hop group */ if (!addNextHopGroup(nextHops)) { + /* If the nexthop is a srv6 nexthop, not create tempRoute + * retry to add route */ + if (nextHops.is_srv6_nexthop()) { + return false; + } for(auto it = nextHops.getNextHops().begin(); it != nextHops.getNextHops().end(); ++it) { const NextHopKey& nextHop = *it; @@ -1915,6 +1962,7 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) copy(route_entry.destination, ipPrefix); sai_attribute_t route_attr; + vector<_sai_attribute_t> route_attrs; auto& object_statuses = ctx.object_statuses; /* If the prefix is not in m_syncdRoutes, then we need to create the route @@ -1938,11 +1986,19 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) { route_attr.id = SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID; route_attr.value.oid = next_hop_id; + route_attrs.push_back(route_attr); + } + + if (nextHops.is_srv6_vpn()) + { + route_attr.id = SAI_ROUTE_ENTRY_ATTR_PREFIX_AGG_ID; + route_attr.value.u32 = m_srv6Orch->getAggId(nextHops); + route_attrs.push_back(route_attr); } /* Default SAI_ROUTE_ATTR_PACKET_ACTION is SAI_PACKET_ACTION_FORWARD */ object_statuses.emplace_back(); - sai_status_t status = gRouteBulker.create_entry(&object_statuses.back(), &route_entry, 1, &route_attr); + sai_status_t status = gRouteBulker.create_entry(&object_statuses.back(), &route_entry, (uint32_t)route_attrs.size(), route_attrs.data()); if (status == SAI_STATUS_ITEM_ALREADY_EXISTS) { SWSS_LOG_ERROR("Failed to create route %s with next hop(s) %s: already exists in bulker", @@ -1989,6 +2045,15 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) gRouteBulker.set_entry_attribute(&object_statuses.back(), &route_entry, &route_attr); } + // Set update preifx agg id if need + if (nextHops.is_srv6_vpn()) + { + route_attr.id = SAI_ROUTE_ENTRY_ATTR_PREFIX_AGG_ID; + route_attr.value.u32 = m_srv6Orch->getAggId(nextHops); + object_statuses.emplace_back(); + gRouteBulker.set_entry_attribute(&object_statuses.back(), &route_entry, &route_attr); + } + route_attr.id = SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID; route_attr.value.oid = next_hop_id; @@ -2143,6 +2208,7 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey /* Clean up the newly created next hop group entry */ removeNextHopGroup(nextHops); } + task_process_status handle_status = handleSaiCreateStatus(SAI_API_ROUTE, status); if (handle_status != task_success) { @@ -2214,6 +2280,12 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey { decreaseNextHopRefCount(it_route->second.nhg_key); auto ol_nextHops = it_route->second.nhg_key; + + if (ol_nextHops.is_srv6_nexthop()) + { + m_bulkSrv6NhgReducedVec.emplace_back(ol_nextHops); + } + if (ol_nextHops.getSize() > 1) { if (m_syncdNextHopGroups[ol_nextHops].ref_count == 0) @@ -2231,10 +2303,6 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey m_bulkNhgReducedRefCnt.emplace(ol_nextHops, vrf_id); } } - else if (ol_nextHops.is_srv6_nexthop()) - { - m_srv6Orch->removeSrv6Nexthops(ol_nextHops); - } else if (ol_nextHops.getSize() == 1) { RouteKey r_key = { vrf_id, ipPrefix }; @@ -2481,6 +2549,11 @@ bool RouteOrch::removeRoutePost(const RouteBulkContext& ctx) decreaseNextHopRefCount(it_route->second.nhg_key); auto ol_nextHops = it_route->second.nhg_key; + if (ol_nextHops.is_srv6_nexthop()) + { + m_bulkSrv6NhgReducedVec.emplace_back(ol_nextHops); + } + MuxOrch* mux_orch = gDirectory.get(); if (it_route->second.nhg_key.getSize() > 1) { @@ -2525,11 +2598,6 @@ bool RouteOrch::removeRoutePost(const RouteBulkContext& ctx) { m_neighOrch->removeMplsNextHop(nexthop); } - else if (nexthop.isSrv6NextHop() && - (m_neighOrch->getNextHopRefCount(nexthop) == 0)) - { - m_srv6Orch->removeSrv6Nexthops(it_route->second.nhg_key); - } RouteKey r_key = { vrf_id, ipPrefix }; removeNextHopRoute(nexthop, r_key); diff --git a/orchagent/routeorch.h b/orchagent/routeorch.h index b232137766..36c2632142 100644 --- a/orchagent/routeorch.h +++ b/orchagent/routeorch.h @@ -249,6 +249,7 @@ class RouteOrch : public Orch, public Subject std::set> m_bulkNhgReducedRefCnt; /* m_bulkNhgReducedRefCnt: nexthop, vrf_id */ + std::vector m_bulkSrv6NhgReducedVec; NextHopObserverTable m_nextHopObservers; diff --git a/orchagent/srv6orch.cpp b/orchagent/srv6orch.cpp index 5a80576e3a..c71c9a45a2 100644 --- a/orchagent/srv6orch.cpp +++ b/orchagent/srv6orch.cpp @@ -130,6 +130,47 @@ bool Srv6Orch::srv6NexthopExists(const NextHopKey &nhKey) } } +bool Srv6Orch::removeSrv6Nexthops(const std::vector &nhgv) +{ + SWSS_LOG_ENTER(); + + // 1. remove vpn_sid first + for (auto& it_nhg : nhgv) + { + for (auto &sr_nh : it_nhg.getNextHops()) + { + if (sr_nh.isSrv6Vpn()) + { + if (!deleteSrv6Vpn(sr_nh, getAggId(it_nhg))) + { + SWSS_LOG_ERROR("Failed to delete SRV6 vpn %s", sr_nh.to_string(false, true).c_str()); + return false; + } + } + } + } + + // 2. delete nexthop & prefix agg id + for (auto& nhg : nhgv) + { + for (auto &sr_nh : nhg.getNextHops()) + { + if (!deleteSrv6Nexthop(sr_nh)) + { + SWSS_LOG_ERROR("Failed to delete SRV6 nexthop %s", sr_nh.to_string(false,true).c_str()); + return false; + } + } + + if (nhg.is_srv6_vpn()) + { + deleteAggId(nhg); + } + } + + return true; +} + bool Srv6Orch::removeSrv6Nexthops(const NextHopGroupKey &nhg) { SWSS_LOG_ENTER(); @@ -198,20 +239,31 @@ bool Srv6Orch::createSrv6Nexthop(const NextHopKey &nh) SWSS_LOG_INFO("SRV6 nexthop already created for %s", nh.to_string(false,true).c_str()); return true; } - sai_object_id_t srv6_object_id = sid_table_[srv6_segment].sid_object_id; - sai_object_id_t srv6_tunnel_id = srv6_tunnel_table_[srv6_source].tunnel_object_id; - if (srv6_object_id == SAI_NULL_OBJECT_ID) + sai_object_id_t srv6_object_id; + sai_object_id_t srv6_tunnel_id; + + if (srv6_segment == "") { - SWSS_LOG_ERROR("segment object doesn't exist for segment %s", srv6_segment.c_str()); - return false; + srv6_object_id = SAI_NULL_OBJECT_ID; + } + else + { + srv6_object_id = sid_table_[srv6_segment].sid_object_id; } - if (srv6_tunnel_id == SAI_NULL_OBJECT_ID) + if (nh.ip_address.to_string() == "0.0.0.0") { - SWSS_LOG_ERROR("tunnel object doesn't exist for source %s", srv6_source.c_str()); - return false; + srv6_tunnel_id = srv6_tunnel_table_[srv6_source].tunnel_object_id; + } + else + { + P2pTunnelKey k; + k.src_ip = nh.srv6_source; + k.endpoint = nh.ip_address.to_string(); + srv6_tunnel_id = srv6_p2p_tunnel_table_[k].tunnel_id; } + SWSS_LOG_INFO("Create srv6 nh for tunnel src %s with seg %s", srv6_source.c_str(), srv6_segment.c_str()); vector nh_attrs; sai_object_id_t nexthop_id; @@ -240,8 +292,110 @@ bool Srv6Orch::createSrv6Nexthop(const NextHopKey &nh) } m_neighOrch->updateSrv6Nexthop(nh, nexthop_id); srv6_nexthop_table_[nh] = nexthop_id; - sid_table_[srv6_segment].nexthops.insert(nh); - srv6TunnelUpdateNexthops(srv6_source, nh, true); + if (srv6_segment != "") + { + sid_table_[srv6_segment].nexthops.insert(nh); + } + + if (nh.ip_address.to_string() == "0.0.0.0") + { + srv6TunnelUpdateNexthops(srv6_source, nh, true); + } + else + { + P2pTunnelKey k; + k.src_ip = nh.srv6_source; + k.endpoint = nh.ip_address.to_string(); + srv6P2ptunnelUpdateNexthops(k, nh, true); + } + return true; +} + +bool Srv6Orch::deleteSrv6Nexthop(const NextHopKey &nh) +{ + SWSS_LOG_ENTER(); + + sai_status_t status = SAI_STATUS_SUCCESS; + + if (!srv6NexthopExists(nh)) + { + return true; + } + + SWSS_LOG_DEBUG("SRV6 Nexthop %s refcount %d", nh.to_string(false,true).c_str(), m_neighOrch->getNextHopRefCount(nh)); + if (m_neighOrch->getNextHopRefCount(nh) == 0) + { + status = sai_next_hop_api->remove_next_hop(srv6_nexthop_table_[nh]); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove SRV6 nexthop %s", nh.to_string(false,true).c_str()); + return false; + } + + /* Decrease srv6 segment reference */ + if (nh.srv6_segment != "") + { + /* Update nexthop in SID table after deleting the nexthop */ + SWSS_LOG_INFO("Seg %s nexthop refcount %zu", + nh.srv6_segment.c_str(), + sid_table_[nh.srv6_segment].nexthops.size()); + if (sid_table_[nh.srv6_segment].nexthops.find(nh) != sid_table_[nh.srv6_segment].nexthops.end()) + { + sid_table_[nh.srv6_segment].nexthops.erase(nh); + } + } + m_neighOrch->updateSrv6Nexthop(nh, 0); + srv6_nexthop_table_.erase(nh); + + /* Delete NH from the tunnel map */ + SWSS_LOG_INFO("Delete NH %s from tunnel map", + nh.to_string(false, true).c_str()); + + if (nh.ip_address.to_string() == "0.0.0.0") + { + string srv6_source = nh.srv6_source; + srv6TunnelUpdateNexthops(srv6_source, nh, false); + size_t tunnel_nhs = srv6TunnelNexthopSize(srv6_source); + if (tunnel_nhs == 0) + { + status = sai_tunnel_api->remove_tunnel(srv6_tunnel_table_[srv6_source].tunnel_object_id); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove SRV6 tunnel object for source %s", srv6_source.c_str()); + return false; + } + srv6_tunnel_table_.erase(srv6_source); + } + else + { + SWSS_LOG_INFO("Nexthops referencing this tunnel object %s: %zu", srv6_source.c_str(),tunnel_nhs); + } + } + else + { + P2pTunnelKey k; + k.src_ip = nh.srv6_source; + k.endpoint = nh.ip_address.to_string(); + srv6P2ptunnelUpdateNexthops(k, nh, false); + + size_t tunnel_nhs = srv6P2pTunnelNexthopSize(k); + if (tunnel_nhs == 0) + { + if (!deleteSrv6P2pTunnel(k)) + { + SWSS_LOG_ERROR("Failed to remove SRV6 p2p tunnel object for src %s, dst %s,", k.src_ip.c_str(), k.endpoint.c_str()); + return false; + } + srv6_p2p_tunnel_table_.erase(k); + } + else + { + SWSS_LOG_INFO("Nexthops referencing this srv6 p2p tunnel object src %s, dst %s : %zu", k.src_ip.c_str(), k.endpoint.c_str(), tunnel_nhs); + } + } + } + + return true; } @@ -254,12 +408,42 @@ bool Srv6Orch::srv6Nexthops(const NextHopGroupKey &nhgKey, sai_object_id_t &next for (auto nh : nexthops) { - srv6_source = nh.srv6_source; - if (!createSrv6Tunnel(srv6_source)) + // 1. create tunnel + if (nh.ip_address.to_string() == "0.0.0.0") { - SWSS_LOG_ERROR("Failed to create tunnel for source %s", srv6_source.c_str()); - return false; + // create srv6 tunnel + srv6_source = nh.srv6_source; + if (!createSrv6Tunnel(srv6_source)) + { + SWSS_LOG_ERROR("Failed to create tunnel for source %s", srv6_source.c_str()); + return false; + } + } + else + { + // create p2p tunnel + P2pTunnelKey k; + k.src_ip = nh.srv6_source; + k.endpoint = nh.ip_address.to_string(); + + if (!createSrv6P2pTunnel(k)) + { + SWSS_LOG_ERROR("Failed to create SRV6 p2p tunnel %s", nh.to_string(false, true).c_str()); + return false; + } } + + // 2. create vpn if need + if (nh.isSrv6Vpn()) + { + if (!createSrv6Vpn(nh, getAggId(nhgKey))) + { + SWSS_LOG_ERROR("Failed to create SRV6 vpn %s", nh.to_string(false, true).c_str()); + return false; + } + } + + // 3. create nexthop if (!createSrv6Nexthop(nh)) { SWSS_LOG_ERROR("Failed to create SRV6 nexthop %s", nh.to_string(false,true).c_str()); @@ -269,7 +453,7 @@ bool Srv6Orch::srv6Nexthops(const NextHopGroupKey &nhgKey, sai_object_id_t &next if (nhgKey.getSize() == 1) { - NextHopKey nhkey(nhgKey.to_string(), false, true); + NextHopKey nhkey = *nhgKey.getNextHops().begin(); nexthop_id = srv6_nexthop_table_[nhkey]; } return true; @@ -600,6 +784,328 @@ bool Srv6Orch::deleteMysidEntry(const string my_sid_string) return true; } +uint32_t Srv6Orch::getAggId(const NextHopGroupKey &nhg) +{ + SWSS_LOG_ENTER(); + static uint32_t g_agg_id = 1; + uint32_t agg_id; + string agg_id_key = nhg.get_srv6_vpn_key(); + + if (srv6_prefix_agg_id_table_.find(agg_id_key) != srv6_prefix_agg_id_table_.end()) { + agg_id = srv6_prefix_agg_id_table_[agg_id_key].prefix_agg_id; + SWSS_LOG_INFO("Agg id already exist, agg_id_key: %s, agg_id %u", agg_id_key.c_str(), agg_id); + } else { + while (srv6_prefix_agg_id_set_.find(g_agg_id) != srv6_prefix_agg_id_set_.end()) { + SWSS_LOG_INFO("Agg id %d is busy, try next", g_agg_id); + g_agg_id++; + // restart with 1 if flip + if (g_agg_id == 0) { + g_agg_id = 1; + } + } + agg_id = g_agg_id; + srv6_prefix_agg_id_table_[agg_id_key].prefix_agg_id = g_agg_id; + // initual ref_count with 0, will be added in increasePrefixAggIdRefCount() later + srv6_prefix_agg_id_table_[agg_id_key].ref_count = 0; + srv6_prefix_agg_id_set_.insert(g_agg_id); + SWSS_LOG_INFO("Agg id not exist, create agg_id_key: %s, agg_id %u", agg_id_key.c_str(), agg_id); + } + + return agg_id; +} + +void Srv6Orch::deleteAggId(const NextHopGroupKey &nhg) +{ + SWSS_LOG_ENTER(); + + string agg_id_key = nhg.get_srv6_vpn_key(); + uint32_t agg_id; + + if (srv6_prefix_agg_id_table_.find(agg_id_key) == srv6_prefix_agg_id_table_.end()) { + return; + } + + agg_id = srv6_prefix_agg_id_table_[agg_id_key].prefix_agg_id; + if (srv6_prefix_agg_id_table_[agg_id_key].ref_count == 0) { + srv6_prefix_agg_id_table_.erase(agg_id_key); + srv6_prefix_agg_id_set_.erase(agg_id); + SWSS_LOG_INFO("Delete Agg id %d, agg_id_key: %s", agg_id, agg_id_key.c_str()); + } + else + { + SWSS_LOG_INFO("Referencing this prefix agg id %u : %u", agg_id, srv6_prefix_agg_id_table_[agg_id_key].ref_count); + } +} + +void Srv6Orch::increasePrefixAggIdRefCount(const NextHopGroupKey &nhg) +{ + SWSS_LOG_ENTER(); + string k = nhg.get_srv6_vpn_key(); + if (srv6_prefix_agg_id_table_.find(k) == srv6_prefix_agg_id_table_.end()) + { + SWSS_LOG_ERROR("Unexpected prefix agg refcount increase for nexthop %s", nhg.to_string().c_str()); + } + else + { + srv6_prefix_agg_id_table_[k].ref_count++; + } +} + +void Srv6Orch::decreasePrefixAggIdRefCount(const NextHopGroupKey &nhg) +{ + SWSS_LOG_ENTER(); + string k = nhg.get_srv6_vpn_key(); + if (srv6_prefix_agg_id_table_.find(k) == srv6_prefix_agg_id_table_.end()) + { + SWSS_LOG_ERROR("Unexpected prefix agg refcount decrease for nexthop %s", nhg.to_string().c_str()); + } + else + { + srv6_prefix_agg_id_table_[k].ref_count--; + } + +} + +bool Srv6Orch::srv6P2pTunnelExists(const P2pTunnelKey &k) +{ + if (srv6_p2p_tunnel_table_.find(k) != srv6_p2p_tunnel_table_.end()) + { + return true; + } + return false; +} + +bool Srv6Orch::createSrv6P2pTunnel(const P2pTunnelKey &k) +{ + SWSS_LOG_ENTER(); + sai_status_t saistatus; + sai_object_id_t srv6_tunnel_map_id; + + sai_attribute_t tunnel_map_attr; + vector tunnel_map_attrs; + + if (srv6P2pTunnelExists(k)) { + return true; + } + + // 0. create tunnel map + tunnel_map_attr.id = SAI_TUNNEL_MAP_ATTR_TYPE; + tunnel_map_attr.value.u32 = SAI_TUNNEL_MAP_TYPE_PREFIX_AGG_ID_TO_SRV6_VPN_SID; + tunnel_map_attrs.push_back(tunnel_map_attr); + + saistatus = sai_tunnel_api->create_tunnel_map(&srv6_tunnel_map_id, gSwitchId, + (uint32_t)tunnel_map_attrs.size(), tunnel_map_attrs.data()); + if (saistatus != SAI_STATUS_SUCCESS) { + SWSS_LOG_ERROR("Failed to create srv6 p2p tunnel map for src_ip: %s dst_ip: %s", k.src_ip.c_str(), k.endpoint.c_str()); + return false; + } + + // 1. create tunnel + sai_object_id_t tunnel_id; + sai_attribute_t tunnel_attr; + vector tunnel_attrs; + sai_ip_address_t ipaddr; + + tunnel_attr.id = SAI_TUNNEL_ATTR_TYPE; + tunnel_attr.value.s32 = SAI_TUNNEL_TYPE_SRV6; + tunnel_attrs.push_back(tunnel_attr); + + IpAddress src_ip(k.src_ip); + ipaddr.addr_family = SAI_IP_ADDR_FAMILY_IPV6; + memcpy(ipaddr.addr.ip6, src_ip.getV6Addr(), sizeof(ipaddr.addr.ip6)); + tunnel_attr.id = SAI_TUNNEL_ATTR_ENCAP_SRC_IP; + tunnel_attr.value.ipaddr = ipaddr; + tunnel_attrs.push_back(tunnel_attr); + + tunnel_attr.id = SAI_TUNNEL_ATTR_UNDERLAY_INTERFACE; + tunnel_attr.value.oid = gUnderlayIfId; + tunnel_attrs.push_back(tunnel_attr); + + sai_object_id_t tunnel_map_list[1]; + tunnel_map_list[0] = srv6_tunnel_map_id; + tunnel_attr.id = SAI_TUNNEL_ATTR_ENCAP_MAPPERS; + tunnel_attr.value.objlist.count = 1; + tunnel_attr.value.objlist.list = tunnel_map_list; + tunnel_attrs.push_back(tunnel_attr); + + tunnel_attr.id = SAI_TUNNEL_ATTR_PEER_MODE; + tunnel_attr.value.u32 = SAI_TUNNEL_PEER_MODE_P2P; + tunnel_attrs.push_back(tunnel_attr); + + IpAddress dst_ip(k.endpoint); + ipaddr.addr_family = SAI_IP_ADDR_FAMILY_IPV6; + memcpy(ipaddr.addr.ip6, dst_ip.getV6Addr(), sizeof(ipaddr.addr.ip6)); + tunnel_attr.id = SAI_TUNNEL_ATTR_ENCAP_DST_IP; + tunnel_attr.value.ipaddr = ipaddr; + tunnel_attrs.push_back(tunnel_attr); + + saistatus = sai_tunnel_api->create_tunnel( + &tunnel_id, gSwitchId, (uint32_t)tunnel_attrs.size(), tunnel_attrs.data()); + if (saistatus != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to create srv6 p2p tunnel for src ip: %s, dst ip: %s", + k.src_ip.c_str(), k.endpoint.c_str()); + + sai_tunnel_api->remove_tunnel_map(srv6_tunnel_map_id); + return false; + } + + srv6_p2p_tunnel_table_[k].tunnel_id = tunnel_id; + srv6_p2p_tunnel_table_[k].tunnel_map_id = srv6_tunnel_map_id; + return true; +} + +bool Srv6Orch::deleteSrv6P2pTunnel(const P2pTunnelKey &k) +{ + if (srv6_p2p_tunnel_table_.find(k) == srv6_p2p_tunnel_table_.end()) + { + return true; + } + + sai_status_t status; + + // 0. remove tunnel + status = sai_tunnel_api->remove_tunnel(srv6_p2p_tunnel_table_[k].tunnel_id); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove SRV6 p2p tunnel object for src_ip %s, dst_ip: %s", k.src_ip.c_str(), k.endpoint.c_str()); + return false; + } + + // 1. remove tunnel map + status = sai_tunnel_api->remove_tunnel_map(srv6_p2p_tunnel_table_[k].tunnel_map_id); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove SRV6 tunnel map object for src_ip %s, dst_ip: %s", k.src_ip.c_str(), k.endpoint.c_str()); + return false; + } + + srv6_p2p_tunnel_table_.erase(k); + return true; +} + +void Srv6Orch::srv6P2ptunnelUpdateNexthops(const P2pTunnelKey &k, const NextHopKey &nhkey, bool insert) +{ + if (insert) + { + srv6_p2p_tunnel_table_[k].nexthops.insert(nhkey); + } + else + { + srv6_p2p_tunnel_table_[k].nexthops.erase(nhkey); + } +} + +size_t Srv6Orch::srv6P2pTunnelNexthopSize(const P2pTunnelKey &k) +{ + return srv6_p2p_tunnel_table_[k].nexthops.size(); +} + +bool Srv6Orch::createSrv6Vpn(const NextHopKey &nh, const uint32_t prefix_agg_id) +{ + SWSS_LOG_ENTER(); + + sai_status_t status; + + Srv6TunnelMapEntryKey tmek; + tmek.src_ip = nh.srv6_source; + tmek.endpoint = nh.ip_address.to_string(); + tmek.vpn_sid = nh.srv6_vpn_sid; + tmek.prefix_agg_id = prefix_agg_id; + + if (srv6_tunnel_map_entry_table_.find(tmek) != srv6_tunnel_map_entry_table_.end()) + { + srv6_tunnel_map_entry_table_[tmek].ref_count++; + return true; + } + + P2pTunnelKey k; + k.src_ip = nh.srv6_source; + k.endpoint = nh.ip_address.to_string(); + + // Accessibility is guaranteed by external + sai_object_id_t tunnel_map_id = srv6_p2p_tunnel_table_[k].tunnel_map_id; + + // 1. create vpn tunnel_map entry + sai_attribute_t tunnel_map_entry_attr; + vector tunnel_map_entry_attrs; + sai_object_id_t tunnel_entry_id; + + tunnel_map_entry_attr.id = SAI_TUNNEL_MAP_ENTRY_ATTR_TUNNEL_MAP_TYPE; + tunnel_map_entry_attr.value.u32 = SAI_TUNNEL_MAP_TYPE_PREFIX_AGG_ID_TO_SRV6_VPN_SID; + tunnel_map_entry_attrs.push_back(tunnel_map_entry_attr); + + tunnel_map_entry_attr.id = SAI_TUNNEL_MAP_ENTRY_ATTR_TUNNEL_MAP; + tunnel_map_entry_attr.value.oid = tunnel_map_id; + tunnel_map_entry_attrs.push_back(tunnel_map_entry_attr); + + tunnel_map_entry_attr.id = SAI_TUNNEL_MAP_ENTRY_ATTR_PREFIX_AGG_ID_KEY; + tunnel_map_entry_attr.value.u32 = prefix_agg_id; + tunnel_map_entry_attrs.push_back(tunnel_map_entry_attr); + + IpAddress vpn_sid(nh.srv6_vpn_sid); + tunnel_map_entry_attr.id = SAI_TUNNEL_MAP_ENTRY_ATTR_SRV6_VPN_SID_VALUE; + memcpy(tunnel_map_entry_attr.value.ip6, vpn_sid.getV6Addr(), sizeof(sai_ip6_t)); + tunnel_map_entry_attrs.push_back(tunnel_map_entry_attr); + + status = sai_tunnel_api->create_tunnel_map_entry(&tunnel_entry_id, gSwitchId, + (uint32_t)tunnel_map_entry_attrs.size(), + tunnel_map_entry_attrs.data()); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to create vpn tunnel_map entry for vpn_sid: %s", nh.srv6_vpn_sid.c_str()); + return false; + } + + // add reference for tunnel map entry + srv6_tunnel_map_entry_table_[tmek].tunnel_map_entry_id = tunnel_entry_id; + srv6_tunnel_map_entry_table_[tmek].ref_count = 1; + return true; +} + +bool Srv6Orch::deleteSrv6Vpn(const NextHopKey &nh, const uint32_t prefix_agg_id) +{ + SWSS_LOG_ENTER(); + sai_status_t status; + + // 1. remove tunnel_map entry if need + sai_object_id_t tunnel_entry_id; + + Srv6TunnelMapEntryKey tmek; + tmek.src_ip = nh.srv6_source; + tmek.endpoint = nh.ip_address.to_string(); + tmek.vpn_sid = nh.srv6_vpn_sid; + tmek.prefix_agg_id = prefix_agg_id; + + if (srv6_tunnel_map_entry_table_.find(tmek) == srv6_tunnel_map_entry_table_.end()) + { + return true; + } + + srv6_tunnel_map_entry_table_[tmek].ref_count--; + if (srv6_tunnel_map_entry_table_[tmek].ref_count == 0) + { + tunnel_entry_id = srv6_tunnel_map_entry_table_[tmek].tunnel_map_entry_id; + status = sai_tunnel_api->remove_tunnel_map_entry(tunnel_entry_id); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove nexthop tunnel map entry %s", nh.to_string(false, true).c_str()); + return false; + } + srv6_tunnel_map_entry_table_.erase(tmek); + } + else + { + SWSS_LOG_INFO("Nexthops referencing this tunnel map entry srv_ip %s, endpoint %s, vpn_sid %s, prefix_agg_id %u : %u", + tmek.src_ip.c_str(), + tmek.endpoint.c_str(), + tmek.vpn_sid.c_str(), + tmek.prefix_agg_id, + srv6_tunnel_map_entry_table_[tmek].ref_count); + } + return true; +} + void Srv6Orch::doTaskMySidTable(const KeyOpFieldsValuesTuple & tuple) { SWSS_LOG_ENTER(); diff --git a/orchagent/srv6orch.h b/orchagent/srv6orch.h index e24f5e00f5..18934df369 100644 --- a/orchagent/srv6orch.h +++ b/orchagent/srv6orch.h @@ -45,10 +45,84 @@ struct MySidEntry string endVrfString; // Used for END.T, END.DT4, END.DT6 and END.DT46, }; +struct P2pTunnelKey +{ + string src_ip; + string endpoint; + + bool operator==(const P2pTunnelKey &o) const + { + return tie(src_ip, endpoint) == + tie(o.src_ip, o.endpoint); + } + + bool operator<(const P2pTunnelKey &o) const + { + return tie(src_ip, endpoint) < + tie(o.src_ip, o.endpoint); + } + + bool operator!=(const P2pTunnelKey &o) const + { + return !(*this == o); + } +}; + +struct P2pTunnelEntry +{ + sai_object_id_t tunnel_id; + sai_object_id_t tunnel_map_id; + + set nexthops; +}; + +struct Srv6TunnelMapEntryKey +{ + string src_ip; + string endpoint; + string vpn_sid; + uint32_t prefix_agg_id; + + bool operator==(const Srv6TunnelMapEntryKey &o) const + { + return tie(src_ip, endpoint, vpn_sid, prefix_agg_id) == + tie(o.src_ip, o.endpoint, o.vpn_sid, o.prefix_agg_id); + } + + bool operator<(const Srv6TunnelMapEntryKey &o) const + { + return tie(src_ip, endpoint, vpn_sid, prefix_agg_id) < + tie(o.src_ip, o.endpoint, o.vpn_sid, o.prefix_agg_id); + } + + bool operator!=(const Srv6TunnelMapEntryKey &o) const + { + return !(*this == o); + } +}; + +struct Srv6TunnelMapEntryEntry +{ + sai_object_id_t tunnel_map_entry_id; + + uint32_t ref_count; +}; + +struct Srv6PrefixAggIdEntry +{ + uint32_t prefix_agg_id; + + uint32_t ref_count; +}; + typedef unordered_map SidTable; typedef unordered_map Srv6TunnelTable; typedef map Srv6NextHopTable; typedef unordered_map Srv6MySidTable; +typedef map Srv6P2pTunnelTable; +typedef unordered_map Srv6PrefixAggIdTable; +typedef set Srv6PrefixAggIdSet; +typedef map Srv6TunnelMapEntryTable; #define SID_LIST_DELIMITER ',' #define MY_SID_KEY_DELIMITER ':' @@ -68,7 +142,15 @@ class Srv6Orch : public Orch { } + + void increasePrefixAggIdRefCount(const NextHopGroupKey&); + void decreasePrefixAggIdRefCount(const NextHopGroupKey&); + + uint32_t getAggId(const NextHopGroupKey &nhg); + void deleteAggId(const NextHopGroupKey &nhg); + bool srv6Nexthops(const NextHopGroupKey &nextHops, sai_object_id_t &next_hop_id); + bool removeSrv6Nexthops(const std::vector &nhgv); bool removeSrv6Nexthops(const NextHopGroupKey &nhg); void update(SubjectType, void *); @@ -80,8 +162,9 @@ class Srv6Orch : public Orch bool deleteSidList(const string seg_name); bool createSrv6Tunnel(const string srv6_source); bool createSrv6Nexthop(const NextHopKey &nh); + bool deleteSrv6Nexthop(const NextHopKey &nh); bool srv6NexthopExists(const NextHopKey &nh); - bool createUpdateMysidEntry(string my_sid_string, const string vrf, const string end_action); + bool createUpdateMysidEntry(string my_sid_string, const string dt_vrf, const string end_action); bool deleteMysidEntry(const string my_sid_string); bool sidEntryEndpointBehavior(const string action, sai_my_sid_entry_endpoint_behavior_t &end_behavior, sai_my_sid_entry_endpoint_behavior_flavor_t &end_flavor); @@ -90,6 +173,14 @@ class Srv6Orch : public Orch void srv6TunnelUpdateNexthops(const string srv6_source, const NextHopKey nhkey, bool insert); size_t srv6TunnelNexthopSize(const string srv6_source); + bool srv6P2pTunnelExists(const P2pTunnelKey &k); + bool createSrv6P2pTunnel(const P2pTunnelKey &k); + bool deleteSrv6P2pTunnel(const P2pTunnelKey &k); + void srv6P2ptunnelUpdateNexthops(const P2pTunnelKey &k, const NextHopKey &nhkey, bool insert); + size_t srv6P2pTunnelNexthopSize(const P2pTunnelKey &k); + + bool createSrv6Vpn(const NextHopKey &nh, const uint32_t prefix_agg_id); + bool deleteSrv6Vpn(const NextHopKey &nh, const uint32_t prefix_agg_id); ProducerStateTable m_sidTable; ProducerStateTable m_mysidTable; @@ -97,6 +188,10 @@ class Srv6Orch : public Orch Srv6TunnelTable srv6_tunnel_table_; Srv6NextHopTable srv6_nexthop_table_; Srv6MySidTable srv6_my_sid_table_; + Srv6P2pTunnelTable srv6_p2p_tunnel_table_; + Srv6PrefixAggIdTable srv6_prefix_agg_id_table_; + Srv6PrefixAggIdSet srv6_prefix_agg_id_set_; + Srv6TunnelMapEntryTable srv6_tunnel_map_entry_table_; VRFOrch *m_vrfOrch; SwitchOrch *m_switchOrch; NeighOrch *m_neighOrch; From c2b80de2509751193cee625a0e5a6bf050eb4411 Mon Sep 17 00:00:00 2001 From: shuai Date: Sun, 25 Jun 2023 12:07:00 +0800 Subject: [PATCH 2/5] addressing review comment --- doc/swss-schema.md | 1 + orchagent/nexthopkey.h | 32 ++++++++++++++++---------------- orchagent/routeorch.cpp | 4 ++-- orchagent/srv6orch.cpp | 17 ++++++++--------- orchagent/srv6orch.h | 4 +++- 5 files changed, 30 insertions(+), 28 deletions(-) diff --git a/doc/swss-schema.md b/doc/swss-schema.md index 74bfd687b8..3c2a51c554 100644 --- a/doc/swss-schema.md +++ b/doc/swss-schema.md @@ -169,6 +169,7 @@ and reflects the LAG ports into the redis under: `LAG_TABLE::port` nexthop_group = string ; index within the NEXTHOP_GROUP_TABLE, used instead of nexthop and intf fields segment = string ; SRV6 segment name seg_src = string ; ipv6 address for SRV6 tunnel source + vpn_sid = string ; ipv6 address for SRV6 VPN sid --------------------------------------------- diff --git a/orchagent/nexthopkey.h b/orchagent/nexthopkey.h index 49e36fb825..ef70c0c23e 100644 --- a/orchagent/nexthopkey.h +++ b/orchagent/nexthopkey.h @@ -25,30 +25,30 @@ struct NextHopKey string srv6_vpn_sid; // SRV6 vpn sid NextHopKey() : weight(0), - srv6_vpn_sid(""), srv6_source(""), - srv6_segment("") + srv6_segment(""), + srv6_vpn_sid("") {} NextHopKey(const std::string &str, const std::string &alias) : alias(alias), vni(0), mac_address(), weight(0), - srv6_vpn_sid(""), srv6_source(""), - srv6_segment("") + srv6_segment(""), + srv6_vpn_sid("") { std::string ip_str = parseMplsNextHop(str); ip_address = ip_str; } NextHopKey(const IpAddress &ip, const std::string &alias) : ip_address(ip), alias(alias), vni(0), mac_address(), weight(0), - srv6_vpn_sid(""), srv6_source(""), - srv6_segment("") + srv6_segment(""), + srv6_vpn_sid("") {} NextHopKey(const std::string &str) : vni(0), mac_address(), - srv6_vpn_sid(""), srv6_source(""), - srv6_segment("") + srv6_segment(""), + srv6_vpn_sid("") { if (str.find(NHG_DELIMITER) != string::npos) { @@ -96,9 +96,9 @@ struct NextHopKey throw std::invalid_argument(err); } ip_address = keys[0]; - srv6_vpn_sid = keys[1]; - srv6_source = keys[2]; - srv6_segment = keys[3]; + srv6_source = keys[1]; + srv6_segment = keys[2]; + srv6_vpn_sid = keys[3]; } else { @@ -114,9 +114,9 @@ struct NextHopKey vni = static_cast(std::stoul(keys[2])); mac_address = keys[3]; weight = 0; - srv6_vpn_sid = ""; srv6_source = ""; srv6_segment = ""; + srv6_vpn_sid = ""; } } @@ -126,9 +126,9 @@ struct NextHopKey vni(vni), mac_address(mac), weight(0), - srv6_vpn_sid(""), srv6_source(""), - srv6_segment("") + srv6_segment(""), + srv6_vpn_sid("") {} const std::string to_string() const @@ -143,9 +143,9 @@ struct NextHopKey if (srv6_nh) { return ip_address.to_string() + NH_DELIMITER + - srv6_vpn_sid + NH_DELIMITER + srv6_source + NH_DELIMITER + - srv6_segment + NH_DELIMITER; + srv6_segment + NH_DELIMITER + + srv6_vpn_sid + NH_DELIMITER; } std::string str = formatMplsNextHop(); str += (ip_address.to_string() + NH_DELIMITER + alias + NH_DELIMITER + diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index 4e18725651..2a928c3584 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -792,9 +792,9 @@ void RouteOrch::doTask(Consumer& consumer) { if (i) nhg_str += NHG_DELIMITER; nhg_str += ipv[i] + NH_DELIMITER; // ip address - nhg_str += srv6_vpn_sidv[i] + NH_DELIMITER; // srv6 vpn sid nhg_str += srv6_src[i] + NH_DELIMITER; // srv6 source nhg_str += NH_DELIMITER; // srv6 segment (empty) + nhg_str += srv6_vpn_sidv[i] + NH_DELIMITER; // srv6 vpn sid } } else @@ -811,9 +811,9 @@ void RouteOrch::doTask(Consumer& consumer) { if (i) nhg_str += NHG_DELIMITER; nhg_str += ip + NH_DELIMITER; // ip address - nhg_str += NH_DELIMITER; // srv6 vpn sid (empty) nhg_str += srv6_src[i] + NH_DELIMITER; // srv6 source nhg_str += srv6_segv[i] + NH_DELIMITER; // srv6 segment + nhg_str += NH_DELIMITER; // srv6 vpn sid (empty) } } diff --git a/orchagent/srv6orch.cpp b/orchagent/srv6orch.cpp index c71c9a45a2..05102b3962 100644 --- a/orchagent/srv6orch.cpp +++ b/orchagent/srv6orch.cpp @@ -787,7 +787,6 @@ bool Srv6Orch::deleteMysidEntry(const string my_sid_string) uint32_t Srv6Orch::getAggId(const NextHopGroupKey &nhg) { SWSS_LOG_ENTER(); - static uint32_t g_agg_id = 1; uint32_t agg_id; string agg_id_key = nhg.get_srv6_vpn_key(); @@ -795,19 +794,19 @@ uint32_t Srv6Orch::getAggId(const NextHopGroupKey &nhg) agg_id = srv6_prefix_agg_id_table_[agg_id_key].prefix_agg_id; SWSS_LOG_INFO("Agg id already exist, agg_id_key: %s, agg_id %u", agg_id_key.c_str(), agg_id); } else { - while (srv6_prefix_agg_id_set_.find(g_agg_id) != srv6_prefix_agg_id_set_.end()) { - SWSS_LOG_INFO("Agg id %d is busy, try next", g_agg_id); - g_agg_id++; + while (srv6_prefix_agg_id_set_.find(m_srv6_agg_id) != srv6_prefix_agg_id_set_.end()) { + SWSS_LOG_INFO("Agg id %d is busy, try next", m_srv6_agg_id); + m_srv6_agg_id++; // restart with 1 if flip - if (g_agg_id == 0) { - g_agg_id = 1; + if (m_srv6_agg_id == 0) { + m_srv6_agg_id = 1; } } - agg_id = g_agg_id; - srv6_prefix_agg_id_table_[agg_id_key].prefix_agg_id = g_agg_id; + agg_id = m_srv6_agg_id; + srv6_prefix_agg_id_table_[agg_id_key].prefix_agg_id = m_srv6_agg_id; // initual ref_count with 0, will be added in increasePrefixAggIdRefCount() later srv6_prefix_agg_id_table_[agg_id_key].ref_count = 0; - srv6_prefix_agg_id_set_.insert(g_agg_id); + srv6_prefix_agg_id_set_.insert(m_srv6_agg_id); SWSS_LOG_INFO("Agg id not exist, create agg_id_key: %s, agg_id %u", agg_id_key.c_str(), agg_id); } diff --git a/orchagent/srv6orch.h b/orchagent/srv6orch.h index 18934df369..8ac222d4e4 100644 --- a/orchagent/srv6orch.h +++ b/orchagent/srv6orch.h @@ -135,7 +135,8 @@ class Srv6Orch : public Orch m_switchOrch(switchOrch), m_neighOrch(neighOrch), m_sidTable(applDb, APP_SRV6_SID_LIST_TABLE_NAME), - m_mysidTable(applDb, APP_SRV6_MY_SID_TABLE_NAME) + m_mysidTable(applDb, APP_SRV6_MY_SID_TABLE_NAME), + m_srv6_agg_id(1) { } ~Srv6Orch() @@ -192,6 +193,7 @@ class Srv6Orch : public Orch Srv6PrefixAggIdTable srv6_prefix_agg_id_table_; Srv6PrefixAggIdSet srv6_prefix_agg_id_set_; Srv6TunnelMapEntryTable srv6_tunnel_map_entry_table_; + uint32_t m_srv6_agg_id; VRFOrch *m_vrfOrch; SwitchOrch *m_switchOrch; NeighOrch *m_neighOrch; From e74bc24fb09efcb9732838134d37bab8a50c36c4 Mon Sep 17 00:00:00 2001 From: shuai Date: Sun, 25 Jun 2023 13:13:51 +0800 Subject: [PATCH 3/5] addressing review comment --- orchagent/srv6orch.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/orchagent/srv6orch.cpp b/orchagent/srv6orch.cpp index 05102b3962..cb193a716e 100644 --- a/orchagent/srv6orch.cpp +++ b/orchagent/srv6orch.cpp @@ -1052,7 +1052,7 @@ bool Srv6Orch::createSrv6Vpn(const NextHopKey &nh, const uint32_t prefix_agg_id) tunnel_map_entry_attrs.data()); if (status != SAI_STATUS_SUCCESS) { - SWSS_LOG_ERROR("Failed to create vpn tunnel_map entry for vpn_sid: %s", nh.srv6_vpn_sid.c_str()); + SWSS_LOG_ERROR("Failed to create vpn tunnel_map entry for vpn_sid: %s prefix_agg_id %u", nh.srv6_vpn_sid.c_str(), prefix_agg_id); return false; } @@ -1088,7 +1088,7 @@ bool Srv6Orch::deleteSrv6Vpn(const NextHopKey &nh, const uint32_t prefix_agg_id) status = sai_tunnel_api->remove_tunnel_map_entry(tunnel_entry_id); if (status != SAI_STATUS_SUCCESS) { - SWSS_LOG_ERROR("Failed to remove nexthop tunnel map entry %s", nh.to_string(false, true).c_str()); + SWSS_LOG_ERROR("Failed to remove nexthop tunnel map entry %s prefix_agg_id %u", nh.to_string(false, true).c_str(), prefix_agg_id); return false; } srv6_tunnel_map_entry_table_.erase(tmek); From 4da302af43f7d391f4b5dbbdadf72e3181f60b8e Mon Sep 17 00:00:00 2001 From: shuai Date: Sun, 25 Jun 2023 16:05:11 +0800 Subject: [PATCH 4/5] The key of Srv6PrefixAggIdTable should be NextHopGroupKey instead of string. --- orchagent/nexthopgroupkey.h | 17 ----------------- orchagent/srv6orch.cpp | 37 ++++++++++++++++--------------------- orchagent/srv6orch.h | 2 +- 3 files changed, 17 insertions(+), 39 deletions(-) diff --git a/orchagent/nexthopgroupkey.h b/orchagent/nexthopgroupkey.h index 9c176b5819..e67934878a 100644 --- a/orchagent/nexthopgroupkey.h +++ b/orchagent/nexthopgroupkey.h @@ -41,7 +41,6 @@ class NextHopGroupKey m_overlay_nexthops = false; m_srv6_nexthops = true; m_srv6_vpn = srv6_vpn; - m_srv6_str = nexthops; auto nhv = tokenize(nexthops, NHG_DELIMITER); for (const auto &nh_str : nhv) @@ -81,11 +80,6 @@ class NextHopGroupKey inline bool operator<(const NextHopGroupKey &o) const { - if (m_srv6_nexthops) - { - return m_srv6_str < o.m_srv6_str; - } - if (m_nexthops < o.m_nexthops) { return true; @@ -205,9 +199,6 @@ class NextHopGroupKey const std::string to_string() const { - if (m_srv6_nexthops) { - return m_srv6_str; - } string nhs_str; for (auto it = m_nexthops.begin(); it != m_nexthops.end(); ++it) @@ -241,12 +232,6 @@ class NextHopGroupKey return m_srv6_vpn; } - inline string get_srv6_vpn_key() const - { - // use nexthopgroupkey as vpn key - return m_srv6_str; - } - void clear() { m_nexthops.clear(); @@ -256,8 +241,6 @@ class NextHopGroupKey std::set m_nexthops; bool m_overlay_nexthops; bool m_srv6_nexthops; - - string m_srv6_str; bool m_srv6_vpn; }; diff --git a/orchagent/srv6orch.cpp b/orchagent/srv6orch.cpp index cb193a716e..98a943f43f 100644 --- a/orchagent/srv6orch.cpp +++ b/orchagent/srv6orch.cpp @@ -788,11 +788,10 @@ uint32_t Srv6Orch::getAggId(const NextHopGroupKey &nhg) { SWSS_LOG_ENTER(); uint32_t agg_id; - string agg_id_key = nhg.get_srv6_vpn_key(); - if (srv6_prefix_agg_id_table_.find(agg_id_key) != srv6_prefix_agg_id_table_.end()) { - agg_id = srv6_prefix_agg_id_table_[agg_id_key].prefix_agg_id; - SWSS_LOG_INFO("Agg id already exist, agg_id_key: %s, agg_id %u", agg_id_key.c_str(), agg_id); + if (srv6_prefix_agg_id_table_.find(nhg) != srv6_prefix_agg_id_table_.end()) { + agg_id = srv6_prefix_agg_id_table_[nhg].prefix_agg_id; + SWSS_LOG_INFO("Agg id already exist, agg_id_key: %s, agg_id %u", nhg.to_string().c_str(), agg_id); } else { while (srv6_prefix_agg_id_set_.find(m_srv6_agg_id) != srv6_prefix_agg_id_set_.end()) { SWSS_LOG_INFO("Agg id %d is busy, try next", m_srv6_agg_id); @@ -803,11 +802,11 @@ uint32_t Srv6Orch::getAggId(const NextHopGroupKey &nhg) } } agg_id = m_srv6_agg_id; - srv6_prefix_agg_id_table_[agg_id_key].prefix_agg_id = m_srv6_agg_id; + srv6_prefix_agg_id_table_[nhg].prefix_agg_id = m_srv6_agg_id; // initual ref_count with 0, will be added in increasePrefixAggIdRefCount() later - srv6_prefix_agg_id_table_[agg_id_key].ref_count = 0; + srv6_prefix_agg_id_table_[nhg].ref_count = 0; srv6_prefix_agg_id_set_.insert(m_srv6_agg_id); - SWSS_LOG_INFO("Agg id not exist, create agg_id_key: %s, agg_id %u", agg_id_key.c_str(), agg_id); + SWSS_LOG_INFO("Agg id not exist, create agg_id_key: %s, agg_id %u", nhg.to_string().c_str(), agg_id); } return agg_id; @@ -816,51 +815,47 @@ uint32_t Srv6Orch::getAggId(const NextHopGroupKey &nhg) void Srv6Orch::deleteAggId(const NextHopGroupKey &nhg) { SWSS_LOG_ENTER(); - - string agg_id_key = nhg.get_srv6_vpn_key(); uint32_t agg_id; - if (srv6_prefix_agg_id_table_.find(agg_id_key) == srv6_prefix_agg_id_table_.end()) { + if (srv6_prefix_agg_id_table_.find(nhg) == srv6_prefix_agg_id_table_.end()) { return; } - agg_id = srv6_prefix_agg_id_table_[agg_id_key].prefix_agg_id; - if (srv6_prefix_agg_id_table_[agg_id_key].ref_count == 0) { - srv6_prefix_agg_id_table_.erase(agg_id_key); + agg_id = srv6_prefix_agg_id_table_[nhg].prefix_agg_id; + if (srv6_prefix_agg_id_table_[nhg].ref_count == 0) { + srv6_prefix_agg_id_table_.erase(nhg); srv6_prefix_agg_id_set_.erase(agg_id); - SWSS_LOG_INFO("Delete Agg id %d, agg_id_key: %s", agg_id, agg_id_key.c_str()); + SWSS_LOG_INFO("Delete Agg id %d, agg_id_key: %s", agg_id, nhg.to_string().c_str()); } else { - SWSS_LOG_INFO("Referencing this prefix agg id %u : %u", agg_id, srv6_prefix_agg_id_table_[agg_id_key].ref_count); + SWSS_LOG_INFO("Referencing this prefix agg id %u : %u", agg_id, srv6_prefix_agg_id_table_[nhg].ref_count); } } void Srv6Orch::increasePrefixAggIdRefCount(const NextHopGroupKey &nhg) { SWSS_LOG_ENTER(); - string k = nhg.get_srv6_vpn_key(); - if (srv6_prefix_agg_id_table_.find(k) == srv6_prefix_agg_id_table_.end()) + if (srv6_prefix_agg_id_table_.find(nhg) == srv6_prefix_agg_id_table_.end()) { SWSS_LOG_ERROR("Unexpected prefix agg refcount increase for nexthop %s", nhg.to_string().c_str()); } else { - srv6_prefix_agg_id_table_[k].ref_count++; + srv6_prefix_agg_id_table_[nhg].ref_count++; } } void Srv6Orch::decreasePrefixAggIdRefCount(const NextHopGroupKey &nhg) { SWSS_LOG_ENTER(); - string k = nhg.get_srv6_vpn_key(); - if (srv6_prefix_agg_id_table_.find(k) == srv6_prefix_agg_id_table_.end()) + if (srv6_prefix_agg_id_table_.find(nhg) == srv6_prefix_agg_id_table_.end()) { SWSS_LOG_ERROR("Unexpected prefix agg refcount decrease for nexthop %s", nhg.to_string().c_str()); } else { - srv6_prefix_agg_id_table_[k].ref_count--; + srv6_prefix_agg_id_table_[nhg].ref_count--; } } diff --git a/orchagent/srv6orch.h b/orchagent/srv6orch.h index 8ac222d4e4..76c8d3297a 100644 --- a/orchagent/srv6orch.h +++ b/orchagent/srv6orch.h @@ -120,7 +120,7 @@ typedef unordered_map Srv6TunnelTable; typedef map Srv6NextHopTable; typedef unordered_map Srv6MySidTable; typedef map Srv6P2pTunnelTable; -typedef unordered_map Srv6PrefixAggIdTable; +typedef map Srv6PrefixAggIdTable; typedef set Srv6PrefixAggIdSet; typedef map Srv6TunnelMapEntryTable; From 728cec5d739beb5c245264d5f2fb9f7291ecd941 Mon Sep 17 00:00:00 2001 From: shuai Date: Mon, 8 May 2023 17:07:12 +0800 Subject: [PATCH 5/5] swss UTs for srv6 vpn functions --- tests/test_srv6.py | 449 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 449 insertions(+) diff --git a/tests/test_srv6.py b/tests/test_srv6.py index 68bee80a52..0394394350 100644 --- a/tests/test_srv6.py +++ b/tests/test_srv6.py @@ -18,6 +18,13 @@ def get_created_entry(db, table, existed_entries): assert len(new_entries) == 1, "Wrong number of created entries." return new_entries[0] +def get_created_entries(db, table, existed_entries, number): + tbl = swsscommon.Table(db, table) + entries = set(tbl.getKeys()) + new_entries = list(entries - existed_entries) + assert len(new_entries) == number, "Wrong number of created entries." + return new_entries + class TestSrv6Mysid(object): def setup_db(self, dvs): self.pdb = dvs.get_app_db() @@ -313,6 +320,448 @@ def test_srv6(self, dvs, testlog): assert nexthop_entries == get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP") assert route_entries == get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY") +class TestSrv6Vpn(object): + def setup_db(self, dvs): + self.pdb = dvs.get_app_db() + self.adb = dvs.get_asic_db() + self.cdb = dvs.get_config_db() + + def create_srv6_vpn_route(self, routeip, nexthop, segsrc, vpn_sid, ifname): + table = "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY" + existed_entries = get_exist_entries(self.adb.db_connection, table) + + fvs=swsscommon.FieldValuePairs([('seg_src', segsrc), ('nexthop', nexthop), ('vpn_sid', vpn_sid), ('ifname', ifname)]) + routetbl = swsscommon.ProducerStateTable(self.pdb.db_connection, "ROUTE_TABLE") + routetbl.set(routeip,fvs) + + self.adb.wait_for_n_keys(table, len(existed_entries) + 1) + return get_created_entry(self.adb.db_connection, table, existed_entries) + + def create_srv6_vpn_route_with_nhg(self, routeip, nexthops, segsrc_list, vpn_list, ifname_list): + table = "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY" + existed_entries = get_exist_entries(self.adb.db_connection, table) + + fvs=swsscommon.FieldValuePairs([('seg_src', ",".join(segsrc_list)), ('nexthop', ",".join(nexthops)), ('vpn_sid', ",".join(vpn_list)), ('ifname', ",".join(ifname_list))]) + routetbl = swsscommon.ProducerStateTable(self.pdb.db_connection, "ROUTE_TABLE") + routetbl.set(routeip,fvs) + + self.adb.wait_for_n_keys(table, len(existed_entries) + 1) + return get_created_entry(self.adb.db_connection, table, existed_entries) + + def update_srv6_vpn_route_attribute(self, routeip, nexthops, segsrc_list, vpn_list, ifname_list): + fvs=swsscommon.FieldValuePairs([('seg_src', ",".join(segsrc_list)), ('nexthop', ",".join(nexthops)), ('vpn_sid', ",".join(vpn_list)), ('ifname', ",".join(ifname_list))]) + routetbl = swsscommon.ProducerStateTable(self.pdb.db_connection, "ROUTE_TABLE") + routetbl.set(routeip,fvs) + return True + + def remove_srv6_route(self, routeip): + routetbl = swsscommon.ProducerStateTable(self.pdb.db_connection, "ROUTE_TABLE") + routetbl._del(routeip) + + def check_deleted_route_entries(self, destinations): + def _access_function(): + route_entries = self.adb.get_keys("ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY") + route_destinations = [json.loads(route_entry)["dest"] for route_entry in route_entries] + return (all(destination not in route_destinations for destination in destinations), None) + + wait_for_result(_access_function) + + def add_neighbor(self, interface, ip, mac): + fvs=swsscommon.FieldValuePairs([("neigh", mac)]) + neightbl = swsscommon.Table(self.cdb.db_connection, "NEIGH") + neightbl.set(interface + "|" +ip, fvs) + time.sleep(1) + + def remove_neighbor(self, interface,ip): + neightbl = swsscommon.Table(self.cdb.db_connection, "NEIGH") + neightbl._del(interface + "|" + ip) + time.sleep(1) + + def test_srv6_vpn(self, dvs, testlog): + self.setup_db(dvs) + dvs.setup_db() + + # save exist asic db entries + tunnel_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL") + nexthop_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP") + route_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY") + vpn_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_TUNNEL_MAP_TYPE_PREFIX_AGG_ID_TO_SRV6_VPN_SID") + map_entry_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP_ENTRY") + map_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP") + + # create v4 route with vpn sid + route_key = self.create_srv6_vpn_route('5000::/64', '2001::1', '1001:2000::1', '3000::1', 'unknown') + nexthop_id = get_created_entry(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP", nexthop_entries) + tunnel_id = get_created_entry(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL", tunnel_entries) + map_entry_id = get_created_entry(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP_ENTRY", map_entry_entries) + map_id = get_created_entry(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP", map_entries) + prefix_agg_id = "1" + + # check ASIC SAI_OBJECT_TYPE_ROUTE_ENTRY database + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY") + (status, fvs) = tbl.get(route_key) + assert status == True + for fv in fvs: + if fv[0] == "SAI_ROUTE_ENTRY_ATTR_PREFIX_AGG_ID": + assert prefix_agg_id == fv[1] + + # check ASIC SAI_OBJECT_TYPE_TUNNEL_MAP database + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP") + (status, fvs) = tbl.get(map_id) + assert status == True + for fv in fvs: + if fv[0] == "SAI_TUNNEL_MAP_ATTR_TYPE": + assert fv[1] == "SAI_TUNNEL_MAP_TYPE_PREFIX_AGG_ID_TO_SRV6_VPN_SID" + + # check ASIC SAI_OBJECT_TYPE_TUNNEL database + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL") + (status, fvs) = tbl.get(tunnel_id) + assert status == True + for fv in fvs: + if fv[0] == "SAI_TUNNEL_ATTR_PEER_MODE": + assert fv[1] == "SAI_TUNNEL_PEER_MODE_P2P" + + # check vpn sid value in SRv6 route is created + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP_ENTRY") + (status, fvs) = tbl.get(map_entry_id) + assert status == True + for fv in fvs: + if fv[0] == "SAI_TUNNEL_MAP_ENTRY_ATTR_SRV6_VPN_SID_VALUE": + assert fv[1] == "3000::1" + if fv[0] == "SAI_TUNNEL_MAP_ENTRY_ATTR_PREFIX_AGG_ID_KEY": + assert fv[1] == prefix_agg_id + + # check sid list value in ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP is created + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP") + (status, fvs) = tbl.get(nexthop_id) + assert status == True + for fv in fvs: + if fv[0] == "SAI_NEXT_HOP_ATTR_TYPE": + assert fv[1] == "SAI_NEXT_HOP_TYPE_SRV6_SIDLIST" + + self.remove_srv6_route('5000::/64') + self.check_deleted_route_entries('5000::/64') + time.sleep(5) + # check ASIC SAI_OBJECT_TYPE_TUNNEL_MAP is removed + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP") + (status, fvs) = tbl.get(map_id) + assert status == False + + # check ASIC SAI_OBJECT_TYPE_TUNNEL is removed + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL") + (status, fvs) = tbl.get(tunnel_id) + assert status == False + + # check vpn sid value in SRv6 route is removed + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP_ENTRY") + (status, fvs) = tbl.get(map_entry_id) + assert status == False + + # check nexthop id in SRv6 route is removed + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP") + (status, fvs) = tbl.get(nexthop_id) + assert status == False + + # check ASIC SAI_OBJECT_TYPE_ROUTE_ENTRY is removed + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY") + (status, fvs) = tbl.get(route_key) + assert status == False + + def test_srv6_vpn_with_nhg(self, dvs, testlog): + self.setup_db(dvs) + dvs.setup_db() + + segsrc_list = [] + nexthop_list = [] + vpn_list = [] + ifname_list = [] + + # save exist asic db entries + tunnel_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL") + nexthop_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP") + route_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY") + vpn_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_TUNNEL_MAP_TYPE_PREFIX_AGG_ID_TO_SRV6_VPN_SID") + map_entry_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP_ENTRY") + map_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP") + + nexthop_group_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP") + nexthop_group_member_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER") + + segsrc_list.append('1001:2000::1') + segsrc_list.append('1001:2000::1') + + nexthop_list.append('2000::1') + nexthop_list.append('2000::2') + + vpn_list.append('3000::1') + vpn_list.append('3000::2') + + ifname_list.append('unknown') + ifname_list.append('unknown') + + route_key = self.create_srv6_vpn_route_with_nhg('5000::/64', nexthop_list, segsrc_list, vpn_list, ifname_list) + tunnel_ids = get_created_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL", tunnel_entries, 2) + nh_ids = get_created_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP", nexthop_entries, 2) + nhg_id = get_created_entry(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP", nexthop_group_entries) + nhg_mem = get_created_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER", nexthop_group_member_entries, 2) + map_entry_ids = get_created_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP_ENTRY", map_entry_entries, 2) + map_ids = get_created_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP", map_entries, 2) + + nh_ids = sorted(nh_ids) + nhg_mem = sorted(nhg_mem) + prefix_agg_id = "1" + + # check ASIC SAI_OBJECT_TYPE_ROUTE_ENTRY database + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY") + (status, fvs) = tbl.get(route_key) + assert status == True + for fv in fvs: + if fv[0] == "SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID": + assert fv[1] == nhg_id + if fv[0] == "SAI_ROUTE_ENTRY_ATTR_PREFIX_AGG_ID": + assert fv[1] == prefix_agg_id + + # check ASIC SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER database + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER") + (status, fvs) = tbl.get(nhg_mem[0]) + assert status == True + for fv in fvs: + if fv[0] == "SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID": + assert fv[1] == nhg_id + elif fv[0] == "SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID": + assert fv[1] == nh_ids[0] + + # check ASIC SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER database + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER") + (status, fvs) = tbl.get(nhg_mem[1]) + assert status == True + for fv in fvs: + if fv[0] == "SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID": + assert fv[1] == nhg_id + elif fv[0] == "SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID": + assert fv[1] == nh_ids[1] + + # check ASIC SAI_OBJECT_TYPE_TUNNEL_MAP database + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP") + for map_id in map_ids: + (status, fvs) = tbl.get(map_id) + assert status == True + for fv in fvs: + if fv[0] == "SAI_TUNNEL_MAP_ATTR_TYPE": + assert fv[1] == "SAI_TUNNEL_MAP_TYPE_PREFIX_AGG_ID_TO_SRV6_VPN_SID" + + # check ASIC SAI_OBJECT_TYPE_TUNNEL database + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL") + for tunnel_id in tunnel_ids: + (status, fvs) = tbl.get(tunnel_id) + assert status == True + for fv in fvs: + if fv[0] == "SAI_TUNNEL_ATTR_PEER_MODE": + assert fv[1] == "SAI_TUNNEL_PEER_MODE_P2P" + + # check vpn sid value in SRv6 route is created + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP_ENTRY") + for map_entry_id in map_entry_ids: + (status, fvs) = tbl.get(map_entry_id) + assert status == True + for fv in fvs: + if fv[0] == "SAI_TUNNEL_MAP_ENTRY_ATTR_PREFIX_AGG_ID_KEY": + assert fv[1] == prefix_agg_id + + # check sid list value in ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP is created + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP") + for nh_id in nh_ids: + (status, fvs) = tbl.get(nh_id) + assert status == True + for fv in fvs: + if fv[0] == "SAI_NEXT_HOP_ATTR_TYPE": + assert fv[1] == "SAI_NEXT_HOP_TYPE_SRV6_SIDLIST" + + + route_key_new = self.create_srv6_vpn_route_with_nhg('5001::/64', nexthop_list, segsrc_list, vpn_list, ifname_list) + # check ASIC SAI_OBJECT_TYPE_ROUTE_ENTRY database + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY") + (status, fvs) = tbl.get(route_key_new) + assert status == True + for fv in fvs: + if fv[0] == "SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID": + assert fv[1] == nhg_id + if fv[0] == "SAI_ROUTE_ENTRY_ATTR_PREFIX_AGG_ID": + assert fv[1] == prefix_agg_id + + # remove routes + self.remove_srv6_route('5001::/64') + self.check_deleted_route_entries('5001::/64') + + time.sleep(5) + # check ASIC SAI_OBJECT_TYPE_ROUTE_ENTRY is removed + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY") + (status, fvs) = tbl.get(route_key_new) + assert status == False + + # remove routes + self.remove_srv6_route('5000::/64') + self.check_deleted_route_entries('5000::/64') + + time.sleep(5) + # check ASIC SAI_OBJECT_TYPE_ROUTE_ENTRY is removed + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY") + (status, fvs) = tbl.get(route_key) + assert status == False + + # check ASIC SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER is removed + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER") + (status, fvs) = tbl.get(nhg_mem[0]) + assert status == False + + # check ASIC SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER is removed + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER") + (status, fvs) = tbl.get(nhg_mem[1]) + assert status == False + + # check ASIC SAI_OBJECT_TYPE_TUNNEL_MAP is removed + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP") + for map_id in map_ids: + (status, fvs) = tbl.get(map_id) + assert status == False + + # check ASIC SAI_OBJECT_TYPE_TUNNEL is removed + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL") + for tunnel_id in tunnel_ids: + (status, fvs) = tbl.get(tunnel_id) + assert status == False + + # check vpn sid value in SRv6 route is removed + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP_ENTRY") + for map_entry_id in map_entry_ids: + (status, fvs) = tbl.get(map_entry_id) + assert status == False + + # check next hop in ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP is removed + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP") + for nh_id in nh_ids: + (status, fvs) = tbl.get(nh_id) + assert status == False + + def test_srv6_vpn_nh_update(self, dvs, testlog): + self.setup_db(dvs) + dvs.setup_db() + + segsrc_list = [] + nexthop_list = [] + vpn_list = [] + ifname_list = [] + + # save exist asic db entries + tunnel_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL") + nexthop_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP") + route_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY") + vpn_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_TUNNEL_MAP_TYPE_PREFIX_AGG_ID_TO_SRV6_VPN_SID") + map_entry_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP_ENTRY") + map_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP") + + nexthop_group_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP") + nexthop_group_member_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER") + map_entry_prefix_agg_id = "1" + route_entry_prefix_agg_id = "1" + route_entry_next_hop_id = "1" + + # create v4 route with vpn sid + route_key = self.create_srv6_vpn_route('5000::/64', '2000::1', '1001:2000::1', '3000::1', 'unknown') + map_entry_id = get_created_entry(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP_ENTRY", map_entry_entries) + + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP_ENTRY") + (status, fvs) = tbl.get(map_entry_id) + assert status == True + for fv in fvs: + if fv[0] == "SAI_TUNNEL_MAP_ENTRY_ATTR_PREFIX_AGG_ID_KEY": + map_entry_prefix_agg_id = fv[1] + + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY") + (status, fvs) = tbl.get(route_key) + assert status == True + for fv in fvs: + if fv[0] == "SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID": + route_entry_next_hop_id = fv[1] + if fv[0] == "SAI_ROUTE_ENTRY_ATTR_PREFIX_AGG_ID": + route_entry_prefix_agg_id = fv[1] + + segsrc_list.append('1001:2000::1') + segsrc_list.append('1001:2000::1') + + nexthop_list.append('2000::1') + nexthop_list.append('2000::2') + + vpn_list.append('3000::1') + vpn_list.append('3000::2') + + ifname_list.append('unknown') + ifname_list.append('unknown') + + map_entry_entries = get_exist_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP_ENTRY") + self.update_srv6_vpn_route_attribute('5000::/64', nexthop_list, segsrc_list, vpn_list, ifname_list) + + time.sleep(5) + nh_ids = get_created_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP", nexthop_entries, 2) + nhg_id = get_created_entry(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP", nexthop_group_entries) + nhg_mem = get_created_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER", nexthop_group_member_entries, 2) + + map_entry_ids = get_created_entries(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP_ENTRY", map_entry_entries, 2) + map_entry_id_group = "1" + + for map_id in map_entry_ids: + if map_id != map_entry_id: + map_entry_id_group = map_id + break + + nh_ids = sorted(nh_ids) + nhg_mem = sorted(nhg_mem) + + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_MAP_ENTRY") + (status, fvs) = tbl.get(map_entry_id_group) + assert status == True + for fv in fvs: + if fv[0] == "SAI_TUNNEL_MAP_ENTRY_ATTR_PREFIX_AGG_ID_KEY": + assert fv[1] != map_entry_prefix_agg_id + + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP") + (status, fvs) = tbl.get(nhg_id) + assert status == True + + # check ASIC SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER database + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER") + (status, fvs) = tbl.get(nhg_mem[0]) + assert status == True + for fv in fvs: + if fv[0] == "SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID": + assert fv[1] == nhg_id + elif fv[0] == "SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID": + assert fv[1] == nh_ids[0] + + # check ASIC SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER database + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER") + (status, fvs) = tbl.get(nhg_mem[1]) + assert status == True + for fv in fvs: + if fv[0] == "SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID": + assert fv[1] == nhg_id + elif fv[0] == "SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID": + assert fv[1] == nh_ids[1] + + tbl = swsscommon.Table(self.adb.db_connection, "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY") + (status, fvs) = tbl.get(route_key) + assert status == True + for fv in fvs: + if fv[0] == "SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID": + assert fv[1] != route_entry_next_hop_id + if fv[0] == "SAI_ROUTE_ENTRY_ATTR_PREFIX_AGG_ID": + assert fv[1] != route_entry_prefix_agg_id + + # remove routes + self.remove_srv6_route('5000::/64') + self.check_deleted_route_entries('5000::/64') + time.sleep(5) + # Add Dummy always-pass test at end as workaroud # for issue when Flaky fail on final test it invokes module tear-down before retrying def test_nonflaky_dummy():