Merge branch 'Azure:master' into route-flow-counter-1

.azure-pipelines/run-test-template.yml

@@ -31,6 +31,7 @@ steps:
     artifact: sonic-buildimage.vs
     runVersion: 'latestFromBranch'
     runBranch: 'refs/heads/master'
+    allowPartiallySucceededBuilds: true
   displayName: "Download sonic kvm image"
 
 - script: |

SECURITY.md

@@ -0,0 +1,41 @@
+<!-- BEGIN MICROSOFT SECURITY.MD V0.0.7 BLOCK -->
+
+## Security
+
+Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
+
+If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below.
+
+## Reporting Security Issues
+
+**Please do not report security vulnerabilities through public GitHub issues.**
+
+Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report).
+
+If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com).  If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey).
+
+You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 
+
+Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
+
+  * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
+  * Full paths of source file(s) related to the manifestation of the issue
+  * The location of the affected source code (tag/branch/commit or direct URL)
+  * Any special configuration required to reproduce the issue
+  * Step-by-step instructions to reproduce the issue
+  * Proof-of-concept or exploit code (if possible)
+  * Impact of the issue, including how an attacker might exploit the issue
+
+This information will help us triage your report more quickly.
+
+If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs.
+
+## Preferred Languages
+
+We prefer all communications to be in English.
+
+## Policy
+
+Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd).
+
+<!-- END MICROSOFT SECURITY.MD BLOCK -->

ansible/dualtor/nic_simulator/nic_simulator.py

@@ -164,17 +164,21 @@ def to_string(self):
 class OVSFlow(StrObj):
     """Object to represent an OVS flow."""
 
-    __slots__ = ("in_port", "output_ports", "group", "_str_prefix")
+    __slots__ = ("in_port", "output_ports", "group", "priority", "_str_prefix")
 
-    def __init__(self, in_port, packet_filter=None, output_ports=[], group=None):
+    def __init__(self, in_port, packet_filter=None, output_ports=[], group=None, priority=None):
         self.in_port = in_port
         self.packet_filter = packet_filter
         self.output_ports = set(output_ports)
         self.group = group
+        self.priority = priority
+        self._str_prefix = []
+        if self.priority:
+            self._str_prefix.append("priority=%s" % self.priority)
         if self.packet_filter:
-            self._str_prefix = "%s,in_port=%s" % (self.packet_filter, self.in_port)
-        else:
-            self._str_prefix = "in_port=%s" % (self.in_port)
+            self._str_prefix.append(str(self.packet_filter))
+        self._str_prefix.append("in_port=%s" % self.in_port)
+        self._str_prefix = ",".join(self._str_prefix)
 
     def to_string(self):
         flow_parts = [self._str_prefix]
@@ -262,8 +266,8 @@ class UpstreamECMPFlow(OVSFlow):
 
     __slots__ = ()
 
-    def __init__(self, in_port, group):
-        super(UpstreamECMPFlow, self).__init__(in_port, group=group)
+    def __init__(self, in_port, group, priority=None):
+        super(UpstreamECMPFlow, self).__init__(in_port, group=group, priority=priority)
 
     def set_upper_tor_forwarding_state(self, state):
         self.group.set_upper_tor_forwarding_state(state)
@@ -361,17 +365,17 @@ def _init_flows(self):
         self._del_flows()
         self._del_groups()
         # downstream flows
-        self._add_flow(self.upper_tor_port, output_ports=[self.ptf_port, self.server_nic])
-        self._add_flow(self.lower_tor_port, output_ports=[self.ptf_port, self.server_nic])
+        self._add_flow(self.upper_tor_port, output_ports=[self.ptf_port, self.server_nic], priority=10)
+        self._add_flow(self.lower_tor_port, output_ports=[self.ptf_port, self.server_nic], priority=10)
 
         # upstream flows
         # upstream packet from server NiC should be directed to both ToRs
-        self._add_flow(self.server_nic, output_ports=[self.upper_tor_port, self.lower_tor_port])
+        self._add_flow(self.server_nic, output_ports=[self.upper_tor_port, self.lower_tor_port], priority=9)
         # upstream icmp packet from ptf port should be directed to both ToRs
-        self._add_flow(self.ptf_port, packet_filter="icmp", output_ports=[self.upper_tor_port, self.lower_tor_port])
+        self._add_flow(self.ptf_port, packet_filter="icmp", output_ports=[self.upper_tor_port, self.lower_tor_port], priority=8)
         # upstream packet from ptf port should be ECMP directed to active ToRs
         self.upstream_ecmp_group = self._add_upstream_ecmp_group(1, self.upper_tor_port, self.lower_tor_port)
-        self.upstream_ecmp_flow = self._add_upstream_ecmp_flow(self.ptf_port, self.upstream_ecmp_group)
+        self.upstream_ecmp_flow = self._add_upstream_ecmp_flow(self.ptf_port, self.upstream_ecmp_group, priority=7)
 
     def _get_ports(self):
         result = OVSCommand.ovs_vsctl_list_ports(self.bridge_name)
@@ -387,8 +391,8 @@ def _del_groups(self):
         self.upstream_ecmp_group = None
         self.groups.clear()
 
-    def _add_flow(self, in_port, packet_filter=None, output_ports=[], group=None):
-        flow = OVSFlow(in_port, packet_filter=packet_filter, output_ports=output_ports, group=group)
+    def _add_flow(self, in_port, packet_filter=None, output_ports=[], group=None, priority=None):
+        flow = OVSFlow(in_port, packet_filter=packet_filter, output_ports=output_ports, group=group, priority=priority)
         logging.info("Add flow to bridge %s: %s", self.bridge_name, flow)
         OVSCommand.ovs_ofctl_add_flow(self.bridge_name, flow)
         self.flows.append(flow)
@@ -401,8 +405,8 @@ def _add_upstream_ecmp_group(self, group_id, upper_tor_port, lower_tor_port):
         self.groups.append(group)
         return group
 
-    def _add_upstream_ecmp_flow(self, in_port, group):
-        flow = UpstreamECMPFlow(in_port, group)
+    def _add_upstream_ecmp_flow(self, in_port, group, priority=None):
+        flow = UpstreamECMPFlow(in_port, group, priority=priority)
         logging.info("Add upstream ecmp flow to bridge %s: %s", self.bridge_name, flow)
         OVSCommand.ovs_ofctl_add_flow(self.bridge_name, flow)
         self.flows.append(flow)

ansible/group_vars/all/nic_simulator_grpc_port_map.yml

@@ -2,4 +2,4 @@
 # mapping from testbed name to nic simulator gRPC binding port
 
 nic_simulator_grpc_port:
-  vms-kvm-dual-mixed: 8900
+  vms-kvm-dual-mixed: 50075

ansible/group_vars/sonic/variables

@@ -23,8 +23,8 @@ barefoot_hwskus: [ "montara", "mavericks", "Arista-7170-64C", "newport", "Arista
 
 marvell_hwskus: [ "et6448m" ]
 
-cisco_hwskus: ["64x100Gb"]
-cisco-8000_gb_hwskus: ["64x100Gb"]
+cisco_hwskus: ["Cisco-8102-C64"]
+cisco-8000_gb_hwskus: ["Cisco-8102-C64"]
 
 ## Note:
 ## Docker volumes should be list instead of dict. However, if we want to keep code DRY, we

ansible/library/dual_tor_facts.py

@@ -1,4 +1,26 @@
+import os
+import yaml
+
 from collections import defaultdict
+
+try:
+    from ansible.module_utils.dualtor_utils import generate_mux_cable_facts
+except ImportError:
+    # Add parent dir for using outside Ansible
+    import sys
+    sys.path.append('..')
+    from ansible.module_utils.dualtor_utils import generate_mux_cable_facts
+
+
+def load_topo_file(topo_name):
+    """Load topo definition yaml file."""
+    topo_file = "vars/topo_%s.yml" % topo_name
+    if not os.path.exists(topo_file):
+        raise ValueError("Topo file %s not exists" % topo_file)
+    with open(topo_file) as fd:
+        return yaml.safe_load(fd)
+
+
 class DualTorParser:
 
     def __init__(self, hostname, testbed_facts, host_vars, vm_config, port_alias, vlan_intfs):
@@ -64,6 +86,14 @@ def generate_cable_names(self):
 
         self.dual_tor_facts['cables'] = cables
 
+    def generate_mux_cable_facts(self):
+        topo_name = self.testbed_facts["topo"]
+        # use mux_cable_facts only for dualtor mixed topologies
+        if "mixed" in topo_name:
+            topology = load_topo_file(topo_name)["topology"]
+            mux_cable_facts = generate_mux_cable_facts(topology=topology)
+            self.dual_tor_facts["mux_cable_facts"] = mux_cable_facts
+
     def get_dual_tor_facts(self):
         '''
         Gathers facts related to a dual ToR configuration
@@ -73,6 +103,7 @@ def get_dual_tor_facts(self):
             self.parse_tor_position()
             self.generate_cable_names()
             self.parse_loopback_ips()
+            self.generate_mux_cable_facts()
 
         return self.dual_tor_facts
 

ansible/module_utils/port_utils.py

@@ -199,7 +199,7 @@ def get_port_alias_to_name_map(hwsku, asic_name=None):
         elif hwsku == "36x100Gb":
             for i in range(0, 36):
                 port_alias_to_name_map["Ethernet%d" % i] = "Ethernet%d" % i
-        elif hwsku == "64x100Gb":
+        elif hwsku == "Cisco-8102-C64":
             for i in range(0, 64):
                 port_alias_to_name_map["Ethernet%d" % i] = "Ethernet%d" % i
         elif hwsku in ["8800-LC-48H-O", "88-LC0-36FH-MO"]:

ansible/roles/test/files/ptftests/fib.py

@@ -1,4 +1,6 @@
 import re
+import six
+
 from ipaddress import ip_address, ip_network
 from lpm import LpmDict
 
@@ -58,22 +60,22 @@ def __init__(self, file_path):
             for line in f.readlines():
                 if pattern.match(line): continue
                 entry = line.split(' ', 1)
-                prefix = ip_network(unicode(entry[0]))
+                prefix = ip_network(six.text_type(entry[0]))
                 next_hop = self.NextHop(entry[1])
                 if prefix.version is 4:
                     self._ipv4_lpm_dict[str(prefix)] = next_hop
                 elif prefix.version is 6:
                     self._ipv6_lpm_dict[str(prefix)] = next_hop
 
     def __getitem__(self, ip):
-        ip = ip_address(unicode(ip))
+        ip = ip_address(six.text_type(ip))
         if ip.version is 4:
             return self._ipv4_lpm_dict[str(ip)]
         elif ip.version is 6:
             return self._ipv6_lpm_dict[str(ip)]
 
     def __contains__(self, ip):
-        ip_obj = ip_address(unicode(ip))
+        ip_obj = ip_address(six.text_type(ip))
         if ip_obj.version == 4:
             return self._ipv4_lpm_dict.contains(ip)
         elif ip_obj.version == 6:

ansible/roles/test/files/ptftests/fib_test.py

@@ -32,6 +32,7 @@
 from ptf.testutils import verify_no_packet_any
 
 import fib
+import macsec
 
 class FibTest(BaseTest):
     '''
@@ -175,6 +176,11 @@ def get_src_and_exp_ports(self, dst_ip):
             exp_port_list = next_hop.get_next_hop_list()
             if src_port in exp_port_list:
                 continue
+            # MACsec link only receive encrypted packets
+            # It's hard to simulate encrypted packets on the injected port
+            # Because the MACsec is session based channel but the injected ports are stateless ports
+            if src_port in macsec.MACSEC_INFOS.keys():
+                continue
             logging.info('src_port={}, exp_port_list={}, active_dut_index={}'.format(src_port, exp_port_list, active_dut_index))
             break
         return src_port, exp_port_list, next_hop
@@ -183,9 +189,9 @@ def check_ip_range(self, ip_range, dut_index, ipv4=True):
 
         dst_ips = []
         dst_ips.append(ip_range.get_first_ip())
-        if ip_range.length > 1:
+        if ip_range.length() > 1:
             dst_ips.append(ip_range.get_last_ip())
-        if ip_range.length > 2:
+        if ip_range.length() > 2:
             dst_ips.append(ip_range.get_random_ip())
 
         for dst_ip in dst_ips:

ansible/roles/test/files/ptftests/lpm.py

@@ -1,4 +1,5 @@
 import random
+import six
 
 from ipaddress import ip_address, ip_network
 from SubnetTree import SubnetTree
@@ -62,7 +63,7 @@ def __init__(self, ipv4=True):
         self._boundaries = { ip_address(u'0.0.0.0') : 1} if ipv4 else { ip_address(u'::') : 1}
 
     def __setitem__(self, key, value):
-        prefix = ip_network(unicode(key))
+        prefix = ip_network(six.text_type(key))
         # add the current key to self._prefix_set only when it is not the default route and it is not a duplicate key
         if prefix.prefixlen and key not in self._prefix_set:
             boundary = prefix[0]
@@ -78,7 +79,7 @@ def __getitem__(self, key):
 
     def __delitem__(self, key):
         if '/0' not in key:
-            prefix = ip_network(unicode(key))
+            prefix = ip_network(six.text_type(key))
             boundary = prefix[0]
             next_boundary = prefix[-1] + 1
             self._boundaries[boundary] = self._boundaries.get(boundary) - 1

ansible/roles/test/files/ptftests/macsec.py

@@ -0,0 +1,76 @@
+import os
+import pickle
+import cryptography.exceptions
+import time
+
+import ptf
+import scapy.all as scapy
+MACSEC_SUPPORTED = False
+if hasattr(scapy, "VERSION") and tuple(map(int, scapy.VERSION.split('.'))) >= (2, 4, 5):
+    MACSEC_SUPPORTED = True
+if MACSEC_SUPPORTED:
+    import scapy.contrib.macsec as scapy_macsec
+
+MACSEC_INFO_FILE = "macsec_info.pickle"
+
+MACSEC_INFOS = {}
+
+
+def __decap_macsec_pkt(macsec_pkt, sci, an, sak, encrypt, send_sci, pn, xpn_en=False, ssci=None, salt=None):
+    sa = scapy_macsec.MACsecSA(sci=sci,
+                               an=an,
+                               pn=pn,
+                               key=sak,
+                               icvlen=16,
+                               encrypt=encrypt,
+                               send_sci=send_sci,
+                               xpn_en=xpn_en,
+                               ssci=ssci,
+                               salt=salt)
+    try:
+        pkt = sa.decrypt(macsec_pkt)
+    except cryptography.exceptions.InvalidTag:
+        # Invalid MACsec packets
+        return None
+    pkt = sa.decap(pkt)
+    return pkt
+
+
+def __macsec_dp_poll(test, device_number=0, port_number=None, timeout=None, exp_pkt=None):
+    recent_packets = []
+    packet_count = 0
+    if timeout is None:
+        timeout = ptf.ptfutils.default_timeout
+    while True:
+        start_time = time.time()
+        ret = __origin_dp_poll(
+            test, device_number=device_number, port_number=port_number, timeout=timeout, exp_pkt=None)
+        timeout -= time.time() - start_time
+        # The device number of PTF host is 0, if the target port isn't a injected port(belong to ptf host), Don't need to do MACsec further.
+        if isinstance(ret, test.dataplane.PollFailure) or exp_pkt is None or ret.device != 0:
+            return ret
+        pkt = scapy.Ether(ret.packet)
+        if pkt[scapy.Ether].type != 0x88e5:
+            if ptf.dataplane.match_exp_pkt(exp_pkt, pkt):
+                return ret
+            else:
+                continue
+        if ret.port in MACSEC_INFOS and MACSEC_INFOS[ret.port]:
+            encrypt, send_sci, xpn_en, sci, an, sak, ssci, salt = MACSEC_INFOS[ret.port]
+            pkt = __decap_macsec_pkt(pkt, sci, an, sak, encrypt,
+                                send_sci, 0, xpn_en, ssci, salt)
+            if pkt is not None and ptf.dataplane.match_exp_pkt(exp_pkt, pkt):
+                return ret
+        recent_packets.append(pkt)
+        packet_count += 1
+        if timeout <= 0:
+            break
+    return test.dataplane.PollFailure(exp_pkt, recent_packets,packet_count)
+
+
+if MACSEC_SUPPORTED and os.path.exists(MACSEC_INFO_FILE):
+    with open(MACSEC_INFO_FILE, "rb") as f:
+        MACSEC_INFOS = pickle.load(f, encoding="bytes")
+        if MACSEC_INFOS:
+            __origin_dp_poll = ptf.testutils.dp_poll
+            ptf.testutils.dp_poll = __macsec_dp_poll