diff --git a/plugins/meta/bandwidth/bandwidth_config_test.go b/plugins/meta/bandwidth/bandwidth_config_test.go new file mode 100644 index 000000000..66b8ebaad --- /dev/null +++ b/plugins/meta/bandwidth/bandwidth_config_test.go @@ -0,0 +1,563 @@ +// Copyright 2018 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "math" + "net" + "syscall" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/vishvananda/netlink" + + "github.com/containernetworking/cni/pkg/skel" + types100 "github.com/containernetworking/cni/pkg/types/100" + "github.com/containernetworking/plugins/pkg/ns" + "github.com/containernetworking/plugins/pkg/testutils" +) + +var _ = Describe("bandwidth config test", func() { + var ( + hostNs ns.NetNS + containerNs ns.NetNS + ifbDeviceName string + hostIfname string + containerIfname string + hostIP net.IP + containerIP net.IP + hostIfaceMTU int + ) + + BeforeEach(func() { + var err error + + hostIfname = "host-veth" + containerIfname = "container-veth" + + hostNs, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + + containerNs, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + + hostIP = net.IP{169, 254, 0, 1} + containerIP = net.IP{10, 254, 0, 1} + hostIfaceMTU = 1024 + ifbDeviceName = "bwpa8eda89404b7" + + createVeth(hostNs, hostIfname, containerNs, containerIfname, hostIP, containerIP, hostIfaceMTU) + }) + + AfterEach(func() { + Expect(containerNs.Close()).To(Succeed()) + Expect(testutils.UnmountNS(containerNs)).To(Succeed()) + Expect(hostNs.Close()).To(Succeed()) + Expect(testutils.UnmountNS(hostNs)).To(Succeed()) + }) + + // Bandwidth requires host-side interface info, and thus only + // supports 0.3.0 and later CNI versions + for _, ver := range []string{"0.3.0", "0.3.1", "0.4.0", "1.0.0"} { + // Redefine ver inside for scope so real value is picked up by each dynamically defined It() + // See Gingkgo's "Patterns for dynamically generating tests" documentation. + ver := ver + + Describe("cmdADD", func() { + It(fmt.Sprintf("[%s] fails with invalid UnshapedSubnets", ver), func() { + conf := fmt.Sprintf(`{ + "cniVersion": "%s", + "name": "cni-plugin-bandwidth-test", + "type": "bandwidth", + "ingressRate": 123, + "ingressBurst": 123, + "egressRate": 123, + "egressBurst": 123, + "unshapedSubnets": ["10.0.0.0/8", "hello"], + "prevResult": { + "interfaces": [ + { + "name": "%s", + "sandbox": "" + }, + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "4", + "address": "%s/24", + "gateway": "10.0.0.1", + "interface": 1 + } + ], + "routes": [] + } + }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) + + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: containerNs.Path(), + IfName: "eth0", + StdinData: []byte(conf), + } + + Expect(hostNs.Do(func(netNS ns.NetNS) error { + defer GinkgoRecover() + + _, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) + Expect(err).To(MatchError("bad subnet \"hello\" provided, details invalid CIDR address: hello")) + return nil + })).To(Succeed()) + }) + + It(fmt.Sprintf("[%s] fails with invalid ShapedSubnets", ver), func() { + conf := fmt.Sprintf(`{ + "cniVersion": "%s", + "name": "cni-plugin-bandwidth-test", + "type": "bandwidth", + "ingressRate": 123, + "ingressBurst": 123, + "egressRate": 123, + "egressBurst": 123, + "ShapedSubnets": ["10.0.0.0/8", "hello"], + "prevResult": { + "interfaces": [ + { + "name": "%s", + "sandbox": "" + }, + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "4", + "address": "%s/24", + "gateway": "10.0.0.1", + "interface": 1 + } + ], + "routes": [] + } + }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) + + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: containerNs.Path(), + IfName: "eth0", + StdinData: []byte(conf), + } + + Expect(hostNs.Do(func(netNS ns.NetNS) error { + defer GinkgoRecover() + + _, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) + Expect(err).To(MatchError("bad subnet \"hello\" provided, details invalid CIDR address: hello")) + return nil + })).To(Succeed()) + }) + + It(fmt.Sprintf("[%s] fails with both ShapedSubnets and UnShapedSubnets specified", ver), func() { + conf := fmt.Sprintf(`{ + "cniVersion": "%s", + "name": "cni-plugin-bandwidth-test", + "type": "bandwidth", + "ingressRate": 123, + "ingressBurst": 123, + "egressRate": 123, + "egressBurst": 123, + "ShapedSubnets": ["10.0.0.0/8"], + "UnShapedSubnets": ["192.168.0.0/16"], + "prevResult": { + "interfaces": [ + { + "name": "%s", + "sandbox": "" + }, + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "4", + "address": "%s/24", + "gateway": "10.0.0.1", + "interface": 1 + } + ], + "routes": [] + } + }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) + + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: containerNs.Path(), + IfName: "eth0", + StdinData: []byte(conf), + } + + Expect(hostNs.Do(func(netNS ns.NetNS) error { + defer GinkgoRecover() + + _, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) + Expect(err).To(MatchError("unshapedSubnets and shapedSubnets cannot be both specified, one of them should be discarded")) + return nil + })).To(Succeed()) + }) + + It(fmt.Sprintf("[%s] fails an invalid ingress config", ver), func() { + conf := fmt.Sprintf(`{ + "cniVersion": "%s", + "name": "cni-plugin-bandwidth-test", + "type": "bandwidth", + "ingressRate": 0, + "ingressBurst": 123, + "egressRate": 123, + "egressBurst": 123, + "prevResult": { + "interfaces": [ + { + "name": "%s", + "sandbox": "" + }, + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "4", + "address": "%s/24", + "gateway": "10.0.0.1", + "interface": 1 + } + ], + "routes": [] + } + }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) + + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: containerNs.Path(), + IfName: "eth0", + StdinData: []byte(conf), + } + + Expect(hostNs.Do(func(netNS ns.NetNS) error { + defer GinkgoRecover() + + _, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) + Expect(err).To(MatchError("if burst is set, rate must also be set")) + return nil + })).To(Succeed()) + }) + + It(fmt.Sprintf("[%s] fails an invalid egress config", ver), func() { + conf := fmt.Sprintf(`{ + "cniVersion": "%s", + "name": "cni-plugin-bandwidth-test", + "type": "bandwidth", + "ingressRate": 123, + "ingressBurst": 123, + "egressRate": 0, + "egressBurst": 123, + "prevResult": { + "interfaces": [ + { + "name": "%s", + "sandbox": "" + }, + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "4", + "address": "%s/24", + "gateway": "10.0.0.1", + "interface": 1 + } + ], + "routes": [] + } + }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) + + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: containerNs.Path(), + IfName: "eth0", + StdinData: []byte(conf), + } + + Expect(hostNs.Do(func(netNS ns.NetNS) error { + defer GinkgoRecover() + + _, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) + Expect(err).To(MatchError("if burst is set, rate must also be set")) + return nil + })).To(Succeed()) + }) + + // Runtime config parameters are expected to be preempted by the global config ones whenever specified + It(fmt.Sprintf("[%s] should apply static config when both static config and runtime config exist", ver), func() { + conf := fmt.Sprintf(`{ + "cniVersion": "%s", + "name": "cni-plugin-bandwidth-test", + "type": "bandwidth", + "ingressRate": 0, + "ingressBurst": 0, + "egressRate": 123, + "egressBurst": 123, + "unshapedSubnets": ["192.168.0.0/24"], + "runtimeConfig": { + "bandWidth": { + "ingressRate": 8, + "ingressBurst": 8, + "egressRate": 16, + "egressBurst": 9, + "unshapedSubnets": ["10.0.0.0/8", "fd00:db8:abcd:1234:e000::/68"] + } + }, + "prevResult": { + "interfaces": [ + { + "name": "%s", + "sandbox": "" + }, + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "4", + "address": "%s/24", + "gateway": "10.0.0.1", + "interface": 1 + } + ], + "routes": [] + } + }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) + + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: containerNs.Path(), + IfName: containerIfname, + StdinData: []byte(conf), + } + + Expect(hostNs.Do(func(netNS ns.NetNS) error { + defer GinkgoRecover() + r, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) + Expect(err).NotTo(HaveOccurred(), string(out)) + result, err := types100.GetResult(r) + Expect(err).NotTo(HaveOccurred()) + + Expect(result.Interfaces).To(HaveLen(3)) + Expect(result.Interfaces[2].Name).To(Equal(ifbDeviceName)) + Expect(result.Interfaces[2].Sandbox).To(Equal("")) + + ifbLink, err := netlink.LinkByName(ifbDeviceName) + Expect(err).NotTo(HaveOccurred()) + Expect(ifbLink.Attrs().MTU).To(Equal(hostIfaceMTU)) + + qdiscs, err := netlink.QdiscList(ifbLink) + Expect(err).NotTo(HaveOccurred()) + + Expect(qdiscs).To(HaveLen(1)) + Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) + Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{})) + Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID))) + + classes, err := netlink.ClassList(ifbLink, qdiscs[0].Attrs().Handle) + + Expect(err).NotTo(HaveOccurred()) + Expect(classes).To(HaveLen(2)) + + // Uncapped class + Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) + Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1))) + Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate)) + Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0))) + Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate)) + Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) + + // Class with traffic shapping settings + Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) + Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls)))) + Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(15))) + // Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(7812500))) + Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(30))) + // Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) + + filters, err := netlink.FilterList(ifbLink, qdiscs[0].Attrs().Handle) + Expect(err).NotTo(HaveOccurred()) + Expect(filters).To(HaveLen(1)) + + // traffic to 192.168.0.0/24 redirected to uncapped class + Expect(filters[0]).To(BeAssignableToTypeOf(&netlink.U32{})) + Expect(filters[0].(*netlink.U32).Actions).To(BeEmpty()) + Expect(filters[0].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IP))) + Expect(filters[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) + Expect(filters[0].Attrs().Priority).To(Equal(uint16(16))) + Expect(filters[0].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle)) + Expect(filters[0].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, 1))) + + filterSel := filters[0].(*netlink.U32).Sel + Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{})) + Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL))) + Expect(filterSel.Keys).To(HaveLen(1)) + Expect(filterSel.Nkeys).To(Equal(uint8(1))) + + // The filter should match to 192.168.0.0/24 dst address in other words it should be: + // match c0a80000/ffffff00 at 16 + selKey := filterSel.Keys[0] + Expect(selKey.Val).To(Equal(uint32(192*math.Pow(256, 3) + 168*math.Pow(256, 2)))) + Expect(selKey.Off).To(Equal(int32(16))) + Expect(selKey.Mask).To(Equal(uint32(255*math.Pow(256, 3) + 255*math.Pow(256, 2) + 255*256))) + + hostVethLink, err := netlink.LinkByName(hostIfname) + Expect(err).NotTo(HaveOccurred()) + + qdiscFilters, err := netlink.FilterList(hostVethLink, netlink.MakeHandle(0xffff, 0)) + Expect(err).NotTo(HaveOccurred()) + + Expect(qdiscFilters).To(HaveLen(1)) + Expect(qdiscFilters[0].(*netlink.U32).Actions[0].(*netlink.MirredAction).Ifindex).To(Equal(ifbLink.Attrs().Index)) + + return nil + })).To(Succeed()) + + // Container ingress (host egress) + Expect(hostNs.Do(func(n ns.NetNS) error { + defer GinkgoRecover() + + vethLink, err := netlink.LinkByName(hostIfname) + Expect(err).NotTo(HaveOccurred()) + + qdiscs, err := netlink.QdiscList(vethLink) + Expect(err).NotTo(HaveOccurred()) + + // No ingress QoS just mirroring + Expect(qdiscs).To(HaveLen(2)) + Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index)) + Expect(qdiscs[0]).NotTo(BeAssignableToTypeOf(&netlink.Htb{})) + Expect(qdiscs[1]).NotTo(BeAssignableToTypeOf(&netlink.Htb{})) + + return nil + })).To(Succeed()) + }) + + It(fmt.Sprintf("[%s] should apply static config when both static config and runtime config exist (bad config example)", ver), func() { + conf := fmt.Sprintf(`{ + "cniVersion": "%s", + "name": "cni-plugin-bandwidth-test", + "type": "bandwidth", + "ingressRate": 0, + "ingressBurst": 123, + "egressRate": 123, + "egressBurst": 123, + "runtimeConfig": { + "bandWidth": { + "ingressRate": 8, + "ingressBurst": 8, + "egressRate": 16, + "egressBurst": 9 + } + }, + "prevResult": { + "interfaces": [ + { + "name": "%s", + "sandbox": "" + }, + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "4", + "address": "%s/24", + "gateway": "10.0.0.1", + "interface": 1 + } + ], + "routes": [] + } + }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) + + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: containerNs.Path(), + IfName: "eth0", + StdinData: []byte(conf), + } + + Expect(hostNs.Do(func(netNS ns.NetNS) error { + defer GinkgoRecover() + + _, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) + Expect(err).To(MatchError("if burst is set, rate must also be set")) + return nil + })).To(Succeed()) + }) + }) + } + + Describe("Validating input", func() { + It("Should allow only 4GB burst rate", func() { + err := validateRateAndBurst(5000, 4*1024*1024*1024*8-16) // 2 bytes less than the max should pass + Expect(err).NotTo(HaveOccurred()) + err = validateRateAndBurst(5000, 4*1024*1024*1024*8) // we're 1 bit above MaxUint32 + Expect(err).To(HaveOccurred()) + err = validateRateAndBurst(0, 1) + Expect(err).To(HaveOccurred()) + err = validateRateAndBurst(1, 0) + Expect(err).To(HaveOccurred()) + err = validateRateAndBurst(0, 0) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Should fail if both ShapedSubnets and UnshapedSubnets are specified", func() { + err := validateSubnets([]string{"10.0.0.0/8"}, []string{"192.168.0.0/24"}) + Expect(err).To(HaveOccurred()) + }) + + It("Should fail if specified UnshapedSubnets are not valid CIDRs", func() { + err := validateSubnets([]string{"10.0.0.0/8", "hello"}, []string{}) + Expect(err).To(HaveOccurred()) + }) + + It("Should fail if specified ShapedSubnets are not valid CIDRs", func() { + err := validateSubnets([]string{}, []string{"10.0.0.0/8", "hello"}) + Expect(err).To(HaveOccurred()) + }) + }) +}) diff --git a/plugins/meta/bandwidth/bandwidth_linux_test.go b/plugins/meta/bandwidth/bandwidth_linux_test.go index b3c251bc0..e785cbde2 100644 --- a/plugins/meta/bandwidth/bandwidth_linux_test.go +++ b/plugins/meta/bandwidth/bandwidth_linux_test.go @@ -15,75 +15,21 @@ package main import ( - "context" - "encoding/json" "fmt" - "log" "math" "net" - "os" "syscall" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/onsi/gomega/gexec" "github.com/vishvananda/netlink" - "github.com/containernetworking/cni/pkg/invoke" "github.com/containernetworking/cni/pkg/skel" - "github.com/containernetworking/cni/pkg/types" types100 "github.com/containernetworking/cni/pkg/types/100" "github.com/containernetworking/plugins/pkg/ns" "github.com/containernetworking/plugins/pkg/testutils" ) -func buildOneConfig(cniVersion string, orig *PluginConf, prevResult types.Result) ([]byte, error) { - var err error - - name := "myBWnet" - - inject := map[string]interface{}{ - "name": name, - "cniVersion": cniVersion, - } - // Add previous plugin result - if prevResult != nil { - r, err := prevResult.GetAsVersion(cniVersion) - Expect(err).NotTo(HaveOccurred()) - inject["prevResult"] = r - } - - // Ensure every config uses the same name and version - config := make(map[string]interface{}) - - confBytes, err := json.Marshal(orig) - if err != nil { - return nil, err - } - - err = json.Unmarshal(confBytes, &config) - if err != nil { - return nil, fmt.Errorf("unmarshal existing network bytes: %s", err) - } - - for key, value := range inject { - config[key] = value - } - - newBytes, err := json.Marshal(config) - if err != nil { - return nil, err - } - - conf := &PluginConf{} - if err := json.Unmarshal(newBytes, &conf); err != nil { - return nil, fmt.Errorf("error parsing configuration: %s", err) - } - - return newBytes, nil -} - var _ = Describe("bandwidth test", func() { var ( hostNs ns.NetNS @@ -1060,250 +1006,6 @@ var _ = Describe("bandwidth test", func() { })).To(Succeed()) }) - It(fmt.Sprintf("[%s] fails with invalid UnshapedSubnets", ver), func() { - conf := fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "cni-plugin-bandwidth-test", - "type": "bandwidth", - "ingressRate": 123, - "ingressBurst": 123, - "egressRate": 123, - "egressBurst": 123, - "unshapedSubnets": ["10.0.0.0/8", "hello"], - "prevResult": { - "interfaces": [ - { - "name": "%s", - "sandbox": "" - }, - { - "name": "%s", - "sandbox": "%s" - } - ], - "ips": [ - { - "version": "4", - "address": "%s/24", - "gateway": "10.0.0.1", - "interface": 1 - } - ], - "routes": [] - } - }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) - - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: containerNs.Path(), - IfName: "eth0", - StdinData: []byte(conf), - } - - Expect(hostNs.Do(func(netNS ns.NetNS) error { - defer GinkgoRecover() - - _, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) - Expect(err).To(MatchError("bad subnet \"hello\" provided, details invalid CIDR address: hello")) - return nil - })).To(Succeed()) - }) - - It(fmt.Sprintf("[%s] fails with invalid ShapedSubnets", ver), func() { - conf := fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "cni-plugin-bandwidth-test", - "type": "bandwidth", - "ingressRate": 123, - "ingressBurst": 123, - "egressRate": 123, - "egressBurst": 123, - "ShapedSubnets": ["10.0.0.0/8", "hello"], - "prevResult": { - "interfaces": [ - { - "name": "%s", - "sandbox": "" - }, - { - "name": "%s", - "sandbox": "%s" - } - ], - "ips": [ - { - "version": "4", - "address": "%s/24", - "gateway": "10.0.0.1", - "interface": 1 - } - ], - "routes": [] - } - }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) - - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: containerNs.Path(), - IfName: "eth0", - StdinData: []byte(conf), - } - - Expect(hostNs.Do(func(netNS ns.NetNS) error { - defer GinkgoRecover() - - _, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) - Expect(err).To(MatchError("bad subnet \"hello\" provided, details invalid CIDR address: hello")) - return nil - })).To(Succeed()) - }) - - It(fmt.Sprintf("[%s] fails with both ShapedSubnets and UnShapedSubnets specified", ver), func() { - conf := fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "cni-plugin-bandwidth-test", - "type": "bandwidth", - "ingressRate": 123, - "ingressBurst": 123, - "egressRate": 123, - "egressBurst": 123, - "ShapedSubnets": ["10.0.0.0/8"], - "UnShapedSubnets": ["192.168.0.0/16"], - "prevResult": { - "interfaces": [ - { - "name": "%s", - "sandbox": "" - }, - { - "name": "%s", - "sandbox": "%s" - } - ], - "ips": [ - { - "version": "4", - "address": "%s/24", - "gateway": "10.0.0.1", - "interface": 1 - } - ], - "routes": [] - } - }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) - - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: containerNs.Path(), - IfName: "eth0", - StdinData: []byte(conf), - } - - Expect(hostNs.Do(func(netNS ns.NetNS) error { - defer GinkgoRecover() - - _, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) - Expect(err).To(MatchError("unshapedSubnets and shapedSubnets cannot be both specified, one of them should be discarded")) - return nil - })).To(Succeed()) - }) - - It(fmt.Sprintf("[%s] fails an invalid ingress config", ver), func() { - conf := fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "cni-plugin-bandwidth-test", - "type": "bandwidth", - "ingressRate": 0, - "ingressBurst": 123, - "egressRate": 123, - "egressBurst": 123, - "prevResult": { - "interfaces": [ - { - "name": "%s", - "sandbox": "" - }, - { - "name": "%s", - "sandbox": "%s" - } - ], - "ips": [ - { - "version": "4", - "address": "%s/24", - "gateway": "10.0.0.1", - "interface": 1 - } - ], - "routes": [] - } - }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) - - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: containerNs.Path(), - IfName: "eth0", - StdinData: []byte(conf), - } - - Expect(hostNs.Do(func(netNS ns.NetNS) error { - defer GinkgoRecover() - - _, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) - Expect(err).To(MatchError("if burst is set, rate must also be set")) - return nil - })).To(Succeed()) - }) - - It(fmt.Sprintf("[%s] fails an invalid egress config", ver), func() { - conf := fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "cni-plugin-bandwidth-test", - "type": "bandwidth", - "ingressRate": 123, - "ingressBurst": 123, - "egressRate": 0, - "egressBurst": 123, - "prevResult": { - "interfaces": [ - { - "name": "%s", - "sandbox": "" - }, - { - "name": "%s", - "sandbox": "%s" - } - ], - "ips": [ - { - "version": "4", - "address": "%s/24", - "gateway": "10.0.0.1", - "interface": 1 - } - ], - "routes": [] - } - }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) - - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: containerNs.Path(), - IfName: "eth0", - StdinData: []byte(conf), - } - - Expect(hostNs.Do(func(netNS ns.NetNS) error { - defer GinkgoRecover() - - _, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) - Expect(err).To(MatchError("if burst is set, rate must also be set")) - return nil - })).To(Succeed()) - }) - It(fmt.Sprintf("[%s] works with a Veth pair using runtime config", ver), func() { conf := fmt.Sprintf(`{ "cniVersion": "%s", @@ -1495,223 +1197,15 @@ var _ = Describe("bandwidth test", func() { })).To(Succeed()) }) - // Runtime config parameters are expected to be preempted by the global config ones whenever specified - It(fmt.Sprintf("[%s] should apply static config when both static config and runtime config exist", ver), func() { - conf := fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "cni-plugin-bandwidth-test", - "type": "bandwidth", - "ingressRate": 0, - "ingressBurst": 0, - "egressRate": 123, - "egressBurst": 123, - "unshapedSubnets": ["192.168.0.0/24"], - "runtimeConfig": { - "bandWidth": { + Describe("cmdDEL", func() { + It(fmt.Sprintf("[%s] works with a Veth pair using 0.3.0 config", ver), func() { + conf := fmt.Sprintf(`{ + "cniVersion": "%s", + "name": "cni-plugin-bandwidth-test", + "type": "bandwidth", "ingressRate": 8, "ingressBurst": 8, - "egressRate": 16, - "egressBurst": 9, - "unshapedSubnets": ["10.0.0.0/8", "fd00:db8:abcd:1234:e000::/68"] - } - }, - "prevResult": { - "interfaces": [ - { - "name": "%s", - "sandbox": "" - }, - { - "name": "%s", - "sandbox": "%s" - } - ], - "ips": [ - { - "version": "4", - "address": "%s/24", - "gateway": "10.0.0.1", - "interface": 1 - } - ], - "routes": [] - } - }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) - - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: containerNs.Path(), - IfName: containerIfname, - StdinData: []byte(conf), - } - - Expect(hostNs.Do(func(netNS ns.NetNS) error { - defer GinkgoRecover() - r, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) - Expect(err).NotTo(HaveOccurred(), string(out)) - result, err := types100.GetResult(r) - Expect(err).NotTo(HaveOccurred()) - - Expect(result.Interfaces).To(HaveLen(3)) - Expect(result.Interfaces[2].Name).To(Equal(ifbDeviceName)) - Expect(result.Interfaces[2].Sandbox).To(Equal("")) - - ifbLink, err := netlink.LinkByName(ifbDeviceName) - Expect(err).NotTo(HaveOccurred()) - Expect(ifbLink.Attrs().MTU).To(Equal(hostIfaceMTU)) - - qdiscs, err := netlink.QdiscList(ifbLink) - Expect(err).NotTo(HaveOccurred()) - - Expect(qdiscs).To(HaveLen(1)) - Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) - Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{})) - Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID))) - - classes, err := netlink.ClassList(ifbLink, qdiscs[0].Attrs().Handle) - - Expect(err).NotTo(HaveOccurred()) - Expect(classes).To(HaveLen(2)) - - // Uncapped class - Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1))) - Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0))) - Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - // Class with traffic shapping settings - Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls)))) - Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(15))) - // Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(7812500))) - Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(30))) - // Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - filters, err := netlink.FilterList(ifbLink, qdiscs[0].Attrs().Handle) - Expect(err).NotTo(HaveOccurred()) - Expect(filters).To(HaveLen(1)) - - // traffic to 192.168.0.0/24 redirected to uncapped class - Expect(filters[0]).To(BeAssignableToTypeOf(&netlink.U32{})) - Expect(filters[0].(*netlink.U32).Actions).To(BeEmpty()) - Expect(filters[0].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IP))) - Expect(filters[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) - Expect(filters[0].Attrs().Priority).To(Equal(uint16(16))) - Expect(filters[0].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle)) - Expect(filters[0].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, 1))) - - filterSel := filters[0].(*netlink.U32).Sel - Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{})) - Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL))) - Expect(filterSel.Keys).To(HaveLen(1)) - Expect(filterSel.Nkeys).To(Equal(uint8(1))) - - // The filter should match to 192.168.0.0/24 dst address in other words it should be: - // match c0a80000/ffffff00 at 16 - selKey := filterSel.Keys[0] - Expect(selKey.Val).To(Equal(uint32(192*math.Pow(256, 3) + 168*math.Pow(256, 2)))) - Expect(selKey.Off).To(Equal(int32(16))) - Expect(selKey.Mask).To(Equal(uint32(255*math.Pow(256, 3) + 255*math.Pow(256, 2) + 255*256))) - - hostVethLink, err := netlink.LinkByName(hostIfname) - Expect(err).NotTo(HaveOccurred()) - - qdiscFilters, err := netlink.FilterList(hostVethLink, netlink.MakeHandle(0xffff, 0)) - Expect(err).NotTo(HaveOccurred()) - - Expect(qdiscFilters).To(HaveLen(1)) - Expect(qdiscFilters[0].(*netlink.U32).Actions[0].(*netlink.MirredAction).Ifindex).To(Equal(ifbLink.Attrs().Index)) - - return nil - })).To(Succeed()) - - // Container ingress (host egress) - Expect(hostNs.Do(func(n ns.NetNS) error { - defer GinkgoRecover() - - vethLink, err := netlink.LinkByName(hostIfname) - Expect(err).NotTo(HaveOccurred()) - - qdiscs, err := netlink.QdiscList(vethLink) - Expect(err).NotTo(HaveOccurred()) - - // No ingress QoS just mirroring - Expect(qdiscs).To(HaveLen(2)) - Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index)) - Expect(qdiscs[0]).NotTo(BeAssignableToTypeOf(&netlink.Htb{})) - Expect(qdiscs[1]).NotTo(BeAssignableToTypeOf(&netlink.Htb{})) - - return nil - })).To(Succeed()) - }) - - It(fmt.Sprintf("[%s] should apply static config when both static config and runtime config exist (bad config example)", ver), func() { - conf := fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "cni-plugin-bandwidth-test", - "type": "bandwidth", - "ingressRate": 0, - "ingressBurst": 123, - "egressRate": 123, - "egressBurst": 123, - "runtimeConfig": { - "bandWidth": { - "ingressRate": 8, - "ingressBurst": 8, - "egressRate": 16, - "egressBurst": 9 - } - }, - "prevResult": { - "interfaces": [ - { - "name": "%s", - "sandbox": "" - }, - { - "name": "%s", - "sandbox": "%s" - } - ], - "ips": [ - { - "version": "4", - "address": "%s/24", - "gateway": "10.0.0.1", - "interface": 1 - } - ], - "routes": [] - } - }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) - - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: containerNs.Path(), - IfName: "eth0", - StdinData: []byte(conf), - } - - Expect(hostNs.Do(func(netNS ns.NetNS) error { - defer GinkgoRecover() - - _, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) - Expect(err).To(MatchError("if burst is set, rate must also be set")) - return nil - })).To(Succeed()) - }) - - Describe("cmdDEL", func() { - It(fmt.Sprintf("[%s] works with a Veth pair using 0.3.0 config", ver), func() { - conf := fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "cni-plugin-bandwidth-test", - "type": "bandwidth", - "ingressRate": 8, - "ingressBurst": 8, - "egressRate": 9, + "egressRate": 9, "egressBurst": 9, "prevResult": { "interfaces": [ @@ -2137,781 +1631,5 @@ var _ = Describe("bandwidth test", func() { })).To(Succeed()) }) }) - - Describe(fmt.Sprintf("[%s] QoS effective", ver), func() { - Context(fmt.Sprintf("[%s] when chaining bandwidth plugin with PTP", ver), func() { - var ptpConf string - var rateInBits uint64 - var burstInBits uint64 - var packetInBytes int - var containerWithoutQoSNS ns.NetNS - var containerWithQoSNS ns.NetNS - var portServerWithQoS int - var portServerWithoutQoS int - - var containerWithQoSRes types.Result - var containerWithoutQoSRes types.Result - var echoServerWithQoS *gexec.Session - var echoServerWithoutQoS *gexec.Session - var dataDir string - - BeforeEach(func() { - rateInBytes := 1000 - rateInBits = uint64(rateInBytes * 8) - burstInBits = rateInBits * 2 - - // NOTE: Traffic shapping is not that precise at low rates, would be better to use higher rates + simple time+netcat for data transfer, rather than the provided - // client/server bin (limited to small amount of data) - packetInBytes = rateInBytes * 3 - - var err error - dataDir, err = os.MkdirTemp("", "bandwidth_linux_test") - Expect(err).NotTo(HaveOccurred()) - - ptpConf = fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "myBWnet", - "type": "ptp", - "ipMasq": true, - "mtu": 512, - "ipam": { - "type": "host-local", - "subnet": "10.1.2.0/24", - "dataDir": "%s" - } - }`, ver, dataDir) - - const ( - containerWithQoSIFName = "ptp0" - containerWithoutQoSIFName = "ptp1" - ) - - containerWithQoSNS, err = testutils.NewNS() - Expect(err).NotTo(HaveOccurred()) - - containerWithoutQoSNS, err = testutils.NewNS() - Expect(err).NotTo(HaveOccurred()) - - By("create two containers, and use the bandwidth plugin on one of them") - Expect(hostNs.Do(func(ns.NetNS) error { - defer GinkgoRecover() - - containerWithQoSRes, _, err = testutils.CmdAdd(containerWithQoSNS.Path(), "dummy", containerWithQoSIFName, []byte(ptpConf), func() error { - r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) - Expect(err).NotTo(HaveOccurred()) - Expect(r.Print()).To(Succeed()) - - return err - }) - Expect(err).NotTo(HaveOccurred()) - - containerWithoutQoSRes, _, err = testutils.CmdAdd(containerWithoutQoSNS.Path(), "dummy2", containerWithoutQoSIFName, []byte(ptpConf), func() error { - r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) - Expect(err).NotTo(HaveOccurred()) - Expect(r.Print()).To(Succeed()) - - return err - }) - Expect(err).NotTo(HaveOccurred()) - - containerWithQoSResult, err := types100.GetResult(containerWithQoSRes) - Expect(err).NotTo(HaveOccurred()) - - bandwidthPluginConf := &PluginConf{} - err = json.Unmarshal([]byte(ptpConf), &bandwidthPluginConf) - Expect(err).NotTo(HaveOccurred()) - - bandwidthPluginConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ - IngressBurst: burstInBits, - IngressRate: rateInBits, - EgressBurst: burstInBits, - EgressRate: rateInBits, - } - bandwidthPluginConf.Type = "bandwidth" - newConfBytes, err := buildOneConfig(ver, bandwidthPluginConf, containerWithQoSResult) - Expect(err).NotTo(HaveOccurred()) - - args := &skel.CmdArgs{ - ContainerID: "dummy3", - Netns: containerWithQoSNS.Path(), - IfName: containerWithQoSIFName, - StdinData: newConfBytes, - } - - result, out, err := testutils.CmdAdd(containerWithQoSNS.Path(), args.ContainerID, "", newConfBytes, func() error { return cmdAdd(args) }) - Expect(err).NotTo(HaveOccurred(), string(out)) - - if testutils.SpecVersionHasCHECK(ver) { - // Do CNI Check - checkConf := &PluginConf{} - err = json.Unmarshal([]byte(ptpConf), &checkConf) - Expect(err).NotTo(HaveOccurred()) - - checkConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ - IngressBurst: burstInBits, - IngressRate: rateInBits, - EgressBurst: burstInBits, - EgressRate: rateInBits, - } - checkConf.Type = "bandwidth" - - newCheckBytes, err := buildOneConfig(ver, checkConf, result) - Expect(err).NotTo(HaveOccurred()) - - args = &skel.CmdArgs{ - ContainerID: "dummy3", - Netns: containerWithQoSNS.Path(), - IfName: containerWithQoSIFName, - StdinData: newCheckBytes, - } - - err = testutils.CmdCheck(containerWithQoSNS.Path(), args.ContainerID, "", func() error { return cmdCheck(args) }) - Expect(err).NotTo(HaveOccurred()) - } - - return nil - })).To(Succeed()) - - By("starting a tcp server on both containers") - portServerWithQoS, echoServerWithQoS = startEchoServerInNamespace(containerWithQoSNS) - portServerWithoutQoS, echoServerWithoutQoS = startEchoServerInNamespace(containerWithoutQoSNS) - }) - - AfterEach(func() { - Expect(os.RemoveAll(dataDir)).To(Succeed()) - - Expect(containerWithQoSNS.Close()).To(Succeed()) - Expect(testutils.UnmountNS(containerWithQoSNS)).To(Succeed()) - Expect(containerWithoutQoSNS.Close()).To(Succeed()) - Expect(testutils.UnmountNS(containerWithoutQoSNS)).To(Succeed()) - - if echoServerWithoutQoS != nil { - echoServerWithoutQoS.Kill() - } - if echoServerWithQoS != nil { - echoServerWithQoS.Kill() - } - }) - - It("limits ingress traffic on veth device", func() { - var runtimeWithLimit time.Duration - var runtimeWithoutLimit time.Duration - - By("gather timing statistics about both containers") - - By("sending tcp traffic to the container that has traffic shaped", func() { - start := time.Now() - result, err := types100.GetResult(containerWithQoSRes) - Expect(err).NotTo(HaveOccurred()) - makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithQoS, packetInBytes) - end := time.Now() - runtimeWithLimit = end.Sub(start) - log.Printf("Elapsed with qos %.2f", runtimeWithLimit.Seconds()) - }) - - By("sending tcp traffic to the container that does not have traffic shaped", func() { - start := time.Now() - result, err := types100.GetResult(containerWithoutQoSRes) - Expect(err).NotTo(HaveOccurred()) - makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutQoS, packetInBytes) - end := time.Now() - runtimeWithoutLimit = end.Sub(start) - log.Printf("Elapsed without qos %.2f", runtimeWithoutLimit.Seconds()) - }) - - Expect(runtimeWithLimit).To(BeNumerically(">", runtimeWithoutLimit+1000*time.Millisecond)) - }) - }) - }) - - Context(fmt.Sprintf("[%s] when chaining bandwidth plugin with PTP and excluding specific subnets from traffic", ver), func() { - var ptpConf string - var rateInBits uint64 - var burstInBits uint64 - var packetInBytes int - var containerWithoutQoSNS ns.NetNS - var containerWithQoSNS ns.NetNS - var portServerWithQoS int - var portServerWithoutQoS int - - var containerWithQoSRes types.Result - var containerWithoutQoSRes types.Result - var echoServerWithQoS *gexec.Session - var echoServerWithoutQoS *gexec.Session - var dataDir string - - BeforeEach(func() { - rateInBytes := 1000 - rateInBits = uint64(rateInBytes * 8) - burstInBits = rateInBits * 2 - unshapedSubnets := []string{"10.1.2.0/24"} - // NOTE: Traffic shapping is not that precise at low rates, would be better to use higher rates + simple time+netcat for data transfer, rather than the provided - // client/server bin (limited to small amount of data) - packetInBytes = rateInBytes * 3 - - var err error - dataDir, err = os.MkdirTemp("", "bandwidth_linux_test") - Expect(err).NotTo(HaveOccurred()) - - ptpConf = fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "myBWnet", - "type": "ptp", - "ipMasq": true, - "mtu": 512, - "ipam": { - "type": "host-local", - "subnet": "10.1.2.0/24", - "dataDir": "%s" - } - }`, ver, dataDir) - - const ( - containerWithQoSIFName = "ptp0" - containerWithoutQoSIFName = "ptp1" - ) - - containerWithQoSNS, err = testutils.NewNS() - Expect(err).NotTo(HaveOccurred()) - - containerWithoutQoSNS, err = testutils.NewNS() - Expect(err).NotTo(HaveOccurred()) - - By("create two containers, and use the bandwidth plugin on one of them") - Expect(hostNs.Do(func(ns.NetNS) error { - defer GinkgoRecover() - - containerWithQoSRes, _, err = testutils.CmdAdd(containerWithQoSNS.Path(), "dummy", containerWithQoSIFName, []byte(ptpConf), func() error { - r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) - Expect(err).NotTo(HaveOccurred()) - Expect(r.Print()).To(Succeed()) - - return err - }) - Expect(err).NotTo(HaveOccurred()) - - containerWithoutQoSRes, _, err = testutils.CmdAdd(containerWithoutQoSNS.Path(), "dummy2", containerWithoutQoSIFName, []byte(ptpConf), func() error { - r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) - Expect(err).NotTo(HaveOccurred()) - Expect(r.Print()).To(Succeed()) - - return err - }) - Expect(err).NotTo(HaveOccurred()) - - containerWithQoSResult, err := types100.GetResult(containerWithQoSRes) - Expect(err).NotTo(HaveOccurred()) - - bandwidthPluginConf := &PluginConf{} - err = json.Unmarshal([]byte(ptpConf), &bandwidthPluginConf) - Expect(err).NotTo(HaveOccurred()) - - bandwidthPluginConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ - IngressBurst: burstInBits, - IngressRate: rateInBits, - EgressBurst: burstInBits, - EgressRate: rateInBits, - UnshapedSubnets: unshapedSubnets, - } - bandwidthPluginConf.Type = "bandwidth" - newConfBytes, err := buildOneConfig(ver, bandwidthPluginConf, containerWithQoSResult) - Expect(err).NotTo(HaveOccurred()) - - args := &skel.CmdArgs{ - ContainerID: "dummy3", - Netns: containerWithQoSNS.Path(), - IfName: containerWithQoSIFName, - StdinData: newConfBytes, - } - - result, out, err := testutils.CmdAdd(containerWithQoSNS.Path(), args.ContainerID, "", newConfBytes, func() error { return cmdAdd(args) }) - Expect(err).NotTo(HaveOccurred(), string(out)) - - if testutils.SpecVersionHasCHECK(ver) { - // Do CNI Check - checkConf := &PluginConf{} - err = json.Unmarshal([]byte(ptpConf), &checkConf) - Expect(err).NotTo(HaveOccurred()) - - checkConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ - IngressBurst: burstInBits, - IngressRate: rateInBits, - EgressBurst: burstInBits, - EgressRate: rateInBits, - UnshapedSubnets: unshapedSubnets, - } - checkConf.Type = "bandwidth" - - newCheckBytes, err := buildOneConfig(ver, checkConf, result) - Expect(err).NotTo(HaveOccurred()) - - args = &skel.CmdArgs{ - ContainerID: "dummy3", - Netns: containerWithQoSNS.Path(), - IfName: containerWithQoSIFName, - StdinData: newCheckBytes, - } - - err = testutils.CmdCheck(containerWithQoSNS.Path(), args.ContainerID, "", func() error { return cmdCheck(args) }) - Expect(err).NotTo(HaveOccurred()) - } - - return nil - })).To(Succeed()) - - By("starting a tcp server on both containers") - portServerWithQoS, echoServerWithQoS = startEchoServerInNamespace(containerWithQoSNS) - portServerWithoutQoS, echoServerWithoutQoS = startEchoServerInNamespace(containerWithoutQoSNS) - }) - - AfterEach(func() { - Expect(os.RemoveAll(dataDir)).To(Succeed()) - - Expect(containerWithQoSNS.Close()).To(Succeed()) - Expect(testutils.UnmountNS(containerWithQoSNS)).To(Succeed()) - Expect(containerWithoutQoSNS.Close()).To(Succeed()) - Expect(testutils.UnmountNS(containerWithoutQoSNS)).To(Succeed()) - - if echoServerWithoutQoS != nil { - echoServerWithoutQoS.Kill() - } - if echoServerWithQoS != nil { - echoServerWithQoS.Kill() - } - }) - - It("does not limits ingress traffic on veth device coming from 10.1.2.0/24", func() { - var runtimeWithLimit time.Duration - var runtimeWithoutLimit time.Duration - - By("gather timing statistics about both containers") - - By("sending tcp traffic to the container that has traffic shaped", func() { - start := time.Now() - result, err := types100.GetResult(containerWithQoSRes) - Expect(err).NotTo(HaveOccurred()) - makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithQoS, packetInBytes) - end := time.Now() - runtimeWithLimit = end.Sub(start) - log.Printf("Elapsed with qos %.2f", runtimeWithLimit.Seconds()) - }) - - By("sending tcp traffic to the container that does not have traffic shaped", func() { - start := time.Now() - result, err := types100.GetResult(containerWithoutQoSRes) - Expect(err).NotTo(HaveOccurred()) - makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutQoS, packetInBytes) - end := time.Now() - runtimeWithoutLimit = end.Sub(start) - log.Printf("Elapsed without qos %.2f", runtimeWithoutLimit.Seconds()) - }) - - Expect(runtimeWithLimit - runtimeWithoutLimit).To(BeNumerically("<", 100*time.Millisecond)) - }) - }) - - Context(fmt.Sprintf("[%s] when chaining bandwidth plugin with PTP and only including specific subnets in traffic shapping (not including the main ns one)", ver), func() { - var ptpConf string - var rateInBits uint64 - var burstInBits uint64 - var packetInBytes int - var containerWithoutQoSNS ns.NetNS - var containerWithQoSNS ns.NetNS - var portServerWithQoS int - var portServerWithoutQoS int - - var containerWithQoSRes types.Result - var containerWithoutQoSRes types.Result - var echoServerWithQoS *gexec.Session - var echoServerWithoutQoS *gexec.Session - var dataDir string - - BeforeEach(func() { - rateInBytes := 1000 - rateInBits = uint64(rateInBytes * 8) - burstInBits = rateInBits * 2 - shapedSubnets := []string{"10.2.2.0/24"} - // NOTE: Traffic shapping is not that precise at low rates, would be better to use higher rates + simple time+netcat for data transfer, rather than the provided - // client/server bin (limited to small amount of data) - packetInBytes = rateInBytes * 3 - - var err error - dataDir, err = os.MkdirTemp("", "bandwidth_linux_test") - Expect(err).NotTo(HaveOccurred()) - - ptpConf = fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "myBWnet", - "type": "ptp", - "ipMasq": true, - "mtu": 512, - "ipam": { - "type": "host-local", - "subnet": "10.1.2.0/24", - "dataDir": "%s" - } - }`, ver, dataDir) - - const ( - containerWithQoSIFName = "ptp0" - containerWithoutQoSIFName = "ptp1" - ) - - containerWithQoSNS, err = testutils.NewNS() - Expect(err).NotTo(HaveOccurred()) - - containerWithoutQoSNS, err = testutils.NewNS() - Expect(err).NotTo(HaveOccurred()) - - By("create two containers, and use the bandwidth plugin on one of them") - - Expect(hostNs.Do(func(ns.NetNS) error { - defer GinkgoRecover() - - containerWithQoSRes, _, err = testutils.CmdAdd(containerWithQoSNS.Path(), "dummy", containerWithQoSIFName, []byte(ptpConf), func() error { - r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) - Expect(err).NotTo(HaveOccurred()) - Expect(r.Print()).To(Succeed()) - - return err - }) - Expect(err).NotTo(HaveOccurred()) - - containerWithoutQoSRes, _, err = testutils.CmdAdd(containerWithoutQoSNS.Path(), "dummy2", containerWithoutQoSIFName, []byte(ptpConf), func() error { - r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) - Expect(err).NotTo(HaveOccurred()) - Expect(r.Print()).To(Succeed()) - - return err - }) - Expect(err).NotTo(HaveOccurred()) - - containerWithQoSResult, err := types100.GetResult(containerWithQoSRes) - Expect(err).NotTo(HaveOccurred()) - - bandwidthPluginConf := &PluginConf{} - err = json.Unmarshal([]byte(ptpConf), &bandwidthPluginConf) - Expect(err).NotTo(HaveOccurred()) - - bandwidthPluginConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ - IngressBurst: burstInBits, - IngressRate: rateInBits, - EgressBurst: burstInBits, - EgressRate: rateInBits, - ShapedSubnets: shapedSubnets, - } - bandwidthPluginConf.Type = "bandwidth" - newConfBytes, err := buildOneConfig(ver, bandwidthPluginConf, containerWithQoSResult) - Expect(err).NotTo(HaveOccurred()) - - args := &skel.CmdArgs{ - ContainerID: "dummy3", - Netns: containerWithQoSNS.Path(), - IfName: containerWithQoSIFName, - StdinData: newConfBytes, - } - - result, out, err := testutils.CmdAdd(containerWithQoSNS.Path(), args.ContainerID, "", newConfBytes, func() error { return cmdAdd(args) }) - Expect(err).NotTo(HaveOccurred(), string(out)) - - if testutils.SpecVersionHasCHECK(ver) { - // Do CNI Check - checkConf := &PluginConf{} - err = json.Unmarshal([]byte(ptpConf), &checkConf) - Expect(err).NotTo(HaveOccurred()) - - checkConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ - IngressBurst: burstInBits, - IngressRate: rateInBits, - EgressBurst: burstInBits, - EgressRate: rateInBits, - ShapedSubnets: shapedSubnets, - } - checkConf.Type = "bandwidth" - - newCheckBytes, err := buildOneConfig(ver, checkConf, result) - Expect(err).NotTo(HaveOccurred()) - - args = &skel.CmdArgs{ - ContainerID: "dummy3", - Netns: containerWithQoSNS.Path(), - IfName: containerWithQoSIFName, - StdinData: newCheckBytes, - } - - err = testutils.CmdCheck(containerWithQoSNS.Path(), args.ContainerID, "", func() error { return cmdCheck(args) }) - Expect(err).NotTo(HaveOccurred()) - } - - return nil - })).To(Succeed()) - - By("starting a tcp server on both containers") - portServerWithQoS, echoServerWithQoS = startEchoServerInNamespace(containerWithQoSNS) - portServerWithoutQoS, echoServerWithoutQoS = startEchoServerInNamespace(containerWithoutQoSNS) - }) - - AfterEach(func() { - Expect(os.RemoveAll(dataDir)).To(Succeed()) - - Expect(containerWithQoSNS.Close()).To(Succeed()) - Expect(testutils.UnmountNS(containerWithQoSNS)).To(Succeed()) - Expect(containerWithoutQoSNS.Close()).To(Succeed()) - Expect(testutils.UnmountNS(containerWithoutQoSNS)).To(Succeed()) - - if echoServerWithoutQoS != nil { - echoServerWithoutQoS.Kill() - } - if echoServerWithQoS != nil { - echoServerWithQoS.Kill() - } - }) - - It("does not limit ingress traffic on veth device coming from non included subnets", func() { - var runtimeWithLimit time.Duration - var runtimeWithoutLimit time.Duration - - By("gather timing statistics about both containers") - - By("sending tcp traffic to the container that has traffic shaped", func() { - start := time.Now() - result, err := types100.GetResult(containerWithQoSRes) - Expect(err).NotTo(HaveOccurred()) - makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithQoS, packetInBytes) - end := time.Now() - runtimeWithLimit = end.Sub(start) - log.Printf("Elapsed with qos %.2f", runtimeWithLimit.Seconds()) - }) - - By("sending tcp traffic to the container that does not have traffic shaped", func() { - start := time.Now() - result, err := types100.GetResult(containerWithoutQoSRes) - Expect(err).NotTo(HaveOccurred()) - makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutQoS, packetInBytes) - end := time.Now() - runtimeWithoutLimit = end.Sub(start) - log.Printf("Elapsed without qos %.2f", runtimeWithoutLimit.Seconds()) - }) - - Expect(runtimeWithLimit - runtimeWithoutLimit).To(BeNumerically("<", 100*time.Millisecond)) - }) - }) - - Context(fmt.Sprintf("[%s] when chaining bandwidth plugin with PTP and only including specific subnets in traffic shapping (including the main ns one)", ver), func() { - var ptpConf string - var rateInBits uint64 - var burstInBits uint64 - var packetInBytes int - var containerWithoutQoSNS ns.NetNS - var containerWithQoSNS ns.NetNS - var portServerWithQoS int - var portServerWithoutQoS int - - var containerWithQoSRes types.Result - var containerWithoutQoSRes types.Result - var echoServerWithQoS *gexec.Session - var echoServerWithoutQoS *gexec.Session - var dataDir string - - BeforeEach(func() { - rateInBytes := 1000 - rateInBits = uint64(rateInBytes * 8) - burstInBits = rateInBits * 2 - shapedSubnets := []string{"10.1.2.1/32"} - // NOTE: Traffic shapping is not that precise at low rates, would be better to use higher rates + simple time+netcat for data transfer, rather than the provided - // client/server bin (limited to small amount of data) - packetInBytes = rateInBytes * 3 - - var err error - dataDir, err = os.MkdirTemp("", "bandwidth_linux_test") - Expect(err).NotTo(HaveOccurred()) - - ptpConf = fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "myBWnet", - "type": "ptp", - "ipMasq": true, - "mtu": 512, - "ipam": { - "type": "host-local", - "subnet": "10.1.2.0/24", - "dataDir": "%s" - } - }`, ver, dataDir) - - const ( - containerWithQoSIFName = "ptp0" - containerWithoutQoSIFName = "ptp1" - ) - - containerWithQoSNS, err = testutils.NewNS() - Expect(err).NotTo(HaveOccurred()) - - containerWithoutQoSNS, err = testutils.NewNS() - Expect(err).NotTo(HaveOccurred()) - - By("create two containers, and use the bandwidth plugin on one of them") - - Expect(hostNs.Do(func(ns.NetNS) error { - defer GinkgoRecover() - - containerWithQoSRes, _, err = testutils.CmdAdd(containerWithQoSNS.Path(), "dummy", containerWithQoSIFName, []byte(ptpConf), func() error { - r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) - Expect(err).NotTo(HaveOccurred()) - Expect(r.Print()).To(Succeed()) - - return err - }) - Expect(err).NotTo(HaveOccurred()) - - containerWithoutQoSRes, _, err = testutils.CmdAdd(containerWithoutQoSNS.Path(), "dummy2", containerWithoutQoSIFName, []byte(ptpConf), func() error { - r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) - Expect(err).NotTo(HaveOccurred()) - Expect(r.Print()).To(Succeed()) - - return err - }) - Expect(err).NotTo(HaveOccurred()) - - containerWithQoSResult, err := types100.GetResult(containerWithQoSRes) - Expect(err).NotTo(HaveOccurred()) - - bandwidthPluginConf := &PluginConf{} - err = json.Unmarshal([]byte(ptpConf), &bandwidthPluginConf) - Expect(err).NotTo(HaveOccurred()) - - bandwidthPluginConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ - IngressBurst: burstInBits, - IngressRate: rateInBits, - EgressBurst: burstInBits, - EgressRate: rateInBits, - ShapedSubnets: shapedSubnets, - } - bandwidthPluginConf.Type = "bandwidth" - newConfBytes, err := buildOneConfig(ver, bandwidthPluginConf, containerWithQoSResult) - Expect(err).NotTo(HaveOccurred()) - - args := &skel.CmdArgs{ - ContainerID: "dummy3", - Netns: containerWithQoSNS.Path(), - IfName: containerWithQoSIFName, - StdinData: newConfBytes, - } - - result, out, err := testutils.CmdAdd(containerWithQoSNS.Path(), args.ContainerID, "", newConfBytes, func() error { return cmdAdd(args) }) - Expect(err).NotTo(HaveOccurred(), string(out)) - - if testutils.SpecVersionHasCHECK(ver) { - // Do CNI Check - checkConf := &PluginConf{} - err = json.Unmarshal([]byte(ptpConf), &checkConf) - Expect(err).NotTo(HaveOccurred()) - - checkConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ - IngressBurst: burstInBits, - IngressRate: rateInBits, - EgressBurst: burstInBits, - EgressRate: rateInBits, - ShapedSubnets: shapedSubnets, - } - checkConf.Type = "bandwidth" - - newCheckBytes, err := buildOneConfig(ver, checkConf, result) - Expect(err).NotTo(HaveOccurred()) - - args = &skel.CmdArgs{ - ContainerID: "dummy3", - Netns: containerWithQoSNS.Path(), - IfName: containerWithQoSIFName, - StdinData: newCheckBytes, - } - - err = testutils.CmdCheck(containerWithQoSNS.Path(), args.ContainerID, "", func() error { return cmdCheck(args) }) - Expect(err).NotTo(HaveOccurred()) - } - - return nil - })).To(Succeed()) - - By("starting a tcp server on both containers") - portServerWithQoS, echoServerWithQoS = startEchoServerInNamespace(containerWithQoSNS) - portServerWithoutQoS, echoServerWithoutQoS = startEchoServerInNamespace(containerWithoutQoSNS) - }) - - AfterEach(func() { - Expect(os.RemoveAll(dataDir)).To(Succeed()) - - Expect(containerWithQoSNS.Close()).To(Succeed()) - Expect(testutils.UnmountNS(containerWithQoSNS)).To(Succeed()) - Expect(containerWithoutQoSNS.Close()).To(Succeed()) - Expect(testutils.UnmountNS(containerWithoutQoSNS)).To(Succeed()) - - if echoServerWithoutQoS != nil { - echoServerWithoutQoS.Kill() - } - if echoServerWithQoS != nil { - echoServerWithQoS.Kill() - } - }) - - It("limits ingress traffic on veth device coming from included subnets", func() { - var runtimeWithLimit time.Duration - var runtimeWithoutLimit time.Duration - - By("gather timing statistics about both containers") - - By("sending tcp traffic to the container that has traffic shaped", func() { - start := time.Now() - result, err := types100.GetResult(containerWithQoSRes) - Expect(err).NotTo(HaveOccurred()) - makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithQoS, packetInBytes) - end := time.Now() - runtimeWithLimit = end.Sub(start) - log.Printf("Elapsed with qos %.2f", runtimeWithLimit.Seconds()) - }) - - By("sending tcp traffic to the container that does not have traffic shaped", func() { - start := time.Now() - result, err := types100.GetResult(containerWithoutQoSRes) - Expect(err).NotTo(HaveOccurred()) - makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutQoS, packetInBytes) - end := time.Now() - runtimeWithoutLimit = end.Sub(start) - log.Printf("Elapsed without qos %.2f", runtimeWithoutLimit.Seconds()) - }) - - Expect(runtimeWithLimit).To(BeNumerically(">", runtimeWithoutLimit+1000*time.Millisecond)) - }) - }) } - - Describe("Validating input", func() { - It("Should allow only 4GB burst rate", func() { - err := validateRateAndBurst(5000, 4*1024*1024*1024*8-16) // 2 bytes less than the max should pass - Expect(err).NotTo(HaveOccurred()) - err = validateRateAndBurst(5000, 4*1024*1024*1024*8) // we're 1 bit above MaxUint32 - Expect(err).To(HaveOccurred()) - err = validateRateAndBurst(0, 1) - Expect(err).To(HaveOccurred()) - err = validateRateAndBurst(1, 0) - Expect(err).To(HaveOccurred()) - err = validateRateAndBurst(0, 0) - Expect(err).NotTo(HaveOccurred()) - }) - - It("Should fail if both ShapedSubnets and UnshapedSubnets are specified", func() { - err := validateSubnets([]string{"10.0.0.0/8"}, []string{"192.168.0.0/24"}) - Expect(err).To(HaveOccurred()) - }) - - It("Should fail if specified UnshapedSubnets are not valid CIDRs", func() { - err := validateSubnets([]string{"10.0.0.0/8", "hello"}, []string{}) - Expect(err).To(HaveOccurred()) - }) - - It("Should fail if specified ShapedSubnets are not valid CIDRs", func() { - err := validateSubnets([]string{}, []string{"10.0.0.0/8", "hello"}) - Expect(err).To(HaveOccurred()) - }) - }) }) diff --git a/plugins/meta/bandwidth/bandwidth_measure_linux_test.go b/plugins/meta/bandwidth/bandwidth_measure_linux_test.go new file mode 100644 index 000000000..4160ea92a --- /dev/null +++ b/plugins/meta/bandwidth/bandwidth_measure_linux_test.go @@ -0,0 +1,824 @@ +// Copyright 2018 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + "net" + "os" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" + + "github.com/containernetworking/cni/pkg/invoke" + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + types100 "github.com/containernetworking/cni/pkg/types/100" + "github.com/containernetworking/plugins/pkg/ns" + "github.com/containernetworking/plugins/pkg/testutils" +) + +var _ = Describe("bandwidth measure test", func() { + var ( + hostNs ns.NetNS + containerNs ns.NetNS + hostIfname string + containerIfname string + hostIP net.IP + containerIP net.IP + hostIfaceMTU int + ) + + BeforeEach(func() { + var err error + + hostIfname = "host-veth" + containerIfname = "container-veth" + + hostNs, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + + containerNs, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + + hostIP = net.IP{169, 254, 0, 1} + containerIP = net.IP{10, 254, 0, 1} + hostIfaceMTU = 1024 + + createVeth(hostNs, hostIfname, containerNs, containerIfname, hostIP, containerIP, hostIfaceMTU) + }) + + AfterEach(func() { + Expect(containerNs.Close()).To(Succeed()) + Expect(testutils.UnmountNS(containerNs)).To(Succeed()) + Expect(hostNs.Close()).To(Succeed()) + Expect(testutils.UnmountNS(hostNs)).To(Succeed()) + }) + + // Bandwidth requires host-side interface info, and thus only + // supports 0.3.0 and later CNI versions + for _, ver := range []string{"0.3.0", "0.3.1", "0.4.0", "1.0.0"} { + Describe(fmt.Sprintf("[%s] QoS effective", ver), func() { + Context(fmt.Sprintf("[%s] when chaining bandwidth plugin with PTP", ver), func() { + var ptpConf string + var rateInBits uint64 + var burstInBits uint64 + var packetInBytes int + var containerWithoutQoSNS ns.NetNS + var containerWithQoSNS ns.NetNS + var portServerWithQoS int + var portServerWithoutQoS int + + var containerWithQoSRes types.Result + var containerWithoutQoSRes types.Result + var echoServerWithQoS *gexec.Session + var echoServerWithoutQoS *gexec.Session + var dataDir string + + BeforeEach(func() { + rateInBytes := 1000 + rateInBits = uint64(rateInBytes * 8) + burstInBits = rateInBits * 2 + + // NOTE: Traffic shapping is not that precise at low rates, would be better to use higher rates + simple time+netcat for data transfer, rather than the provided + // client/server bin (limited to small amount of data) + packetInBytes = rateInBytes * 3 + + var err error + dataDir, err = os.MkdirTemp("", "bandwidth_linux_test") + Expect(err).NotTo(HaveOccurred()) + + ptpConf = fmt.Sprintf(`{ + "cniVersion": "%s", + "name": "myBWnet", + "type": "ptp", + "ipMasq": true, + "mtu": 512, + "ipam": { + "type": "host-local", + "subnet": "10.1.2.0/24", + "dataDir": "%s" + } + }`, ver, dataDir) + + const ( + containerWithQoSIFName = "ptp0" + containerWithoutQoSIFName = "ptp1" + ) + + containerWithQoSNS, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + + containerWithoutQoSNS, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + + By("create two containers, and use the bandwidth plugin on one of them") + Expect(hostNs.Do(func(ns.NetNS) error { + defer GinkgoRecover() + + containerWithQoSRes, _, err = testutils.CmdAdd(containerWithQoSNS.Path(), "dummy", containerWithQoSIFName, []byte(ptpConf), func() error { + r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) + Expect(err).NotTo(HaveOccurred()) + Expect(r.Print()).To(Succeed()) + + return err + }) + Expect(err).NotTo(HaveOccurred()) + + containerWithoutQoSRes, _, err = testutils.CmdAdd(containerWithoutQoSNS.Path(), "dummy2", containerWithoutQoSIFName, []byte(ptpConf), func() error { + r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) + Expect(err).NotTo(HaveOccurred()) + Expect(r.Print()).To(Succeed()) + + return err + }) + Expect(err).NotTo(HaveOccurred()) + + containerWithQoSResult, err := types100.GetResult(containerWithQoSRes) + Expect(err).NotTo(HaveOccurred()) + + bandwidthPluginConf := &PluginConf{} + err = json.Unmarshal([]byte(ptpConf), &bandwidthPluginConf) + Expect(err).NotTo(HaveOccurred()) + + bandwidthPluginConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ + IngressBurst: burstInBits, + IngressRate: rateInBits, + EgressBurst: burstInBits, + EgressRate: rateInBits, + } + bandwidthPluginConf.Type = "bandwidth" + newConfBytes, err := buildOneConfig(ver, bandwidthPluginConf, containerWithQoSResult) + Expect(err).NotTo(HaveOccurred()) + + args := &skel.CmdArgs{ + ContainerID: "dummy3", + Netns: containerWithQoSNS.Path(), + IfName: containerWithQoSIFName, + StdinData: newConfBytes, + } + + result, out, err := testutils.CmdAdd(containerWithQoSNS.Path(), args.ContainerID, "", newConfBytes, func() error { return cmdAdd(args) }) + Expect(err).NotTo(HaveOccurred(), string(out)) + + if testutils.SpecVersionHasCHECK(ver) { + // Do CNI Check + checkConf := &PluginConf{} + err = json.Unmarshal([]byte(ptpConf), &checkConf) + Expect(err).NotTo(HaveOccurred()) + + checkConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ + IngressBurst: burstInBits, + IngressRate: rateInBits, + EgressBurst: burstInBits, + EgressRate: rateInBits, + } + checkConf.Type = "bandwidth" + + newCheckBytes, err := buildOneConfig(ver, checkConf, result) + Expect(err).NotTo(HaveOccurred()) + + args = &skel.CmdArgs{ + ContainerID: "dummy3", + Netns: containerWithQoSNS.Path(), + IfName: containerWithQoSIFName, + StdinData: newCheckBytes, + } + + err = testutils.CmdCheck(containerWithQoSNS.Path(), args.ContainerID, "", func() error { return cmdCheck(args) }) + Expect(err).NotTo(HaveOccurred()) + } + + return nil + })).To(Succeed()) + + By("starting a tcp server on both containers") + portServerWithQoS, echoServerWithQoS = startEchoServerInNamespace(containerWithQoSNS) + portServerWithoutQoS, echoServerWithoutQoS = startEchoServerInNamespace(containerWithoutQoSNS) + }) + + AfterEach(func() { + Expect(os.RemoveAll(dataDir)).To(Succeed()) + + Expect(containerWithQoSNS.Close()).To(Succeed()) + Expect(testutils.UnmountNS(containerWithQoSNS)).To(Succeed()) + Expect(containerWithoutQoSNS.Close()).To(Succeed()) + Expect(testutils.UnmountNS(containerWithoutQoSNS)).To(Succeed()) + + if echoServerWithoutQoS != nil { + echoServerWithoutQoS.Kill() + } + if echoServerWithQoS != nil { + echoServerWithQoS.Kill() + } + }) + + It("limits ingress traffic on veth device", func() { + var runtimeWithLimit time.Duration + var runtimeWithoutLimit time.Duration + + By("gather timing statistics about both containers") + + By("sending tcp traffic to the container that has traffic shaped", func() { + start := time.Now() + result, err := types100.GetResult(containerWithQoSRes) + Expect(err).NotTo(HaveOccurred()) + makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithQoS, packetInBytes) + end := time.Now() + runtimeWithLimit = end.Sub(start) + log.Printf("Elapsed with qos %.2f", runtimeWithLimit.Seconds()) + }) + + By("sending tcp traffic to the container that does not have traffic shaped", func() { + start := time.Now() + result, err := types100.GetResult(containerWithoutQoSRes) + Expect(err).NotTo(HaveOccurred()) + makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutQoS, packetInBytes) + end := time.Now() + runtimeWithoutLimit = end.Sub(start) + log.Printf("Elapsed without qos %.2f", runtimeWithoutLimit.Seconds()) + }) + + Expect(runtimeWithLimit).To(BeNumerically(">", runtimeWithoutLimit+1000*time.Millisecond)) + }) + }) + }) + + Context(fmt.Sprintf("[%s] when chaining bandwidth plugin with PTP and excluding specific subnets from traffic", ver), func() { + var ptpConf string + var rateInBits uint64 + var burstInBits uint64 + var packetInBytes int + var containerWithoutQoSNS ns.NetNS + var containerWithQoSNS ns.NetNS + var portServerWithQoS int + var portServerWithoutQoS int + + var containerWithQoSRes types.Result + var containerWithoutQoSRes types.Result + var echoServerWithQoS *gexec.Session + var echoServerWithoutQoS *gexec.Session + var dataDir string + + BeforeEach(func() { + rateInBytes := 1000 + rateInBits = uint64(rateInBytes * 8) + burstInBits = rateInBits * 2 + unshapedSubnets := []string{"10.1.2.0/24"} + // NOTE: Traffic shapping is not that precise at low rates, would be better to use higher rates + simple time+netcat for data transfer, rather than the provided + // client/server bin (limited to small amount of data) + packetInBytes = rateInBytes * 3 + + var err error + dataDir, err = os.MkdirTemp("", "bandwidth_linux_test") + Expect(err).NotTo(HaveOccurred()) + + ptpConf = fmt.Sprintf(`{ + "cniVersion": "%s", + "name": "myBWnet", + "type": "ptp", + "ipMasq": true, + "mtu": 512, + "ipam": { + "type": "host-local", + "subnet": "10.1.2.0/24", + "dataDir": "%s" + } + }`, ver, dataDir) + + const ( + containerWithQoSIFName = "ptp0" + containerWithoutQoSIFName = "ptp1" + ) + + containerWithQoSNS, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + + containerWithoutQoSNS, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + + By("create two containers, and use the bandwidth plugin on one of them") + Expect(hostNs.Do(func(ns.NetNS) error { + defer GinkgoRecover() + + containerWithQoSRes, _, err = testutils.CmdAdd(containerWithQoSNS.Path(), "dummy", containerWithQoSIFName, []byte(ptpConf), func() error { + r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) + Expect(err).NotTo(HaveOccurred()) + Expect(r.Print()).To(Succeed()) + + return err + }) + Expect(err).NotTo(HaveOccurred()) + + containerWithoutQoSRes, _, err = testutils.CmdAdd(containerWithoutQoSNS.Path(), "dummy2", containerWithoutQoSIFName, []byte(ptpConf), func() error { + r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) + Expect(err).NotTo(HaveOccurred()) + Expect(r.Print()).To(Succeed()) + + return err + }) + Expect(err).NotTo(HaveOccurred()) + + containerWithQoSResult, err := types100.GetResult(containerWithQoSRes) + Expect(err).NotTo(HaveOccurred()) + + bandwidthPluginConf := &PluginConf{} + err = json.Unmarshal([]byte(ptpConf), &bandwidthPluginConf) + Expect(err).NotTo(HaveOccurred()) + + bandwidthPluginConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ + IngressBurst: burstInBits, + IngressRate: rateInBits, + EgressBurst: burstInBits, + EgressRate: rateInBits, + UnshapedSubnets: unshapedSubnets, + } + bandwidthPluginConf.Type = "bandwidth" + newConfBytes, err := buildOneConfig(ver, bandwidthPluginConf, containerWithQoSResult) + Expect(err).NotTo(HaveOccurred()) + + args := &skel.CmdArgs{ + ContainerID: "dummy3", + Netns: containerWithQoSNS.Path(), + IfName: containerWithQoSIFName, + StdinData: newConfBytes, + } + + result, out, err := testutils.CmdAdd(containerWithQoSNS.Path(), args.ContainerID, "", newConfBytes, func() error { return cmdAdd(args) }) + Expect(err).NotTo(HaveOccurred(), string(out)) + + if testutils.SpecVersionHasCHECK(ver) { + // Do CNI Check + checkConf := &PluginConf{} + err = json.Unmarshal([]byte(ptpConf), &checkConf) + Expect(err).NotTo(HaveOccurred()) + + checkConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ + IngressBurst: burstInBits, + IngressRate: rateInBits, + EgressBurst: burstInBits, + EgressRate: rateInBits, + UnshapedSubnets: unshapedSubnets, + } + checkConf.Type = "bandwidth" + + newCheckBytes, err := buildOneConfig(ver, checkConf, result) + Expect(err).NotTo(HaveOccurred()) + + args = &skel.CmdArgs{ + ContainerID: "dummy3", + Netns: containerWithQoSNS.Path(), + IfName: containerWithQoSIFName, + StdinData: newCheckBytes, + } + + err = testutils.CmdCheck(containerWithQoSNS.Path(), args.ContainerID, "", func() error { return cmdCheck(args) }) + Expect(err).NotTo(HaveOccurred()) + } + + return nil + })).To(Succeed()) + + By("starting a tcp server on both containers") + portServerWithQoS, echoServerWithQoS = startEchoServerInNamespace(containerWithQoSNS) + portServerWithoutQoS, echoServerWithoutQoS = startEchoServerInNamespace(containerWithoutQoSNS) + }) + + AfterEach(func() { + Expect(os.RemoveAll(dataDir)).To(Succeed()) + + Expect(containerWithQoSNS.Close()).To(Succeed()) + Expect(testutils.UnmountNS(containerWithQoSNS)).To(Succeed()) + Expect(containerWithoutQoSNS.Close()).To(Succeed()) + Expect(testutils.UnmountNS(containerWithoutQoSNS)).To(Succeed()) + + if echoServerWithoutQoS != nil { + echoServerWithoutQoS.Kill() + } + if echoServerWithQoS != nil { + echoServerWithQoS.Kill() + } + }) + + It("does not limits ingress traffic on veth device coming from 10.1.2.0/24", func() { + var runtimeWithLimit time.Duration + var runtimeWithoutLimit time.Duration + + By("gather timing statistics about both containers") + + By("sending tcp traffic to the container that has traffic shaped", func() { + start := time.Now() + result, err := types100.GetResult(containerWithQoSRes) + Expect(err).NotTo(HaveOccurred()) + makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithQoS, packetInBytes) + end := time.Now() + runtimeWithLimit = end.Sub(start) + log.Printf("Elapsed with qos %.2f", runtimeWithLimit.Seconds()) + }) + + By("sending tcp traffic to the container that does not have traffic shaped", func() { + start := time.Now() + result, err := types100.GetResult(containerWithoutQoSRes) + Expect(err).NotTo(HaveOccurred()) + makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutQoS, packetInBytes) + end := time.Now() + runtimeWithoutLimit = end.Sub(start) + log.Printf("Elapsed without qos %.2f", runtimeWithoutLimit.Seconds()) + }) + + Expect(runtimeWithLimit - runtimeWithoutLimit).To(BeNumerically("<", 100*time.Millisecond)) + }) + }) + + Context(fmt.Sprintf("[%s] when chaining bandwidth plugin with PTP and only including specific subnets in traffic shapping (not including the main ns one)", ver), func() { + var ptpConf string + var rateInBits uint64 + var burstInBits uint64 + var packetInBytes int + var containerWithoutQoSNS ns.NetNS + var containerWithQoSNS ns.NetNS + var portServerWithQoS int + var portServerWithoutQoS int + + var containerWithQoSRes types.Result + var containerWithoutQoSRes types.Result + var echoServerWithQoS *gexec.Session + var echoServerWithoutQoS *gexec.Session + var dataDir string + + BeforeEach(func() { + rateInBytes := 1000 + rateInBits = uint64(rateInBytes * 8) + burstInBits = rateInBits * 2 + shapedSubnets := []string{"10.2.2.0/24"} + // NOTE: Traffic shapping is not that precise at low rates, would be better to use higher rates + simple time+netcat for data transfer, rather than the provided + // client/server bin (limited to small amount of data) + packetInBytes = rateInBytes * 3 + + var err error + dataDir, err = os.MkdirTemp("", "bandwidth_linux_test") + Expect(err).NotTo(HaveOccurred()) + + ptpConf = fmt.Sprintf(`{ + "cniVersion": "%s", + "name": "myBWnet", + "type": "ptp", + "ipMasq": true, + "mtu": 512, + "ipam": { + "type": "host-local", + "subnet": "10.1.2.0/24", + "dataDir": "%s" + } + }`, ver, dataDir) + + const ( + containerWithQoSIFName = "ptp0" + containerWithoutQoSIFName = "ptp1" + ) + + containerWithQoSNS, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + + containerWithoutQoSNS, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + + By("create two containers, and use the bandwidth plugin on one of them") + + Expect(hostNs.Do(func(ns.NetNS) error { + defer GinkgoRecover() + + containerWithQoSRes, _, err = testutils.CmdAdd(containerWithQoSNS.Path(), "dummy", containerWithQoSIFName, []byte(ptpConf), func() error { + r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) + Expect(err).NotTo(HaveOccurred()) + Expect(r.Print()).To(Succeed()) + + return err + }) + Expect(err).NotTo(HaveOccurred()) + + containerWithoutQoSRes, _, err = testutils.CmdAdd(containerWithoutQoSNS.Path(), "dummy2", containerWithoutQoSIFName, []byte(ptpConf), func() error { + r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) + Expect(err).NotTo(HaveOccurred()) + Expect(r.Print()).To(Succeed()) + + return err + }) + Expect(err).NotTo(HaveOccurred()) + + containerWithQoSResult, err := types100.GetResult(containerWithQoSRes) + Expect(err).NotTo(HaveOccurred()) + + bandwidthPluginConf := &PluginConf{} + err = json.Unmarshal([]byte(ptpConf), &bandwidthPluginConf) + Expect(err).NotTo(HaveOccurred()) + + bandwidthPluginConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ + IngressBurst: burstInBits, + IngressRate: rateInBits, + EgressBurst: burstInBits, + EgressRate: rateInBits, + ShapedSubnets: shapedSubnets, + } + bandwidthPluginConf.Type = "bandwidth" + newConfBytes, err := buildOneConfig(ver, bandwidthPluginConf, containerWithQoSResult) + Expect(err).NotTo(HaveOccurred()) + + args := &skel.CmdArgs{ + ContainerID: "dummy3", + Netns: containerWithQoSNS.Path(), + IfName: containerWithQoSIFName, + StdinData: newConfBytes, + } + + result, out, err := testutils.CmdAdd(containerWithQoSNS.Path(), args.ContainerID, "", newConfBytes, func() error { return cmdAdd(args) }) + Expect(err).NotTo(HaveOccurred(), string(out)) + + if testutils.SpecVersionHasCHECK(ver) { + // Do CNI Check + checkConf := &PluginConf{} + err = json.Unmarshal([]byte(ptpConf), &checkConf) + Expect(err).NotTo(HaveOccurred()) + + checkConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ + IngressBurst: burstInBits, + IngressRate: rateInBits, + EgressBurst: burstInBits, + EgressRate: rateInBits, + ShapedSubnets: shapedSubnets, + } + checkConf.Type = "bandwidth" + + newCheckBytes, err := buildOneConfig(ver, checkConf, result) + Expect(err).NotTo(HaveOccurred()) + + args = &skel.CmdArgs{ + ContainerID: "dummy3", + Netns: containerWithQoSNS.Path(), + IfName: containerWithQoSIFName, + StdinData: newCheckBytes, + } + + err = testutils.CmdCheck(containerWithQoSNS.Path(), args.ContainerID, "", func() error { return cmdCheck(args) }) + Expect(err).NotTo(HaveOccurred()) + } + + return nil + })).To(Succeed()) + + By("starting a tcp server on both containers") + portServerWithQoS, echoServerWithQoS = startEchoServerInNamespace(containerWithQoSNS) + portServerWithoutQoS, echoServerWithoutQoS = startEchoServerInNamespace(containerWithoutQoSNS) + }) + + AfterEach(func() { + Expect(os.RemoveAll(dataDir)).To(Succeed()) + + Expect(containerWithQoSNS.Close()).To(Succeed()) + Expect(testutils.UnmountNS(containerWithQoSNS)).To(Succeed()) + Expect(containerWithoutQoSNS.Close()).To(Succeed()) + Expect(testutils.UnmountNS(containerWithoutQoSNS)).To(Succeed()) + + if echoServerWithoutQoS != nil { + echoServerWithoutQoS.Kill() + } + if echoServerWithQoS != nil { + echoServerWithQoS.Kill() + } + }) + + It("does not limit ingress traffic on veth device coming from non included subnets", func() { + var runtimeWithLimit time.Duration + var runtimeWithoutLimit time.Duration + + By("gather timing statistics about both containers") + + By("sending tcp traffic to the container that has traffic shaped", func() { + start := time.Now() + result, err := types100.GetResult(containerWithQoSRes) + Expect(err).NotTo(HaveOccurred()) + makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithQoS, packetInBytes) + end := time.Now() + runtimeWithLimit = end.Sub(start) + log.Printf("Elapsed with qos %.2f", runtimeWithLimit.Seconds()) + }) + + By("sending tcp traffic to the container that does not have traffic shaped", func() { + start := time.Now() + result, err := types100.GetResult(containerWithoutQoSRes) + Expect(err).NotTo(HaveOccurred()) + makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutQoS, packetInBytes) + end := time.Now() + runtimeWithoutLimit = end.Sub(start) + log.Printf("Elapsed without qos %.2f", runtimeWithoutLimit.Seconds()) + }) + + Expect(runtimeWithLimit - runtimeWithoutLimit).To(BeNumerically("<", 100*time.Millisecond)) + }) + }) + + Context(fmt.Sprintf("[%s] when chaining bandwidth plugin with PTP and only including specific subnets in traffic shapping (including the main ns one)", ver), func() { + var ptpConf string + var rateInBits uint64 + var burstInBits uint64 + var packetInBytes int + var containerWithoutQoSNS ns.NetNS + var containerWithQoSNS ns.NetNS + var portServerWithQoS int + var portServerWithoutQoS int + + var containerWithQoSRes types.Result + var containerWithoutQoSRes types.Result + var echoServerWithQoS *gexec.Session + var echoServerWithoutQoS *gexec.Session + var dataDir string + + BeforeEach(func() { + rateInBytes := 1000 + rateInBits = uint64(rateInBytes * 8) + burstInBits = rateInBits * 2 + shapedSubnets := []string{"10.1.2.1/32"} + // NOTE: Traffic shapping is not that precise at low rates, would be better to use higher rates + simple time+netcat for data transfer, rather than the provided + // client/server bin (limited to small amount of data) + packetInBytes = rateInBytes * 3 + + var err error + dataDir, err = os.MkdirTemp("", "bandwidth_linux_test") + Expect(err).NotTo(HaveOccurred()) + + ptpConf = fmt.Sprintf(`{ + "cniVersion": "%s", + "name": "myBWnet", + "type": "ptp", + "ipMasq": true, + "mtu": 512, + "ipam": { + "type": "host-local", + "subnet": "10.1.2.0/24", + "dataDir": "%s" + } + }`, ver, dataDir) + + const ( + containerWithQoSIFName = "ptp0" + containerWithoutQoSIFName = "ptp1" + ) + + containerWithQoSNS, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + + containerWithoutQoSNS, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + + By("create two containers, and use the bandwidth plugin on one of them") + + Expect(hostNs.Do(func(ns.NetNS) error { + defer GinkgoRecover() + + containerWithQoSRes, _, err = testutils.CmdAdd(containerWithQoSNS.Path(), "dummy", containerWithQoSIFName, []byte(ptpConf), func() error { + r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) + Expect(err).NotTo(HaveOccurred()) + Expect(r.Print()).To(Succeed()) + + return err + }) + Expect(err).NotTo(HaveOccurred()) + + containerWithoutQoSRes, _, err = testutils.CmdAdd(containerWithoutQoSNS.Path(), "dummy2", containerWithoutQoSIFName, []byte(ptpConf), func() error { + r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) + Expect(err).NotTo(HaveOccurred()) + Expect(r.Print()).To(Succeed()) + + return err + }) + Expect(err).NotTo(HaveOccurred()) + + containerWithQoSResult, err := types100.GetResult(containerWithQoSRes) + Expect(err).NotTo(HaveOccurred()) + + bandwidthPluginConf := &PluginConf{} + err = json.Unmarshal([]byte(ptpConf), &bandwidthPluginConf) + Expect(err).NotTo(HaveOccurred()) + + bandwidthPluginConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ + IngressBurst: burstInBits, + IngressRate: rateInBits, + EgressBurst: burstInBits, + EgressRate: rateInBits, + ShapedSubnets: shapedSubnets, + } + bandwidthPluginConf.Type = "bandwidth" + newConfBytes, err := buildOneConfig(ver, bandwidthPluginConf, containerWithQoSResult) + Expect(err).NotTo(HaveOccurred()) + + args := &skel.CmdArgs{ + ContainerID: "dummy3", + Netns: containerWithQoSNS.Path(), + IfName: containerWithQoSIFName, + StdinData: newConfBytes, + } + + result, out, err := testutils.CmdAdd(containerWithQoSNS.Path(), args.ContainerID, "", newConfBytes, func() error { return cmdAdd(args) }) + Expect(err).NotTo(HaveOccurred(), string(out)) + + if testutils.SpecVersionHasCHECK(ver) { + // Do CNI Check + checkConf := &PluginConf{} + err = json.Unmarshal([]byte(ptpConf), &checkConf) + Expect(err).NotTo(HaveOccurred()) + + checkConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ + IngressBurst: burstInBits, + IngressRate: rateInBits, + EgressBurst: burstInBits, + EgressRate: rateInBits, + ShapedSubnets: shapedSubnets, + } + checkConf.Type = "bandwidth" + + newCheckBytes, err := buildOneConfig(ver, checkConf, result) + Expect(err).NotTo(HaveOccurred()) + + args = &skel.CmdArgs{ + ContainerID: "dummy3", + Netns: containerWithQoSNS.Path(), + IfName: containerWithQoSIFName, + StdinData: newCheckBytes, + } + + err = testutils.CmdCheck(containerWithQoSNS.Path(), args.ContainerID, "", func() error { return cmdCheck(args) }) + Expect(err).NotTo(HaveOccurred()) + } + + return nil + })).To(Succeed()) + + By("starting a tcp server on both containers") + portServerWithQoS, echoServerWithQoS = startEchoServerInNamespace(containerWithQoSNS) + portServerWithoutQoS, echoServerWithoutQoS = startEchoServerInNamespace(containerWithoutQoSNS) + }) + + AfterEach(func() { + Expect(os.RemoveAll(dataDir)).To(Succeed()) + + Expect(containerWithQoSNS.Close()).To(Succeed()) + Expect(testutils.UnmountNS(containerWithQoSNS)).To(Succeed()) + Expect(containerWithoutQoSNS.Close()).To(Succeed()) + Expect(testutils.UnmountNS(containerWithoutQoSNS)).To(Succeed()) + + if echoServerWithoutQoS != nil { + echoServerWithoutQoS.Kill() + } + if echoServerWithQoS != nil { + echoServerWithQoS.Kill() + } + }) + + It("limits ingress traffic on veth device coming from included subnets", func() { + var runtimeWithLimit time.Duration + var runtimeWithoutLimit time.Duration + + By("gather timing statistics about both containers") + + By("sending tcp traffic to the container that has traffic shaped", func() { + start := time.Now() + result, err := types100.GetResult(containerWithQoSRes) + Expect(err).NotTo(HaveOccurred()) + makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithQoS, packetInBytes) + end := time.Now() + runtimeWithLimit = end.Sub(start) + log.Printf("Elapsed with qos %.2f", runtimeWithLimit.Seconds()) + }) + + By("sending tcp traffic to the container that does not have traffic shaped", func() { + start := time.Now() + result, err := types100.GetResult(containerWithoutQoSRes) + Expect(err).NotTo(HaveOccurred()) + makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutQoS, packetInBytes) + end := time.Now() + runtimeWithoutLimit = end.Sub(start) + log.Printf("Elapsed without qos %.2f", runtimeWithoutLimit.Seconds()) + }) + + Expect(runtimeWithLimit).To(BeNumerically(">", runtimeWithoutLimit+1000*time.Millisecond)) + }) + }) + } +}) diff --git a/plugins/meta/bandwidth/bandwidth_suite_test.go b/plugins/meta/bandwidth/bandwidth_suite_test.go index bfb2410e2..ce7a94ce4 100644 --- a/plugins/meta/bandwidth/bandwidth_suite_test.go +++ b/plugins/meta/bandwidth/bandwidth_suite_test.go @@ -15,6 +15,7 @@ package main import ( "bytes" + "encoding/json" "fmt" "io" "net" @@ -30,6 +31,7 @@ import ( "github.com/onsi/gomega/gexec" "github.com/vishvananda/netlink" + "github.com/containernetworking/cni/pkg/types" "github.com/containernetworking/plugins/pkg/ns" ) @@ -243,3 +245,47 @@ func createMacvlan(netNS ns.NetNS, master, macvlanName string) { }) Expect(err).NotTo(HaveOccurred()) } + +func buildOneConfig(cniVersion string, orig *PluginConf, prevResult types.Result) ([]byte, error) { + var err error + + inject := map[string]interface{}{ + "name": "myBWnet", + "cniVersion": cniVersion, + } + // Add previous plugin result + if prevResult != nil { + r, err := prevResult.GetAsVersion(cniVersion) + Expect(err).NotTo(HaveOccurred()) + inject["prevResult"] = r + } + + // Ensure every config uses the same name and version + config := make(map[string]interface{}) + + confBytes, err := json.Marshal(orig) + if err != nil { + return nil, err + } + + err = json.Unmarshal(confBytes, &config) + if err != nil { + return nil, fmt.Errorf("unmarshal existing network bytes: %s", err) + } + + for key, value := range inject { + config[key] = value + } + + newBytes, err := json.Marshal(config) + if err != nil { + return nil, err + } + + conf := &PluginConf{} + if err := json.Unmarshal(newBytes, &conf); err != nil { + return nil, fmt.Errorf("error parsing configuration: %s", err) + } + + return newBytes, nil +} diff --git a/plugins/meta/bandwidth/ifb_creator.go b/plugins/meta/bandwidth/ifb_creator.go index 5600aa24f..88b8549e9 100644 --- a/plugins/meta/bandwidth/ifb_creator.go +++ b/plugins/meta/bandwidth/ifb_creator.go @@ -217,12 +217,10 @@ func createHTB(rateInBits, burstInBits uint64, linkIndex int, subnets []string, } // Now add filters to redirect subnets to the class 1 if excluded instead of the default one (30) - for _, subnet := range subnets { // cmd = exec.Command("/usr/sbin/tc", "filter", "add", "dev", interfaceName, "parent", "1:", "protocol", protocol, // "prio", "16", "u32", "match", "ip", "dst", subnet, "flowid", "1:1") - _, nw, err := net.ParseCIDR(subnet) if err != nil { return fmt.Errorf("bad subnet %s: %s", subnet, err) @@ -246,7 +244,6 @@ func createHTB(rateInBits, burstInBits uint64, linkIndex int, subnets []string, keepBytes = 4 // prio/pref needs to be changed if we change the protocol, looks like we cannot mix protocols with the same pref prio = 16 - } if len(maskBytes) < keepBytes {