From 088d1c631be85c49610c1383667cef52ffe6c506 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Mon, 5 Jun 2023 14:15:17 -0700 Subject: [PATCH] Test Coverage Reports for E2E tests (#7526) * Move coverage writer into agent and server * Add coverage report to E2E PR tests * Add codecov upload to drone Signed-off-by: Derek Nola --- .drone.yml | 29 +++++++++++++++++-- Dockerfile.test | 6 ++-- pkg/cli/agent/agent.go | 2 ++ pkg/cli/cmds/cover_default.go | 7 +++++ .../cover.go => cli/cmds/cover_linux.go} | 6 ++-- pkg/cli/server/server.go | 2 ++ pkg/server/no_cover.go | 7 ----- pkg/server/server.go | 1 - tests/e2e/secretsencryption/Vagrantfile | 5 ++++ .../secretsencryption_test.go | 3 ++ tests/e2e/startup/Vagrantfile | 7 ++++- tests/e2e/startup/startup_test.go | 3 ++ tests/e2e/testutils.go | 23 +++++++++++++++ tests/e2e/upgradecluster/Vagrantfile | 5 ++++ .../e2e/upgradecluster/upgradecluster_test.go | 3 ++ tests/e2e/vagrantdefaults.rb | 10 +++++++ tests/e2e/validatecluster/Vagrantfile | 6 +++- .../validatecluster/validatecluster_test.go | 3 ++ .../integration/longhorn/longhorn_int_test.go | 2 +- 19 files changed, 111 insertions(+), 19 deletions(-) create mode 100644 pkg/cli/cmds/cover_default.go rename pkg/{server/cover.go => cli/cmds/cover_linux.go} (88%) delete mode 100644 pkg/server/no_cover.go diff --git a/.drone.yml b/.drone.yml index 1395d72b2416..327a03b40d87 100644 --- a/.drone.yml +++ b/.drone.yml @@ -709,7 +709,7 @@ steps: image: rancher/dapper:v0.5.0 commands: - DOCKER_BUILDKIT=1 docker build --target test-e2e -t test-e2e -f Dockerfile.test . - - SKIP_VALIDATE=true SKIP_AIRGAP=true dapper ci + - SKIP_VALIDATE=true SKIP_AIRGAP=true GOCOVER=1 dapper ci - cp dist/artifacts/* /tmp/artifacts/ volumes: - name: cache @@ -725,6 +725,7 @@ steps: memory: 10Gi environment: E2E_REGISTRY: 'true' + E2E_GOCOVER: 'true' commands: - mkdir -p dist/artifacts - cp /tmp/artifacts/* dist/artifacts/ @@ -743,20 +744,24 @@ steps: - cd tests/e2e/validatecluster - vagrant destroy -f - go test -v -timeout=45m ./validatecluster_test.go -ci -local + - cp ./coverage.out /tmp/artifacts/validate-coverage.out - cd ../secretsencryption - vagrant destroy -f - go test -v -timeout=30m ./secretsencryption_test.go -ci -local + - cp ./coverage.out /tmp/artifacts/se-coverage.out - cd ../startup - vagrant destroy -f - go test -v -timeout=30m ./startup_test.go -ci -local + - cp ./coverage.out /tmp/artifacts/startup-coverage.out - | if [ "$DRONE_BUILD_EVENT" = "pull_request" ]; then cd ../upgradecluster vagrant destroy -f E2E_RELEASE_CHANNEL="latest" go test -v -timeout=45m ./upgradecluster_test.go -ci -local + cp ./coverage.out /tmp/artifacts/upgrade-coverage.out fi - docker stop registry && docker rm registry - + volumes: - name: libvirt path: /var/run/libvirt/ @@ -765,6 +770,26 @@ steps: - name: cache path: /tmp/artifacts +- name: upload to codecov + image: robertstettner/drone-codecov + settings: + token: + from_secret: codecov_token + files: + - /tmp/artifacts/validate-coverage.out + - /tmp/artifacts/se-coverage.out + - /tmp/artifacts/startup-coverage.out + - /tmp/artifacts/upgrade-coverage.out + flags: + - e2etests + when: + event: + - pull_request + + volumes: + - name: cache + path: /tmp/artifacts + volumes: - name: docker host: diff --git a/Dockerfile.test b/Dockerfile.test index 43f954c0d799..e2b4d1cbd238 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -44,10 +44,10 @@ RUN vagrant box add generic/ubuntu2004 --provider libvirt --force RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"; \ chmod +x ./kubectl; \ mv ./kubectl /usr/local/bin/kubectl -ENV GO_VERSION 1.19.2 -RUN curl -O -L "https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz"; \ +RUN GO_VERSION=$(curl https://go.dev/VERSION?m=text); \ + curl -O -L "https://golang.org/dl/${GO_VERSION}.linux-amd64.tar.gz"; \ rm -rf /usr/local/go; \ - tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz; + tar -C /usr/local -xzf ${GO_VERSION}.linux-amd64.tar.gz; ENV PATH="${PATH}:/usr/local/go/bin" diff --git a/pkg/cli/agent/agent.go b/pkg/cli/agent/agent.go index 501ac1bc0c1b..a3e5189c9d18 100644 --- a/pkg/cli/agent/agent.go +++ b/pkg/cli/agent/agent.go @@ -76,5 +76,7 @@ func Run(ctx *cli.Context) error { contextCtx := signals.SetupSignalContext() + go cmds.WriteCoverage(contextCtx) + return agent.Run(contextCtx, cfg) } diff --git a/pkg/cli/cmds/cover_default.go b/pkg/cli/cmds/cover_default.go new file mode 100644 index 000000000000..38cc9f82916b --- /dev/null +++ b/pkg/cli/cmds/cover_default.go @@ -0,0 +1,7 @@ +//go:build !linux || !cover + +package cmds + +import "context" + +func WriteCoverage(ctx context.Context) {} diff --git a/pkg/server/cover.go b/pkg/cli/cmds/cover_linux.go similarity index 88% rename from pkg/server/cover.go rename to pkg/cli/cmds/cover_linux.go index 8290db063378..82e4cf090325 100644 --- a/pkg/server/cover.go +++ b/pkg/cli/cmds/cover_linux.go @@ -1,6 +1,6 @@ -//go:build cover +//go:build linux && cover -package server +package cmds import ( "context" @@ -13,7 +13,7 @@ import ( // writeCoverage checks if GOCOVERDIR is set on startup and writes coverage files to that directory // every 20 seconds. This is done to ensure that the coverage files are written even if the process is killed. -func writeCoverage(ctx context.Context) { +func WriteCoverage(ctx context.Context) { if k, ok := os.LookupEnv("GOCOVERDIR"); ok { for { select { diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index 75bc39ea966d..84a53feeb16d 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -441,6 +441,8 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont return err } + go cmds.WriteCoverage(ctx) + go func() { if !serverConfig.ControlConfig.DisableAPIServer { <-serverConfig.ControlConfig.Runtime.APIServerReady diff --git a/pkg/server/no_cover.go b/pkg/server/no_cover.go deleted file mode 100644 index 7cfc44c29728..000000000000 --- a/pkg/server/no_cover.go +++ /dev/null @@ -1,7 +0,0 @@ -//go:build !cover - -package server - -import "context" - -func writeCoverage(ctx context.Context) {} diff --git a/pkg/server/server.go b/pkg/server/server.go index 7c8899177c1e..a64e5da73849 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -80,7 +80,6 @@ func StartServer(ctx context.Context, config *Config, cfg *cmds.Server) error { return errors.Wrap(err, "startup hook") } } - go writeCoverage(ctx) go startOnAPIServerReady(ctx, config) if err := printTokens(&config.ControlConfig); err != nil { diff --git a/tests/e2e/secretsencryption/Vagrantfile b/tests/e2e/secretsencryption/Vagrantfile index b98322e76156..9170a607a291 100644 --- a/tests/e2e/secretsencryption/Vagrantfile +++ b/tests/e2e/secretsencryption/Vagrantfile @@ -5,6 +5,7 @@ NODE_BOXES = (ENV['E2E_NODE_BOXES'] || ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") +GOCOVER = (ENV['E2E_GOCOVER'] || "") NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 2048).to_i # Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks @@ -25,6 +26,10 @@ def provision(vm, role, role_num, node_num) vm.provision "shell", inline: "ping -c 2 k3s.io" + if !GOCOVER.empty? + addCoverageDir(vm, role) + end + if role.include?("server") && role_num == 0 vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| k3s.args = %W[server --cluster-init --node-external-ip=#{NETWORK_PREFIX}.100 --flannel-iface=eth1 --secrets-encryption] diff --git a/tests/e2e/secretsencryption/secretsencryption_test.go b/tests/e2e/secretsencryption/secretsencryption_test.go index 5c6d86639d24..db8ea649708a 100644 --- a/tests/e2e/secretsencryption/secretsencryption_test.go +++ b/tests/e2e/secretsencryption/secretsencryption_test.go @@ -306,6 +306,9 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { + if os.Getenv("E2E_GOCOVER") != "" { + Expect(e2e.GetCoverageReport(serverNodeNames)).To(Succeed()) + } if failed && !*ci { fmt.Println("FAILED!") } else { diff --git a/tests/e2e/startup/Vagrantfile b/tests/e2e/startup/Vagrantfile index 37b441e0649d..68aa8cad8d3b 100644 --- a/tests/e2e/startup/Vagrantfile +++ b/tests/e2e/startup/Vagrantfile @@ -5,6 +5,7 @@ NODE_BOXES = (ENV['E2E_NODE_BOXES'] || ['generic/ubuntu2004', 'generic/ubuntu2004']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") +GOCOVER = (ENV['E2E_GOCOVER'] || "") NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 2048).to_i # Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks @@ -26,7 +27,7 @@ def provision(vm, role, role_num, node_num) node_ip = "#{NETWORK_PREFIX}.#{100+node_num}" vm.provision "shell", inline: "ping -c 2 k3s.io" - + if role.include?("server") vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| k3s.args = "server " @@ -52,6 +53,10 @@ def provision(vm, role, role_num, node_num) end end + if !GOCOVER.empty? + addCoverageDir(vm, role) + end + if vm.box.to_s.include?("microos") vm.provision 'k3s-reload', type: 'reload', run: 'once' end diff --git a/tests/e2e/startup/startup_test.go b/tests/e2e/startup/startup_test.go index c806beab7135..651522756f80 100644 --- a/tests/e2e/startup/startup_test.go +++ b/tests/e2e/startup/startup_test.go @@ -274,6 +274,9 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { + if os.Getenv("E2E_GOCOVER") != "" { + Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) + } if failed && !*ci { fmt.Println("FAILED!") } else { diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go index 0e4956c00bd2..9aa7c1a4fa4b 100644 --- a/tests/e2e/testutils.go +++ b/tests/e2e/testutils.go @@ -489,6 +489,29 @@ func UpgradeCluster(nodeNames []string, local bool) error { return nil } +func GetCoverageReport(nodeNames []string) error { + covDirs := []string{} + for _, nodeName := range nodeNames { + covDir := nodeName + "-cov" + covDirs = append(covDirs, covDir) + os.MkdirAll(covDir, 0755) + cmd := "vagrant scp " + nodeName + ":/tmp/k3scov/* " + covDir + if _, err := RunCommand(cmd); err != nil { + return err + } + } + cmd := "go tool covdata textfmt -i " + strings.Join(covDirs, ",") + " -o coverage.out" + if out, err := RunCommand(cmd); err != nil { + return fmt.Errorf("failed to generate coverage report: %s, %v", out, err) + } + for _, covDir := range covDirs { + if err := os.RemoveAll(covDir); err != nil { + return err + } + } + return nil +} + // getPodIPs returns the IPs of all pods func GetPodIPs(kubeConfigFile string) ([]ObjIP, error) { cmd := `kubectl get pods -A -o=jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.podIPs[*].ip}{"\n"}{end}' --kubeconfig=` + kubeConfigFile diff --git a/tests/e2e/upgradecluster/Vagrantfile b/tests/e2e/upgradecluster/Vagrantfile index 33c51e841e7e..35c8b6544f8c 100644 --- a/tests/e2e/upgradecluster/Vagrantfile +++ b/tests/e2e/upgradecluster/Vagrantfile @@ -7,6 +7,7 @@ RELEASE_CHANNEL = (ENV['E2E_RELEASE_CHANNEL'] || "latest") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "etcd") REGISTRY = (ENV['E2E_REGISTRY'] || "") +GOCOVER = (ENV['E2E_GOCOVER'] || "") NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 2048).to_i # Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks @@ -46,6 +47,10 @@ def provision(vm, role, role_num, node_num) vm.provision "Set private registry", type: "shell", path: scripts_location + "/registry.sh", args: [ "#{NETWORK_PREFIX}.1" ] end + if !GOCOVER.empty? + addCoverageDir(vm, role) + end + if role.include?("server") && role_num == 0 vm.provision 'k3s-primary-server', type: 'k3s', run: 'once' do |k3s| k3s.args = "server " diff --git a/tests/e2e/upgradecluster/upgradecluster_test.go b/tests/e2e/upgradecluster/upgradecluster_test.go index d4acd7d6b027..35ed556c7765 100644 --- a/tests/e2e/upgradecluster/upgradecluster_test.go +++ b/tests/e2e/upgradecluster/upgradecluster_test.go @@ -385,6 +385,9 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { + if os.Getenv("E2E_GOCOVER") != "" { + Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) + } if failed && !*ci { fmt.Println("FAILED!") } else { diff --git a/tests/e2e/vagrantdefaults.rb b/tests/e2e/vagrantdefaults.rb index 95e0ae6744a3..960bbd60183d 100644 --- a/tests/e2e/vagrantdefaults.rb +++ b/tests/e2e/vagrantdefaults.rb @@ -34,6 +34,16 @@ def getInstallType(vm, release_version, branch) end end +def addCoverageDir(vm, role) + service = role.include?("agent") ? "k3s-agent" : "k3s" + script = <<~SHELL + mkdir -p /tmp/k3scov + echo -e 'GOCOVERDIR=/tmp/k3scov' >> /etc/default/#{service} + systemctl daemon-reload + SHELL + vm.provision "go coverage", type: "shell", inline: script +end + def getHardenedArg(vm, hardened, scripts_location) if hardened.empty? return "" diff --git a/tests/e2e/validatecluster/Vagrantfile b/tests/e2e/validatecluster/Vagrantfile index 18409d3baf79..0f369ebb1bca 100644 --- a/tests/e2e/validatecluster/Vagrantfile +++ b/tests/e2e/validatecluster/Vagrantfile @@ -9,6 +9,7 @@ EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "etcd") HARDENED = (ENV['E2E_HARDENED'] || "") REGISTRY = (ENV['E2E_REGISTRY'] || "") RANCHER = (ENV['E2E_RANCHER'] || "") +GOCOVER = (ENV['E2E_GOCOVER'] || "") NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 2048).to_i # Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks @@ -39,6 +40,9 @@ def provision(vm, role, role_num, node_num) vm.provision "Set private registry", type: "shell", path: scripts_location + "/registry.sh", args: [ "#{NETWORK_PREFIX}.1" ] end + if !GOCOVER.empty? + addCoverageDir(vm, role) + end if role.include?("server") && role_num == 0 vm.provision 'k3s-primary-server', type: 'k3s', run: 'once' do |k3s| @@ -133,7 +137,7 @@ def getDBType(role, role_num, vm) end Vagrant.configure("2") do |config| - config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload"] + config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload", "vagrant-scp"] # Default provider is libvirt, virtualbox is only provided as a backup config.vm.provider "libvirt" do |v| v.cpus = NODE_CPUS diff --git a/tests/e2e/validatecluster/validatecluster_test.go b/tests/e2e/validatecluster/validatecluster_test.go index 786f187e6289..0e932d52f411 100644 --- a/tests/e2e/validatecluster/validatecluster_test.go +++ b/tests/e2e/validatecluster/validatecluster_test.go @@ -404,6 +404,9 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { + if os.Getenv("E2E_GOCOVER") != "" { + Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) + } if failed && !*ci { fmt.Println("FAILED!") } else { diff --git a/tests/integration/longhorn/longhorn_int_test.go b/tests/integration/longhorn/longhorn_int_test.go index 7bade6cc222a..e4c475e71605 100644 --- a/tests/integration/longhorn/longhorn_int_test.go +++ b/tests/integration/longhorn/longhorn_int_test.go @@ -139,7 +139,7 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if !testutil.IsExistingServer() { + if !testutil.IsExistingServer() && server != nil { if failed { testutil.K3sSaveLog(server, false) }