Merge pull request #1053 from alvaroaleman/junit

🏃 Generate junit in CI
kubernetes-sigs · Aug 3, 2020 · de782a3 · de782a3
2 parents 201e747 + 6633846
commit de782a3
Show file tree

Hide file tree

Showing 32 changed files with 189 additions and 34 deletions.
diff --git a/hack/test-all.sh b/hack/test-all.sh
@@ -21,3 +21,7 @@ source $(dirname ${BASH_SOURCE})/common.sh
 header_text "running go test"
 
 go test -race ${MOD_OPT} ./...
+
+if [[ -n ${ARTIFACTS:-} ]]; then
+  if grep -Rin '<failure type="Failure">' ${ARTIFACTS}/*; then exit 1; fi
+fi
diff --git a/pkg/builder/builder_suite_test.go b/pkg/builder/builder_suite_test.go
@@ -38,7 +38,8 @@ import (
 
 func TestBuilder(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecsWithDefaultAndCustomReporters(t, "application Suite", []Reporter{printer.NewlineReporter{}})
+	suiteName := "application Suite"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
 
 var testenv *envtest.Environment

diff --git a/pkg/cache/cache_suite_test.go b/pkg/cache/cache_suite_test.go
@@ -31,7 +31,8 @@ import (
 
 func TestSource(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecsWithDefaultAndCustomReporters(t, "Cache Suite", []Reporter{printer.NewlineReporter{}})
+	suiteName := "Cache Suite"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
 
 var testenv *envtest.Environment

diff --git a/pkg/client/apiutil/apiutil_suite_test.go b/pkg/client/apiutil/apiutil_suite_test.go
@@ -30,7 +30,8 @@ import (
 
 func TestSource(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecsWithDefaultAndCustomReporters(t, "API Utilities Test Suite", []Reporter{printer.NewlineReporter{}})
+	suiteName := "API Utilities Test Suite"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
 
 var cfg *rest.Config

diff --git a/pkg/client/client_suite_test.go b/pkg/client/client_suite_test.go
@@ -32,7 +32,8 @@ import (
 
 func TestSource(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecsWithDefaultAndCustomReporters(t, "Controller Integration Suite", []Reporter{printer.NewlineReporter{}})
+	suiteName := "Client Suite"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
 
 var testenv *envtest.Environment

diff --git a/pkg/client/config/config_suite_test.go b/pkg/client/config/config_suite_test.go
@@ -29,7 +29,8 @@ import (
 
 func TestConfig(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecsWithDefaultAndCustomReporters(t, "Client Config Test Suite", []Reporter{printer.NewlineReporter{}})
+	suiteName := "Client Config Test Suite"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
 
 var _ = BeforeSuite(func(done Done) {

diff --git a/pkg/client/fake/client_suite_test.go b/pkg/client/fake/client_suite_test.go
@@ -29,7 +29,8 @@ import (
 
 func TestSource(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecsWithDefaultAndCustomReporters(t, "Controller Integration Suite", []Reporter{printer.NewlineReporter{}})
+	suiteName := "Fake client Suite"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
 
 var _ = BeforeSuite(func(done Done) {

diff --git a/pkg/controller/controller_suite_test.go b/pkg/controller/controller_suite_test.go
@@ -37,7 +37,8 @@ import (
 
 func TestSource(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecsWithDefaultAndCustomReporters(t, "Controller Integration Suite", []Reporter{printer.NewlineReporter{}})
+	suiteName := "Controller Integration Suite"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
 
 var testenv *envtest.Environment

diff --git a/pkg/controller/controllerutil/controllerutil_suite_test.go b/pkg/controller/controllerutil/controllerutil_suite_test.go
@@ -25,11 +25,13 @@ import (
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
+	"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
 )
 
 func TestControllerutil(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Controllerutil Suite")
+	suiteName := "Controllerutil Suite"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
 
 var t *envtest.Environment

diff --git a/pkg/envtest/envtest_suite_test.go b/pkg/envtest/envtest_suite_test.go
@@ -25,13 +25,15 @@ import (
 	admissionv1beta1 "k8s.io/api/admissionregistration/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 )
 
 func TestSource(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecsWithDefaultAndCustomReporters(t, "EnvTest Suite", []Reporter{NewlineReporter{}})
+	suiteName := "Envtest Suite"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
 
 var env *Environment

diff --git a/pkg/envtest/printer/prow.go b/pkg/envtest/printer/prow.go
@@ -0,0 +1,109 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package printer
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/config"
+	"github.com/onsi/ginkgo/reporters"
+	"github.com/onsi/ginkgo/types"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+var (
+	allRegisteredSuites     = sets.String{}
+	allRegisteredSuitesLock = &sync.Mutex{}
+)
+
+type prowReporter struct {
+	junitReporter *reporters.JUnitReporter
+}
+
+// NewProwReporter returns a prowReporter that will write out junit if running in Prow and do
+// nothing otherwise.
+// WARNING: It seems this does not always properly fail the test runs when there are failures,
+// see https://github.com/onsi/ginkgo/issues/706
+// When using this you must make sure to grep for failures in your junit xmls and fail the run
+// if there are any.
+func NewProwReporter(suiteName string) ginkgo.Reporter {
+	allRegisteredSuitesLock.Lock()
+	if allRegisteredSuites.Has(suiteName) {
+		panic(fmt.Sprintf("Suite named %q registered more than once", suiteName))
+	}
+	allRegisteredSuites.Insert(suiteName)
+	allRegisteredSuitesLock.Unlock()
+
+	if os.Getenv("CI") == "" {
+		return &prowReporter{}
+	}
+	artifactsDir := os.Getenv("ARTIFACTS")
+	if artifactsDir == "" {
+		return &prowReporter{}
+	}
+
+	path := filepath.Join(artifactsDir, fmt.Sprintf("junit_%s_%d.xml", suiteName, config.GinkgoConfig.ParallelNode))
+	return &prowReporter{
+		junitReporter: reporters.NewJUnitReporter(path),
+	}
+}
+
+func (pr *prowReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, summary *types.SuiteSummary) {
+	if pr.junitReporter != nil {
+		pr.junitReporter.SpecSuiteWillBegin(config, summary)
+	}
+}
+
+// BeforeSuiteDidRun implements ginkgo.Reporter
+func (pr *prowReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) {
+	if pr.junitReporter != nil {
+		pr.junitReporter.BeforeSuiteDidRun(setupSummary)
+	}
+}
+
+// AfterSuiteDidRun implements ginkgo.Reporter
+func (pr *prowReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) {
+	if pr.junitReporter != nil {
+		pr.junitReporter.AfterSuiteDidRun(setupSummary)
+	}
+}
+
+// SpecWillRun implements ginkgo.Reporter
+func (pr *prowReporter) SpecWillRun(specSummary *types.SpecSummary) {
+	if pr.junitReporter != nil {
+		pr.junitReporter.SpecWillRun(specSummary)
+	}
+}
+
+// SpecDidComplete implements ginkgo.Reporter
+func (pr *prowReporter) SpecDidComplete(specSummary *types.SpecSummary) {
+	if pr.junitReporter != nil {
+		pr.junitReporter.SpecDidComplete(specSummary)
+	}
+}
+
+// SpecSuiteDidEnd Prints a newline between "35 Passed | 0 Failed | 0 Pending | 0 Skipped" and "--- PASS:"
+func (pr *prowReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) {
+	if pr.junitReporter != nil {
+		pr.junitReporter.SpecSuiteDidEnd(summary)
+	}
+}
diff --git a/pkg/handler/eventhandler_suite_test.go b/pkg/handler/eventhandler_suite_test.go
@@ -30,7 +30,8 @@ import (
 
 func TestEventhandler(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecsWithDefaultAndCustomReporters(t, "Eventhandler Suite", []Reporter{printer.NewlineReporter{}})
+	suiteName := "Eventhandler Suite"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
 
 var testenv *envtest.Environment

diff --git a/pkg/healthz/healthz_suite_test.go b/pkg/healthz/healthz_suite_test.go
@@ -28,7 +28,8 @@ import (
 
 func TestHealthz(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecsWithDefaultAndCustomReporters(t, "Healthz Suite", []Reporter{printer.NewlineReporter{}})
+	suiteName := "Healthz Suite"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
 
 var _ = BeforeSuite(func() {

diff --git a/pkg/internal/controller/controller_suite_test.go b/pkg/internal/controller/controller_suite_test.go
@@ -31,7 +31,8 @@ import (
 
 func TestSource(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecsWithDefaultAndCustomReporters(t, "Controller Integration Suite", []Reporter{printer.NewlineReporter{}})
+	suiteName := "Controller internal Suite"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
 
 var testenv *envtest.Environment

diff --git a/pkg/internal/recorder/recorder_suite_test.go b/pkg/internal/recorder/recorder_suite_test.go
@@ -31,7 +31,8 @@ import (
 
 func TestRecorder(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecsWithDefaultAndCustomReporters(t, "Recorder Integration Suite", []Reporter{printer.NewlineReporter{}})
+	suiteName := "Recorder Integration Suite"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
 
 var testenv *envtest.Environment

diff --git a/pkg/internal/testing/integration/addr/addr_suite_test.go b/pkg/internal/testing/integration/addr/addr_suite_test.go
@@ -1,14 +1,17 @@
 package addr_test
 
 import (
+	"testing"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"testing"
+	"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
 )
 
 func TestAddr(t *testing.T) {
 	t.Parallel()
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Addr Suite")
+	suiteName := "Addr Suite"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
diff --git a/pkg/internal/testing/integration/integration_suite_test.go b/pkg/internal/testing/integration/integration_suite_test.go
@@ -1,14 +1,17 @@
 package integration_test
 
 import (
+	"testing"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"testing"
+	"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
 )
 
 func TestIntegration(t *testing.T) {
 	t.Parallel()
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Integration Framework Unit Tests")
+	suiteName := "Integration Framework Unit Tests"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
diff --git a/pkg/internal/testing/integration/internal/integration_tests/integration_suite_test.go b/pkg/internal/testing/integration/internal/integration_tests/integration_suite_test.go
@@ -1,14 +1,17 @@
 package integrationtests
 
 import (
+	"testing"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"testing"
+	"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
 )
 
 func TestIntegration(t *testing.T) {
 	t.Parallel()
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Integration Framework Integration Tests")
+	suiteName := "Integration Framework Integration Tests"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
diff --git a/pkg/internal/testing/integration/internal/internal_suite_test.go b/pkg/internal/testing/integration/internal/internal_suite_test.go
@@ -1,14 +1,17 @@
 package internal_test
 
 import (
+	"testing"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"testing"
+	"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
 )
 
 func TestInternal(t *testing.T) {
 	t.Parallel()
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Internal Suite")
+	suiteName := "Internal Suite"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
diff --git a/pkg/log/log_suite_test.go b/pkg/log/log_suite_test.go
@@ -26,5 +26,6 @@ import (
 
 func TestSource(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecsWithDefaultAndCustomReporters(t, "Log Suite", []Reporter{printer.NewlineReporter{}})
+	suiteName := "Log Suite"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
diff --git a/pkg/log/zap/zap_suite_test.go b/pkg/log/zap/zap_suite_test.go
@@ -26,5 +26,6 @@ import (
 
 func TestSource(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecsWithDefaultAndCustomReporters(t, "Zap Log Suite", []Reporter{printer.NewlineReporter{}})
+	suiteName := "Zap Log Suite"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
diff --git a/pkg/manager/manager_suite_test.go b/pkg/manager/manager_suite_test.go
@@ -32,7 +32,8 @@ import (
 
 func TestSource(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecsWithDefaultAndCustomReporters(t, "Controller Integration Suite", []Reporter{printer.NewlineReporter{}})
+	suiteName := "Manager Suite"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
 
 var testenv *envtest.Environment

diff --git a/pkg/manager/signals/signals_suite_test.go b/pkg/manager/signals/signals_suite_test.go
@@ -27,7 +27,8 @@ import (
 
 func TestSource(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecsWithDefaultAndCustomReporters(t, "Runtime Signal Suite", []Reporter{printer.NewlineReporter{}})
+	suiteName := "Runtime Signal Suite"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
 
 var _ = BeforeSuite(func() {

diff --git a/pkg/predicate/predicate_suite_test.go b/pkg/predicate/predicate_suite_test.go
@@ -28,7 +28,8 @@ import (
 
 func TestPredicate(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecsWithDefaultAndCustomReporters(t, "Predicate Suite", []Reporter{printer.NewlineReporter{}})
+	suiteName := "Predicate Suite"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
 
 var _ = BeforeSuite(func() {

diff --git a/pkg/reconcile/reconcile_suite_test.go b/pkg/reconcile/reconcile_suite_test.go
@@ -28,7 +28,8 @@ import (
 
 func TestReconcile(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecsWithDefaultAndCustomReporters(t, "reconcile Suite", []Reporter{printer.NewlineReporter{}})
+	suiteName := "Reconcile Suite"
+	RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)})
 }
 
 var _ = BeforeSuite(func() {