From 83ff808f0def7d05c24b7f67024cee07467d2921 Mon Sep 17 00:00:00 2001 From: Spencer Schrock Date: Tue, 30 Jan 2024 12:40:41 -0800 Subject: [PATCH] :seedling: Enhance test output and management in ValidateTestReturn (#3810) * test failures should print the details they receive this makes debugging failing tests easier. Signed-off-by: Spencer Schrock * use GinkgoTB so the test helpers work instead of panicing Signed-off-by: Spencer Schrock * ValidateTestReturn will fail the test directly, no need for the bool return Signed-off-by: Spencer Schrock * clarify diff details Signed-off-by: Spencer Schrock --------- Signed-off-by: Spencer Schrock --- checks/branch_protection_test.go | 4 +- checks/cii_best_practices_test.go | 4 +- checks/dependency_update_tool_test.go | 4 +- checks/evaluation/binary_artifacts_test.go | 4 +- checks/evaluation/branch_protection_test.go | 4 +- checks/evaluation/ci_tests_test.go | 4 +- checks/evaluation/cii_best_practices_test.go | 4 +- checks/evaluation/code_review_test.go | 4 +- checks/evaluation/contributors_test.go | 4 +- checks/evaluation/dangerous_workflow_test.go | 4 +- .../evaluation/dependency_update_tool_test.go | 4 +- checks/evaluation/fuzzing_test.go | 4 +- checks/evaluation/license_test.go | 4 +- checks/evaluation/maintained_test.go | 4 +- checks/evaluation/packaging_test.go | 4 +- checks/evaluation/pinned_dependencies_test.go | 4 +- checks/evaluation/sast_test.go | 4 +- checks/evaluation/security_policy_test.go | 4 +- checks/evaluation/signed_releases_test.go | 4 +- checks/evaluation/vulnerabilities_test.go | 4 +- checks/evaluation/webhooks_test.go | 4 +- checks/fuzzing_test.go | 4 +- checks/license_test.go | 4 +- checks/permissions_test.go | 4 +- checks/raw/security_policy_test.go | 4 +- checks/raw/vulnerabilities_test.go | 4 +- checks/raw/webhooks_test.go | 4 +- checks/security_policy_test.go | 4 +- e2e/binary_artifacts_test.go | 12 +++--- e2e/branch_protection_test.go | 8 ++-- e2e/ci_tests_test.go | 8 ++-- e2e/cii_best_practices_test.go | 2 +- e2e/code_review_test.go | 10 ++--- e2e/contributors_test.go | 2 +- e2e/dangerous_workflow_test.go | 6 +-- e2e/dependency_update_tool_test.go | 4 +- e2e/fuzzing_test.go | 10 ++--- e2e/license_test.go | 18 +++----- e2e/maintained_test.go | 4 +- e2e/packaging_test.go | 2 +- e2e/permissions_test.go | 6 +-- e2e/pinned_dependencies_test.go | 6 +-- e2e/sast_test.go | 2 +- e2e/security_policy_test.go | 10 ++--- e2e/signedreleases_test.go | 2 +- e2e/vulnerabilities_test.go | 6 +-- utests/utlib.go | 42 +++++++++++-------- 47 files changed, 109 insertions(+), 163 deletions(-) diff --git a/checks/branch_protection_test.go b/checks/branch_protection_test.go index 165d3b8d56e..d56a4cce2bd 100644 --- a/checks/branch_protection_test.go +++ b/checks/branch_protection_test.go @@ -431,9 +431,7 @@ func TestReleaseAndDevBranchProtected(t *testing.T) { RepoClient: mockRepoClient, } r := BranchProtection(&req) - if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &r, &dl) { - t.Fail() - } + scut.ValidateTestReturn(t, tt.name, &tt.expected, &r, &dl) ctrl.Finish() }) } diff --git a/checks/cii_best_practices_test.go b/checks/cii_best_practices_test.go index 943e52a7aef..4c9546de8ea 100644 --- a/checks/cii_best_practices_test.go +++ b/checks/cii_best_practices_test.go @@ -122,9 +122,7 @@ func TestCIIBestPractices(t *testing.T) { } res := CIIBestPractices(&req) dl := scut.TestDetailLogger{} - if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, &dl) { - t.Fail() - } + scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, &dl) ctrl.Finish() }) } diff --git a/checks/dependency_update_tool_test.go b/checks/dependency_update_tool_test.go index e2f75babee5..6cccdb7f111 100644 --- a/checks/dependency_update_tool_test.go +++ b/checks/dependency_update_tool_test.go @@ -155,9 +155,7 @@ func TestDependencyUpdateTool(t *testing.T) { } res := DependencyUpdateTool(c) - if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, &dl) { - t.Fail() - } + scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, &dl) }) } } diff --git a/checks/evaluation/binary_artifacts_test.go b/checks/evaluation/binary_artifacts_test.go index d2f9c4f8727..59cf25dbf0c 100644 --- a/checks/evaluation/binary_artifacts_test.go +++ b/checks/evaluation/binary_artifacts_test.go @@ -142,9 +142,7 @@ func TestBinaryArtifacts(t *testing.T) { t.Parallel() dl := scut.TestDetailLogger{} got := BinaryArtifacts(tt.name, tt.findings, &dl) - if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) { - t.Errorf("got %v, expected %v", got, tt.result) - } + scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) }) } } diff --git a/checks/evaluation/branch_protection_test.go b/checks/evaluation/branch_protection_test.go index 86c34c3e5e6..0084e4e0a34 100644 --- a/checks/evaluation/branch_protection_test.go +++ b/checks/evaluation/branch_protection_test.go @@ -573,9 +573,7 @@ func TestIsBranchProtected(t *testing.T) { Score: score, Error: err, } - if !scut.ValidateTestReturn(t, tt.name, &tt.expected, actual, &dl) { - t.Fail() - } + scut.ValidateTestReturn(t, tt.name, &tt.expected, actual, &dl) }) } } diff --git a/checks/evaluation/ci_tests_test.go b/checks/evaluation/ci_tests_test.go index 825ad5c6142..a8fc4ecb9ae 100644 --- a/checks/evaluation/ci_tests_test.go +++ b/checks/evaluation/ci_tests_test.go @@ -111,9 +111,7 @@ func TestCITests(t *testing.T) { t.Parallel() dl := scut.TestDetailLogger{} got := CITests(tt.name, tt.findings, &dl) - if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) { - t.Errorf("got %v, expected %v", got, tt.result) - } + scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) }) } } diff --git a/checks/evaluation/cii_best_practices_test.go b/checks/evaluation/cii_best_practices_test.go index 50b15ad1b5f..5cfb08c2142 100644 --- a/checks/evaluation/cii_best_practices_test.go +++ b/checks/evaluation/cii_best_practices_test.go @@ -143,9 +143,7 @@ func TestCIIBestPractices(t *testing.T) { t.Parallel() dl := scut.TestDetailLogger{} got := CIIBestPractices(tt.name, tt.findings, &dl) - if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) { - t.Errorf("got %v, expected %v", got, tt.result) - } + scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) }) } } diff --git a/checks/evaluation/code_review_test.go b/checks/evaluation/code_review_test.go index bb17732e24d..b916224509d 100644 --- a/checks/evaluation/code_review_test.go +++ b/checks/evaluation/code_review_test.go @@ -216,9 +216,7 @@ func TestCodeReview(t *testing.T) { dl := &scut.TestDetailLogger{} res := CodeReview(tt.name, dl, tt.rawData) - if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, dl) { - t.Error() - } + scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, dl) }) } } diff --git a/checks/evaluation/contributors_test.go b/checks/evaluation/contributors_test.go index d5962e1328e..e00db8f9957 100644 --- a/checks/evaluation/contributors_test.go +++ b/checks/evaluation/contributors_test.go @@ -83,9 +83,7 @@ func TestContributors(t *testing.T) { t.Parallel() dl := scut.TestDetailLogger{} got := Contributors(tt.name, tt.findings, &dl) - if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) { - t.Error(tt.name) - } + scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) }) } } diff --git a/checks/evaluation/dangerous_workflow_test.go b/checks/evaluation/dangerous_workflow_test.go index 261ba0e7509..847c67694c2 100644 --- a/checks/evaluation/dangerous_workflow_test.go +++ b/checks/evaluation/dangerous_workflow_test.go @@ -252,9 +252,7 @@ func TestDangerousWorkflow(t *testing.T) { t.Parallel() dl := scut.TestDetailLogger{} got := DangerousWorkflow(tt.name, tt.findings, &dl) - if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) { - t.Errorf("got %v, expected %v", got, tt.result) - } + scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) }) } } diff --git a/checks/evaluation/dependency_update_tool_test.go b/checks/evaluation/dependency_update_tool_test.go index 1063929aace..bda157c4eff 100644 --- a/checks/evaluation/dependency_update_tool_test.go +++ b/checks/evaluation/dependency_update_tool_test.go @@ -164,9 +164,7 @@ func TestDependencyUpdateTool(t *testing.T) { dl := scut.TestDetailLogger{} got := DependencyUpdateTool(tt.name, tt.findings, &dl) - if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) { - t.Errorf("got %v, expected %v", got, tt.result) - } + scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) }) } } diff --git a/checks/evaluation/fuzzing_test.go b/checks/evaluation/fuzzing_test.go index 5c06f49da73..042b18b24a6 100644 --- a/checks/evaluation/fuzzing_test.go +++ b/checks/evaluation/fuzzing_test.go @@ -217,9 +217,7 @@ func TestFuzzing(t *testing.T) { t.Parallel() dl := scut.TestDetailLogger{} got := Fuzzing(tt.name, tt.findings, &dl) - if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) { - t.Errorf("got %v, expected %v", got, tt.result) - } + scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) }) } } diff --git a/checks/evaluation/license_test.go b/checks/evaluation/license_test.go index 8cf611f32e1..66857014f60 100644 --- a/checks/evaluation/license_test.go +++ b/checks/evaluation/license_test.go @@ -155,9 +155,7 @@ func TestLicense(t *testing.T) { t.Parallel() dl := scut.TestDetailLogger{} got := License(tt.name, tt.findings, &dl) - if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) { - t.Errorf("got %v, expected %v", got, tt.result) - } + scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) }) } } diff --git a/checks/evaluation/maintained_test.go b/checks/evaluation/maintained_test.go index 420955e5cd1..d504a0c0864 100644 --- a/checks/evaluation/maintained_test.go +++ b/checks/evaluation/maintained_test.go @@ -127,9 +127,7 @@ func TestMaintained(t *testing.T) { t.Parallel() dl := scut.TestDetailLogger{} got := Maintained(tt.name, tt.findings, &dl) - if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) { - t.Errorf("got %v, expected %v", got, tt.result) - } + scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) }) } } diff --git a/checks/evaluation/packaging_test.go b/checks/evaluation/packaging_test.go index 459552e35be..cfa52196f6d 100644 --- a/checks/evaluation/packaging_test.go +++ b/checks/evaluation/packaging_test.go @@ -88,9 +88,7 @@ func TestPackaging(t *testing.T) { t.Parallel() dl := scut.TestDetailLogger{} got := Packaging(tt.name, tt.findings, &dl) - if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) { - t.Errorf("got %v, expected %v", got, tt.result) - } + scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) }) } } diff --git a/checks/evaluation/pinned_dependencies_test.go b/checks/evaluation/pinned_dependencies_test.go index 50c38c12282..ef3fefb7edd 100644 --- a/checks/evaluation/pinned_dependencies_test.go +++ b/checks/evaluation/pinned_dependencies_test.go @@ -841,9 +841,7 @@ func Test_PinningDependencies(t *testing.T) { ProcessingErrors: tt.processingErrors, }) - if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &actual, &dl) { - t.Fail() - } + scut.ValidateTestReturn(t, tt.name, &tt.expected, &actual, &dl) }) } } diff --git a/checks/evaluation/sast_test.go b/checks/evaluation/sast_test.go index c58ced752df..656ebe9afdc 100644 --- a/checks/evaluation/sast_test.go +++ b/checks/evaluation/sast_test.go @@ -301,9 +301,7 @@ func TestSAST(t *testing.T) { t.Parallel() dl := scut.TestDetailLogger{} got := SAST(tt.name, tt.findings, &dl) - if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) { - t.Errorf("got %v, expected %v", got, tt.result) - } + scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) }) } } diff --git a/checks/evaluation/security_policy_test.go b/checks/evaluation/security_policy_test.go index 4a57164077f..ac55f7073c4 100644 --- a/checks/evaluation/security_policy_test.go +++ b/checks/evaluation/security_policy_test.go @@ -190,9 +190,7 @@ func TestSecurityPolicy(t *testing.T) { t.Parallel() dl := scut.TestDetailLogger{} got := SecurityPolicy(tt.name, tt.findings, &dl) - if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) { - t.Errorf("got %v, expected %v", got, tt.result) - } + scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) }) } } diff --git a/checks/evaluation/signed_releases_test.go b/checks/evaluation/signed_releases_test.go index 478154497c0..e02516f4032 100644 --- a/checks/evaluation/signed_releases_test.go +++ b/checks/evaluation/signed_releases_test.go @@ -303,9 +303,7 @@ func TestSignedReleases(t *testing.T) { t.Parallel() dl := scut.TestDetailLogger{} got := SignedReleases(tt.name, tt.findings, &dl) - if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) { - t.Errorf("got %v, expected %v", got, tt.result) - } + scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) }) } } diff --git a/checks/evaluation/vulnerabilities_test.go b/checks/evaluation/vulnerabilities_test.go index 3ed902d1175..382a44b308a 100644 --- a/checks/evaluation/vulnerabilities_test.go +++ b/checks/evaluation/vulnerabilities_test.go @@ -139,9 +139,7 @@ func TestVulnerabilities(t *testing.T) { t.Parallel() dl := scut.TestDetailLogger{} got := Vulnerabilities(tt.name, tt.findings, &dl) - if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) { - t.Errorf("got %v, expected %v", got, tt.result) - } + scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) }) } } diff --git a/checks/evaluation/webhooks_test.go b/checks/evaluation/webhooks_test.go index a698e732115..a6de4ec5dc6 100644 --- a/checks/evaluation/webhooks_test.go +++ b/checks/evaluation/webhooks_test.go @@ -229,9 +229,7 @@ func TestWebhooks(t *testing.T) { t.Parallel() dl := scut.TestDetailLogger{} got := Webhooks(tt.name, tt.findings, &dl) - if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) { - t.Errorf("got %v, expected %v", got, tt.result) - } + scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) }) } } diff --git a/checks/fuzzing_test.go b/checks/fuzzing_test.go index 30004a08ccb..852b0172ff4 100644 --- a/checks/fuzzing_test.go +++ b/checks/fuzzing_test.go @@ -169,9 +169,7 @@ func TestFuzzing(t *testing.T) { return } - if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &result, &dl) { - t.Fatalf(tt.name, tt.expected) - } + scut.ValidateTestReturn(t, tt.name, &tt.expected, &result, &dl) }) } } diff --git a/checks/license_test.go b/checks/license_test.go index 1963643b727..7726d1c7372 100644 --- a/checks/license_test.go +++ b/checks/license_test.go @@ -92,9 +92,7 @@ func TestLicenseFileSubdirectory(t *testing.T) { res := License(&req) - if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, &dl) { - t.Fail() - } + scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, &dl) ctrl.Finish() }) diff --git a/checks/permissions_test.go b/checks/permissions_test.go index 9261fb2792d..2d80ef33f1c 100644 --- a/checks/permissions_test.go +++ b/checks/permissions_test.go @@ -458,9 +458,7 @@ func TestGithubTokenPermissions(t *testing.T) { res := TokenPermissions(&c) - if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, &dl) { - t.Errorf("test failed: log message not present: %+v\n%+v", tt.expected, dl) - } + scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, &dl) }) } } diff --git a/checks/raw/security_policy_test.go b/checks/raw/security_policy_test.go index 24a6cb6eb19..d311d85e283 100644 --- a/checks/raw/security_policy_test.go +++ b/checks/raw/security_policy_test.go @@ -165,9 +165,7 @@ func TestSecurityPolicy(t *testing.T) { res, err := SecurityPolicy(&c) - if !scut.ValidateTestReturn(t, tt.name, &tt.want, &checker.CheckResult{}, &dl) { - t.Errorf("test failed: log message not present: %+v , for test %v", tt.want, tt.name) - } + scut.ValidateTestReturn(t, tt.name, &tt.want, &checker.CheckResult{}, &dl) if (err != nil) != tt.wantErr { t.Errorf("SecurityPolicy() error = %v, wantErr %v", err, tt.wantErr) diff --git a/checks/raw/vulnerabilities_test.go b/checks/raw/vulnerabilities_test.go index 0df35e35557..4b9ec8ddf42 100644 --- a/checks/raw/vulnerabilities_test.go +++ b/checks/raw/vulnerabilities_test.go @@ -114,9 +114,7 @@ func TestVulnerabilities(t *testing.T) { } } - if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &checker.CheckResult{}, &dl) { - t.Fatalf("Test %s failed", tt.name) - } + scut.ValidateTestReturn(t, tt.name, &tt.expected, &checker.CheckResult{}, &dl) }) } } diff --git a/checks/raw/webhooks_test.go b/checks/raw/webhooks_test.go index 90f7d45e0a5..931b0eaec45 100644 --- a/checks/raw/webhooks_test.go +++ b/checks/raw/webhooks_test.go @@ -130,9 +130,7 @@ func TestWebhooks(t *testing.T) { } } - if !scut.ValidateTestReturn(t, tt.name, &tt.expected, &checker.CheckResult{}, &dl) { - t.Fatalf("Test %s failed", tt.name) - } + scut.ValidateTestReturn(t, tt.name, &tt.expected, &checker.CheckResult{}, &dl) }) } } diff --git a/checks/security_policy_test.go b/checks/security_policy_test.go index 4a2596b8189..1517c780589 100644 --- a/checks/security_policy_test.go +++ b/checks/security_policy_test.go @@ -197,9 +197,7 @@ func TestSecurityPolicy(t *testing.T) { res := SecurityPolicy(c) - if !scut.ValidateTestReturn(t, tt.name, &tt.want, &res, &dl) { - t.Errorf("test failed: log message not present: %+v on %+v", tt.want, res) - } + scut.ValidateTestReturn(t, tt.name, &tt.want, &res, &dl) }) } } diff --git a/e2e/binary_artifacts_test.go b/e2e/binary_artifacts_test.go index 1905e1f662f..1f3dfb02741 100644 --- a/e2e/binary_artifacts_test.go +++ b/e2e/binary_artifacts_test.go @@ -57,7 +57,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() { } result := checks.BinaryArtifacts(&req) - Expect(scut.ValidateTestReturn(nil, "no binary artifacts", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "no binary artifacts", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should return binary artifacts present in source code", func() { @@ -84,7 +84,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() { NumberOfDebug: 0, } result := checks.BinaryArtifacts(&req) - Expect(scut.ValidateTestReturn(nil, "binary artifacts", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "binary artifacts", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should return binary artifacts present at commit in source code", func() { @@ -111,7 +111,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() { NumberOfDebug: 0, } result := checks.BinaryArtifacts(&req) - Expect(scut.ValidateTestReturn(nil, "binary artifacts", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "binary artifacts", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should return no binary artifacts present in source code", func() { @@ -139,7 +139,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() { } result := checks.BinaryArtifacts(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "binary artifacts", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "binary artifacts", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should return binary artifacts present at commit in source code", func() { @@ -167,7 +167,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() { } result := checks.BinaryArtifacts(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "binary artifacts", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "binary artifacts", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should return binary artifacts present at commit in source code when using local repoClient", func() { @@ -206,7 +206,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() { } result := checks.BinaryArtifacts(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "binary artifacts", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "binary artifacts", &expected, &result, &dl) Expect(x.Close()).Should(BeNil()) }) }) diff --git a/e2e/branch_protection_test.go b/e2e/branch_protection_test.go index 07f617333f1..0696903de5c 100644 --- a/e2e/branch_protection_test.go +++ b/e2e/branch_protection_test.go @@ -56,7 +56,7 @@ var _ = Describe("E2E TEST PAT:"+checks.CheckBranchProtection, func() { // Old version. // New version. - Expect(scut.ValidateTestReturn(nil, "branch protection accessible", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "branch protection accessible", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should fail to return branch protection on other repositories", func() { @@ -84,7 +84,7 @@ var _ = Describe("E2E TEST PAT:"+checks.CheckBranchProtection, func() { result := checks.BranchProtection(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "branch protection accessible none", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "branch protection accessible none", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should fail to return branch protection on other repositories patch", func() { @@ -112,7 +112,7 @@ var _ = Describe("E2E TEST PAT:"+checks.CheckBranchProtection, func() { result := checks.BranchProtection(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "branch protection accessible patch", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "branch protection accessible patch", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) }) @@ -169,7 +169,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBranchProtection+" (repo rules)", func( } result := checks.BranchProtection(&req) Expect(result.Error).Should(BeNil()) - Expect(scut.ValidateTestReturn(nil, "repo rules accessible", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "repo rules accessible", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) }) diff --git a/e2e/ci_tests_test.go b/e2e/ci_tests_test.go index 69c3699eb0e..b4cb3d76d47 100644 --- a/e2e/ci_tests_test.go +++ b/e2e/ci_tests_test.go @@ -51,7 +51,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() { NumberOfDebug: 0, } result := checks.CITests(&req) - Expect(scut.ValidateTestReturn(nil, "CI tests run", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "CI tests run", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should return use of CI tests at commit", func() { @@ -75,7 +75,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() { NumberOfDebug: 0, } result := checks.CITests(&req) - Expect(scut.ValidateTestReturn(nil, "CI tests run", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "CI tests run", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should return absence of CI tests in a repo with unsquashed merges", func() { @@ -99,7 +99,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() { NumberOfDebug: 12, } result := checks.CITests(&req) - Expect(scut.ValidateTestReturn(nil, "CI tests run", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "CI tests run", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should return use of CI tests at commit - GitLab", func() { @@ -127,7 +127,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() { NumberOfDebug: 22, } result := checks.CITests(&req) - Expect(scut.ValidateTestReturn(nil, "CI tests at commit - GitLab", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "CI tests at commit - GitLab", &expected, &result, &dl) Expect(result.Error).Should(BeNil()) Expect(repoClient.Close()).Should(BeNil()) }) diff --git a/e2e/cii_best_practices_test.go b/e2e/cii_best_practices_test.go index a15d7237a3a..352a17c8a87 100644 --- a/e2e/cii_best_practices_test.go +++ b/e2e/cii_best_practices_test.go @@ -51,7 +51,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCIIBestPractices, func() { } result := checks.CIIBestPractices(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "passing badge", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "passing badge", &expected, &result, &dl) }) }) }) diff --git a/e2e/code_review_test.go b/e2e/code_review_test.go index c4eb25f17cb..38560e14080 100644 --- a/e2e/code_review_test.go +++ b/e2e/code_review_test.go @@ -54,7 +54,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCodeReview, func() { NumberOfDebug: 0, } result := checks.CodeReview(&req) - Expect(scut.ValidateTestReturn(nil, "use code reviews", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "use code reviews", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should return use of implicit code reviews at commit", func() { @@ -97,7 +97,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCodeReview, func() { NumberOfDebug: 18, } result := checks.CodeReview(&req) - Expect(scut.ValidateTestReturn(nil, "use code reviews", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "use code reviews", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should return partial score for a single-maintainer project with some unreviewed human changesets", func() { @@ -119,7 +119,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCodeReview, func() { NumberOfDebug: 10, } result := checks.CodeReview(&req) - Expect(scut.ValidateTestReturn(nil, "use code reviews", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "use code reviews", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) }) @@ -149,7 +149,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCodeReview, func() { NumberOfDebug: 1, } result := checks.CodeReview(&req) - Expect(scut.ValidateTestReturn(nil, "use code reviews", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "use code reviews", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) // GitLab doesn't seem to preserve merge requests (pull requests in github) and some users had data lost in @@ -179,7 +179,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCodeReview, func() { NumberOfDebug: 1, } result := checks.CodeReview(&req) - Expect(scut.ValidateTestReturn(nil, "use code reviews", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "use code reviews", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) }) diff --git a/e2e/contributors_test.go b/e2e/contributors_test.go index e3cce8aab11..4c42e225ea9 100644 --- a/e2e/contributors_test.go +++ b/e2e/contributors_test.go @@ -54,7 +54,7 @@ var _ = Describe("E2E TEST:"+checks.CheckContributors, func() { } result := checks.Contributors(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "several contributors", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "several contributors", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) diff --git a/e2e/dangerous_workflow_test.go b/e2e/dangerous_workflow_test.go index ecdb21e0727..b0ea5f822ec 100644 --- a/e2e/dangerous_workflow_test.go +++ b/e2e/dangerous_workflow_test.go @@ -53,7 +53,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() { } result := checks.DangerousWorkflow(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "dangerous workflow", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "dangerous workflow", &expected, &result, &dl) }) It("Should return dangerous workflow at commit", func() { dl := scut.TestDetailLogger{} @@ -77,7 +77,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() { } result := checks.DangerousWorkflow(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "dangerous workflow", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "dangerous workflow", &expected, &result, &dl) }) It("Should return dangerous workflow for local repoClient", func() { dl := scut.TestDetailLogger{} @@ -113,7 +113,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() { } result := checks.DangerousWorkflow(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "dangerous workflow", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "dangerous workflow", &expected, &result, &dl) }) }) }) diff --git a/e2e/dependency_update_tool_test.go b/e2e/dependency_update_tool_test.go index a580a6bb933..1cb9bc681fb 100644 --- a/e2e/dependency_update_tool_test.go +++ b/e2e/dependency_update_tool_test.go @@ -55,7 +55,7 @@ var _ = Describe("E2E TEST:"+checks.CheckDependencyUpdateTool, func() { result := checks.DependencyUpdateTool(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "dependabot", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "dependabot", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should return repo uses renovatebot", func() { @@ -81,7 +81,7 @@ var _ = Describe("E2E TEST:"+checks.CheckDependencyUpdateTool, func() { } result := checks.DependencyUpdateTool(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "renovatebot", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "renovatebot", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) }) diff --git a/e2e/fuzzing_test.go b/e2e/fuzzing_test.go index c93c951e497..bbd15b2362a 100644 --- a/e2e/fuzzing_test.go +++ b/e2e/fuzzing_test.go @@ -57,7 +57,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() { NumberOfDebug: 0, } result := checks.Fuzzing(&req) - Expect(scut.ValidateTestReturn(nil, "use fuzzing", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "use fuzzing", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) Expect(ossFuzzRepoClient.Close()).Should(BeNil()) }) @@ -89,7 +89,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() { NumberOfDebug: 0, } result := checks.Fuzzing(&req) - Expect(scut.ValidateTestReturn(nil, "use fuzzing", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "use fuzzing", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) Expect(ossFuzzRepoClient.Close()).Should(BeNil()) }) @@ -118,7 +118,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() { NumberOfDebug: 0, } result := checks.Fuzzing(&req) - Expect(scut.ValidateTestReturn(nil, "use fuzzing", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "use fuzzing", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) Expect(ossFuzzRepoClient.Close()).Should(BeNil()) }) @@ -147,7 +147,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() { NumberOfDebug: 0, } result := checks.Fuzzing(&req) - Expect(scut.ValidateTestReturn(nil, "use fuzzing", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "use fuzzing", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) Expect(ossFuzzRepoClient.Close()).Should(BeNil()) }) @@ -197,7 +197,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() { NumberOfDebug: 0, } result := checks.Fuzzing(&req) - Expect(scut.ValidateTestReturn(nil, "no fuzzing", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "no fuzzing", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) Expect(ossFuzzRepoClient.Close()).Should(BeNil()) }) diff --git a/e2e/license_test.go b/e2e/license_test.go index 61cd12ce910..38a13ec1a1d 100644 --- a/e2e/license_test.go +++ b/e2e/license_test.go @@ -54,8 +54,7 @@ var _ = Describe("E2E TEST:"+checks.CheckLicense, func() { } result := checks.License(&req) - Expect(scut.ValidateTestReturn(nil, "license found", &expected, &result, - &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "license found", &expected, &result, &dl) }) It("Should return license check works at commitSHA", func() { dl := scut.TestDetailLogger{} @@ -79,8 +78,7 @@ var _ = Describe("E2E TEST:"+checks.CheckLicense, func() { } result := checks.License(&req) - Expect(scut.ValidateTestReturn(nil, "license found", &expected, &result, - &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "license found", &expected, &result, &dl) }) It("Should return license check works for the local repoclient", func() { dl := scut.TestDetailLogger{} @@ -116,8 +114,7 @@ var _ = Describe("E2E TEST:"+checks.CheckLicense, func() { } result := checks.License(&req) - Expect(scut.ValidateTestReturn(nil, "license found", &expected, &result, - &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "license found", &expected, &result, &dl) }) It("Should return license check works - GitLab", func() { skipIfTokenIsNot(gitlabPATTokenType, "GitLab only") @@ -142,8 +139,7 @@ var _ = Describe("E2E TEST:"+checks.CheckLicense, func() { } result := checks.License(&req) - Expect(scut.ValidateTestReturn(nil, "license found", &expected, &result, - &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "license found", &expected, &result, &dl) }) It("Should return license check works for unrecognized license type - GitLab", func() { skipIfTokenIsNot(gitlabPATTokenType, "GitLab only") @@ -170,8 +166,7 @@ var _ = Describe("E2E TEST:"+checks.CheckLicense, func() { } result := checks.License(&req) - Expect(scut.ValidateTestReturn(nil, "license found", &expected, &result, - &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "license found", &expected, &result, &dl) }) It("Should return license check works at commitSHA - GitLab", func() { skipIfTokenIsNot(gitlabPATTokenType, "GitLab only") @@ -196,8 +191,7 @@ var _ = Describe("E2E TEST:"+checks.CheckLicense, func() { } result := checks.License(&req) - Expect(scut.ValidateTestReturn(nil, "license found", &expected, &result, - &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "license found", &expected, &result, &dl) }) }) }) diff --git a/e2e/maintained_test.go b/e2e/maintained_test.go index a20bcaf602f..f4158138ca5 100644 --- a/e2e/maintained_test.go +++ b/e2e/maintained_test.go @@ -53,7 +53,7 @@ var _ = Describe("E2E TEST:"+checks.CheckMaintained, func() { } result := checks.Maintained(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "active repo", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "active repo", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should return valid maintained status - GitLab", func() { @@ -82,7 +82,7 @@ var _ = Describe("E2E TEST:"+checks.CheckMaintained, func() { } result := checks.Maintained(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "active repo", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "active repo", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) }) diff --git a/e2e/packaging_test.go b/e2e/packaging_test.go index 9c6eb42b269..73bde3e550a 100644 --- a/e2e/packaging_test.go +++ b/e2e/packaging_test.go @@ -50,7 +50,7 @@ var _ = Describe("E2E TEST:"+checks.CheckPackaging, func() { NumberOfDebug: 0, } result := checks.Packaging(&req) - Expect(scut.ValidateTestReturn(nil, "use packaging", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "use packaging", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) }) diff --git a/e2e/permissions_test.go b/e2e/permissions_test.go index 3ded0dfbec2..440c689aca7 100644 --- a/e2e/permissions_test.go +++ b/e2e/permissions_test.go @@ -53,7 +53,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() { } result := checks.TokenPermissions(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "token permissions", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "token permissions", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should return token permission at commit", func() { @@ -78,7 +78,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() { } result := checks.TokenPermissions(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "token permissions", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "token permissions", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should return token permission for a local repo client", func() { @@ -115,7 +115,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() { } result := checks.TokenPermissions(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "token permissions", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "token permissions", &expected, &result, &dl) Expect(x.Close()).Should(BeNil()) }) }) diff --git a/e2e/pinned_dependencies_test.go b/e2e/pinned_dependencies_test.go index 924a43a1ce1..64d27463aea 100644 --- a/e2e/pinned_dependencies_test.go +++ b/e2e/pinned_dependencies_test.go @@ -55,7 +55,7 @@ var _ = Describe("E2E TEST:"+checks.CheckPinnedDependencies, func() { NumberOfDebug: 0, } result := checks.PinningDependencies(&req) - Expect(scut.ValidateTestReturn(nil, "dependencies check", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "dependencies check", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should return dependencies check at commit", func() { @@ -80,7 +80,7 @@ var _ = Describe("E2E TEST:"+checks.CheckPinnedDependencies, func() { NumberOfDebug: 0, } result := checks.PinningDependencies(&req) - Expect(scut.ValidateTestReturn(nil, "dependencies check", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "dependencies check", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should return dependencies check for a local repoClient", func() { @@ -116,7 +116,7 @@ var _ = Describe("E2E TEST:"+checks.CheckPinnedDependencies, func() { NumberOfDebug: 0, } result := checks.PinningDependencies(&req) - Expect(scut.ValidateTestReturn(nil, "dependencies check", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "dependencies check", &expected, &result, &dl) Expect(x.Close()).Should(BeNil()) }) }) diff --git a/e2e/sast_test.go b/e2e/sast_test.go index 484547f81b1..c3b1b3d8564 100644 --- a/e2e/sast_test.go +++ b/e2e/sast_test.go @@ -51,7 +51,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSAST, func() { } result := checks.SAST(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "sast used", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "sast used", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) }) diff --git a/e2e/security_policy_test.go b/e2e/security_policy_test.go index f52386f7836..b42c406ddc2 100644 --- a/e2e/security_policy_test.go +++ b/e2e/security_policy_test.go @@ -55,7 +55,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSecurityPolicy, func() { } result := checks.SecurityPolicy(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "policy found", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "policy found", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should return valid security policy at commitSHA", func() { @@ -81,7 +81,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSecurityPolicy, func() { } result := checks.SecurityPolicy(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "policy found", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "policy found", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should return valid security policy for local repoClient at head", func() { @@ -118,7 +118,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSecurityPolicy, func() { } result := checks.SecurityPolicy(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "policy found", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "policy found", &expected, &result, &dl) Expect(x.Close()).Should(BeNil()) }) It("Should return valid security policy - GitLab", func() { @@ -149,7 +149,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSecurityPolicy, func() { } result := checks.SecurityPolicy(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "policy found", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "policy found", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should return valid security policy at commitSHA - GitLab", func() { @@ -181,7 +181,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSecurityPolicy, func() { } result := checks.SecurityPolicy(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "policy found", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "policy found", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) }) diff --git a/e2e/signedreleases_test.go b/e2e/signedreleases_test.go index 3f6e88d308a..7d629513467 100644 --- a/e2e/signedreleases_test.go +++ b/e2e/signedreleases_test.go @@ -51,7 +51,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSignedReleases, func() { } result := checks.SignedReleases(&req) // New version. - Expect(scut.ValidateTestReturn(nil, "verified release", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "verified release", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) }) diff --git a/e2e/vulnerabilities_test.go b/e2e/vulnerabilities_test.go index ad145706e75..a5274bb0dd7 100644 --- a/e2e/vulnerabilities_test.go +++ b/e2e/vulnerabilities_test.go @@ -53,7 +53,7 @@ var _ = Describe("E2E TEST:"+checks.CheckVulnerabilities, func() { } result := checks.Vulnerabilities(&checkRequest) // New version. - Expect(scut.ValidateTestReturn(nil, "osv vulnerabilities", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "osv vulnerabilities", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should return that there are vulnerabilities at commit", func() { @@ -80,7 +80,7 @@ var _ = Describe("E2E TEST:"+checks.CheckVulnerabilities, func() { } result := checks.Vulnerabilities(&checkRequest) // New version. - Expect(scut.ValidateTestReturn(nil, "osv vulnerabilities", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "osv vulnerabilities", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) It("Should return that there are vulnerable packages", func() { @@ -107,7 +107,7 @@ var _ = Describe("E2E TEST:"+checks.CheckVulnerabilities, func() { } result := checks.Vulnerabilities(&checkRequest) // New version. - Expect(scut.ValidateTestReturn(nil, "osv vulnerabilities", &expected, &result, &dl)).Should(BeTrue()) + scut.ValidateTestReturn(GinkgoTB(), "osv vulnerabilities", &expected, &result, &dl) Expect(repoClient.Close()).Should(BeNil()) }) }) diff --git a/utests/utlib.go b/utests/utlib.go index c59c185943a..53e5ae78c0c 100644 --- a/utests/utlib.go +++ b/utests/utlib.go @@ -16,12 +16,12 @@ package utests import ( - "errors" "fmt" "log" "testing" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/ossf/scorecard/v4/checker" ) @@ -74,7 +74,18 @@ func (l *TestDetailLogger) Flush() []checker.CheckDetail { return ret } -func getTestReturn(cr *checker.CheckResult, logger *TestDetailLogger) (*TestReturn, error) { +//nolint:gocritic // not worried about test helper param size +func logDetail(tb testing.TB, level string, detail checker.CheckDetail) { + tb.Helper() + if detail.Msg.Finding != nil { + tb.Logf("%s: %s", level, detail.Msg.Finding.Message) + } else { + tb.Logf("%s: %s", level, detail.Msg.Text) + } +} + +func getTestReturn(tb testing.TB, cr *checker.CheckResult, logger *TestDetailLogger) (*TestReturn, error) { + tb.Helper() ret := new(TestReturn) for _, v := range logger.messages { switch v.Type { @@ -83,10 +94,13 @@ func getTestReturn(cr *checker.CheckResult, logger *TestDetailLogger) (*TestRetu return nil, fmt.Errorf("invalid type %v", v.Type) case checker.DetailInfo: ret.NumberOfInfo++ + logDetail(tb, "INFO", v) case checker.DetailDebug: ret.NumberOfDebug++ + logDetail(tb, "DEBUG", v) case checker.DetailWarn: ret.NumberOfWarn++ + logDetail(tb, "WARN", v) } } ret.Score = cr.Score @@ -94,29 +108,23 @@ func getTestReturn(cr *checker.CheckResult, logger *TestDetailLogger) (*TestRetu return ret, nil } -func errCmp(e1, e2 error) bool { - return errors.Is(e1, e2) || errors.Is(e2, e1) -} - -// ValidateTestReturn validates expected TestReturn with actual checker.CheckResult values. -// -//nolint:thelper +// ValidateTestReturn validates expected [TestReturn] with actual [checker.CheckResult] values. +// All test management is handled by this function. func ValidateTestReturn( - t *testing.T, + tb testing.TB, name string, expected *TestReturn, actual *checker.CheckResult, logger *TestDetailLogger, -) bool { - actualTestReturn, err := getTestReturn(actual, logger) +) { + tb.Helper() + actualTestReturn, err := getTestReturn(tb, actual, logger) if err != nil { - panic(err) + tb.Fatal(err) } - if !cmp.Equal(*expected, *actualTestReturn, cmp.Comparer(errCmp)) { - log.Println(name+":", cmp.Diff(*expected, *actualTestReturn)) - return false + if diff := cmp.Diff(*expected, *actualTestReturn, cmpopts.EquateErrors()); diff != "" { + tb.Error(name + ": (-expected +actual)" + diff) } - return true } // ValidatePinningDependencies tests that at least one entry returns true for isExpectedMessage.