✨ Gitlab: CI-Tests check

.github/workflows/gitlab.yml

@@ -0,0 +1,63 @@
+# Copyright 2023 OpenSSF Scorecard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: gitlab-tests
+
+permissions: read-all
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  gitlab-integration-trusted:
+    runs-on: ubuntu-latest
+    environment: gitlab
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@03bee3930647ebbf994244c21ddbc0d4933aab4f # v1
+        with:
+          egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+      - name: Clone the code
+        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab
+        with:
+           fetch-depth: 0
+
+      - name: setup-go
+        uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v2.2.0
+        with:
+          go-version: '1.19'
+          check-latest: true
+
+      - name: Prepare test env
+        run: |
+            go mod download
+
+      - name: Run GitLab PAT E2E  #using retry because the GitHub token is being throttled.
+        uses: nick-invision/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd
+        env:
+          GITLAB_AUTH_TOKEN: ${{ secrets.GITLAB_TOKEN }}
+        with:
+          max_attempts: 3
+          retry_on: error
+          timeout_minutes: 30
+          command: make e2e-gitlab-token
+
+      - name: codecov
+        uses: codecov/codecov-action@40a12dcee2df644d47232dde008099a3e9e4f865 # 2.1.0
+        with:
+         files: ./e2e-coverage.out
+         verbose: true

.github/workflows/integration.yml

@@ -56,32 +56,35 @@ jobs:
         run: |
             go mod download
 
-      - name: Run GITHUB_TOKEN E2E  #using retry because the GitHub token is being throttled.
+      - name: Run GitLab E2E  #using retry because the GitHub token is being throttled.
         uses: nick-invision/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd
         env:
-          GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITLAB_AUTH_TOKEN: ${{ secrets.GITLAB_TOKEN }}
         with:
           max_attempts: 3
           retry_on: error
           timeout_minutes: 30
-          command: make e2e-gh-token
+          command: make e2e-gitlab
 
       - name: codecov
         uses: codecov/codecov-action@894ff025c7b54547a9a2a1e9f228beae737ad3c2 # 2.1.0
         with:
          files: ./e2e-coverage.out
          verbose: true
-      
-      - name: Run GitLab E2E  #using retry because the GitHub token is being throttled.
+
+      - name: Run GITHUB_TOKEN E2E  #using retry because the GitHub token is being throttled.
         uses: nick-invision/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd
+        env:
+          GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           max_attempts: 3
           retry_on: error
           timeout_minutes: 30
-          command: make e2e-gitlab
+          command: make e2e-gh-token
 
       - name: codecov
         uses: codecov/codecov-action@894ff025c7b54547a9a2a1e9f228beae737ad3c2 # 2.1.0
         with:
          files: ./e2e-coverage.out
          verbose: true
+

Makefile

@@ -324,6 +324,11 @@ ifndef GITHUB_AUTH_TOKEN
 	$(error GITHUB_AUTH_TOKEN is undefined)
 endif
 
+check-env-gitlab:
+ifndef GITLAB_AUTH_TOKEN
+	$(error GITLAB_AUTH_TOKEN is undefined)
+endif
+
 e2e-pat: ## Runs e2e tests. Requires GITHUB_AUTH_TOKEN env var to be set to GitHub personal access token
 e2e-pat: build-scorecard check-env | $(GINKGO)
 	# Run e2e tests. GITHUB_AUTH_TOKEN with personal access token must be exported to run this
@@ -335,10 +340,12 @@ e2e-gh-token: build-scorecard check-env | $(GINKGO)
 	TOKEN_TYPE="GITHUB_TOKEN" $(GINKGO) --race -p -v -cover -coverprofile=e2e-coverage.out --keep-separate-coverprofiles ./...
 
 e2e-gitlab-token: ## Runs e2e tests that require a GITLAB_TOKEN
-	TEST_GITLAB_EXTERNAL=1 TOKEN_TYPE="GITLAB_PAT" $(GINKGO) --race -p -vv --focus '.*GitLab Token' ./...
+e2e-gitlab-token: build-scorecard check-env-gitlab | $(GINKGO)
+	TEST_GITLAB_EXTERNAL=1 TOKEN_TYPE="GITLAB_PAT" $(GINKGO) --race -p -vv --focus '.*GitLab' ./...
 
 e2e-gitlab: ## Runs e2e tests for GitLab only. TOKEN_TYPE is not used (since these are public APIs), but must be set to something
-	TOKEN_TYPE="GITLAB_PAT" $(GINKGO) --race -p -vv --focus '.*GitLab' ./...
+e2e-gitlab: build-scorecard | $(GINKGO)
+	TEST_GITLAB_EXTERNAL=1 TOKEN_TYPE="PAT" $(GINKGO) --race -p -vv --focus ".*GitLab" ./...
 
 e2e-attestor: ## Runs e2e tests for scorecard-attestor
 	cd attestor/e2e; go test -covermode=atomic -coverprofile=e2e-coverage.out; cd ../..
@@ -439,4 +446,4 @@ cron-github-server-ko: | $(KO) $(KOCACHE_PATH)
 			   --tags latest,$(GIT_VERSION),$(GIT_HASH) \
 			   github.com/ossf/scorecard/v4/clients/githubrepo/roundtripper/tokens/server
 
-###############################################################################
+###############################################################################

checks/evaluation/ci_tests.go

@@ -61,6 +61,7 @@ func CITests(name string, c *checker.CITestData, dl checker.DetailLogger) checke
 		if !foundCI {
 			// Log message says commit, but really we only care about PRs, and
 			// use only one commit (branch HEAD) to refer to all commits in a PR
+
 			dl.Debug(&checker.LogMessage{
 				Text: fmt.Sprintf("merged PR %d without CI test at HEAD: %s", r.PullRequestNumber, r.HeadSHA),
 			})

clients/gitlabrepo/checkruns.go

@@ -47,11 +47,11 @@ func checkRunsFrom(data []*gitlab.PipelineInfo, ref string) []clients.CheckRun {
 	var checkRuns []clients.CheckRun
 	for _, pipelineInfo := range data {
 		if strings.EqualFold(pipelineInfo.Ref, ref) {
+			// TODO: Can get more info from GitLab API here (e.g. pipeline name, URL)
+			// https://docs.gitlab.com/ee/api/pipelines.html#get-a-pipelines-test-report
 			checkRuns = append(checkRuns, clients.CheckRun{
-				Status:     pipelineInfo.Status,
-				Conclusion: "",
-				URL:        pipelineInfo.WebURL,
-				App:        clients.CheckRunApp{Slug: pipelineInfo.Source},
+				Status: pipelineInfo.Status,
+				URL:    pipelineInfo.WebURL,
 			})
 		}
 	}

clients/gitlabrepo/statuses.go

@@ -46,7 +46,7 @@ func statusFromData(commitStatuses []*gitlab.CommitStatus) []clients.Status {
 	for _, commitStatus := range commitStatuses {
 		statuses = append(statuses, clients.Status{
 			State:     commitStatus.Status,
-			Context:   fmt.Sprint(commitStatus.ID),
+			Context:   commitStatus.Name,
 			URL:       commitStatus.TargetURL,
 			TargetURL: commitStatus.TargetURL,
 		})

docs/checks.md

@@ -130,7 +130,8 @@ Risk: `Low` (possible unknown vulnerabilities)
 
 This check tries to determine if the project runs tests before pull requests are
 merged. It is currently limited to repositories hosted on GitHub, and does not
-support other source hosting repositories (i.e., Forges).
+support other source hosting repositories (i.e., Forges). All commits that are
+part of a PR must be tested by a CI Test for the check to pass.
 
 Running tests helps developers catch mistakes early on, which can reduce the
 number of vulnerabilities that find their way into a project.

e2e/ci_tests_test.go

@@ -16,6 +16,7 @@ package e2e
 
 import (
 	"context"
+	"os"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -24,6 +25,7 @@ import (
 	"github.com/ossf/scorecard/v4/checks"
 	"github.com/ossf/scorecard/v4/clients"
 	"github.com/ossf/scorecard/v4/clients/githubrepo"
+	"github.com/ossf/scorecard/v4/clients/gitlabrepo"
 	scut "github.com/ossf/scorecard/v4/utests"
 )
 
@@ -101,5 +103,63 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() {
 			Expect(scut.ValidateTestReturn(nil, "CI tests run", &expected, &result, &dl)).Should(BeTrue())
 			Expect(repoClient.Close()).Should(BeNil())
 		})
+		It("Should return use of CI tests at commit - GitLab", func() {
+			skipIfTokenIsNot(gitlabPATTokenType, "GitLab only")
+
+			dl := scut.TestDetailLogger{}
+			repo, err := gitlabrepo.MakeGitlabRepo("gitlab.com/gitlab-org/gitlab")
+			Expect(err).Should(BeNil())
+			repoClient, err := gitlabrepo.CreateGitlabClientWithToken(context.Background(), os.Getenv("GITLAB_AUTH_TOKEN"), repo)
+			Expect(err).Should(BeNil())
+			// url to commit is https://gitlab.com/gitlab-org/gitlab/-/commit/8ae23fa220d73fa07501aabd94214c9e83fe61a0
+			err = repoClient.InitRepo(repo, "8ae23fa220d73fa07501aabd94214c9e83fe61a0", 0)
+			Expect(err).Should(BeNil())
+			req := checker.CheckRequest{
+				Ctx:        context.Background(),
+				RepoClient: repoClient,
+				Repo:       repo,
+				Dlogger:    &dl,
+			}
+			expected := scut.TestReturn{
+				Error:         nil,
+				Score:         0,
+				NumberOfWarn:  0,
+				NumberOfInfo:  0,
+				NumberOfDebug: 13,
+			}
+			result := checks.CITests(&req)
+			Expect(result.Score).Should(BeNumerically("==", expected.Score))
+			Expect(result.Error).Should(BeNil())
+			Expect(repoClient.Close()).Should(BeNil())
+		})
+		It("Should return use of CI tests at commit - GitLab", func() {
+			skipIfTokenIsNot(gitlabPATTokenType, "GitLab only")
+
+			dl := scut.TestDetailLogger{}
+			repo, err := gitlabrepo.MakeGitlabRepo("gitlab.com/fdroid/fdroidclient")
+			Expect(err).Should(BeNil())
+			repoClient, err := gitlabrepo.CreateGitlabClientWithToken(context.Background(), os.Getenv("GITLAB_AUTH_TOKEN"), repo)
+			Expect(err).Should(BeNil())
+			// url to commit is https://gitlab.com/fdroid/fdroidclient/-/commit/a1d33881902cee33586a4fd4ee1538042a7bdedf
+			err = repoClient.InitRepo(repo, "a1d33881902cee33586a4fd4ee1538042a7bdedf", 0)
+			Expect(err).Should(BeNil())
+			req := checker.CheckRequest{
+				Ctx:        context.Background(),
+				RepoClient: repoClient,
+				Repo:       repo,
+				Dlogger:    &dl,
+			}
+			expected := scut.TestReturn{
+				Error:         nil,
+				Score:         2,
+				NumberOfWarn:  0,
+				NumberOfInfo:  0,
+				NumberOfDebug: 1,
+			}
+			result := checks.CITests(&req)
+			Expect(result.Score).Should(BeNumerically("==", expected.Score))
+			Expect(result.Error).Should(BeNil())
+			Expect(repoClient.Close()).Should(BeNil())
+		})
 	})
 })