Skip to content

Commit

Permalink
Initial implementation of go-git client (#2720)
Browse files Browse the repository at this point in the history
Signed-off-by: Azeem Shaikh <azeemshaikh38@gmail.com>
  • Loading branch information
azeemshaikh38 authored Mar 15, 2023
1 parent 603263c commit 8966abd
Show file tree
Hide file tree
Showing 5 changed files with 425 additions and 9 deletions.
209 changes: 209 additions & 0 deletions clients/git/client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,209 @@
// Copyright 2021 OpenSSF Scorecard Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package git defines helper functions for clients.RepoClient interface.
package git

import (
"errors"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"strings"
"sync"

"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
cp "github.com/otiai10/copy"

"github.com/ossf/scorecard/v4/clients"
)

const repoDir = "repo*"

var (
errNilCommitFound = errors.New("nil commit found")
errEmptyQuery = errors.New("query is empty")
)

type Client struct {
gitRepo *git.Repository
worktree *git.Worktree
listCommits *sync.Once
tempDir string
errListCommits error
commits []clients.Commit
commitDepth int
}

func (c *Client) InitRepo(uri, commitSHA string, commitDepth int) error {
// cleanup previous state, if any.
c.Close()
c.listCommits = new(sync.Once)
c.commits = nil

// init
c.commitDepth = commitDepth
tempDir, err := os.MkdirTemp("", repoDir)
if err != nil {
return fmt.Errorf("os.MkdirTemp: %w", err)
}

// git clone
const filePrefix = "file://"
if strings.HasPrefix(uri, filePrefix) {
if err := cp.Copy(strings.TrimPrefix(uri, filePrefix), tempDir); err != nil {
return fmt.Errorf("cp.Copy: %w", err)
}
c.gitRepo, err = git.PlainOpen(tempDir)
if err != nil {
return fmt.Errorf("git.PlainOpen: %w", err)
}
} else {
c.gitRepo, err = git.PlainClone(tempDir, false /*isBare*/, &git.CloneOptions{
URL: uri,
Progress: os.Stdout,
})
if err != nil {
return fmt.Errorf("git.PlainClone: %w", err)
}
}
c.tempDir = tempDir
c.worktree, err = c.gitRepo.Worktree()
if err != nil {
return fmt.Errorf("git.Worktree: %w", err)
}

// git checkout
if commitSHA != clients.HeadSHA {
if err := c.worktree.Checkout(&git.CheckoutOptions{
Hash: plumbing.NewHash(commitSHA),
Force: true, // throw away any unsaved changes.
}); err != nil {
return fmt.Errorf("git.Worktree: %w", err)
}
}

return nil
}

func (c *Client) ListCommits() ([]clients.Commit, error) {
c.listCommits.Do(func() {
commitIter, err := c.gitRepo.Log(&git.LogOptions{
Order: git.LogOrderCommitterTime,
})
if err != nil {
c.errListCommits = fmt.Errorf("git.CommitObjects: %w", err)
return
}
c.commits = make([]clients.Commit, 0, c.commitDepth)
for i := 0; i < c.commitDepth; i++ {
commit, err := commitIter.Next()
if err != nil && !errors.Is(err, io.EOF) {
c.errListCommits = fmt.Errorf("commitIter.Next: %w", err)
return
}
// No more commits.
if errors.Is(err, io.EOF) {
break
}

if commit == nil {
// Not sure in what case a nil commit is returned. Fail explicitly.
c.errListCommits = fmt.Errorf("%w", errNilCommitFound)
return
}

c.commits = append(c.commits, clients.Commit{
SHA: commit.Hash.String(),
Message: commit.Message,
CommittedDate: commit.Committer.When,
Committer: clients.User{
Login: commit.Committer.Email,
},
})
}
})
return c.commits, c.errListCommits
}

func (c *Client) Search(request clients.SearchRequest) (clients.SearchResponse, error) {
// Pattern
if request.Query == "" {
return clients.SearchResponse{}, errEmptyQuery
}
queryRegexp, err := regexp.Compile(request.Query)
if err != nil {
return clients.SearchResponse{}, fmt.Errorf("regexp.Compile: %w", err)
}
grepOpts := &git.GrepOptions{
Patterns: []*regexp.Regexp{queryRegexp},
}

// path/filename
var pathExpr string
switch {
case request.Path != "" && request.Filename != "":
pathExpr = filepath.Join(fmt.Sprintf("^%s", request.Path),
fmt.Sprintf(".*%s$", request.Filename))
case request.Path != "":
pathExpr = fmt.Sprintf("^%s", request.Path)
case request.Filename != "":
pathExpr = filepath.Join(".*", fmt.Sprintf("%s$", request.Filename))
}
if pathExpr != "" {
pathRegexp, err := regexp.Compile(pathExpr)
if err != nil {
return clients.SearchResponse{}, fmt.Errorf("regexp.Compile: %w", err)
}
grepOpts.PathSpecs = append(grepOpts.PathSpecs, pathRegexp)
}

// Grep
grepResults, err := c.worktree.Grep(grepOpts)
if err != nil {
return clients.SearchResponse{}, fmt.Errorf("git.Grep: %w", err)
}

ret := clients.SearchResponse{}
for _, grepResult := range grepResults {
ret.Results = append(ret.Results, clients.SearchResult{
Path: grepResult.FileName,
})
}
ret.Hits = len(grepResults)
return ret, nil
}

// TODO(#1709): Implement below fns using go-git.
func (c *Client) SearchCommits(request clients.SearchCommitsOptions) ([]clients.Commit, error) {
return nil, nil
}

func (c *Client) ListFiles(predicate func(string) (bool, error)) ([]string, error) {
return nil, nil
}

func (c *Client) GetFileContent(filename string) ([]byte, error) {
return nil, nil
}

func (c *Client) Close() error {
if err := os.RemoveAll(c.tempDir); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("os.RemoveAll: %w", err)
}
return nil
}
155 changes: 155 additions & 0 deletions clients/git/e2e_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
// Copyright 2021 OpenSSF Scorecard Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package git

import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

"github.com/ossf/scorecard/v4/clients"
)

var _ = DescribeTable("Test ListCommits commit-depth for HEAD",
func(uri string) {
const commitSHA = clients.HeadSHA
const commitDepth = 1
client := &Client{}
Expect(client.InitRepo(uri, commitSHA, commitDepth)).To(BeNil())
commits, err := client.ListCommits()
Expect(err).To(BeNil())
Expect(len(commits)).Should(BeEquivalentTo(commitDepth))
Expect(client.Close()).To(BeNil())
},
Entry("GitHub", "https://github.com/ossf/scorecard"),
Entry("Local", "file://../../"),
Entry("GitLab", "https://gitlab.haskell.org/haskell/filepath"),
)

var _ = DescribeTable("Test ListCommits commit-depth and latest commit at [0]",
func(uri, commitSHA string) {
const commitDepth = 10
client := &Client{}
Expect(client.InitRepo(uri, commitSHA, commitDepth)).To(BeNil())
commits, err := client.ListCommits()
Expect(err).To(BeNil())
Expect(len(commits)).Should(BeEquivalentTo(commitDepth))
Expect(commits[0].SHA).Should(BeEquivalentTo(commitSHA))
Expect(client.Close()).To(BeNil())
},
Entry("GitHub", "https://github.com/ossf/scorecard", "c06ac740cc49fea404c54c036000731d5ea6ebe3"),
Entry("Local", "file://../../", "c06ac740cc49fea404c54c036000731d5ea6ebe3"),
Entry("GitLab", "https://gitlab.haskell.org/haskell/filepath", "98f8bba9eac8c7183143d290d319be7df76c258b"),
)

var _ = DescribeTable("Test ListCommits without enough commits",
func(uri string) {
const commitSHA = "dc1835b7ffe526969d65436b621e171e3386771e"
const commitDepth = 10
client := &Client{}
Expect(client.InitRepo(uri, commitSHA, commitDepth)).To(BeNil())
commits, err := client.ListCommits()
Expect(err).To(BeNil())
Expect(len(commits)).Should(BeEquivalentTo(3))
Expect(commits[0].SHA).Should(BeEquivalentTo(commitSHA))
Expect(client.Close()).To(BeNil())
},
Entry("GitHub", "https://github.com/ossf/scorecard"),
Entry("Local", "file://../../"),
// TODO(#1709): Add equivalent test for GitLab.
)

var _ = DescribeTable("Test Search across a repo",
func(uri string) {
const (
commitSHA = "c06ac740cc49fea404c54c036000731d5ea6ebe3"
commitDepth = 10
)
client := &Client{}
Expect(client.InitRepo(uri, commitSHA, commitDepth)).To(BeNil())
resp, err := client.Search(clients.SearchRequest{
Query: "github/codeql-action/analyze",
})
Expect(err).To(BeNil())
Expect(resp.Hits).Should(BeNumerically(">=", 1))
Expect(client.Close()).To(BeNil())
},
Entry("GitHub", "https://github.com/ossf/scorecard"),
Entry("Local", "file://../../"),
// TODO(#1709): Add equivalent test for GitLab.
)

var _ = DescribeTable("Test Search within a path",
func(uri string) {
const (
commitSHA = "c06ac740cc49fea404c54c036000731d5ea6ebe3"
commitDepth = 10
)
client := &Client{}
Expect(client.InitRepo(uri, commitSHA, commitDepth)).To(BeNil())
resp, err := client.Search(clients.SearchRequest{
Query: "github/codeql-action/analyze",
Path: ".github/workflows",
})
Expect(err).To(BeNil())
Expect(resp.Hits).Should(BeEquivalentTo(1))
Expect(client.Close()).To(BeNil())
},
Entry("GitHub", "https://github.com/ossf/scorecard"),
Entry("Local", "file://../../"),
// TODO(#1709): Add equivalent test for GitLab.
)

var _ = DescribeTable("Test Search within a filename",
func(uri string) {
const (
commitSHA = "c06ac740cc49fea404c54c036000731d5ea6ebe3"
commitDepth = 10
)
client := &Client{}
Expect(client.InitRepo(uri, commitSHA, commitDepth)).To(BeNil())
resp, err := client.Search(clients.SearchRequest{
Query: "github/codeql-action/analyze",
Filename: "codeql-analysis.yml",
})
Expect(err).To(BeNil())
Expect(resp.Hits).Should(BeEquivalentTo(1))
Expect(client.Close()).To(BeNil())
},
Entry("GitHub", "https://github.com/ossf/scorecard"),
Entry("Local", "file://../../"),
// TODO(#1709): Add equivalent test for GitLab.
)

var _ = DescribeTable("Test Search within path and filename",
func(uri string) {
const (
commitSHA = "c06ac740cc49fea404c54c036000731d5ea6ebe3"
commitDepth = 10
)
client := &Client{}
Expect(client.InitRepo(uri, commitSHA, commitDepth)).To(BeNil())
resp, err := client.Search(clients.SearchRequest{
Query: "github/codeql-action/analyze",
Path: ".github/workflows",
Filename: "codeql-analysis.yml",
})
Expect(err).To(BeNil())
Expect(resp.Hits).Should(BeEquivalentTo(1))
Expect(client.Close()).To(BeNil())
},
Entry("GitHub", "https://github.com/ossf/scorecard"),
Entry("Local", "file://../../"),
// TODO(#1709): Add equivalent test for GitLab.
)
Loading

0 comments on commit 8966abd

Please sign in to comment.