Skip to content

Commit

Permalink
🐛 Check OSS Fuzz build file for Fuzzing check (#2719)
Browse files Browse the repository at this point in the history
* Check OSS-Fuzz using project list

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Use clients.RepoClient interface to perform the new OSS Fuzz check

Signed-off-by: Spencer Schrock <sschrock@google.com>

* wip: add eager client for better repeated lookup of projects

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Split lazy and eager behavior into different implementations.

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Add tests and benchmarks

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Switch to always parsing JSON to determine if a project is present. The other approach of looking for a substring match would lead to false positives.

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Add eager constructor to surface status file errors sooner.

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Switch existing users to new OSS Fuzz client

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Mark old method as deprecated in the godoc

Signed-off-by: Spencer Schrock <sschrock@google.com>

* remove unused comment.

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Use new OSS Fuzz client in e2e test.

Signed-off-by: Spencer Schrock <sschrock@google.com>

* fix typo.

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Fix potential path bug with test server.

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Force include the two JSON files which were being ignored by .gitignore

Signed-off-by: Spencer Schrock <sschrock@google.com>

* trim the status json file

Signed-off-by: Spencer Schrock <sschrock@google.com>

---------

Signed-off-by: Spencer Schrock <sschrock@google.com>
  • Loading branch information
spencerschrock authored Mar 4, 2023
1 parent c06ac74 commit 61866a0
Show file tree
Hide file tree
Showing 9 changed files with 492 additions and 18 deletions.
11 changes: 3 additions & 8 deletions checker/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/ossf/scorecard/v4/clients"
ghrepo "github.com/ossf/scorecard/v4/clients/githubrepo"
"github.com/ossf/scorecard/v4/clients/localdir"
"github.com/ossf/scorecard/v4/clients/ossfuzz"
"github.com/ossf/scorecard/v4/log"
)

Expand Down Expand Up @@ -59,16 +60,10 @@ func GetClients(ctx context.Context, repoURI, localURI string, logger *log.Logge
fmt.Errorf("getting local directory client: %w", errGitHub)
}

ossFuzzRepoClient, errOssFuzz := ghrepo.CreateOssFuzzRepoClient(ctx, logger)
var retErr error
if errOssFuzz != nil {
retErr = fmt.Errorf("getting OSS-Fuzz repo client: %w", errOssFuzz)
}
// TODO(repo): Should we be handling the OSS-Fuzz client error like this?
return githubRepo, /*repo*/
ghrepo.CreateGithubRepoClient(ctx, logger), /*repoClient*/
ossFuzzRepoClient, /*ossFuzzClient*/
ossfuzz.CreateOSSFuzzClient(ossfuzz.StatusURL), /*ossFuzzClient*/
clients.DefaultCIIBestPracticesClient(), /*ciiClient*/
clients.DefaultVulnerabilitiesClient(), /*vulnClient*/
retErr
nil
}
3 changes: 3 additions & 0 deletions clients/githubrepo/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,9 @@ func CreateGithubRepoClient(ctx context.Context, logger *log.Logger) clients.Rep

// CreateOssFuzzRepoClient returns a RepoClient implementation
// intialized to `google/oss-fuzz` GitHub repository.
//
// Deprecated: Searching the github.com/google/oss-fuzz repo for projects is flawed. Use a constructor
// from clients/ossfuzz instead. https://github.com/ossf/scorecard/issues/2670
func CreateOssFuzzRepoClient(ctx context.Context, logger *log.Logger) (clients.RepoClient, error) {
ossFuzzRepo, err := MakeGithubRepo("google/oss-fuzz")
if err != nil {
Expand Down
259 changes: 259 additions & 0 deletions clients/ossfuzz/client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,259 @@
// Copyright 2023 OpenSSF Scorecard Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ossfuzz

import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"sync"
"time"

"github.com/ossf/scorecard/v4/clients"
)

const (
StatusURL = "https://oss-fuzz-build-logs.storage.googleapis.com/status.json"
)

var (
errUnreachableStatusFile = errors.New("could not fetch OSS Fuzz status file")
errMalformedURL = errors.New("malformed repo url")
)

type client struct {
err error
projects map[string]bool
statusURL string
once sync.Once
}

type ossFuzzStatus struct {
Projects []struct {
RepoURI string `json:"main_repo"`
} `json:"projects"`
}

// CreateOSSFuzzClient returns a client which implements RepoClient interface.
func CreateOSSFuzzClient(ossFuzzStatusURL string) clients.RepoClient {
return &client{
statusURL: ossFuzzStatusURL,
projects: map[string]bool{},
}
}

// CreateOSSFuzzClientEager returns a OSS Fuzz Client which has already fetched and parsed the status file.
func CreateOSSFuzzClientEager(ossFuzzStatusURL string) (clients.RepoClient, error) {
c := client{
statusURL: ossFuzzStatusURL,
projects: map[string]bool{},
}
c.once.Do(func() {
c.init()
})
if c.err != nil {
return nil, c.err
}
return &c, nil
}

// Search implements RepoClient.Search.
func (c *client) Search(request clients.SearchRequest) (clients.SearchResponse, error) {
c.once.Do(func() {
c.init()
})
var sr clients.SearchResponse
if c.err != nil {
return sr, c.err
}
if c.projects[request.Query] {
sr.Hits = 1
}
return sr, nil
}

func (c *client) init() {
b, err := fetchStatusFile(c.statusURL)
if err != nil {
c.err = err
return
}
if err = parseStatusFile(b, c.projects); err != nil {
c.err = err
return
}
}

func parseStatusFile(contents []byte, m map[string]bool) error {
status := ossFuzzStatus{}
if err := json.Unmarshal(contents, &status); err != nil {
return fmt.Errorf("parse status file: %w", err)
}
for i := range status.Projects {
repoURI := status.Projects[i].RepoURI
normalizedRepoURI, err := normalize(repoURI)
if err != nil {
continue
}
m[normalizedRepoURI] = true
}
return nil
}

func fetchStatusFile(uri string) ([]byte, error) {
//nolint:gosec // URI comes from a constant or a test HTTP server, not user input
resp, err := http.Get(uri)
if err != nil {
return nil, fmt.Errorf("http.Get: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return nil, fmt.Errorf("%s: %w", resp.Status, errUnreachableStatusFile)
}
b, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("io.ReadAll: %w", err)
}
return b, nil
}

func normalize(rawURL string) (string, error) {
u, err := url.Parse(rawURL)
if err != nil {
return "", fmt.Errorf("url.Parse: %w", err)
}
const splitLen = 2
split := strings.SplitN(strings.Trim(u.Path, "/"), "/", splitLen)
if len(split) != splitLen {
return "", fmt.Errorf("%s: %w", rawURL, errMalformedURL)
}
org := split[0]
repo := strings.TrimSuffix(split[1], ".git")
return fmt.Sprintf("%s/%s/%s", u.Host, org, repo), nil
}

// URI implements RepoClient.URI.
func (c *client) URI() string {
return c.statusURL
}

// InitRepo implements RepoClient.InitRepo.
func (c *client) InitRepo(inputRepo clients.Repo, commitSHA string, commitDepth int) error {
return fmt.Errorf("InitRepo: %w", clients.ErrUnsupportedFeature)
}

// IsArchived implements RepoClient.IsArchived.
func (c *client) IsArchived() (bool, error) {
return false, fmt.Errorf("IsArchived: %w", clients.ErrUnsupportedFeature)
}

// LocalPath implements RepoClient.LocalPath.
func (c *client) LocalPath() (string, error) {
return "", fmt.Errorf("LocalPath: %w", clients.ErrUnsupportedFeature)
}

// ListFiles implements RepoClient.ListFiles.
func (c *client) ListFiles(predicate func(string) (bool, error)) ([]string, error) {
return nil, fmt.Errorf("ListFiles: %w", clients.ErrUnsupportedFeature)
}

// GetFileContent implements RepoClient.GetFileContent.
func (c *client) GetFileContent(filename string) ([]byte, error) {
return nil, fmt.Errorf("GetFileContent: %w", clients.ErrUnsupportedFeature)
}

// GetBranch implements RepoClient.GetBranch.
func (c *client) GetBranch(branch string) (*clients.BranchRef, error) {
return nil, fmt.Errorf("GetBranch: %w", clients.ErrUnsupportedFeature)
}

// GetDefaultBranch implements RepoClient.GetDefaultBranch.
func (c *client) GetDefaultBranch() (*clients.BranchRef, error) {
return nil, fmt.Errorf("GetDefaultBranch: %w", clients.ErrUnsupportedFeature)
}

// GetDefaultBranchName implements RepoClient.GetDefaultBranchName.
func (c *client) GetDefaultBranchName() (string, error) {
return "", fmt.Errorf("GetDefaultBranchName: %w", clients.ErrUnsupportedFeature)
}

// ListCommits implements RepoClient.ListCommits.
func (c *client) ListCommits() ([]clients.Commit, error) {
return nil, fmt.Errorf("ListCommits: %w", clients.ErrUnsupportedFeature)
}

// ListIssues implements RepoClient.ListIssues.
func (c *client) ListIssues() ([]clients.Issue, error) {
return nil, fmt.Errorf("ListIssues: %w", clients.ErrUnsupportedFeature)
}

// ListReleases implements RepoClient.ListReleases.
func (c *client) ListReleases() ([]clients.Release, error) {
return nil, fmt.Errorf("ListReleases: %w", clients.ErrUnsupportedFeature)
}

// ListContributors implements RepoClient.ListContributors.
func (c *client) ListContributors() ([]clients.User, error) {
return nil, fmt.Errorf("ListContributors: %w", clients.ErrUnsupportedFeature)
}

// ListSuccessfulWorkflowRuns implements RepoClient.ListSuccessfulWorkflowRuns.
func (c *client) ListSuccessfulWorkflowRuns(filename string) ([]clients.WorkflowRun, error) {
return nil, fmt.Errorf("ListSuccessfulWorkflowRuns: %w", clients.ErrUnsupportedFeature)
}

// ListCheckRunsForRef implements RepoClient.ListCheckRunsForRef.
func (c *client) ListCheckRunsForRef(ref string) ([]clients.CheckRun, error) {
return nil, fmt.Errorf("ListCheckRunsForRef: %w", clients.ErrUnsupportedFeature)
}

// ListStatuses implements RepoClient.ListStatuses.
func (c *client) ListStatuses(ref string) ([]clients.Status, error) {
return nil, fmt.Errorf("ListStatuses: %w", clients.ErrUnsupportedFeature)
}

// ListWebhooks implements RepoClient.ListWebhooks.
func (c *client) ListWebhooks() ([]clients.Webhook, error) {
return nil, fmt.Errorf("ListWebhooks: %w", clients.ErrUnsupportedFeature)
}

// SearchCommits implements RepoClient.SearchCommits.
func (c *client) SearchCommits(request clients.SearchCommitsOptions) ([]clients.Commit, error) {
return nil, fmt.Errorf("SearchCommits: %w", clients.ErrUnsupportedFeature)
}

// Close implements RepoClient.Close.
func (c *client) Close() error {
return nil
}

// ListProgrammingLanguages implements RepoClient.ListProgrammingLanguages.
func (c *client) ListProgrammingLanguages() ([]clients.Language, error) {
return nil, fmt.Errorf("ListProgrammingLanguages: %w", clients.ErrUnsupportedFeature)
}

// ListLicenses implements RepoClient.ListLicenses.
func (c *client) ListLicenses() ([]clients.License, error) {
return nil, fmt.Errorf("ListLicenses: %w", clients.ErrUnsupportedFeature)
}

// GetCreatedAt implements RepoClient.GetCreatedAt.
func (c *client) GetCreatedAt() (time.Time, error) {
return time.Time{}, fmt.Errorf("GetCreatedAt: %w", clients.ErrUnsupportedFeature)
}
Loading

0 comments on commit 61866a0

Please sign in to comment.