Skip to content

Commit

Permalink
🌱 SAST: dedupe and add Pysa and Qodana probe (#3743)
Browse files Browse the repository at this point in the history 
* Add SAST Pysa probe

Signed-off-by: David Korczynski <david@adalogics.com>

* Add Pysa positive unit test

Signed-off-by: David Korczynski <david@adalogics.com>

* Add Qodana as well

Signed-off-by: David Korczynski <david@adalogics.com>

* fix some styling

Signed-off-by: David Korczynski <david@adalogics.com>

* fix some messaging

Signed-off-by: David Korczynski <david@adalogics.com>

* checks: raw: sast: dedup by way of regex

Ref: #3745

Signed-off-by: David Korczynski <david@adalogics.com>

* deduplicate SAST score checker

Signed-off-by: David Korczynski <david@adalogics.com>

* fix styling

Signed-off-by: David Korczynski <david@adalogics.com>

* fix styling

Signed-off-by: David Korczynski <david@adalogics.com>

* Rename variables appropriately

Signed-off-by: David Korczynski <david@adalogics.com>

* fix error message

Signed-off-by: David Korczynski <david@adalogics.com>

* rename useRegex to usesRegex and add comment

Signed-off-by: David Korczynski <david@adalogics.com>

* Force regex to compile

Signed-off-by: David Korczynski <david@adalogics.com>

---------

Signed-off-by: David Korczynski <david@adalogics.com>
  • Loading branch information
@DavidKorczynski
DavidKorczynski committed Jan 2, 2024
1 parent 6c2a266 commit 99c455b
Show file tree
Hide file tree
Showing 14 changed files with 674 additions and 99 deletions.
4 changes: 4 additions & 0 deletions checker/raw_result.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,10 @@ const (
SonarWorkflow SASTWorkflowType = "Sonar"
// SnykWorkflow represents a workflow that runs Snyk.
SnykWorkflow SASTWorkflowType = "Snyk"
// PysaWorkflow represents a workflow that runs Pysa.
PysaWorkflow SASTWorkflowType = "Pysa"
// QodanaWorkflow represents a workflow that runs Qodana.
QodanaWorkflow SASTWorkflowType = "Qodana"
)

// SASTWorkflow represents a SAST workflow.
Expand Down
40 changes: 20 additions & 20 deletions checks/evaluation/sast.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ import (
sce "github.com/ossf/scorecard/v4/errors"
"github.com/ossf/scorecard/v4/finding"
"github.com/ossf/scorecard/v4/probes/sastToolCodeQLInstalled"
"github.com/ossf/scorecard/v4/probes/sastToolPysaInstalled"
"github.com/ossf/scorecard/v4/probes/sastToolQodanaInstalled"
"github.com/ossf/scorecard/v4/probes/sastToolRunsOnAllCommits"
"github.com/ossf/scorecard/v4/probes/sastToolSnykInstalled"
"github.com/ossf/scorecard/v4/probes/sastToolSonarInstalled"
Expand All @@ -31,6 +33,8 @@ func SAST(name string,
// We have 3 unique probes, each should have a finding.
expectedProbes := []string{
sastToolCodeQLInstalled.Probe,
sastToolPysaInstalled.Probe,
sastToolQodanaInstalled.Probe,
sastToolRunsOnAllCommits.Probe,
sastToolSonarInstalled.Probe,
sastToolSnykInstalled.Probe,
Expand All @@ -41,17 +45,21 @@ func SAST(name string,
return checker.CreateRuntimeErrorResult(name, e)
}

var sastScore, codeQlScore, snykScore, sonarScore int
var sastScore, codeQlScore, pysaScore, qodanaScore, snykScore, sonarScore int
// Assign sastScore, codeQlScore and sonarScore
for i := range findings {
f := &findings[i]
switch f.Probe {
case sastToolRunsOnAllCommits.Probe:
sastScore = getSASTScore(f, dl)
case sastToolCodeQLInstalled.Probe:
codeQlScore = getCodeQLScore(f, dl)
codeQlScore = getSastToolScore(f, dl)
case sastToolSnykInstalled.Probe:
snykScore = getSnykScore(f, dl)
snykScore = getSastToolScore(f, dl)
case sastToolPysaInstalled.Probe:
pysaScore = getSastToolScore(f, dl)
case sastToolQodanaInstalled.Probe:
qodanaScore = getSastToolScore(f, dl)
case sastToolSonarInstalled.Probe:
if f.Outcome == finding.OutcomePositive {
sonarScore = checker.MaxResultScore
Expand All @@ -75,6 +83,12 @@ func SAST(name string,
if snykScore == checker.MaxResultScore {
return checker.CreateMaxScoreResult(name, "SAST tool detected: Snyk")
}
if pysaScore == checker.MaxResultScore {
return checker.CreateMaxScoreResult(name, "SAST tool detected: Pysa")
}
if qodanaScore == checker.MaxResultScore {
return checker.CreateMaxScoreResult(name, "SAST tool detected: Qodana")
}

if sastScore == checker.InconclusiveResultScore &&
codeQlScore == checker.InconclusiveResultScore {
Expand Down Expand Up @@ -154,23 +168,9 @@ func getSASTScore(f *finding.Finding, dl checker.DetailLogger) int {
return checker.CreateProportionalScore(f.Values["totalPullRequestsAnalyzed"], f.Values["totalPullRequestsMerged"])
}

// getCodeQLScore returns positive the project runs CodeQL and negative
// if it doesn't.
func getCodeQLScore(f *finding.Finding, dl checker.DetailLogger) int {
switch f.Outcome {
case finding.OutcomePositive:
dl.Info(&checker.LogMessage{
Text: f.Message,
})
return checker.MaxResultScore
case finding.OutcomeNegative:
return checker.MinResultScore
default:
panic("Should not happen")
}
}

func getSnykScore(f *finding.Finding, dl checker.DetailLogger) int {
// getSastToolScore returns positive if the project runs the Sast tool
// and negative if it doesn't.
func getSastToolScore(f *finding.Finding, dl checker.DetailLogger) int {
switch f.Outcome {
case finding.OutcomePositive:
dl.Info(&checker.LogMessage{
Expand Down
116 changes: 112 additions & 4 deletions checks/evaluation/sast_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ func TestSAST(t *testing.T) {
},
},
{
name: "Sonar and codeQL is installed",
name: "Sonar and codeQL is installed. Snyk, Qodana and Pysa are not installed.",
findings: []finding.Finding{
{
Probe: "sastToolCodeQLInstalled",
Expand All @@ -64,6 +64,14 @@ func TestSAST(t *testing.T) {
Probe: "sastToolSnykInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sastToolPysaInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sastToolQodanaInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sastToolRunsOnAllCommits",
Outcome: finding.OutcomePositive,
Expand Down Expand Up @@ -91,7 +99,45 @@ func TestSAST(t *testing.T) {
},
},
{
name: `Sonar is installed. CodeQL is not installed.
name: "Pysa is installed. CodeQL, Snyk, Qodana and Sonar are not installed.",
findings: []finding.Finding{
{
Probe: "sastToolCodeQLInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sastToolSnykInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sastToolPysaInstalled",
Outcome: finding.OutcomePositive,
},
{
Probe: "sastToolQodanaInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sastToolRunsOnAllCommits",
Outcome: finding.OutcomePositive,
Values: map[string]int{
"totalPullRequestsAnalyzed": 1,
"totalPullRequestsMerged": 2,
},
},
{
Probe: "sastToolSonarInstalled",
Outcome: finding.OutcomeNegative,
},
},
result: scut.TestReturn{
Score: 10,
NumberOfInfo: 2,
NumberOfWarn: 0,
},
},
{
name: `Sonar is installed. CodeQL, Snyk, Pysa, Qodana are not installed.
Does not have info about whether SAST runs
on every commit.`,
findings: []finding.Finding{
Expand All @@ -103,6 +149,14 @@ func TestSAST(t *testing.T) {
Probe: "sastToolSnykInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sastToolQodanaInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sastToolPysaInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sastToolRunsOnAllCommits",
Outcome: finding.OutcomeNotApplicable,
Expand All @@ -126,7 +180,7 @@ func TestSAST(t *testing.T) {
},
},
{
name: "Sonar and CodeQL are not installed",
name: "Sonar, CodeQL, Snyk, Qodana and Pysa are not installed",
findings: []finding.Finding{
{
Probe: "sastToolCodeQLInstalled",
Expand All @@ -136,6 +190,14 @@ func TestSAST(t *testing.T) {
Probe: "sastToolSnykInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sastToolPysaInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sastToolQodanaInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sastToolRunsOnAllCommits",
Outcome: finding.OutcomeNegative,
Expand All @@ -156,7 +218,7 @@ func TestSAST(t *testing.T) {
},
},
{
name: "Snyk is installed, Sonar and CodeQL are not installed",
name: "Snyk is installed, Sonar, Qodana and CodeQL are not installed",
findings: []finding.Finding{
{
Probe: "sastToolCodeQLInstalled",
Expand All @@ -178,6 +240,52 @@ func TestSAST(t *testing.T) {
Probe: "sastToolSonarInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sastToolPysaInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sastToolQodanaInstalled",
Outcome: finding.OutcomeNegative,
},
},
result: scut.TestReturn{
Score: 10,
NumberOfWarn: 0,
NumberOfInfo: 2,
},
},
{
name: "Qodana is installed, Snyk, Sonar, and CodeQL are not installed",
findings: []finding.Finding{
{
Probe: "sastToolCodeQLInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sastToolSnykInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sastToolRunsOnAllCommits",
Outcome: finding.OutcomePositive,
Values: map[string]int{
"totalPullRequestsAnalyzed": 1,
"totalPullRequestsMerged": 3,
},
},
{
Probe: "sastToolSonarInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sastToolPysaInstalled",
Outcome: finding.OutcomeNegative,
},
{
Probe: "sastToolQodanaInstalled",
Outcome: finding.OutcomePositive,
},
},
result: scut.TestReturn{
Score: 10,
Expand Down

0 comments on commit 99c455b

Please sign in to comment.