🌱 Combine fuzzing probes (#3877)

* single fuzz probe boilerplate Signed-off-by: Spencer Schrock <sschrock@google.com> * initial implementation Signed-off-by: Spencer Schrock <sschrock@google.com> * connect fuzzing probe to eval code Signed-off-by: Spencer Schrock <sschrock@google.com> * include fuzzer name as tool Signed-off-by: Spencer Schrock <sschrock@google.com> * connect to probes flag Signed-off-by: Spencer Schrock <sschrock@google.com> * remove old probes from list Signed-off-by: Spencer Schrock <sschrock@google.com> * remove old probes Signed-off-by: Spencer Schrock <sschrock@google.com> * fix failing test Signed-off-by: Spencer Schrock <sschrock@google.com> * add tool value to test Signed-off-by: Spencer Schrock <sschrock@google.com> * add fuzz tool helper Signed-off-by: Spencer Schrock <sschrock@google.com> * specify supported tools Signed-off-by: Spencer Schrock <sschrock@google.com> * update e2e test Signed-off-by: Spencer Schrock <sschrock@google.com> * check for no raw data Signed-off-by: Spencer Schrock <sschrock@google.com> * add basic tests Signed-off-by: Spencer Schrock <sschrock@google.com> * add test to ensure fuzzer location is propagated Signed-off-by: Spencer Schrock <sschrock@google.com> * expand detailed tests to include other info like tool value Signed-off-by: Spencer Schrock <sschrock@google.com> --------- Signed-off-by: Spencer Schrock <sschrock@google.com>
ossf · Mar 11, 2024 · f1e703f · f1e703f
1 parent e1f5483
commit f1e703f
Show file tree

Hide file tree

Showing 44 changed files with 205 additions and 2,722 deletions.
diff --git a/checks/evaluation/fuzzing.go b/checks/evaluation/fuzzing.go
@@ -18,38 +18,15 @@ import (
 	"github.com/ossf/scorecard/v4/checker"
 	sce "github.com/ossf/scorecard/v4/errors"
 	"github.com/ossf/scorecard/v4/finding"
-	"github.com/ossf/scorecard/v4/probes/fuzzedWithCLibFuzzer"
-	"github.com/ossf/scorecard/v4/probes/fuzzedWithClusterFuzzLite"
-	"github.com/ossf/scorecard/v4/probes/fuzzedWithCppLibFuzzer"
-	"github.com/ossf/scorecard/v4/probes/fuzzedWithGoNative"
-	"github.com/ossf/scorecard/v4/probes/fuzzedWithJavaJazzerFuzzer"
-	"github.com/ossf/scorecard/v4/probes/fuzzedWithOSSFuzz"
-	"github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedHaskell"
-	"github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedJavascript"
-	"github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedTypescript"
-	"github.com/ossf/scorecard/v4/probes/fuzzedWithPythonAtheris"
-	"github.com/ossf/scorecard/v4/probes/fuzzedWithRustCargofuzz"
-	"github.com/ossf/scorecard/v4/probes/fuzzedWithSwiftLibFuzzer"
+	"github.com/ossf/scorecard/v4/probes/fuzzed"
 )
 
 // Fuzzing applies the score policy for the Fuzzing check.
 func Fuzzing(name string,
 	findings []finding.Finding, dl checker.DetailLogger,
 ) checker.CheckResult {
-	// We have 7 unique probes, each should have a finding.
 	expectedProbes := []string{
-		fuzzedWithClusterFuzzLite.Probe,
-		fuzzedWithGoNative.Probe,
-		fuzzedWithPythonAtheris.Probe,
-		fuzzedWithCLibFuzzer.Probe,
-		fuzzedWithCppLibFuzzer.Probe,
-		fuzzedWithRustCargofuzz.Probe,
-		fuzzedWithSwiftLibFuzzer.Probe,
-		fuzzedWithJavaJazzerFuzzer.Probe,
-		fuzzedWithOSSFuzz.Probe,
-		fuzzedWithPropertyBasedHaskell.Probe,
-		fuzzedWithPropertyBasedJavascript.Probe,
-		fuzzedWithPropertyBasedTypescript.Probe,
+		fuzzed.Probe,
 	}
 	// TODO: other packages to consider:
 	// - github.com/google/fuzztest

diff --git a/checks/evaluation/fuzzing_test.go b/checks/evaluation/fuzzing_test.go
@@ -19,6 +19,8 @@ import (
 	"github.com/ossf/scorecard/v4/checker"
 	sce "github.com/ossf/scorecard/v4/errors"
 	"github.com/ossf/scorecard/v4/finding"
+	"github.com/ossf/scorecard/v4/internal/fuzzers"
+	"github.com/ossf/scorecard/v4/probes/fuzzed"
 	scut "github.com/ossf/scorecard/v4/utests"
 )
 
@@ -30,180 +32,48 @@ func TestFuzzing(t *testing.T) {
 		result   scut.TestReturn
 	}{
 		{
-			name: "Fuzzing - no fuzzing",
+			name: "no fuzzers",
 			findings: []finding.Finding{
 				{
-					Probe:   "fuzzedWithClusterFuzzLite",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithGoNative",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithPythonAtheris",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithCLibFuzzer",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithCppLibFuzzer",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithRustCargofuzz",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithSwiftLibFuzzer",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithJavaJazzerFuzzer",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithOSSFuzz",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithPropertyBasedHaskell",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithPropertyBasedJavascript",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithPropertyBasedTypescript",
+					Probe:   fuzzed.Probe,
 					Outcome: finding.OutcomeNegative,
 				},
 			},
 			result: scut.TestReturn{
 				Score:        checker.MinResultScore,
-				NumberOfWarn: 12,
+				NumberOfWarn: 1,
 			},
 		},
 		{
-			name: "Fuzzing - fuzzing GoNative",
+			name: "single fuzzer gives max score",
 			findings: []finding.Finding{
-				{
-					Probe:   "fuzzedWithClusterFuzzLite",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithGoNative",
-					Outcome: finding.OutcomePositive,
-				},
-				{
-					Probe:   "fuzzedWithPythonAtheris",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithCLibFuzzer",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithCppLibFuzzer",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithRustCargofuzz",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithSwiftLibFuzzer",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithJavaJazzerFuzzer",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithOSSFuzz",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithPropertyBasedHaskell",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithPropertyBasedJavascript",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithPropertyBasedTypescript",
-					Outcome: finding.OutcomeNegative,
-				},
+				fuzzTool(fuzzers.BuiltInGo),
 			},
 			result: scut.TestReturn{
 				Score:        checker.MaxResultScore,
 				NumberOfInfo: 1,
 			},
 		},
-
 		{
-			name: "Fuzzing - fuzzing missing GoNative finding",
+			name: "one info per fuzzer",
 			findings: []finding.Finding{
-				{
-					Probe:   "fuzzedWithClusterFuzzLite",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithOSSFuzz",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithPropertyBasedHaskell",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithPropertyBasedJavascript",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithPropertyBasedTypescript",
-					Outcome: finding.OutcomeNegative,
-				},
+				fuzzTool(fuzzers.BuiltInGo),
+				fuzzTool(fuzzers.OSSFuzz),
+				fuzzTool(fuzzers.ClusterFuzzLite),
 			},
 			result: scut.TestReturn{
-				Score: checker.InconclusiveResultScore,
-				Error: sce.ErrScorecardInternal,
+				Score:        checker.MaxResultScore,
+				NumberOfInfo: 3,
 			},
 		},
 		{
-			name: "Fuzzing - fuzzing invalid probe name",
+			name: "extra probe not part of check",
 			findings: []finding.Finding{
 				{
-					Probe:   "fuzzedWithClusterFuzzLite",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithGoNative",
-					Outcome: finding.OutcomePositive,
-				},
-				{
-					Probe:   "fuzzedWithOSSFuzz",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithPropertyBasedHaskell",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithPropertyBasedJavascript",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "fuzzedWithPropertyBasedTypescript",
+					Probe:   "someUnrelatedProbe",
 					Outcome: finding.OutcomeNegative,
 				},
-				{
-					Probe:   "fuzzedWithInvalidProbeName",
-					Outcome: finding.OutcomePositive,
-				},
+				fuzzTool(fuzzers.RustCargoFuzz),
 			},
 			result: scut.TestReturn{
 				Score: checker.InconclusiveResultScore,
@@ -221,3 +91,13 @@ func TestFuzzing(t *testing.T) {
 		})
 	}
 }
+
+func fuzzTool(name string) finding.Finding {
+	return finding.Finding{
+		Probe:   fuzzed.Probe,
+		Outcome: finding.OutcomePositive,
+		Values: map[string]string{
+			fuzzed.ToolKey: name,
+		},
+	}
+}
diff --git a/checks/fuzzing_test.go b/checks/fuzzing_test.go
@@ -54,7 +54,7 @@ func TestFuzzing(t *testing.T) {
 			wantErr: false,
 			expected: scut.TestReturn{
 				Error:         nil,
-				NumberOfWarn:  12,
+				NumberOfWarn:  1,
 				NumberOfDebug: 0,
 				NumberOfInfo:  0,
 				Score:         0,
@@ -111,7 +111,7 @@ func TestFuzzing(t *testing.T) {
 			wantFuzzErr: false,
 			expected: scut.TestReturn{
 				Error:         nil,
-				NumberOfWarn:  12,
+				NumberOfWarn:  1,
 				NumberOfDebug: 0,
 				NumberOfInfo:  0,
 				Score:         0,
@@ -122,7 +122,7 @@ func TestFuzzing(t *testing.T) {
 			wantFuzzErr: true,
 			expected: scut.TestReturn{
 				Error:         nil,
-				NumberOfWarn:  12,
+				NumberOfWarn:  1,
 				NumberOfDebug: 0,
 				NumberOfInfo:  0,
 				Score:         0,

diff --git a/docs/checks/fuzzing/README.md b/docs/checks/fuzzing/README.md
@@ -0,0 +1,30 @@
+# Supported Fuzzers
+* [LibFuzzer](https://llvm.org/docs/LibFuzzer.html)
+  * Detection is based on usages of a function named `LLVMFuzzerTestOneInput` in C, C++, or Swift files.
+* [ClusterFuzzLite](https://github.com/google/clusterfuzzlite)
+  * Detection is based on a file called `.clusterfuzzlite/Dockerfile`.
+* [Native Go Fuzzing](https://go.dev/doc/security/fuzz/)
+  * Looks for functions of the form `func FuzzXxx(*testing.F)` in Go files.
+* [Jazzer](https://github.com/CodeIntelligenceTesting/jazzer)
+  * Detection based on the import of `com.code_intelligence.jazzer.api.FuzzedDataProvider` in Java files.
+* [OSS-Fuzz](https://github.com/google/oss-fuzz)
+  * Detection based on the presence of integrated projects in the [google/oss-fuzz GitHub repo](https://github.com/google/oss-fuzz/tree/master/projects).
+* Property-based Haskell Fuzzers
+  * Detected based on imports of various testing frameworks:
+    * [QuickCheck](https://hackage.haskell.org/package/QuickCheck)
+    * [hedgehog]( https://hedgehog.qa/)
+    * [validity](https://github.com/NorfairKing/validity)
+    * [smallcheck](https://hackage.haskell.org/package/smallcheck)
+    * [hspec](https://hspec.github.io/)
+    * [tasty](https://hackage.haskell.org/package/tasty)
+* [fast-check](https://github.com/dubzzz/fast-check)
+  * Detection based on import statements in JavaScript and TypeScript files.
+* [Atheris](https://github.com/google/atheris)
+  * Detection based on the presence of `import atheris` in Python files.
+* [cargo-fuzz](https://rust-fuzz.github.io/book/cargo-fuzz.html)
+  * Detection based on presence of `libfuzzer_sys` in Rust files.
+
+# Add Support
+
+Don't see your fuzzing tool listed? 
+Search for an existing issue, or create one, to discuss adding support.
diff --git a/e2e/fuzzing_test.go b/e2e/fuzzing_test.go
@@ -192,7 +192,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() {
 			expected := scut.TestReturn{
 				Error:         nil,
 				Score:         checker.MinResultScore,
-				NumberOfWarn:  12,
+				NumberOfWarn:  1,
 				NumberOfInfo:  0,
 				NumberOfDebug: 0,
 			}

diff --git a/pkg/scorecard_test.go b/pkg/scorecard_test.go
@@ -28,6 +28,7 @@ import (
 	"github.com/ossf/scorecard/v4/finding"
 	"github.com/ossf/scorecard/v4/finding/probe"
 	"github.com/ossf/scorecard/v4/log"
+	"github.com/ossf/scorecard/v4/probes/fuzzed"
 )
 
 func Test_getRepoCommitHash(t *testing.T) {
@@ -210,7 +211,7 @@ func TestExperimentalRunProbes(t *testing.T) {
 			args: args{
 				uri:       "github.com/ossf/scorecard",
 				commitSHA: "1a17bb812fb2ac23e9d09e86e122f8b67563aed7",
-				probes:    []string{"fuzzedWithOSSFuzz"},
+				probes:    []string{fuzzed.Probe},
 			},
 			want: ScorecardResult{
 				Repo: RepoInfo{
@@ -233,10 +234,10 @@ func TestExperimentalRunProbes(t *testing.T) {
 				},
 				Findings: []finding.Finding{
 					{
-						Probe:   "fuzzedWithOSSFuzz",
-						Message: "no OSSFuzz integration found",
+						Probe:   fuzzed.Probe,
+						Message: "no fuzzer integrations found",
 						Remediation: &probe.Remediation{
-							Effort: 3,
+							Effort: probe.RemediationEffortHigh,
 						},
 					},
 				},