diff --git a/checks/raw/fuzzing.go b/checks/raw/fuzzing.go index e76ba04f3ee..8f01595881b 100644 --- a/checks/raw/fuzzing.go +++ b/checks/raw/fuzzing.go @@ -28,10 +28,11 @@ import ( ) const ( - fuzzerOSSFuzz = "OSSFuzz" - fuzzerClusterFuzzLite = "ClusterFuzzLite" - oneFuzz = "OneFuzz" - fuzzerBuiltInGo = "GoBuiltInFuzzer" + fuzzerOSSFuzz = "OSSFuzz" + fuzzerClusterFuzzLite = "ClusterFuzzLite" + oneFuzz = "OneFuzz" + fuzzerBuiltInGo = "GoBuiltInFuzzer" + fuzzerPropertyBasedHaskell = "HaskellPropertyBasedTesting" // TODO: add more fuzzing check supports. ) @@ -42,8 +43,12 @@ type filesWithPatternStr struct { // Configurations for language-specified fuzzers. type languageFuzzConfig struct { - URL, Desc *string - filePattern, funcPattern, Name string + URL, Desc *string + + // Pattern is according to path.Match. + filePattern string + + funcPattern, Name string // TODO: add more language fuzzing-related fields. } @@ -59,6 +64,29 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{ Desc: asPointer( "Go fuzzing intelligently walks through the source code to report failures and find vulnerabilities."), }, + // Fuzz patterns for Haskell based on property-based testing. + // + // Based on the import of one of these packages: + // * https://hackage.haskell.org/package/QuickCheck + // * https://hedgehog.qa/ + // * https://github.com/NorfairKing/validity + // * https://hackage.haskell.org/package/smallcheck + // + // They can also be imported indirectly through these test frameworks: + // * https://hspec.github.io/ + // * https://hackage.haskell.org/package/tasty + // + // This is not an exhaustive list. + clients.Haskell: { + filePattern: "*.hs", + // Look for direct imports of QuickCheck, Hedgehog, validity, or SmallCheck, + // or their indirect imports through the higher-level Hspec or Tasty testing frameworks. + funcPattern: `import\s+(qualified\s+)?Test\.((Hspec|Tasty)\.)?(QuickCheck|Hedgehog|Validity|SmallCheck)`, + Name: fuzzerPropertyBasedHaskell, + Desc: asPointer( + "Property-based testing in Haskell generates test instances randomly or exhaustively " + + "and test that specific properties are satisfied."), + }, // TODO: add more language-specific fuzz patterns & configs. } diff --git a/checks/raw/fuzzing_test.go b/checks/raw/fuzzing_test.go index b6d2ccc4740..5804a30c62a 100644 --- a/checks/raw/fuzzing_test.go +++ b/checks/raw/fuzzing_test.go @@ -316,6 +316,103 @@ func Test_checkFuzzFunc(t *testing.T) { }, fileContent: "func TestFoo (t *testing.T)", }, + { + name: "Haskell QuickCheck", + want: true, + fileName: []string{"ModuleSpec.hs"}, + langs: []clients.Language{ + { + Name: clients.Haskell, + NumLines: 50, + }, + }, + fileContent: "import Test.QuickCheck", + }, + { + name: "Haskell Hedgehog", + want: true, + fileName: []string{"TestSpec.hs"}, + langs: []clients.Language{ + { + Name: clients.Haskell, + NumLines: 50, + }, + }, + fileContent: "import Test.Hedgehog", + }, + { + name: "Haskell Validity", + want: true, + fileName: []string{"validity_test.hs"}, + langs: []clients.Language{ + { + Name: clients.Haskell, + NumLines: 50, + }, + }, + fileContent: "import Test.Validity", + }, + { + name: "Haskell SmallCheck", + want: true, + fileName: []string{"SmallSpec.hs"}, + langs: []clients.Language{ + { + Name: clients.Haskell, + NumLines: 50, + }, + }, + fileContent: "import Test.SmallCheck", + }, + { + name: "Haskell QuickCheck with qualified import", + want: true, + fileName: []string{"QualifiedSpec.hs"}, + langs: []clients.Language{ + { + Name: clients.Haskell, + NumLines: 50, + }, + }, + fileContent: "import qualified Test.QuickCheck", + }, + { + name: "Haskell QuickCheck through Hspec", + want: true, + fileName: []string{"ArrowSpec.hs"}, + langs: []clients.Language{ + { + Name: clients.Haskell, + NumLines: 50, + }, + }, + fileContent: "import Test.Hspec.QuickCheck", + }, + { + name: "Haskell QuickCheck through Tasty", + want: true, + fileName: []string{"test.hs"}, + langs: []clients.Language{ + { + Name: clients.Haskell, + NumLines: 50, + }, + }, + fileContent: "import Test.Tasty.QuickCheck", + }, + { + name: "Haskell with no property-based testing", + want: false, + fileName: []string{"PropertySpec.hs"}, + wantErr: true, + langs: []clients.Language{ + { + Name: clients.Haskell, + NumLines: 50, + }, + }, + fileContent: "import Test.Hspec", + }, } for _, tt := range tests { tt := tt @@ -325,12 +422,12 @@ func Test_checkFuzzFunc(t *testing.T) { defer ctrl.Finish() mockClient := mockrepo.NewMockRepoClient(ctrl) mockClient.EXPECT().ListFiles(gomock.Any()).Return(tt.fileName, nil).AnyTimes() - mockClient.EXPECT().GetFileContent(gomock.Any()).DoAndReturn(func(f string) (string, error) { + mockClient.EXPECT().GetFileContent(gomock.Any()).DoAndReturn(func(f string) ([]byte, error) { if tt.wantErr { //nolint - return "", errors.New("error") + return nil, errors.New("error") } - return tt.fileContent, nil + return []byte(tt.fileContent), nil }).AnyTimes() req := checker.CheckRequest{ RepoClient: mockClient, diff --git a/checks/write.md b/checks/write.md index fc0bc9e6c95..2a517daa696 100644 --- a/checks/write.md +++ b/checks/write.md @@ -75,7 +75,7 @@ The steps to writing a check are as follows: 8. Create e2e tests in `e2e/mycheck_test.go`. Use a dedicated repo that will not change over time, so that it's reliable for the tests. -9. Update the `checks/checks.yaml` with a description of your check. +9. Update the `docs/checks/internal/checks.yaml` with a description of your check. 10. Generate `docs/check.md` using `make generate-docs`. This will validate and generate `docs/check.md`. diff --git a/clients/languages.go b/clients/languages.go index c9778b04648..5009e6775c2 100644 --- a/clients/languages.go +++ b/clients/languages.go @@ -71,6 +71,9 @@ const ( // Dockerfile: https://docs.docker.com/engine/reference/builder/ Dockerfile LanguageName = "dockerfile" + // Haskell: https://www.haskell.org/ + Haskell LanguageName = "haskell" + // Other indicates other languages not listed by the GitHub API. Other LanguageName = "other" diff --git a/docs/checks.md b/docs/checks.md index 3d744a660e1..2883446a91e 100644 --- a/docs/checks.md +++ b/docs/checks.md @@ -336,7 +336,8 @@ This check tries to determine if the project uses [fuzzing](https://owasp.org/www-community/Fuzzing) by checking: 1. if the repository name is included in the [OSS-Fuzz](https://github.com/google/oss-fuzz) project list; 2. if [ClusterFuzzLite](https://google.github.io/clusterfuzzlite/) is deployed in the repository; -3. if there are user-defined language-specified fuzzing functions (currently only supports [Go fuzzing](https://go.dev/doc/fuzz/)) in the repository. +3. if there are user-defined language-specified fuzzing functions in the repository. + - currently only supports [Go fuzzing](https://go.dev/doc/fuzz/) and a limited set of property-based testing libraries for Haskell. 4. if it contains a [OneFuzz](https://github.com/microsoft/onefuzz) integration [detection file](https://github.com/microsoft/onefuzz/blob/main/docs/getting-started.md#detecting-the-use-of-onefuzz); Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data diff --git a/docs/checks/internal/checks.yaml b/docs/checks/internal/checks.yaml index a8ba343a069..f8709cd6559 100644 --- a/docs/checks/internal/checks.yaml +++ b/docs/checks/internal/checks.yaml @@ -396,7 +396,8 @@ checks: [fuzzing](https://owasp.org/www-community/Fuzzing) by checking: 1. if the repository name is included in the [OSS-Fuzz](https://github.com/google/oss-fuzz) project list; 2. if [ClusterFuzzLite](https://google.github.io/clusterfuzzlite/) is deployed in the repository; - 3. if there are user-defined language-specified fuzzing functions (currently only supports [Go fuzzing](https://go.dev/doc/fuzz/)) in the repository. + 3. if there are user-defined language-specified fuzzing functions in the repository. + - currently only supports [Go fuzzing](https://go.dev/doc/fuzz/) and a limited set of property-based testing libraries for Haskell. 4. if it contains a [OneFuzz](https://github.com/microsoft/onefuzz) integration [detection file](https://github.com/microsoft/onefuzz/blob/main/docs/getting-started.md#detecting-the-use-of-onefuzz); Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data