From 9b2b0f2ba3a9e747c8c758d4478eee9d472de564 Mon Sep 17 00:00:00 2001 From: Yoo Chung Date: Sat, 8 Apr 2023 14:55:36 +0000 Subject: [PATCH 1/4] Add Haskell as a language. Signed-off-by: Yoo Chung --- clients/languages.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/clients/languages.go b/clients/languages.go index c9778b04648..5009e6775c2 100644 --- a/clients/languages.go +++ b/clients/languages.go @@ -71,6 +71,9 @@ const ( // Dockerfile: https://docs.docker.com/engine/reference/builder/ Dockerfile LanguageName = "dockerfile" + // Haskell: https://www.haskell.org/ + Haskell LanguageName = "haskell" + // Other indicates other languages not listed by the GitHub API. Other LanguageName = "other" From 5b2d337cbf1dfe413931462479e749e66a586aee Mon Sep 17 00:00:00 2001 From: Yoo Chung Date: Mon, 10 Apr 2023 15:18:00 +0000 Subject: [PATCH 2/4] Detect fuzzing in Haskell using presence of property-based testing. Signed-off-by: Yoo Chung --- checks/raw/fuzzing.go | 32 +++++++++++++++++++--- checks/raw/fuzzing_test.go | 55 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/checks/raw/fuzzing.go b/checks/raw/fuzzing.go index e76ba04f3ee..752f98bf6d0 100644 --- a/checks/raw/fuzzing.go +++ b/checks/raw/fuzzing.go @@ -28,10 +28,11 @@ import ( ) const ( - fuzzerOSSFuzz = "OSSFuzz" - fuzzerClusterFuzzLite = "ClusterFuzzLite" - oneFuzz = "OneFuzz" - fuzzerBuiltInGo = "GoBuiltInFuzzer" + fuzzerOSSFuzz = "OSSFuzz" + fuzzerClusterFuzzLite = "ClusterFuzzLite" + oneFuzz = "OneFuzz" + fuzzerBuiltInGo = "GoBuiltInFuzzer" + fuzzerPropertyBasedHaskell = "HaskellPropertyBasedTesting" // TODO: add more fuzzing check supports. ) @@ -59,6 +60,29 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{ Desc: asPointer( "Go fuzzing intelligently walks through the source code to report failures and find vulnerabilities."), }, + // Fuzz patterns for Haskell based on property-based testing. + // + // Based on the import of one of these packages: + // * https://hackage.haskell.org/package/QuickCheck + // * https://hedgehog.qa/ + // * https://github.com/NorfairKing/validity + // * https://hackage.haskell.org/package/smallcheck + // + // They can also be imported indirectly through these test frameworks: + // * https://hspec.github.io/ + // * https://hackage.haskell.org/package/tasty + // + // This is not an exhaustive list. + clients.Haskell: { + filePattern: "*.hs,*.lhs", + // Look for direct imports of QuickCheck, Hedgehog, validity, or SmallCheck, + // or their indirect imports through the higher-level Hspec or Tasty testing frameworks. + funcPattern: `import\s+(qualified\s+)?Test\.(Hspec|Tasty\.)?(QuickCheck|Hedgehog|Validity|SmallCheck)`, + Name: fuzzerPropertyBasedHaskell, + Desc: asPointer( + "Property-based testing in Haskell generates test instances randomly or exhaustively " + + "and test that specific properties are satisfied."), + }, // TODO: add more language-specific fuzz patterns & configs. } diff --git a/checks/raw/fuzzing_test.go b/checks/raw/fuzzing_test.go index b6d2ccc4740..094a32a5774 100644 --- a/checks/raw/fuzzing_test.go +++ b/checks/raw/fuzzing_test.go @@ -316,6 +316,61 @@ func Test_checkFuzzFunc(t *testing.T) { }, fileContent: "func TestFoo (t *testing.T)", }, + { + name: "Haskell QuickCheck", + fileName: []string{ "TestSpec.hs" }, + langs: []clients.Language{ + { + Name: clients.Haskell, + NumLines: 50, + }, + }, + fileContent: "import Test.QuickCheck", + }, + { + name: "Haskell Hedgehog", + fileName: []string{ "TestSpec.hs" }, + langs: []clients.Language{ + { + Name: clients.Haskell, + NumLines: 50, + }, + }, + fileContent: "import Test.Hedgehog", + }, + { + name: "Haskell Validity", + fileName: []string{"validity_test.hs"}, + langs: []clients.Language{ + { + Name: clients.Haskell, + NumLines: 50, + }, + }, + fileContent: "import Test.Validity", + }, + { + name: "Haskell SmallCheck", + fileName: []string{"SmallSpec.hs"}, + langs: []clients.Language{ + { + Name: clients.Haskell, + NumLines: 50, + }, + }, + fileContent: "import Test.SmallCheck", + }, + { + name: "Haskell QuickCheck through Hspec", + fileName: []string{"ArrowSpec.hs"}, + langs: []clients.Language{ + { + Name: clients.Haskell, + NumLines: 50, + }, + }, + fileContent: "import Test.Hspec.QuickCheck", + }, } for _, tt := range tests { tt := tt From 7dac2447638f0303203c6352ddbc31d82cb42a98 Mon Sep 17 00:00:00 2001 From: Yoo Chung Date: Mon, 10 Apr 2023 16:53:53 +0000 Subject: [PATCH 3/4] Mention fuzzing detection for Haskell in documentation. Signed-off-by: Yoo Chung --- checks/raw/fuzzing_test.go | 31 +++++++++++++++++++++++++++---- checks/write.md | 2 +- docs/checks.md | 3 ++- docs/checks/internal/checks.yaml | 3 ++- 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/checks/raw/fuzzing_test.go b/checks/raw/fuzzing_test.go index 094a32a5774..fccc3b66866 100644 --- a/checks/raw/fuzzing_test.go +++ b/checks/raw/fuzzing_test.go @@ -317,11 +317,11 @@ func Test_checkFuzzFunc(t *testing.T) { fileContent: "func TestFoo (t *testing.T)", }, { - name: "Haskell QuickCheck", - fileName: []string{ "TestSpec.hs" }, + name: "Haskell QuickCheck", + fileName: []string{"ModuleSpec.hs"}, langs: []clients.Language{ { - Name: clients.Haskell, + Name: clients.Haskell, NumLines: 50, }, }, @@ -365,12 +365,35 @@ func Test_checkFuzzFunc(t *testing.T) { fileName: []string{"ArrowSpec.hs"}, langs: []clients.Language{ { - Name: clients.Haskell, + Name: clients.Haskell, NumLines: 50, }, }, fileContent: "import Test.Hspec.QuickCheck", }, + { + name: "Haskell QuickCheck through Tasty", + fileName: []string{"test.hs"}, + langs: []clients.Language{ + { + Name: clients.Haskell, + NumLines: 50, + }, + }, + fileContent: "import Test.Tasty.QuickCheck", + }, + { + name: "Haskell with no property-based testing", + fileName: []string{"PropertySpec.hs"}, + wantErr: true, + langs: []clients.Language{ + { + Name: clients.Haskell, + NumLines: 50, + }, + }, + fileContent: "import Test.Hspec", + }, } for _, tt := range tests { tt := tt diff --git a/checks/write.md b/checks/write.md index fc0bc9e6c95..2a517daa696 100644 --- a/checks/write.md +++ b/checks/write.md @@ -75,7 +75,7 @@ The steps to writing a check are as follows: 8. Create e2e tests in `e2e/mycheck_test.go`. Use a dedicated repo that will not change over time, so that it's reliable for the tests. -9. Update the `checks/checks.yaml` with a description of your check. +9. Update the `docs/checks/internal/checks.yaml` with a description of your check. 10. Generate `docs/check.md` using `make generate-docs`. This will validate and generate `docs/check.md`. diff --git a/docs/checks.md b/docs/checks.md index 3d744a660e1..2883446a91e 100644 --- a/docs/checks.md +++ b/docs/checks.md @@ -336,7 +336,8 @@ This check tries to determine if the project uses [fuzzing](https://owasp.org/www-community/Fuzzing) by checking: 1. if the repository name is included in the [OSS-Fuzz](https://github.com/google/oss-fuzz) project list; 2. if [ClusterFuzzLite](https://google.github.io/clusterfuzzlite/) is deployed in the repository; -3. if there are user-defined language-specified fuzzing functions (currently only supports [Go fuzzing](https://go.dev/doc/fuzz/)) in the repository. +3. if there are user-defined language-specified fuzzing functions in the repository. + - currently only supports [Go fuzzing](https://go.dev/doc/fuzz/) and a limited set of property-based testing libraries for Haskell. 4. if it contains a [OneFuzz](https://github.com/microsoft/onefuzz) integration [detection file](https://github.com/microsoft/onefuzz/blob/main/docs/getting-started.md#detecting-the-use-of-onefuzz); Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data diff --git a/docs/checks/internal/checks.yaml b/docs/checks/internal/checks.yaml index a8ba343a069..f8709cd6559 100644 --- a/docs/checks/internal/checks.yaml +++ b/docs/checks/internal/checks.yaml @@ -396,7 +396,8 @@ checks: [fuzzing](https://owasp.org/www-community/Fuzzing) by checking: 1. if the repository name is included in the [OSS-Fuzz](https://github.com/google/oss-fuzz) project list; 2. if [ClusterFuzzLite](https://google.github.io/clusterfuzzlite/) is deployed in the repository; - 3. if there are user-defined language-specified fuzzing functions (currently only supports [Go fuzzing](https://go.dev/doc/fuzz/)) in the repository. + 3. if there are user-defined language-specified fuzzing functions in the repository. + - currently only supports [Go fuzzing](https://go.dev/doc/fuzz/) and a limited set of property-based testing libraries for Haskell. 4. if it contains a [OneFuzz](https://github.com/microsoft/onefuzz) integration [detection file](https://github.com/microsoft/onefuzz/blob/main/docs/getting-started.md#detecting-the-use-of-onefuzz); Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data From 530c13991cae88f91be11ef301ef0967752dc71c Mon Sep 17 00:00:00 2001 From: Yoo Chung Date: Tue, 11 Apr 2023 23:41:04 +0000 Subject: [PATCH 4/4] Fix pattern and test. Add test case. Signed-off-by: Yoo Chung --- checks/raw/fuzzing.go | 12 ++++++++---- checks/raw/fuzzing_test.go | 29 ++++++++++++++++++++++++----- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/checks/raw/fuzzing.go b/checks/raw/fuzzing.go index 752f98bf6d0..8f01595881b 100644 --- a/checks/raw/fuzzing.go +++ b/checks/raw/fuzzing.go @@ -43,8 +43,12 @@ type filesWithPatternStr struct { // Configurations for language-specified fuzzers. type languageFuzzConfig struct { - URL, Desc *string - filePattern, funcPattern, Name string + URL, Desc *string + + // Pattern is according to path.Match. + filePattern string + + funcPattern, Name string // TODO: add more language fuzzing-related fields. } @@ -74,10 +78,10 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{ // // This is not an exhaustive list. clients.Haskell: { - filePattern: "*.hs,*.lhs", + filePattern: "*.hs", // Look for direct imports of QuickCheck, Hedgehog, validity, or SmallCheck, // or their indirect imports through the higher-level Hspec or Tasty testing frameworks. - funcPattern: `import\s+(qualified\s+)?Test\.(Hspec|Tasty\.)?(QuickCheck|Hedgehog|Validity|SmallCheck)`, + funcPattern: `import\s+(qualified\s+)?Test\.((Hspec|Tasty)\.)?(QuickCheck|Hedgehog|Validity|SmallCheck)`, Name: fuzzerPropertyBasedHaskell, Desc: asPointer( "Property-based testing in Haskell generates test instances randomly or exhaustively " + diff --git a/checks/raw/fuzzing_test.go b/checks/raw/fuzzing_test.go index fccc3b66866..5804a30c62a 100644 --- a/checks/raw/fuzzing_test.go +++ b/checks/raw/fuzzing_test.go @@ -318,6 +318,7 @@ func Test_checkFuzzFunc(t *testing.T) { }, { name: "Haskell QuickCheck", + want: true, fileName: []string{"ModuleSpec.hs"}, langs: []clients.Language{ { @@ -328,8 +329,9 @@ func Test_checkFuzzFunc(t *testing.T) { fileContent: "import Test.QuickCheck", }, { - name: "Haskell Hedgehog", - fileName: []string{ "TestSpec.hs" }, + name: "Haskell Hedgehog", + want: true, + fileName: []string{"TestSpec.hs"}, langs: []clients.Language{ { Name: clients.Haskell, @@ -340,6 +342,7 @@ func Test_checkFuzzFunc(t *testing.T) { }, { name: "Haskell Validity", + want: true, fileName: []string{"validity_test.hs"}, langs: []clients.Language{ { @@ -351,6 +354,7 @@ func Test_checkFuzzFunc(t *testing.T) { }, { name: "Haskell SmallCheck", + want: true, fileName: []string{"SmallSpec.hs"}, langs: []clients.Language{ { @@ -360,8 +364,21 @@ func Test_checkFuzzFunc(t *testing.T) { }, fileContent: "import Test.SmallCheck", }, + { + name: "Haskell QuickCheck with qualified import", + want: true, + fileName: []string{"QualifiedSpec.hs"}, + langs: []clients.Language{ + { + Name: clients.Haskell, + NumLines: 50, + }, + }, + fileContent: "import qualified Test.QuickCheck", + }, { name: "Haskell QuickCheck through Hspec", + want: true, fileName: []string{"ArrowSpec.hs"}, langs: []clients.Language{ { @@ -373,6 +390,7 @@ func Test_checkFuzzFunc(t *testing.T) { }, { name: "Haskell QuickCheck through Tasty", + want: true, fileName: []string{"test.hs"}, langs: []clients.Language{ { @@ -384,6 +402,7 @@ func Test_checkFuzzFunc(t *testing.T) { }, { name: "Haskell with no property-based testing", + want: false, fileName: []string{"PropertySpec.hs"}, wantErr: true, langs: []clients.Language{ @@ -403,12 +422,12 @@ func Test_checkFuzzFunc(t *testing.T) { defer ctrl.Finish() mockClient := mockrepo.NewMockRepoClient(ctrl) mockClient.EXPECT().ListFiles(gomock.Any()).Return(tt.fileName, nil).AnyTimes() - mockClient.EXPECT().GetFileContent(gomock.Any()).DoAndReturn(func(f string) (string, error) { + mockClient.EXPECT().GetFileContent(gomock.Any()).DoAndReturn(func(f string) ([]byte, error) { if tt.wantErr { //nolint - return "", errors.New("error") + return nil, errors.New("error") } - return tt.fileContent, nil + return []byte(tt.fileContent), nil }).AnyTimes() req := checker.CheckRequest{ RepoClient: mockClient,