diff --git a/fse/fuzz_test.go b/fse/fuzz_test.go new file mode 100644 index 0000000000..64b670ee9a --- /dev/null +++ b/fse/fuzz_test.go @@ -0,0 +1,44 @@ +package fse + +import ( + "bytes" + "fmt" + "testing" + + "github.com/klauspost/compress/internal/fuzz" +) + +func FuzzCompress(f *testing.F) { + fuzz.AddFromZip(f, "testdata/fse_compress.zip", fuzz.TypeRaw, false) + f.Fuzz(func(t *testing.T, buf0 []byte) { + var s, s2 Scratch + b, err := Compress(buf0, &s) + if err != nil || b == nil { + return + } + err = s.validateNorm() + if err != nil { + return + } + //Decompress + got, err := Decompress(b, &s2) + if err != nil || len(got) == 0 { + return + } + if !bytes.Equal(buf0, got) { + t.Fatal(fmt.Sprintln("FuzzCompress output mismatch\n", len(got), "org: \n", len(buf0))) + } + }) +} + +func FuzzDecompress(f *testing.F) { + fuzz.AddFromZip(f, "testdata/fse_decompress.zip", fuzz.TypeRaw, false) + f.Fuzz(func(t *testing.T, buf0 []byte) { + var s2 Scratch + //Decompress + got, err := Decompress(buf0, &s2) + if err != nil || len(got) == 0 { + return + } + }) +} diff --git a/fse/testdata/fse_compress.zip b/fse/testdata/fse_compress.zip new file mode 100644 index 0000000000..9250408c9a Binary files /dev/null and b/fse/testdata/fse_compress.zip differ diff --git a/fse/testdata/fse_decompress.zip b/fse/testdata/fse_decompress.zip new file mode 100644 index 0000000000..7d59ed2e1c Binary files /dev/null and b/fse/testdata/fse_decompress.zip differ diff --git a/huff0/fuzz_test.go b/huff0/fuzz_test.go new file mode 100644 index 0000000000..fabb12e3fe --- /dev/null +++ b/huff0/fuzz_test.go @@ -0,0 +1,90 @@ +package huff0 + +import ( + "bytes" + "fmt" + "testing" + + "github.com/klauspost/compress/internal/fuzz" +) + +func FuzzCompress(f *testing.F) { + fuzz.AddFromZip(f, "testdata/fse_compress.zip", fuzz.TypeRaw, false) + f.Fuzz(func(t *testing.T, buf0 []byte) { + //use of Compress1X + var s Scratch + if len(buf0) > BlockSizeMax { + buf0 = buf0[:BlockSizeMax] + } + EstimateSizes(buf0, &s) + b, re, err := Compress1X(buf0, &s) + s.validateTable(s.cTable) + s.canUseTable(s.cTable) + if err != nil || b == nil { + return + } + + min := s.minSize(len(buf0)) + + if len(s.OutData) < min { + t.Errorf("FuzzCompress: output data length (%d) below shannon limit (%d)", len(s.OutData), min) + } + if len(s.OutTable) == 0 { + t.Error("FuzzCompress: got no table definition") + } + if re { + t.Error("FuzzCompress: claimed to have re-used.") + } + if len(s.OutData) == 0 { + t.Error("FuzzCompress: got no data output") + } + + dec, remain, err := ReadTable(b, nil) + + //use of Decompress1X + out, err := dec.Decompress1X(remain) + if err != nil || len(out) == 0 { + return + } + if !bytes.Equal(out, buf0) { + t.Fatal(fmt.Sprintln("FuzzCompressX1 output mismatch\n", len(out), "org: \n", len(buf0))) + } + + //use of Compress4X + s.Reuse = ReusePolicyAllow + b, reUsed, err := Compress4X(buf0, &s) + if err != nil || b == nil { + return + } + remain = b + if !reUsed { + dec, remain, err = ReadTable(b, dec) + if err != nil { + return + } + } + //use of Decompress4X + out, err = dec.Decompress4X(remain, len(buf0)) + if err != nil || out == nil { + return + } + if !bytes.Equal(out, buf0) { + t.Fatal(fmt.Sprintln("FuzzCompressX4 output mismatch: ", len(out), ", org: ", len(buf0))) + } + }) +} + +func FuzzDecompress1x(f *testing.F) { + fuzz.AddFromZip(f, "testdata/huff0_decompress1x.zip", fuzz.TypeRaw, false) + f.Fuzz(func(t *testing.T, buf0 []byte) { + var s Scratch + _, remain, err := ReadTable(buf0, &s) + if err != nil { + return + } + out, err := s.Decompress1X(remain) + if err != nil || out == nil { + return + } + }) +} diff --git a/huff0/testdata/fse_compress.zip b/huff0/testdata/fse_compress.zip new file mode 100644 index 0000000000..9250408c9a Binary files /dev/null and b/huff0/testdata/fse_compress.zip differ diff --git a/huff0/testdata/huff0_decompress1x.zip b/huff0/testdata/huff0_decompress1x.zip new file mode 100644 index 0000000000..ab062c3cc8 Binary files /dev/null and b/huff0/testdata/huff0_decompress1x.zip differ diff --git a/ossfuzz/cmd/setup_dicts.go b/ossfuzz/cmd/setup_dicts.go new file mode 100644 index 0000000000..0183f29966 --- /dev/null +++ b/ossfuzz/cmd/setup_dicts.go @@ -0,0 +1,100 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "flag" + "fmt" + "io" + "os" + "strings" + "text/template" + + "github.com/klauspost/compress/zip" +) + +var ( + dictPath = flag.String("dict-path", "", "dict path") + outputFile = flag.String("output-file", "", "output file") +) + +func main() { + flag.Parse() + if *dictPath == "" { + panic("Need a dict path") + } + if *outputFile == "" { + panic("Need an output file") + } + dicts := getFuzzDicts(*dictPath) + + t, err := template.New("todos").Parse(` +package zstd +var fuzzDicts = make([][]byte, 0) +func init() { +{{range $val := .}} + fuzzDicts = append(fuzzDicts, {{$val}}) +{{end}} +} +`) + if err != nil { + panic(err) + } + f, err := os.Create(*outputFile) + err = t.Execute(f, dicts) + if err != nil { + panic(err) + } + f.Close() +} + +func getFuzzDicts(path string) []string { + data, err := os.ReadFile(path) + if err != nil { + panic(err) + } + zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + panic(err) + } + var dicts [][]byte + for _, tt := range zr.File { + if !strings.HasSuffix(tt.Name, ".dict") { + continue + } + func() { + r, err := tt.Open() + if err != nil { + panic(err) + } + defer r.Close() + in, err := io.ReadAll(r) + if err != nil { + panic(err) + } + dicts = append(dicts, in) + }() + } + stringDicts := make([]string, 0) + for _, d := range dicts { + stringedArray := fmt.Sprintf("%v", d) + withComma := strings.Replace(stringedArray, " ", ", ", -1) + withClosingBracket := strings.Replace(withComma, "]", "}", -1) + withOpenBracket := strings.Replace(withClosingBracket, "[", "[]byte{", -1) + stringDicts = append(stringDicts, withOpenBracket) + } + return stringDicts +} diff --git a/ossfuzz/ossfuzz.sh b/ossfuzz/ossfuzz.sh new file mode 100755 index 0000000000..7df263b8c7 --- /dev/null +++ b/ossfuzz/ossfuzz.sh @@ -0,0 +1,96 @@ +#!/bin/bash -eu +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +# This script is meant to be run by +# https://github.com/google/oss-fuzz/tree/master/projects/compress + +# In one of the Zstd fuzzers, the "dict" variable is created by reading a series of files. +# These files are not available at runtime in the OSS-FUzz environment, +# so we add the contents of these files to a variable and create a new file +# that is included when we build the fuzzers in OSS-Fuzz. +mkdir $SRC/setupdicts +cp $SRC/compress/ossfuzz/cmd/setup_dicts.go $SRC/setupdicts/main.go +cd $SRC/setupdicts +go mod init setupdicts +go mod tidy +go run main.go --dict-path=$SRC/compress/zstd/testdata/dict-tests-small.zip --output-file=$SRC/compress/zstd/fuzzDicts.go +cp $SRC/compress/zstd/fuzzDicts.go $OUT/ +# Done creating "dicts" variable. + +cd $SRC/compress + +# Modify some files. This would be better done upstream. +sed -i '38 a\ + if fi == nil { return }' $SRC/compress/internal/fuzz/helpers.go +printf "package compress\nimport _ \"github.com/AdamKorcz/go-118-fuzz-build/testing\"\n" > registerfuzzdependency.go +sed -i 's/zr := testCreateZipReader/\/\/zr := testCreateZipReader/g' "${SRC}"/compress/zstd/fuzz_test.go +sed -i 's/dicts = readDicts(f, zr)/dicts = fuzzDicts/g' "${SRC}"/compress/zstd/fuzz_test.go + +if [ "$SANITIZER" != "coverage" ]; then + sed -i 's/\"testing\"/\"github.com\/AdamKorcz\/go-118-fuzz-build\/testing\"/g' "${SRC}"/compress/internal/fuzz/helpers.go +fi + +# OSS-Fuzz uses 'go build' to build the fuzzers, so we move the tests +# we need into scope. +mv $SRC/compress/zstd/decoder_test.go $SRC/compress/zstd/decoder_test_fuzz.go +mv $SRC/compress/zstd/zstd_test.go $SRC/compress/zstd/zstd_test_fuzz.go +mv $SRC/compress/zstd/seqdec_test.go $SRC/compress/zstd/seqdec_test_fuzz.go +mv $SRC/compress/zstd/dict_test.go $SRC/compress/zstd/dict_test_fuzz.go +mv $SRC/compress/s2/s2_test.go $SRC/compress/s2/s2_test_fuzz.go +go mod tidy + +# Build fuzzers +compile_native_go_fuzzer $SRC/compress/flate FuzzEncoding FuzzFlateEncoding +compile_native_go_fuzzer $SRC/compress/zstd FuzzDecodeAll FuzzDecodeAll +compile_native_go_fuzzer $SRC/compress/zstd FuzzDecoder FuzzDecoder +compile_native_go_fuzzer $SRC/compress/zstd FuzzEncoding FuzzZstdEncoding +compile_native_go_fuzzer $SRC/compress/zstd FuzzDecAllNoBMI2 FuzzDecAllNoBMI2 +compile_native_go_fuzzer $SRC/compress/zstd FuzzNoBMI2Dec FuzzNoBMI2Dec +compile_native_go_fuzzer $SRC/compress/s2 FuzzDictBlocks FuzzDictBlocks +compile_native_go_fuzzer $SRC/compress/s2 FuzzEncodingBlocks FuzzEncodingBlocks +compile_native_go_fuzzer $SRC/compress/zip FuzzReader FuzzReader +compile_native_go_fuzzer $SRC/compress/snappy/xerial FuzzDecode FuzzDecode +compile_native_go_fuzzer $SRC/compress/fse FuzzCompress FuzzFSECompress +compile_native_go_fuzzer $SRC/compress/fse FuzzDecompress FuzzFSEDecompress +compile_native_go_fuzzer $SRC/compress/huff0 FuzzCompress FuzzHuff0Compress +compile_native_go_fuzzer $SRC/compress/huff0 FuzzDecompress1x FuzzHuff0Decompress1x +compile_native_go_fuzzer $SRC/compress/snappy/xerial FuzzEncode FuzzEncode + +#Add corpora from compress-fuzz dir +cp $SRC/compress-fuzz/zstd/compress/fuzz/encode-corpus-raw.zip $OUT/FuzzZstdEncoding_seed_corpus.zip +cp $SRC/compress-fuzz/zstd/decompress/fuzz/decode-corpus-raw.zip $OUT/FuzzDecodeAll_seed_corpus.zip +cp $SRC/compress-fuzz/zstd/decompress/fuzz/decode-corpus-raw.zip $OUT/FuzzDecoder_seed_corpus.zip +cp $SRC/compress-fuzz/zip/fuzz/FuzzReader-raw.zip $OUT/FuzzReader_seed_corpus.zip +cp $SRC/compress-fuzz/fse/compress/fuzz/fse_compress.zip $OUT/FuzzFSECompress_seed_corpus.zip +cp $SRC/compress-fuzz/fse/decompress/fuzz/fse_decompress.zip $OUT/FuzzFSEDecompress_seed_corpus.zip +cp $SRC/compress-fuzz/huff0/compress/fuzz/huff0_compress.zip $OUT/FuzzHuff0Compress_seed_corpus.zip +cp $SRC/compress-fuzz/huff0/decompress/fuzz/huff0_decompress1x.zip $OUT/FuzzHuff0Decompress1x_seed_corpus.zip +cp $SRC/compress-fuzz/flate/flate/fuzz/encode-raw-corpus.zip $OUT/FuzzFlateEncoding_seed_corpus.zip +cp $SRC/compress-fuzz/s2/compress/fuzz/block-corpus-raw.zip $OUT/FuzzDictBlocks_seed_corpus.zip +cp $SRC/compress-fuzz/s2/compress/fuzz/block-corpus-raw.zip $OUT/FuzzEncodingBlocks_seed_corpus.zip +cp $SRC/compress-fuzz/snappy/fuzz/FuzzDecode_raw.zip $OUT/FuzzDecode_seed_corpus.zip +cp $SRC/compress-fuzz/snappy/fuzz/block-corpus-raw.zip $OUT/FuzzEncode_seed_corpus.zip + +# Add missing test files to avoid errors and have test files for code coverage +cp -r $SRC/compress/zstd/testdata $OUT/ +cp -r $SRC/compress/zip/testdata $OUT/ +cp -r $SRC/compress/flate/testdata $OUT/ +cp -r $SRC/compress/s2/testdata $OUT/ +cp -r $SRC/compress/fse/testdata $OUT/ +cp -r $SRC/compress/snappy/xerial/testdata $OUT/ +cp -r $SRC/compress/huff0/testdata $OUT/ +cp -r $SRC/compress/testdata $OUT \ No newline at end of file diff --git a/s2/fuzz_test.go b/s2/fuzz_test.go index 1cc6c853bc..4acc301e6c 100644 --- a/s2/fuzz_test.go +++ b/s2/fuzz_test.go @@ -126,5 +126,25 @@ func FuzzEncodingBlocks(f *testing.F) { t.Error(fmt.Errorf("MaxEncodedLen Exceed: input: %d, mel: %d, got %d", len(data), mel, len(comp))) return } + + concat, err := ConcatBlocks(nil, data, []byte{0}) + if err != nil || concat == nil { + return + } + + EstimateBlockSize(data) + encoded := make([]byte, MaxEncodedLen(len(data))) + if len(encoded) < MaxEncodedLen(len(data)) || minNonLiteralBlockSize > len(data) || len(data) > maxBlockSize { + return + } + + encodeBlockGo(encoded, data) + encodeBlockBetterGo(encoded, data) + encodeBlockSnappyGo(encoded, data) + encodeBlockBetterSnappyGo(encoded, data) + dst := encodeGo(encoded, data) + if dst == nil { + return + } }) } diff --git a/snappy/xerial/fuzz_test.go b/snappy/xerial/fuzz_test.go index b03e5bb455..ff40443808 100644 --- a/snappy/xerial/fuzz_test.go +++ b/snappy/xerial/fuzz_test.go @@ -21,44 +21,37 @@ func FuzzDecode(f *testing.F) { } func FuzzEncode(f *testing.F) { - fuzz.AddFromZip(f, "../../s2/testdata/enc_regressions.zip", fuzz.TypeRaw, false) - fuzz.AddFromZip(f, "../../s2/testdata/fuzz/block-corpus-raw.zip", fuzz.TypeRaw, testing.Short()) - fuzz.AddFromZip(f, "../../s2/testdata/fuzz/block-corpus-enc.zip", fuzz.TypeGoFuzz, testing.Short()) + fuzz.AddFromZip(f, "testdata/block-corpus-raw.zip", fuzz.TypeRaw, false) f.Fuzz(func(t *testing.T, data []byte) { - t.Run("standard", func(t *testing.T) { - encoded := Encode(make([]byte, 0, len(data)/2), data) - decoded, err := Decode(encoded) - if err != nil { - t.Errorf("input: %+v, encoded: %+v", data, encoded) - t.Fatal(err) - } - if !bytes.Equal(decoded, data) { - t.Fatal("mismatch") - } + encoded := Encode(make([]byte, 0, len(data)/2), data) + decoded, err := Decode(encoded) + if err != nil { + t.Errorf("input: %+v, encoded: %+v", data, encoded) + t.Fatal(err) + } + if !bytes.Equal(decoded, data) { + t.Fatal("mismatch") + } - }) - t.Run("better", func(t *testing.T) { - encoded := EncodeBetter(make([]byte, 0, len(data)/2), data) - decoded, err := Decode(encoded) - if err != nil { - t.Errorf("input: %+v, encoded: %+v", data, encoded) - t.Fatal(err) - } - if !bytes.Equal(decoded, data) { - t.Fatal("mismatch") - } - }) - t.Run("snappy", func(t *testing.T) { - encoded := s2.EncodeSnappy(make([]byte, 0, len(data)/2), data) - decoded, err := Decode(encoded) - if err != nil { - t.Errorf("input: %+v, encoded: %+v", data, encoded) - t.Fatal(err) - } - if !bytes.Equal(decoded, data) { - t.Fatal("mismatch") - } - }) + encoded = EncodeBetter(make([]byte, 0, len(data)/2), data) + decoded, err = Decode(encoded) + if err != nil { + t.Errorf("input: %+v, encoded: %+v", data, encoded) + t.Fatal(err) + } + if !bytes.Equal(decoded, data) { + t.Fatal("mismatch") + } + + encoded = s2.EncodeSnappy(make([]byte, 0, len(data)/2), data) + decoded, err = Decode(encoded) + if err != nil { + t.Errorf("input: %+v, encoded: %+v", data, encoded) + t.Fatal(err) + } + if !bytes.Equal(decoded, data) { + t.Fatal("mismatch") + } }) } diff --git a/snappy/xerial/testdata/FuzzDecode_raw.zip b/snappy/xerial/testdata/FuzzDecode_raw.zip new file mode 100644 index 0000000000..ab97b47d3c Binary files /dev/null and b/snappy/xerial/testdata/FuzzDecode_raw.zip differ diff --git a/snappy/xerial/testdata/block-corpus-raw.zip b/snappy/xerial/testdata/block-corpus-raw.zip new file mode 100644 index 0000000000..11af6e0495 Binary files /dev/null and b/snappy/xerial/testdata/block-corpus-raw.zip differ