diff --git a/.github/workflows/dev-long-tests.yml b/.github/workflows/dev-long-tests.yml new file mode 100644 index 00000000000..773be83fad6 --- /dev/null +++ b/.github/workflows/dev-long-tests.yml @@ -0,0 +1,165 @@ +name: dev-long-tests +# Tests longer than 10mn + +concurrency: + group: long-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + branches: [ dev, release, actionsTest ] + +jobs: + make-test: + runs-on: ubuntu-latest + env: + DEVNULLRIGHTS: 1 + READFROMBLOCKDEVICE: 1 + steps: + - uses: actions/checkout@v2 + - name: make test + run: make test + + make-test-osx: + runs-on: macos-latest + steps: + - uses: actions/checkout@v2 + - name: OS-X test + run: make test # make -c lib all doesn't work because of the fact that it's not a tty + + tsan-zstreamtest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: thread sanitizer zstreamtest + run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream + + tsan-fuzztest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: thread sanitizer fuzztest + run: CC=clang make tsan-fuzztest + + gcc-8-asan-ubsan-testzstd: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: gcc-8 + ASan + UBSan + Test Zstd + run: | + make gcc8install + CC=gcc-8 make -j uasan-test-zstd + msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 + /t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}} + # For reference : icc tests # icc tests are currently failing on Github Actions, likely to issues during installation stage # To be fixed later diff --git a/.github/workflows/generic-release.yml b/.github/workflows/generic-release.yml deleted file mode 100644 index 1e47bba4d5c..00000000000 --- a/.github/workflows/generic-release.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: generic-release - -on: - pull_request: - # This will eventually only be for pushes to release - # but for dogfooding purposes, I'm running it even - # on dev pushes - branches: [ dev, release, actionsTest ] - -jobs: - # missing jobs - # - # ppc64le + fuzz test - # Qemu PPC64 + Fuzz test - # Qemu aarch64 + Fuzz Test (on Xenial) - # versions comp - # meson test - - osx: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - name: OS-X - run: | - make test - # make -c lib all (need to fix. not working right now) - - tsan: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: thread sanitizer - run: | - CC=clang make tsan-test-zstream - CC=clang make tsan-fuzztest - - zlib-wrapper: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: zlib wrapper test - run: | - make valgrindinstall - make -C zlibWrapper test - make -C zlibWrapper valgrindTest - - lz4-threadpool-partial-libs: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: LZ4, thread pool, and partial libs testslib wrapper test - run: | - make lz4install - make -C tests test-lz4 - make check < /dev/null | tee # mess with lz4 console detection - make clean - make -C tests test-pool - make clean - bash tests/libzstd_partial_builds.sh diff --git a/.github/workflows/linux-kernel.yml b/.github/workflows/linux-kernel.yml deleted file mode 100644 index 124f7777828..00000000000 --- a/.github/workflows/linux-kernel.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: linux-kernel - -on: - pull_request: - branches: [ dev, release, actionsTest ] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Create linux kernel library + build + test - run: make -C contrib/linux-kernel test CFLAGS="-Werror -Wunused-const-variable -Wunused-but-set-variable" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 5e5aae11245..00000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: CIFuzz -on: [pull_request] -jobs: - Fuzzing: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - sanitizer: [address, undefined] - steps: - - name: Build Fuzzers (${{ matrix.sanitizer }}) - id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master - with: - oss-fuzz-project-name: 'zstd' - dry-run: false - sanitizer: ${{ matrix.sanitizer }} - - name: Run Fuzzers (${{ matrix.sanitizer }}) - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master - with: - oss-fuzz-project-name: 'zstd' - fuzz-seconds: 600 - dry-run: false - sanitizer: ${{ matrix.sanitizer }} - - name: Upload Crash - uses: actions/upload-artifact@v1 - if: failure() && steps.build.outcome == 'success' - with: - name: ${{ matrix.sanitizer }}-artifacts - path: ./out/artifacts diff --git a/.travis.yml b/.travis.yml index 16d1d42819d..57db8f05274 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,6 @@ -# Medium Tests: Run on all commits/PRs to dev branch - +# Travis CI is used to test platforms that github-actions currently doesn't support +# without either self-hosting or some finnicky work-around. Also, some tests +# are troublesome to migrate since GH Actions runs tests not in a tty. language: c git: @@ -18,11 +19,10 @@ addons: env: global: - - FUZZERTEST=-T2mn - ZSTREAM_TESTTIME=-T2mn + - FUZZERTEST=-T1mn + ZSTREAM_TESTTIME=-T1mn DECODECORPUS_TESTTIME=-T1mn - matrix: fast_finish: true include: @@ -31,7 +31,8 @@ matrix: arch: arm64 script: - make check - + + # TODO: migrate to GH actions once warnings are fixed - name: Minimal Decompressor Macros # ~5mn script: - make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror" @@ -43,17 +44,17 @@ matrix: - make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS" - make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS" + # TODO: migrate to GH actions once newest clang staticanalyze warnings are fixed - name: static analyzer scanbuild # ~26mn dist: trusty # note : it's important to pin down a version of static analyzer, since different versions report different false positives script: - make staticAnalyze - - - name: Valgrind + Fuzz Test Stack Mode # ~ 7mn + + # GH actions can't run this command on OS-X, non-tty issues + - name: OS-X make all lib + os: osx script: - - make valgrindinstall - - make -C tests clean valgrindTest - - make clean - - make -C tests test-fuzzer-stackmode + - make -C lib all # Introduced to check compat with old toolchains, to prevent e.g. #1872 - name: ARM Build Test (on Trusty) @@ -68,58 +69,41 @@ matrix: - make ppcinstall - make ppcfuzz - # check release number (release only) + # check release number (release/new tag only) - name: Tag-Specific Test if: tag =~ ^v[0-9]\.[0-9] script: - make -C tests checkTag - tests/checkTag "$TRAVIS_BRANCH" - # tests for release branch and cron job only - - name: OS-X # ~13mn - if: branch = release - os: osx - script: - - make test - - make -C lib all - - - name: Versions Compatibility Test # 11.5mn - if: branch = release - script: - - make -C tests versionsTest - - - name: thread sanitizer # ~29mn - if: branch = release - script: - - make clang38install - - CC=clang-3.8 make tsan-test-zstream - - CC=clang-3.8 make tsan-fuzztest - - name: PPC64LE + Fuzz test # ~13mn - if: branch = release arch: ppc64le + env: + - FUZZER_FLAGS=--no-big-tests + - MOREFLAGS="-static" script: - cat /proc/cpuinfo - - make test + - make -C tests fuzztest - name: Qemu PPC64 + Fuzz test # ~13mn, presumed Big-Endian (?) dist: trusty # note : PPC64 cross-compilation for Qemu tests seems broken on Xenial - if: branch = release script: - make ppcinstall - make ppc64fuzz # note : we already have aarch64 tests on hardware - name: Qemu aarch64 + Fuzz Test (on Xenial) # ~14mn - if: branch = release dist: xenial script: - make arminstall - make aarch64fuzz + + - name: Versions Compatibility Test # 11.5mn + script: + - make -C tests versionsTest # meson dedicated test - name: Xenial (Meson + clang) # ~15mn - if: branch = release dist: bionic language: cpp compiler: clang diff --git a/appveyor.yml b/appveyor.yml index c6ab786889e..a7eeb3f1f6a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -151,16 +151,6 @@ DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe && MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe && COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe && - ECHO *** && - ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% && - ECHO *** && - msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && - DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe && - MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe && - msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && - DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe && - MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe && - COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\ ) - if [%HOST%]==[cmake-visual] ( @@ -224,23 +214,6 @@ PLATFORM: "x64" SCRIPT: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1" - - COMPILER: "visual" - HOST: "visual" - PLATFORM: "x64" - CONFIGURATION: "Debug" - - COMPILER: "visual" - HOST: "visual" - PLATFORM: "Win32" - CONFIGURATION: "Debug" - - COMPILER: "visual" - HOST: "visual" - PLATFORM: "x64" - CONFIGURATION: "Release" - - COMPILER: "visual" - HOST: "visual" - PLATFORM: "Win32" - CONFIGURATION: "Release" - - COMPILER: "clang-cl" HOST: "cmake-visual" PLATFORM: "x64" @@ -266,9 +239,6 @@ COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe && COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe ) - - IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] ( - SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;" - ) build_script: - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION% @@ -294,19 +264,6 @@ set "CC=%COMPILER%" && sh -c "%SCRIPT%" ) - - if [%HOST%]==[visual] ( - ECHO *** && - ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% && - ECHO *** && - msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && - DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe && - MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe && - msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && - DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe && - MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe && - COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && - COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\ - ) - if [%HOST%]==[cmake-visual] ( ECHO *** && ECHO *** Building %CMAKE_GENERATOR% ^(%CMAKE_GENERATOR_TOOLSET%^) %PLATFORM%\%CONFIGURATION% &&