Refactor regression tests

.github/workflows/ci.yml

@@ -1,5 +1,10 @@
 name: CI
 
+defaults:
+  run:
+    # To load bashrc
+    shell: bash -ieo pipefail {0}
+
 on:
   push:
     branches:
@@ -15,7 +20,23 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        type: ["4", "5", "cli", "data_dependency", "embark", "erc", "etherlime", "find_paths", "kspec", "printers", "simil", "slither_config", "truffle", "upgradability", "prop"]
+        type: ["detectors_4", 
+               "detectors_5", 
+               "detectors_6", 
+               "detectors_7", 
+               "cli", 
+               "data_dependency", 
+               "embark", 
+               "erc", 
+               "etherlime", 
+               "find_paths", 
+               "kspec", 
+               "printers", 
+               "simil", 
+               "slither_config", 
+               "truffle", 
+               "upgradability", 
+               "prop"]
     steps:
     - uses: actions/checkout@v1
     - name: Set up Python 3.6
@@ -25,19 +46,18 @@ jobs:
     - name: Install dependencies
       run: |
         python setup.py install
-        # Used by travis_test.sh
+        # Used by ci_test.sh
         pip install deepdiff
 
-        sudo wget -O /usr/bin/solc-0.4.25 https://github.com/ethereum/solidity/releases/download/v0.4.25/solc-static-linux
-        sudo chmod +x /usr/bin/solc-0.4.25
-        sudo wget -O /usr/bin/solc-0.5.1 https://github.com/ethereum/solidity/releases/download/v0.5.1/solc-static-linux
-        sudo chmod +x /usr/bin/solc-0.5.1
-        sudo wget -O /usr/bin/solc-0.5.0 https://github.com/ethereum/solidity/releases/download/v0.5.0/solc-static-linux
-        sudo chmod +x /usr/bin/solc-0.5.0
-        sudo cp /usr/bin/solc-0.5.1 /usr/bin/solc
+        git clone https://github.com/crytic/solc-select.git
+        ./solc-select/scripts/install.sh
+        export PATH=/home/runner/.solc-select:$PATH
+        echo "export PATH=/home/runner/.solc-select:$PATH" >> ~/.bashrc
+        solc use 0.5.1
+
     - name: Run Tests
       env:
         TEST_TYPE: ${{ matrix.type }}
         GITHUB_ETHERSCAN: ${{ secrets.GITHUB_ETHERSCAN }}
       run: |
-        bash scripts/travis_test_${TEST_TYPE}.sh
+        bash scripts/ci_test_${TEST_TYPE}.sh

CONTRIBUTING.md

@@ -22,3 +22,16 @@ Some pull request guidelines:
 
 ## Development Environment
 Instructions for installing a development version of Slither can be found in our [wiki](https://github.com/crytic/slither/wiki/Developer-installation).
+
+## Detectors regression tests
+
+For each new detector, at least one regression tests must be present. 
+To generate the following scripts, you must have [`solc-select`](https://github.com/crytic/solc-select) installed.
+
+- Create a test in `tests`
+- Update `script/ci_test_detectors_[solc_version].sh`, and add `generate_expected_json tests/YOUR_FILENAME.sol "DETECTOR_NAME"`. Be sure that all the other lines are commented (otherwise you will regenerate the tests for all the detectores)
+- Run `./script/ci_test_detectors_[solc_version].sh`. This will generate the json artifacts in `tests/expected_json`. Add the generated files to git.
+- Update `scripts/ci_test_detectors_[solc_version].sh` with your new tests.
+- Run `scripts/ci_test_detectors_[solc_version].sh` and check that everything worked.
+
+

scripts/tests_generate_expected_json_4.sh → scripts/ci_generate_test_detectors_4.sh

@@ -1,24 +1,10 @@
 #!/usr/bin/env bash
 
-DIR="$(cd "$(dirname "$0")" && pwd)"
-CURRENT_PATH=$(pwd)
-TRAVIS_PATH='/home/travis/build/crytic/slither'
+source "$(dirname "$0")""/ci_test.sh"
 
-# generate_expected_json file.sol detectors
-generate_expected_json(){
-    # generate output filename
-    # e.g. file: uninitialized.sol detector: uninitialized-state
-    # ---> uninitialized.uninitialized-state.json
-    output_filename="$DIR/../tests/expected_json/$(basename "$1" .sol).$2.json"
-    output_filename_txt="$DIR/../tests/expected_json/$(basename "$1" .sol).$2.txt"
+solc use "0.4.25"
 
-    # run slither detector on input file and save output as json
-    slither "$1" --solc-disable-warnings --detect "$2" --json "$output_filename" --solc solc-0.4.25 > "$output_filename_txt" 2>&1
-
-
-    sed "s|$CURRENT_PATH|$TRAVIS_PATH|g" "$output_filename" -i
-    sed "s|$CURRENT_PATH|$TRAVIS_PATH|g" "$output_filename_txt" -i
-}
+# Be sure that only one of the following line is uncommented before running the script
 
 
 #generate_expected_json tests/deprecated_calls.sol "deprecated-standards"
@@ -58,3 +44,4 @@ generate_expected_json(){
 #generate_expected_json tests/solc_version_incorrect.sol "solc-version"
 #generate_expected_json tests/right_to_left_override.sol "rtlo"
 #generate_expected_json tests/unchecked_lowlevel.sol "unchecked-lowlevel"
+

scripts/ci_generate_test_detectors_5.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+source "$(dirname "$0")""/ci_test.sh"
+
+solc use "0.5.1"
+
+# Be sure that only one of the following line is uncommented before running the script
+
+# generate_expected_json tests/void-cst.sol "void-cst"
+# generate_expected_json tests/solc_version_incorrect_05.ast.json "solc-version"
+# generate_expected_json tests/uninitialized-0.5.1.sol "uninitialized-state"
+# generate_expected_json tests/backdoor.sol "backdoor"
+# generate_expected_json tests/backdoor.sol "suicidal"
+# generate_expected_json tests/old_solc.sol.json "solc-version"
+# generate_expected_json tests/reentrancy-0.5.1.sol "reentrancy-eth"
+# generate_expected_json tests/reentrancy-0.5.1-events.sol "reentrancy-events"
+# generate_expected_json tests/tx_origin-0.5.1.sol "tx-origin"
+# generate_expected_json tests/locked_ether-0.5.1.sol "locked-ether"
+# generate_expected_json tests/arbitrary_send-0.5.1.sol "arbitrary-send"
+# generate_expected_json tests/inline_assembly_contract-0.5.1.sol "assembly"
+# generate_expected_json tests/inline_assembly_library-0.5.1.sol "assembly"
+# generate_expected_json tests/constant-0.5.1.sol "constant-function-asm"
+# generate_expected_json tests/constant-0.5.1.sol "constant-function-state"
+# generate_expected_json tests/incorrect_equality.sol "incorrect-equality"
+# generate_expected_json tests/too_many_digits.sol "too-many-digits"
+# generate_expected_json tests/unchecked_lowlevel-0.5.1.sol "unchecked-lowlevel"
+# generate_expected_json tests/unchecked_send-0.5.1.sol "unchecked-send"
+

scripts/ci_generate_test_detectors_6.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+source "$(dirname "$0")""/ci_test.sh"
+
+solc use "0.6.11"
+
+# Be sure that only one of the following line is uncommented before running the script
+
+
+#generate_expected_json tests/filename.sol "detector_name"
+

scripts/ci_generate_test_detectors_7.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+source "$(dirname "$0")""/ci_test.sh"
+
+solc use "0.7.0"
+
+# Be sure that only one of the following line is uncommented before running the script
+
+
+#generate_expected_json tests/filename.sol "detector_name"
+

scripts/ci_test.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+
+### Test Detectors
+
+DIR="$(cd "$(dirname "$0")" && pwd)"
+
+CURRENT_PATH=$(pwd)
+TRAVIS_PATH='/home/travis/build/crytic/slither'
+
+# test_slither file.sol detectors
+test_slither(){
+
+    expected="$DIR/../tests/expected_json/$(basename "$1" .sol).$2.json"
+
+    # run slither detector on input file and save output as json
+    slither "$1" --solc-disable-warnings --detect "$2" --json "$DIR/tmp-test.json" 
+    if [ $? -eq 255 ]
+    then
+        echo "Slither crashed"
+        exit 255
+    fi
+
+    if [ ! -f "$DIR/tmp-test.json" ]; then
+        echo ""
+        echo "Missing generated file"
+        echo ""
+        exit 1
+    fi
+    sed "s|$CURRENT_PATH|$TRAVIS_PATH|g" "$DIR/tmp-test.json" -i
+    result=$(python "$DIR/json_diff.py" "$expected" "$DIR/tmp-test.json")
+
+    rm "$DIR/tmp-test.json"
+    if [ "$result" != "{}" ]; then
+      echo ""
+      echo "failed test of file: $1, detector: $2"
+      echo ""
+      echo "$result"
+      echo ""
+      exit 1
+    fi
+
+    # run slither detector on input file and save output as json
+    slither "$1" --solc-disable-warnings --detect "$2" --legacy-ast --json "$DIR/tmp-test.json" 
+    if [ $? -eq 255 ]
+    then
+        echo "Slither crashed"
+        exit 255
+    fi
+
+    if [ ! -f "$DIR/tmp-test.json" ]; then
+        echo ""
+        echo "Missing generated file"
+        echo ""
+        exit 1
+    fi
+
+    sed "s|$CURRENT_PATH|$TRAVIS_PATH|g" "$DIR/tmp-test.json" -i
+    result=$(python "$DIR/json_diff.py" "$expected" "$DIR/tmp-test.json")
+
+    rm "$DIR/tmp-test.json"
+    if [ "$result" != "{}" ]; then
+      echo ""
+      echo "failed test of file: $1, detector: $2"
+      echo ""
+      echo "$result"
+      echo ""
+      exit 1
+    fi
+}
+
+# generate_expected_json file.sol detectors
+generate_expected_json(){
+    # generate output filename
+    # e.g. file: uninitialized.sol detector: uninitialized-state
+    # ---> uninitialized.uninitialized-state.json
+    output_filename="$DIR/../tests/expected_json/$(basename "$1" .sol).$2.json"
+    output_filename_txt="$DIR/../tests/expected_json/$(basename "$1" .sol).$2.txt"
+
+    # run slither detector on input file and save output as json
+    slither "$1" --solc-disable-warnings --detect "$2" --json "$output_filename" > "$output_filename_txt" 2>&1
+
+
+    sed "s|$CURRENT_PATH|$TRAVIS_PATH|g" "$output_filename" -i
+    sed "s|$CURRENT_PATH|$TRAVIS_PATH|g" "$output_filename_txt" -i
+}
+

scripts/travis_test_4.sh → scripts/ci_test_detectors_4.sh

@@ -1,73 +1,8 @@
 #!/usr/bin/env bash
 
-### Test Detectors
-
-DIR="$(cd "$(dirname "$0")" && pwd)"
-
-CURRENT_PATH=$(pwd)
-TRAVIS_PATH='/home/travis/build/crytic/slither'
-# test_slither file.sol detectors
-test_slither(){
-
-    expected="$DIR/../tests/expected_json/$(basename "$1" .sol).$2.json"
-
-    # run slither detector on input file and save output as json
-    slither "$1" --solc-disable-warnings --detect "$2" --json "$DIR/tmp-test.json" --solc solc-0.4.25
-    if [ $? -eq 255 ]
-    then
-        echo "Slither crashed"
-        exit 255
-    fi
-
-    if [ ! -f "$DIR/tmp-test.json" ]; then
-        echo ""
-        echo "Missing generated file"
-        echo ""
-        exit 1
-    fi
-
-    sed "s|$CURRENT_PATH|$TRAVIS_PATH|g" "$DIR/tmp-test.json" -i
-    result=$(python "$DIR/json_diff.py" "$expected" "$DIR/tmp-test.json")
-
-    rm "$DIR/tmp-test.json"
-    if [ "$result" != "{}" ]; then
-      echo ""
-      echo "failed test of file: $1, detector: $2"
-      echo ""
-      echo "$result"
-      echo ""
-      exit 1
-    fi
-
-    # run slither detector on input file and save output as json
-    slither "$1" --solc-disable-warnings --detect "$2" --legacy-ast --json "$DIR/tmp-test.json" --solc solc-0.4.25
-    if [ $? -eq 255 ]
-    then
-        echo "Slither crashed"
-        exit 255
-    fi
-
-    if [ ! -f "$DIR/tmp-test.json" ]; then
-        echo ""
-        echo "Missing generated file"
-        echo ""
-        exit 1
-    fi
-
-    sed "s|$CURRENT_PATH|$TRAVIS_PATH|g" "$DIR/tmp-test.json" -i
-    result=$(python "$DIR/json_diff.py" "$expected" "$DIR/tmp-test.json")
-
-    rm "$DIR/tmp-test.json"
-    if [ "$result" != "{}" ]; then
-      echo ""
-      echo "failed test of file: $1, detector: $2"
-      echo ""
-      echo "$result"
-      echo ""
-      exit 1
-    fi
-}
+source "$(dirname "$0")""/ci_test.sh"
 
+solc use "0.4.25"
 
 test_slither tests/unchecked_lowlevel.sol "unchecked-lowlevel"
 test_slither tests/deprecated_calls.sol "deprecated-standards"
@@ -92,7 +27,6 @@ test_slither tests/const_state_variables.sol "constable-states"
 test_slither tests/external_function.sol "external-function"
 test_slither tests/external_function_2.sol "external-function"
 test_slither tests/naming_convention.sol "naming-convention"
-#test_slither tests/complex_func.sol "complex-function"
 test_slither tests/controlled_delegatecall.sol "controlled-delegatecall"
 test_slither tests/uninitialized_local_variable.sol "uninitialized-local"
 test_slither tests/constant.sol "constant-function-asm"