diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f33c274b5..b0e52e73f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Setup Build Env + - name: Install Build Tools run: sudo ./scripts/install-build-tools.sh - name: Setup Local Dependencies run: ./scripts/setup-dependencies.sh @@ -38,7 +38,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Setup Build Env + - name: Install Build Tools run: sudo ./scripts/install-build-tools.sh - name: Setup Local Dependencies run: ./scripts/setup-dependencies.sh @@ -50,7 +50,7 @@ jobs: name: Pylint runs-on: ubuntu-22.04 continue-on-error: true - timeout-minutes: 10 + timeout-minutes: 5 strategy: matrix: python-version: ["3.10"] @@ -62,10 +62,23 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Setup Build Env + - name: Install Build Tools run: sudo ./scripts/install-build-tools.sh - name: Lint with Pylint run: ./scripts/pylint.sh + shellcheck: + name: Shellcheck + runs-on: ubuntu-22.04 + continue-on-error: true + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install Build Tools + run: sudo ./scripts/install-build-tools.sh + - name: Lint with Shellcheck + run: ./scripts/shellcheck.sh view unit-and-integration-test: name: Unit and Integration Tests runs-on: ubuntu-22.04 @@ -74,7 +87,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Setup Build Env + - name: Install Build Tools run: sudo ./scripts/install-build-tools.sh - name: Setup Local Dependencies run: ./scripts/setup-dependencies.sh @@ -84,7 +97,7 @@ jobs: run: ./scripts/test.sh - name: Shorten SHA id: vars - run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_ENV - uses: actions/upload-artifact@v4 if: ${{ !env.ACT }} name: Archive Test Results @@ -114,4 +127,3 @@ jobs: name: OpenCBDC Transaction Processor docs for ${{ steps.vars.outputs.sha_short }} path: ./doxygen_generated/html/* retention-days: 7 - diff --git a/scripts/create-e2e-report.sh b/scripts/create-e2e-report.sh index 51214a4d0..456097b1b 100755 --- a/scripts/create-e2e-report.sh +++ b/scripts/create-e2e-report.sh @@ -11,7 +11,7 @@ function readAndFormatLogs() { return fi - for logfile in $(ls $logdir); do + for logfile in "$logdir"/*; do logfile_path="$logdir/$logfile" logfile_content=$(cat $logfile_path) message+="\n
\n$logfile\n\n\`\`\`\n$logfile_content\n\`\`\`\n
\n" diff --git a/scripts/install-build-tools.sh b/scripts/install-build-tools.sh index 523789d99..197ab904b 100755 --- a/scripts/install-build-tools.sh +++ b/scripts/install-build-tools.sh @@ -17,7 +17,7 @@ fi # Supporting these versions for buildflow PYTHON_VERSIONS=("3.10" "3.11" "3.12") -echo "Python3 versions supported: ${PYTHON_VERSIONS[@]}" +echo "Python3 versions supported: ${PYTHON_VERSIONS[*]}" # check if supported version of python3 is already installed, and save the version PY_INSTALLED='' diff --git a/scripts/lint.sh b/scripts/lint.sh index c6e4b0936..2197ab050 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -25,9 +25,12 @@ if [ -n "$whitespace_files" ] || [ -n "$newline_files" ] ; then exit 1 fi -check_format_files=$(git ls-files | grep -E "tools|tests|src|cmake-tests" \ - | grep -E "\..*pp") -clang-format --style=file --Werror --dry-run ${check_format_files[@]} +check_format_files=$(git ls-files | \ + grep -E "tools|tests|src|cmake-tests" | \ + grep -E "\..*pp") + +echo "${check_format_files}" | \ + xargs -n1 -I{} clang-format --style=file --Werror --dry-run {} if ! command -v clang-tidy &>/dev/null; then echo "clang-tidy does not appear to be installed" diff --git a/scripts/native-system-benchmark.sh b/scripts/native-system-benchmark.sh index 1b7923fa2..f295f2861 100755 --- a/scripts/native-system-benchmark.sh +++ b/scripts/native-system-benchmark.sh @@ -151,7 +151,7 @@ on_int() { printf 'Interrupting all components\n' trap '' SIGINT # avoid interrupting ourself for i in $PIDS; do # intentionally unquoted - if [[ -n "RECORD" ]]; then + if [[ -n "$RECORD" ]]; then kill -SIGINT -- "-$i" else kill -SIGINT -- "$i" @@ -194,7 +194,7 @@ on_int() { printf 'Terminating any remaining processes\n' for i in $PIDS; do # intentionally unquoted - if [[ -n "RECORD" ]]; then + if [[ -n "$RECORD" ]]; then kill -SIGTERM -- "-$i" else kill -SIGTERM -- "$i" @@ -253,7 +253,7 @@ run() { COMP= case "$RECORD" in perf) - $@ &> "$PROC_LOG" & + "$@" &> "$PROC_LOG" & COMP="$!" perf record -F 99 -a -g -o "$PNAME".perf -p "$COMP" &> "$PERF_LOG" & PERFS="$PERFS $!";; @@ -261,7 +261,7 @@ run() { ${DBG} "$@" &> "$PROC_LOG" & COMP="$!";; *) - $@ &> "$PROC_LOG" & + "$@" &> "$PROC_LOG" & COMP="$!";; esac @@ -324,7 +324,7 @@ launch() { "$RT"/scripts/wait-for-it.sh -q -t 5 -h localhost -p "$ep" done printf 'Launched logical %s %d, replica %d [PID: %d]\n' "$1" "$id" "$node" "$PID" - if [[ -n "RECORD" ]]; then + if [[ -n "$RECORD" ]]; then PIDS="$PIDS $(getpgid $PID)" else PIDS="$PIDS $PID" @@ -337,7 +337,7 @@ launch() { "$RT"/scripts/wait-for-it.sh -q -t 5 -h localhost -p "$ep" done printf 'Launched %s %d [PID: %d]\n' "$1" "$id" "$PID" - if [[ -n "RECORD" ]]; then + if [[ -n "$RECORD" ]]; then PIDS="$PIDS $(getpgid $PID)" else PIDS="$PIDS $PID" diff --git a/scripts/shellcheck.sh b/scripts/shellcheck.sh new file mode 100755 index 000000000..50b65ec1d --- /dev/null +++ b/scripts/shellcheck.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# Usage: ./scripts/shellcheck.sh +ROOT="$(cd "$(dirname "$0")"/.. && pwd)" +SHELLCHECK_REPORT="${ROOT}/shellcheck-report.txt" + +NUM_CORES=1 +if [[ "${OSTYPE}" == "linux-gnu"* ]]; then + NUM_CORES=$(grep -c ^processor /proc/cpuinfo) +elif [[ "${OSTYPE}" == "darwin"* ]]; then + NUM_CORES=$(sysctl -n hw.ncpu) +fi + +if ! command -v shellcheck &>/dev/null; then + echo "shellcheck is not installed." + echo "Run 'sudo ./scripts/install-build-tools.sh' to install shellcheck." + exit 1 +fi + +# run shellcheck in parallel on all tracked shell scripts +# checking status of this run will give failure if even a warning is found by default +# info/warnings/errors are treated as unsuccessful, so we need to search the report for +# explicit errors messages to determine if there are any true errors +git ls-files '*.sh' | xargs -n 1 -P "${NUM_CORES}" shellcheck > "${SHELLCHECK_REPORT}" + +# if shell check report exists to determine if shellcheck run was successful +if [[ -z "${SHELLCHECK_REPORT}" ]]; then + echo "Shellcheck report ${SHELLCHECK_REPORT} not found. Exiting..." + exit 1 +else + echo "Shellcheck report: ${SHELLCHECK_REPORT}" + if [[ ! -s "${SHELLCHECK_REPORT}" ]]; then + echo "Shellcheck report is empty." + echo "Either there are no info/warning/error messages across all shell scripts," + echo "or shellcheck failed to run successfully." + exit 0 + fi +fi + +# view non-empty shellcheck report, includes info, warnings, errors +if [[ "$#" -eq 1 && "$1" == "view" ]]; then + echo "Shellcheck report: ${SHELLCHECK_REPORT}" + cat "${SHELLCHECK_REPORT}" +fi + +# detect if fatal errors are in shellcheck report +echo "Checking for fatal errors in shellcheck report..." +if ! grep "(error):" "${SHELLCHECK_REPORT}"; then + echo "Shellcheck found no fatal errors in report: ${SHELLCHECK_REPORT}" + echo "Shellcheck passed." + exit 0 +else + echo "Shellcheck found fatal errors in report: ${SHELLCHECK_REPORT}" + echo "Shellcheck failed." + exit 1 +fi diff --git a/scripts/test-e2e-minikube.sh b/scripts/test-e2e-minikube.sh index d25900163..b4dc55759 100755 --- a/scripts/test-e2e-minikube.sh +++ b/scripts/test-e2e-minikube.sh @@ -7,7 +7,7 @@ BUILD_DOCKER=${TESTRUN_BUILD_DOCKER:-1} # Make sure we have the necessary tools installed required_executables=(minikube docker go helm kubectl) -for e in ${required_executables[@]}; do +for e in "${required_executables[@]}"; do if ! command -v $e &> /dev/null; then echo "'$e' command not be found! This is required to run. Please install it." exit 1 diff --git a/scripts/wait-for-it.sh b/scripts/wait-for-it.sh index d990e0d36..df6ad2646 100755 --- a/scripts/wait-for-it.sh +++ b/scripts/wait-for-it.sh @@ -150,7 +150,7 @@ if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then WAITFORIT_ISBUSY=1 # Check if busybox timeout uses -t flag # (recent Alpine versions don't support -t anymore) - if timeout &>/dev/stdout | grep -q -e '-t '; then + if timeout |& tee /dev/stdout | grep -q -e '-t '; then WAITFORIT_BUSYTIMEFLAG="-t" fi else