Merge pull request #1767 from snyk/smoke/improve-shellspec

Improve smoke tests dev experience

.github/workflows/smoke-tests.yml

@@ -184,7 +184,6 @@ jobs:
           export PATH="/usr/local/bin/snyk-mac/docker:$PATH"
           which snyk
           snyk version
-          export EXPECTED_SNYK_VERSION=$(snyk --version)
           shellspec -f d
 
       - name: Run shellspec tests - Windows

package.json

@@ -46,6 +46,7 @@
     "test:packages-unit": "jest \"\/packages\/(.+)\/test\/unit\/((.+)\/)*[^/]+\\.spec\\.ts\"",
     "test:packages-acceptance": "jest \"\/packages\/(.+)\/test\/acceptance\/((.+)\/)*[^/]+\\.spec\\.ts\"",
     "test:test": "tap test/*.test.* -Rspec --timeout=300 --node-arg=-r --node-arg=ts-node/register",
+    "test:smoke": "./scripts/run-smoke-tests-locally.sh",
     "test": "npm run test:acceptance && npm run test:system && npm run test:test && npm run test:jest && test:packages-unit && test:packages-acceptance",
     "test-windows": "npm run test:acceptance-windows && npm run test:system && npm run test:test && npm run test:jest",
     "lint": "run-p --aggregate-output lint:*",

scripts/run-smoke-tests-locally.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -e
+
+echo "Attempting to run Smoke Tests locally. See file 'test/smoke/README.md' for details. This will drop your local 'snyk config'!"
+
+if [ -z "$SNYK_API_TOKEN" ]; then
+  echo "You need to set 'SNYK_API_TOKEN' envvar." >&2
+  exit 1
+fi
+
+if ! [ -x "$(command -v shellspec)" ]; then
+  if ! [ -x "$(command -v brew)" ]; then
+    echo "Error: Shellspec is not installed. See https://shellspec.info for install instructions" >&2
+    exit 1
+  fi
+  echo "Installing shellspec with brew"
+  brew install shellspec
+fi
+
+if ! [ -x "$(command -v jq)" ]; then
+  if ! [ -x "$(command -v brew)" ]; then
+    echo "Error: jq is not installed. See https://stedolan.github.io/jq/ for install instructions" >&2
+    exit 1
+  fi
+  echo "Installing jq with brew"
+  brew install jq
+fi
+
+if ! [ -x "$(command -v timeout)" ]; then
+  if ! [ -x "$(command -v brew)" ]; then
+    echo "Error: 'timeout' command is not installed." >&2
+    exit 1
+  fi
+  echo "Installing coreutils (which contains timeout) with brew"
+  brew install coreutils
+fi
+
+echo "Installing fixture project with npm install"
+npm install --silent --prefix test/fixtures/basic-npm
+
+SNYK_COMMAND="node ${PWD}/dist/cli" REGRESSION_TEST=1 SMOKE_TESTS_SKIP_TEST_THAT_OPENS_BROWSER=1 SMOKE_TESTS_SNYK_TOKEN=$SNYK_API_TOKEN shellspec --chdir test/smoke test/smoke/spec/snyk_auth_spec.sh -f d

test/smoke/README.md

@@ -17,28 +17,31 @@ Before you start adding specs, those files are bash scripts, it's recommended to
 
 It's recommended to have a branch named `smoke/_SOMETHING_`, as [this branch will run the GitHub Action](https://github.com/snyk/snyk/blob/f35f39e96ef7aa69b22a846315dda015b12a4564/.github/workflows/smoke-tests.yml#L3-L5).
 
-To run these tests locally:
+To run these tests locally you may use `npm run test:smoke`:
 
 1. Install:
 
    - [Shellspec](https://shellspec.info)
    - [jq](https://stedolan.github.io/jq/)
    - timeout (if not available on your platform)
 
-2. Run:
+2. Install dependencies for the local fixture `test/fixtures/basic-npm` with `npm install --prefix test/fixtures/basic-npm`
+
+3. Run shellspec locally:
 
 ```sh
-cd test/smoke
-CI=1 SMOKE_TESTS_SNYK_TOKEN=$SNYK_API_TOKEN shellspec -f d
+SMOKE_TESTS_SNYK_TOKEN=$SNYK_API_TOKEN REGRESSION_TEST=1 shellspec --directory test/smoke -f d
 ```
 
-To run the Alpine test in Docker locally:
+### Notes on the local run
 
-```
- docker build -f ./test/smoke/alpine/Dockerfile -t snyk-cli-alpine ./test/ && docker run --rm -eCI=1 -eSMOKE_TESTS_SNYK_TOKEN=$SNYK_API_TOKEN snyk-cli-alpine
-```
+`REGRESSION_TEST=1` enables the extended mode we use for regression testing. For the hourly tests in GitHub Actions we use a limited scope of tested commands.
 
-_Note: Alpine image is not copying/mounting everything, so you might need to add anything new to the `test/smoke/alpine/Dockerfile`_
+You may specify an envvar `SNYK_COMMAND` to any executable that will be used by Smoke tests. E.g. a local exuctable `SNYK_COMMAND="./snyk-macos"` or an `SNYK_COMMAND="npx snyk@1.500.0"` or `SNYK_COMMAND="node ./dist/cli"` for local execution.
+
+This will meddle with your `snyk config` file as Smoke Tests are checking functionality of `snyk config` command.
+
+This will open a browser in one instance unless it's disabled with `SMOKE_TESTS_SKIP_TEST_THAT_OPENS_BROWSER=1` envvar. Opening of a browser is disabled by default when running Smoke tests with npm command `npm run test:smoke`.
 
 ## TODO
 
@@ -71,3 +74,11 @@ _Note: Alpine image is not copying/mounting everything, so you might need to add
 
 - Needs to run in a Docker container because GitHub Actions don't support Alpine as a host OS. Using shellspec container, as it's based on alpine and ready to run the tests.
 - Need to skip a test that normally tries to open browser for login, but that fails horribly on Alpine.
+
+To run the Alpine test in Docker locally (you probably don't want to…):
+
+```
+ docker build -f ./test/smoke/alpine/Dockerfile -t snyk-cli-alpine ./test/ && docker run --rm -eSMOKE_TESTS_SNYK_TOKEN=$SNYK_API_TOKEN snyk-cli-alpine
+```
+
+_Note: Alpine image is not copying/mounting everything, so you might need to add anything new to the `test/smoke/alpine/Dockerfile`_

test/smoke/alpine/entrypoint.sh

@@ -2,11 +2,10 @@
 
 echo "install snyk with binary"
 snyk_cli_dl=$(curl -H "Authorization: token $GITHUB_TOKEN" https://api.github.com/repos/snyk/snyk/releases/latest | jq --raw-output '(.assets[])? | select(.name == "snyk-alpine") | .browser_download_url')
-echo "snyk_cli_dl: ${snyk_cli_dl}"
-curl -Lo ./snyk-cli $snyk_cli_dl
+echo "snyk_cli_dl: $snyk_cli_dl"
+curl -Lo ./snyk-cli "$snyk_cli_dl"
 chmod -R +x ./snyk-cli
 mv ./snyk-cli /usr/local/bin/snyk
 snyk --version
-export EXPECTED_SNYK_VERSION=$(snyk --version)
 
 shellspec -f d

test/smoke/docker-root/entrypoint.sh

@@ -6,6 +6,4 @@ echo "Install Snyk CLI with npm"
 npm install snyk -g
 snyk --version
 
-export EXPECTED_SNYK_VERSION=$(snyk --version)
-
 shellspec -f d

test/smoke/run-shellspec-win.sh

@@ -1,5 +1,3 @@
 echo "run-shellscript-win.sh"
 
-export EXPECTED_SNYK_VERSION=$(snyk --version)
-
 /c/Users/runneradmin/.local/bin/shellspec -f d

test/smoke/spec/snyk_auth_spec.sh

@@ -17,11 +17,15 @@ Describe "Snyk CLI Authorization"
 
     It "fails when run without token set"
       # Alpine can't open browser, misses xdg-open utility and errors out
-      is_alpine() {
+      is_alpine_or_disabled() {
+        if [ -n "$SMOKE_TESTS_SKIP_TEST_THAT_OPENS_BROWSER" ]; then
+          echo "Won't test auth command that opens browser" >&2
+          exit 0
+        fi
         grep "Alpine Linux" /etc/os-release > /dev/null 2>&1
         return $?
       }
-      Skip if "function returns success" is_alpine
+      Skip if "test is disabled" is_alpine_or_disabled
 
       # Using timeout to not wait for browser confirmation
       When run timeout 5 snyk auth

test/smoke/spec/snyk_basic_spec.sh

@@ -4,15 +4,15 @@ Describe "Snyk CLI basics"
   Describe "snyk version"
     It "prints version"
       When run snyk version
-      The output should include "${EXPECTED_SNYK_VERSION}"
+      The output should include "1." # Version should start with a (major) 1
       The status should be success
       # TODO: unusable with our current docker issues
       The stderr should equal ""
     End
 
     It "prints version with --version flag"
       When run snyk --version
-      The output should include "${EXPECTED_SNYK_VERSION}"
+      The output should include "1." # Version should start with a (major) 1
       The status should be success
       # TODO: unusable with our current docker issues
       The stderr should equal ""

test/smoke/spec/spec_helper.sh

@@ -1,44 +1,71 @@
 #shellcheck shell=sh
 set -e
 
-print_snyk_config() {
-  snyk config
+spec_helper_precheck() {
+  setenv CI=1 # This flag influences behavior of `snyk auth` so it needs to be explicitly set
+  setenv ORIGINAL_SNYK_EXECUTABLE="$(which snyk)"
 }
 
-snyk_login() {
-  snyk auth "${SMOKE_TESTS_SNYK_TOKEN}" > /dev/null 2>&1
-}
+spec_helper_configure() {
+  print_snyk_config() {
+    snyk config
+  }
 
-snyk_logout() {
-  snyk config clear > /dev/null 2>&1
-}
+  snyk_login() {
+    snyk auth "${SMOKE_TESTS_SNYK_TOKEN}" > /dev/null 2>&1
+  }
 
-verify_login_url() {
-  # https://snyk.io/login?token=uuid-token&utm_medium=cli&utm_source=cli&utm_campaign=cli&os=darwin&docker=false
-  echo "$1" | grep https | grep -E "^https://(dev\.)?(test\.)?snyk\.io/login\?token=[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}\&.*$"
-}
+  snyk_logout() {
+    snyk config clear > /dev/null 2>&1
+  }
 
-# Consume stdout and checks validates whether it's a valid JSON
-check_valid_json() {
-  printf %s "$1" | jq . > /dev/null
-  echo $?
-}
+  verify_login_url() {
+    # https://snyk.io/login?token=uuid-token&utm_medium=cli&utm_source=cli&utm_campaign=cli&os=darwin&docker=false
+    echo "$1" | grep https | grep -E "^https://(dev\.)?(test\.)?snyk\.io/login\?token=[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}\&.*$"
+  }
 
-# These 2 commands should run in succession, some CLI functionality uses isCI detection
-disable_is_ci_flags() {
-  # save original value and unset
-  if [ -n "${CI}" ]; then CI_BACKUP_VALUE=$CI; unset CI; fi
-  if [ -n "${CIRCLECI}" ]; then CIRCLECI_BACKUP_VALUE=$CIRCLECI; unset CIRCLECI; fi
-}
-restore_is_ci_flags() {
-  # recover the original value
-  if [ -n "${CI}" ]; then CI=$CI_BACKUP_VALUE; unset CI_BACKUP_VALUE; fi
-  if [ -n "${CIRCLECI}" ]; then CIRCLECI=$CIRCLECI_BACKUP_VALUE; unset CIRCLECI_BACKUP_VALUE; fi
-}
+  # Consume stdout and checks validates whether it's a valid JSON
+  check_valid_json() {
+    printf %s "$1" | jq . > /dev/null
+    echo $?
+  }
+
+  # These 2 commands should run in succession, some CLI functionality uses isCI detection
+  disable_is_ci_flags() {
+    # save original value and unset
+    if [ -n "${CI}" ]; then CI_BACKUP_VALUE="$CI"; unset CI; fi
+    if [ -n "${CIRCLECI}" ]; then CIRCLECI_BACKUP_VALUE="$CIRCLECI"; unset CIRCLECI; fi
+  }
+  restore_is_ci_flags() {
+    # recover the original value
+    if [ -n "${CI}" ]; then CI="$CI_BACKUP_VALUE"; unset CI_BACKUP_VALUE; fi
+    if [ -n "${CIRCLECI}" ]; then CIRCLECI="$CIRCLECI_BACKUP_VALUE"; unset CIRCLECI_BACKUP_VALUE; fi
+  }
+
+  check_if_regression_test() { ! [ "${REGRESSION_TEST}" = "1" ]; }
+
+  check_auth_output() {
+    printf %s "$1" | grep -F -e "To authenticate your account, open the below URL in your browser." -e "Now redirecting you to our auth page, go ahead and log in," > /dev/null
+    echo $?
+  }
+
+  echo "
+\033[1mS n y k  C L I\033[0m
+███████╗███╗   ███╗ ██████╗ ██╗  ██╗███████╗    ████████╗███████╗███████╗████████╗███████╗
+██╔════╝████╗ ████║██╔═══██╗██║ ██╔╝██╔════╝    ╚══██╔══╝██╔════╝██╔════╝╚══██╔══╝██╔════╝
+███████╗██╔████╔██║██║   ██║█████╔╝ █████╗         ██║   █████╗  ███████╗   ██║   ███████╗
+╚════██║██║╚██╔╝██║██║   ██║██╔═██╗ ██╔══╝         ██║   ██╔══╝  ╚════██║   ██║   ╚════██║
+███████║██║ ╚═╝ ██║╚██████╔╝██║  ██╗███████╗       ██║   ███████╗███████║   ██║   ███████║
+╚══════╝╚═╝     ╚═╝ ╚═════╝ ╚═╝  ╚═╝╚══════╝       ╚═╝   ╚══════╝╚══════╝   ╚═╝   ╚══════╝
+"
 
-check_if_regression_test() { ! [ "${REGRESSION_TEST}" = "1" ]; }
+  echo "Using this 'snyk' executable:"
+  echo "${SNYK_COMMAND:=$ORIGINAL_SNYK_EXECUTABLE}"
+  echo " "
+  echo "You may override it with envvar SNYK_COMMAND - e.g. SNYK_COMMAND=\"node ./dist/cli\" to test a local build"
+  echo " "
 
-check_auth_output() {
-  printf %s "$1" | grep -F -e "To authenticate your account, open the below URL in your browser." -e "Now redirecting you to our auth page, go ahead and log in," > /dev/null
-  echo $?
+  snyk() {
+    eval "${SNYK_COMMAND:=$ORIGINAL_SNYK_EXECUTABLE}" "$@"
+  }
 }