[SDPA-2705] Turn on regression tests on for release branches and nigh…

…tly (#534) * Setup regression tests against release branches in circle, add cucumber reporter * change folder path to /cucumber * add eslint reporters for circleci * remove reporters from jest config * Move all cucumber reporters to root * use test-results folder - test e2e * convertore cucke json to xml * cache cypress * remove delete step * update lockfile * bump cache version * bump save cache version * restore cache in regression step * Remove smoke tags from example tests * fix command * seperate build and test steps * fix requires key * fix build command * fix build step * parallelize e2e tests * setup prerelease to npm on tests passing * fix example test command * cucumber convertor * Test prerelease step * Remove prerelease step, test branch
dpc-sdp · Sep 18, 2019 · e84c1a0 · e84c1a0
1 parent f5237ca
commit e84c1a0
Show file tree

Hide file tree

Showing 14 changed files with 339 additions and 47 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,53 +1,88 @@
 version: 2.0
 
 jobs:
-  test:
+  build:
     context: Ripple
     docker:
       - image: cypress/base:10
         environment:
           TERM: xterm
           SEARCH_BE: LOCAL-CI
+          CIRCLE_TEST_REPORTS: test-results
     working_directory: ~/app
     steps:
       - checkout
-      - run: yarn --version
       - restore_cache:
           keys:
-            - v1-deps-{{ checksum "yarn.lock" }}
-            - v1-deps-{{ .Branch }}
-            - v1-deps
+            - v2-deps-{{ checksum "yarn.lock" }}
+            - v2-deps-{{ .Branch }}
+            - v2-deps
       - run:
           name: Install Dependencies
           command: yarn install --frozen-lockfile
       - save_cache:
-          key: v1-deps-{{ checksum "yarn.lock" }}
+          key: v2-deps-{{ checksum "yarn.lock" }}
           paths:
             - node_modules
             - packages/ripple-nuxt-tide/node_modules
             - packages/ripple-nuxt-ui/node_modules
             - packages/ripple-create-app/node_modules
             - packages/ripple-test-tools/node_modules
             - examples/vic-gov-au/node_modules
+            - examples/basic-examples/node_modules
             - ~/.cache
+      - run:
+          name: build vic-gov-au nuxt app
+          command: cd examples/vic-gov-au && yarn build:default
+      - persist_to_workspace:
+          root: ~/app
+          paths: .
+  test:
+    context: Ripple
+    docker:
+      - image: cypress/base:10
+        environment:
+          TERM: xterm
+          SEARCH_BE: LOCAL-CI
+          CIRCLE_TEST_REPORTS: test-results
+    working_directory: ~/app
+    steps:
+      - attach_workspace:
+          at: ~/app
+      - restore_cache:
+          keys:
+            - v2-deps-{{ checksum "yarn.lock" }}
+            - v2-deps-{{ .Branch }}
+            - v2-deps
       - run:
           name: Lint code.
-          command: yarn lint     
+          command: yarn lint --format ./node_modules/eslint-junit/index.js 
+          environment:
+             ESLINT_JUNIT_OUTPUT: ./test-results/lint/eslint.xml
       - run:
           name: Unit tests
-          command: yarn test:unit
-      - run: 
+          command: yarn test:unit --ci --runInBand --reporters=default --reporters=jest-junit
+          environment:
+            JEST_JUNIT_OUTPUT_DIR: test-results/unit/
+      - run:
           name: run smoke test
           command: yarn test:smoke
+      - run:
+          name: convert cucumber test output
+          command: node ./scripts/cucumberconvertor.js          
       - persist_to_workspace:
           root: ~/app
           paths: .
+      - store_test_results:
+          path: test-results
+      - store_artifacts:
+          path: test-results
       - store_artifacts:
-          path: test/e2e/videos
+          path: examples/vic-gov-au/test/e2e/videos
       - store_artifacts:
-          path: test/e2e/screenshots
+          path: examples/vic-gov-au/test/e2e/screenshots
 
-  regression:
+  e2e-vic-gov:
     # This runs the full suite of e2e tests 
     docker:
       - image: cypress/base:10
@@ -58,20 +93,119 @@ jobs:
     steps:    
       - attach_workspace:
           at: ~/app
+      - restore_cache:
+          keys:
+            - v2-deps-{{ checksum "yarn.lock" }}
+            - v2-deps-{{ .Branch }}
+            - v2-deps
       - run: 
           name: run full e2e test
-          command: yarn test:e2e
+          command: yarn test:e2e-vic
+      - run:
+          name: convert cucumber test output
+          command: node ./scripts/cucumberconvertor.js          
       - store_artifacts:
-          path: test/e2e/videos
+          path: test-results
+      - store_test_results:
+          path: test-results
       - store_artifacts:
-          path: test/e2e/screenshots
+          path: examples/vic-gov-au/test/e2e/videos
+      - store_artifacts:
+          path: examples/vic-gov-au/test/e2e/screenshots
+
+  e2e-example:
+    # This runs the full suite of e2e tests 
+    docker:
+      - image: cypress/base:10
+        environment:
+          TERM: xterm
+          SEARCH_BE: LOCAL-CI
+    working_directory: ~/app
+    steps:    
+      - attach_workspace:
+          at: ~/app
+      - restore_cache:
+          keys:
+            - v2-deps-{{ checksum "yarn.lock" }}
+            - v2-deps-{{ .Branch }}
+            - v2-deps
+      - run:
+          name: build example nuxt app
+          command: cd examples/basic-examples && yarn build:default
+      - run: 
+          name: run full e2e test
+          command: yarn test:e2e-example
+      - run:
+          name: convert cucumber test output
+          command: node ./scripts/cucumberconvertor.js
+      - store_test_results:
+          path: test-results
+      - store_artifacts:
+          path: test-results
+      - store_artifacts:
+          path: examples/vic-gov-au/test/e2e/videos
+      - store_artifacts:
+          path: examples/vic-gov-au/test/e2e/screenshots
+
+  prerelease:
+    # NOT ENABLED YET - needs further testing
+    docker:
+      - image: cypress/base:10
+        environment:
+          TERM: xterm
+          SEARCH_BE: LOCAL-CI
+    working_directory: ~/app
+    steps:    
+      - attach_workspace:
+          at: ~/app
+      - restore_cache:
+          keys:
+            - v2-deps-{{ checksum "yarn.lock" }}
+            - v2-deps-{{ .Branch }}
+            - v2-deps
+      - run: 
+          name: Set NPM_TOKEN
+          command: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc
+      - run:
+          name: Lerna publish
+          command: yarn publish --dist-tag dev --yes
 
 workflows:
   version: 2
   commit:
     jobs:
-      - test
-  # Run full regression test nightly on master and release branches
+      - build      
+      - test:
+          requires:
+            - build  
+      # Run regression tests on release branches
+      - e2e-vic-gov:
+          requires:
+            - build
+          filters:
+            branches:
+              only:
+                - release/*
+      - e2e-example:
+          requires:
+            - build
+          filters:
+            branches:
+              only:
+                - release/*
+      #  Release prerelease DEV tag to NPM on successful tests passing
+      # - prerelease:
+      #     requires:
+      #       - build
+      #       - test
+      #       - e2e-example
+      #       - e2e-vic-gov
+      #     filters:
+      #       branches:
+      #         only:
+      #           - release/*
+
+  # Run full regression test at midnight UTC on develop branch
   nightly:
     triggers:
       - schedule:
@@ -81,7 +215,21 @@ workflows:
               only:
                 - develop
     jobs:
-      - test
-      # - regression:
-      #     requires:
-      #       - test
+      - build
+      - test:
+          requires:
+            - build  
+      - e2e-vic-gov:
+          requires:
+            - build
+          filters:
+            branches:
+              only:
+                - release/*    
+      - e2e-example:
+          requires:
+            - build
+          filters:
+            branches:
+              only:
+                - release/*
diff --git a/.gitignore b/.gitignore
@@ -22,6 +22,7 @@ yarn-error.log*
 coverage
 examples/**/test/e2e/videos
 examples/**/test/e2e/screenshots
+test-results
 
 # temp test fixtures
 /examples/test*
diff --git a/examples/basic-examples/package.json b/examples/basic-examples/package.json
@@ -21,7 +21,7 @@
     "start:build": "nuxt build && nuxt start"
   },
   "dependencies": {
-    "@dpc-sdp/ripple-nuxt-tide": "1.2.0",
+    "@dpc-sdp/ripple-nuxt-tide": "^1.2.0",
     "dotenv": "^5.0.1",
     "nuxt": "2.6.3",
     "run-script-os": "^1.0.7"
@@ -32,7 +32,7 @@
     "@babel/plugin-transform-runtime": "^7.4.4",
     "@babel/preset-env": "^7.5.0",
     "@babel/runtime": "^7.5.1",
-    "@dpc-sdp/ripple-test-tools": "1.2.0",
+    "@dpc-sdp/ripple-test-tools": "^1.2.0",
     "babel-core": "^7.0.0-bridge.0",
     "babel-eslint": "^10.0.1",
     "babel-jest": "^24.8.0",
@@ -62,7 +62,12 @@
     "vue-jest": "^3.0.4"
   },
   "cypress-cucumber-preprocessor": {
-    "nonGlobalStepDefinitions": true
+    "nonGlobalStepDefinitions": true,
+    "cucumberJson": {
+      "generate": true,
+      "outputFolder": "./../../test-results/cucumber",
+      "filePrefix": "basic-examples-"
+    }
   },
   "browserslist": [
     "> 1%",

diff --git a/.../e2e/integration/smoke/error-page.feature → ...e2e/integration/custom/error-page.feature b/.../e2e/integration/smoke/error-page.feature → ...e2e/integration/custom/error-page.feature
@@ -2,7 +2,6 @@ Feature: Custom error page
 
   Custom error page content should work
 
-  @smoke
   Scenario: Example for adding custom 404 messages
     Given I visit the page "/404"
     And the example 404 error content is added

diff --git a/...ation/smoke/error-page/step_definition.js → ...tion/custom/error-page/step_definition.js b/...ation/smoke/error-page/step_definition.js → ...tion/custom/error-page/step_definition.js
diff --git a/.../e2e/integration/smoke/middleware.feature → ...e2e/integration/custom/middleware.feature b/.../e2e/integration/smoke/middleware.feature → ...e2e/integration/custom/middleware.feature
@@ -5,7 +5,6 @@ Feature: Custom middleware
   Background:
     Given I visit the page "/"
 
-  @smoke
   Scenario: Example for adding component
     And the example component is added
 

diff --git a/...ation/smoke/middleware/step_definition.js → ...tion/custom/middleware/step_definition.js b/...ation/smoke/middleware/step_definition.js → ...tion/custom/middleware/step_definition.js
diff --git a/...est/e2e/integration/smoke/theming.feature → ...st/e2e/integration/custom/theming.feature b/...est/e2e/integration/smoke/theming.feature → ...st/e2e/integration/custom/theming.feature
@@ -5,7 +5,6 @@ Feature: Custom theme should work
   Background:
     Given I visit the page "/"
 
-  @smoke
   Scenario: Custom colour is applied
     And the site header has custom background color
 

diff --git a/...egration/smoke/theming/step_definition.js → ...gration/custom/theming/step_definition.js b/...egration/smoke/theming/step_definition.js → ...gration/custom/theming/step_definition.js
diff --git a/examples/vic-gov-au/package.json b/examples/vic-gov-au/package.json
@@ -12,23 +12,23 @@
     "cy:run": "cypress run -e TAGS='not @skip or @smoke'",
     "cy:run-smoke": "cypress run -e TAGS='@smoke' --spec 'test/e2e/integration/smoke/*'",
     "test:dev": "cross-env NODE_ENV=test start-server-and-test dev http://localhost:3000 cy:open",
-    "test:smoke": "cross-env NODE_ENV=test start-server-and-test start:build http://localhost:3000 cy:run-smoke",
-    "test:e2e": "cross-env NODE_ENV=test start-server-and-test start:build http://localhost:3000 cy:run",
+    "test:smoke": "cross-env TEST=1 start-server-and-test start http://localhost:3000 cy:run-smoke",
+    "test:e2e": "cross-env TEST=1 start-server-and-test start http://localhost:3000 cy:run",
     "test:unit": "cross-env BASIC_AUTH=0 NODE_ENV=test jest",
     "start": "run-script-os",
     "start:default": "if [ \"$NUXT_HOT_RELOADING\" = \"1\" ] ; then nuxt; else nuxt start; fi",
     "start:win32": "IF \"%NUXT_HOT_RELOADING%\" EQU \"1\" ( nuxt ) ELSE ( nuxt start )",
     "start:build": "nuxt build && nuxt start"
   },
   "dependencies": {
-    "@dpc-sdp/ripple-nuxt-tide": "1.2.0",
+    "@dpc-sdp/ripple-nuxt-tide": "^1.2.0",
     "@nuxtjs/robots": "^2.0.0",
     "dotenv": "^5.0.1",
     "nuxt": "2.6.3",
     "run-script-os": "^1.0.7"
   },
   "devDependencies": {
-    "@dpc-sdp/ripple-test-tools": "1.2.0",
+    "@dpc-sdp/ripple-test-tools": "^1.2.0",
     "babel-eslint": "^10.0.1",
     "cross-env": "^5.2.0",
     "cypress": "^3.1.5",
@@ -52,7 +52,12 @@
     "vue-jest": "^3.0.4"
   },
   "cypress-cucumber-preprocessor": {
-    "nonGlobalStepDefinitions": true
+    "nonGlobalStepDefinitions": true,
+    "cucumberJson": {
+      "generate": true,
+      "outputFolder": "./../../test-results/cucumber",
+      "filePrefix": "vic-gov-"
+    }
   },
   "browserslist": [
     "> 1%",