diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE index c06185bc..6ae086a8 100644 --- a/.github/PULL_REQUEST_TEMPLATE +++ b/.github/PULL_REQUEST_TEMPLATE @@ -12,8 +12,8 @@ Draft for a patch: TODO: - [ ] Added patch for [describe issue] -- [ ] Add/update tests -- unit and/or integrated (if needed) - - [ ] Ensure only one function is tested per test file. +- [ ] Add/update tests -- unit/integrated/E2E (if needed) + - [ ] Ensure only one function/functionality is tested per test file. - [ ] Add to, or update, `Scan run detail` report as applicable - [ ] Check status of automated tests - [ ] Ensure `PHPDoc` comments are up to date for functions added or altered @@ -29,8 +29,8 @@ TODO: - [ ] Update `--help` message - [ ] Implement [new feature / logic] - [ ] Add to, or update, `Scan run detail` report as applicable -- [ ] Add/update tests -- unit and/or integrated - - [ ] Ensure only one function is tested per test file. +- [ ] Add/update tests -- unit/integrated/E2E + - [ ] Ensure only one function/functionality is tested per test file. - [ ] Ensure `PHPDoc` comments are up to date for functions added or altered - [ ] Update repository documentation (README.md, RELEASING.md, TESTING.md, TOOLS-UPDATE.md) - [ ] Assign appropriate [priority](https://github.com/Automattic/vip-go-ci/blob/trunk/CONTRIBUTING.md#priorities) and [type of change labels](https://github.com/Automattic/vip-go-ci/blob/trunk/CONTRIBUTING.md#type-of-change-labels). @@ -47,8 +47,10 @@ TODO: - [ ] Add same version number in defines.php - [ ] Assign a milestone corresponding to the version number to this PR and PRs that will form the release - [ ] Assign label ([Changelog and version](https://github.com/Automattic/vip-go-ci/blob/trunk/CONTRIBUTING.md#type-of-change-labels)) -- [ ] Run unit-test suite +- [ ] Unit-test suite run successful +- [ ] Integration-test suite successful (without secrets) - [ ] Run integration-test suite with secrets +- [ ] E2E-test suite run successful - [ ] Manual testing - [ ] Pull request with PHP linting issues - [ ] Pull request with PHPCS issues diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5521312..0ab2a5dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,7 +132,7 @@ jobs: - name: Check out the source code uses: actions/checkout@v3 - - name: Ask git to fetch latest branch + - name: Ask git to fetch latest branch and other branches run: git fetch origin latest && git pull - name: Set up PHP diff --git a/TESTING.md b/TESTING.md index 01439d21..a01a21b2 100644 --- a/TESTING.md +++ b/TESTING.md @@ -2,7 +2,7 @@ ## Introduction -`vip-go-ci` relies on both manual and automated testing. Much of the functionality `vip-go-ci` provides is automatically tested using it's extensive unit and integration test suites. _Most_ of the tests in the test suites are run automatically when code is committed and pushed to the repository, though _some_ integration tests need to be run manually (due to secrets, see below). The manual testing that should be performed is functional, testing the final behaviour of the software. +`vip-go-ci` relies on both manual and automated testing. Much of the functionality `vip-go-ci` provides is automatically tested using it's extensive unit, integration and E2E (End-to-End) test suites. _Most_ of the tests in the test suites are run automatically when code is committed and pushed to the repository, though _some_ integration tests need to be run manually (due to secrets, see below). The manual testing that should be performed is functional, testing the final behaviour of the software. ## Automated testing @@ -80,6 +80,12 @@ Integration tests will execute the scanning utilities — PHPCS, SVG scanner and By using this command, you will run the tests of the test-suite which can be run (depending on tokens and other detail), and get feedback on any errors or warnings. Note that when run, requests will be made to the GitHub API using anonymous calls (unless configured to use an access-token as shown above). It can happen that the GitHub API returns with an error indicating that the maximum limit of API requests has been reached; the solution is to wait and re-run or switch to authenticted calls. +### E2E test suite + +The E2E (End-to-End) tests can be run using the following command: + +> VIPGOCI_TESTING_DEBUG_MODE=true phpunit --testsuite=e2e-tests + ### Test isolation Note that the test suite uses the `@runTestsInSeparateProcesses` and `@preserveGlobalState` PHPUnit flags to avoid any influence of one test on another. Further, tests should include all required files in `setUp()` function to avoid the same function being defined multiple times across multiple tests during the same run. Combining the usage of `@runTestsInSeparateProcesses` and the inclusion of required files in `setUp()` means each test is independent of other tests, which enables functions to be defined for each test easily and avoids leakage between tests. diff --git a/tools-init.sh b/tools-init.sh index de69bfea..9549bbad 100755 --- a/tools-init.sh +++ b/tools-init.sh @@ -104,6 +104,7 @@ function gh_fetch_and_verify() { return 1 ) } +# Put lock file in place. function lock_place() { # Get lock, if that fails, just exit if [ -f "$TMP_LOCK_FILE" ] ; then @@ -112,13 +113,35 @@ function lock_place() { fi # Acquire lock - touch "$TMP_LOCK_FILE" + echo "$$" > "$TMP_LOCK_FILE" + + # Try to detect if two instances run at the same time + # on the same system. Should not happen often. + sleep 1 + + if [ "$$" == `cat "$TMP_LOCK_FILE"` ] ; then + echo "$0: Acquired lock ($TMP_LOCK_FILE)" + else + echo "$0: Someone else got the lock before us. Bailing out" + exit 1 + fi } +# Remove lock file, but only if we acquired it. function lock_remove() { - rm -f "$TMP_LOCK_FILE" + if [ -f "$TMP_LOCK_FILE" ] ; then + if [ "$$" == `cat "$TMP_LOCK_FILE"` ] ; then + echo "$0: Removed lock" + rm -f "$TMP_LOCK_FILE" + else + echo "$0: Someone else got the lock file. Not removing lock file." + fi + fi } +# When exiting, ensure we remove lock file. +trap lock_remove EXIT + lock_place #