Merge pull request #3794 from LinuxForHealth/issue-3676-ci

issue #3676 - Replace db2 with postgres in containerized e2e tests

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: Build and Tests
+name: Build and Test
 
 on:
   pull_request:

.github/workflows/integration.yml

@@ -8,7 +8,7 @@ on:
       - '.github/workflows/site.yml'
 
 jobs:
-  e2e-db2-with-bulkdata:
+  e2e-postgres:
     runs-on: ubuntu-latest
     if: "!contains(github.event.pull_request.labels.*.name, 'ci-skip')"
     strategy:
@@ -30,7 +30,7 @@ jobs:
         mvn -B org.apache.maven.plugins:maven-dependency-plugin:3.1.2:go-offline -f fhir-parent -DexcludeReactor=true -Dmaven.wagon.http.retryHandler.count=3
         mvn -B org.apache.maven.plugins:maven-dependency-plugin:3.1.2:resolve-plugins -f fhir-parent -DexcludeReactor=true -Dmaven.wagon.http.retryHandler.count=3
         mvn -B install --file fhir-parent -DskipTests -P include-fhir-igs,integration --no-transfer-progress -Dmaven.wagon.http.retryHandler.count=3
-        mvn -B dockerfile:build -f fhir-install --no-transfer-progress -Dmaven.wagon.http.retryHandler.count=3
+        mvn -B dockerfile:build -f fhir-install --no-transfer-progress -Dmaven.wagon.http.retryHandler.count=3 -Ddocker.buildArg.VERBOSE=false
     - name: free disk space
       run: |
         # create and remove a 200 MB file to make sure we have the room needed later
@@ -55,7 +55,6 @@ jobs:
         containerId=$(docker ps -a | grep fhir | cut -d ' ' -f 1)
         if [[ -z "${containerId}" ]]; then
             echo "Warning: Could not find fhir container!!!"
-            bash build/docker/docker-cleanup.sh
         else
             echo "fhir container id: $containerId"
 
@@ -66,12 +65,12 @@ jobs:
             docker cp -L $containerId:/logs ${it_results}/server-logs
         fi
         echo "Gathering integration test output"
-        cp -pr ${GITHUB_WORKSPACE}/fhir-server-test/target/surefire-reports/* ${it_results}/fhir-server-test || true
+        cp -r ${GITHUB_WORKSPACE}/fhir-server-test/target/surefire-reports/* ${it_results}/fhir-server-test || true
     - name: Upload logs
       if: always()
       uses: actions/upload-artifact@v2.2.0
       with:
-        name: integration-test-results-db2-${{ matrix.java }}
+        name: integration-test-results-${{ matrix.java }}
         path: integration-test-results
 
 concurrency:

.github/workflows/migration.yml

@@ -42,7 +42,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        datastore: [ 'db2', 'postgres' ]
+        datastore: [ 'postgres' ]
         release: ${{fromJSON(needs.setup.outputs.version_tag)}}
       fail-fast: false
     steps:

.github/workflows/reindex.yml

@@ -14,7 +14,7 @@ jobs:
     if: "!contains(github.event.pull_request.labels.*.name, 'ci-skip')"
     strategy:
       matrix:
-        datastore: [ 'db2', 'derby', 'postgres' ]
+        datastore: [ 'derby', 'postgres' ]
       fail-fast: false
     steps:
     - name: Checkout source code

build/.gitignore

@@ -1,5 +1,5 @@
 logs/
-docker/fhir-server/volumes/
+docker/fhir-server/
 deploy.sh
 *.jar
 *.zip

build/README.md

@@ -0,0 +1,83 @@
+# Build and Test the LinuxForHealth FHIR Server
+
+This `build` directory contains scripts for executing LinuxForHealth FHIR Continous Integration (CI) tests.
+
+Alternatively, LinuxForHealth FHIR can be built manually from the root of the project using Maven:
+```sh
+mvn clean install -f fhir-examples
+
+mvn clean install -f fhir-parent
+```
+
+The latter command will execute the unit tests for each module.
+If you want to skip the tests, add `-DskipTests` to the end of the command.
+
+Once the project is built, the LinuxForHealth FHIR Server container image can be built from the Dockerfile under fhir-install:
+```sh
+docker build fhir-install -t linuxforhealth/fhir-server
+```
+
+## Running the integration tests locally
+
+These commands are expected to work on MacOS and/or Linux. For Windows, use the PowerShell (.ps1) variants instead.
+
+### Prerequisites
+
+- Build the project manually using the Maven commands listed above. This will produce the fhir-server-distribution.zip used
+to install the server.
+
+### Run
+
+From the root of the project (aka the "WORKSPACE"):
+1. Set up and start the server 
+    ```sh
+    build/pre-integration-test.sh
+    ```
+
+2. Run the tests. 
+    ```sh
+    mvn test -DskipTests=false -f fhir-server-test
+    ```
+
+3. Collect logs and clean up
+    ```sh
+    build/post-integration-test.sh
+    ```
+
+## Running the integraiton tests using docker compose
+
+### Prerequisites
+
+- [Docker](https://www.docker.com)
+- `linuxforhealth/fhir-server:latest` (built from the fhir-install module as described above)
+
+### Run
+
+From the root of the project (aka the "WORKSPACE"):
+1. Set up and start the containers 
+    ```sh
+    build/pre-integration-test-docker.sh
+    ```
+
+   Note: If you are testing NATS notifications, invoke the NATS subscriber via `node fhir-server-test/src/test/nodejs/nats-subscriber`.  If this is your first time, install the dependencies first by installing [Node.js](https://nodejs.org/en/download) (if not already installed) and running `(cd fhir-server-test/src/test/nodejs && npm install)`.  
+
+2. Run the tests. 
+    ```sh
+    mvn test -DskipTests=false -f fhir-server-test
+    ```
+
+3. Collect logs and tear down the containers
+    ```sh
+    build/post-integration-test-docker.sh
+    ```
+
+### Details
+
+The `pre-integration-test-docker.sh` script will:
+1. Configure volume mounts under build/docker/fhir-server and add the data and configuration needed for the tests.
+2. Bring up the containers and configure the database.
+3. Wait for the fhir-server healthcheck to pass and then exit.
+
+----
+
+FHIR® is the registered trademark of HL7 and is used with the permission of HL7.

build/audit/bin/gather-logs.sh

@@ -22,7 +22,7 @@ package_logs(){
     mkdir -p ${it_results}/fhir-server-test
 
     # Look for the FHIR Server Container
-    containerId=$(docker ps -a | grep ibm-fhir-server | cut -d ' ' -f 1)
+    containerId=$(docker ps -a | grep fhir-server | cut -d ' ' -f 1)
     if [[ -z "${containerId}" ]]; then
         echo "Warning: Could not find fhir container!!!"
     else

build/audit/bin/integration-test.sh

@@ -1,14 +1,11 @@
 #!/usr/bin/env bash
 
 ###############################################################################
-# (C) Copyright IBM Corp. 2020
+# (C) Copyright IBM Corp. 2020, 2022
 #
 # SPDX-License-Identifier: Apache-2.0
 ###############################################################################
-
-set -o errexit
-set -o nounset
-set -o pipefail
+set -ex
 
 run_tests(){
     # The integration tests may be overriden completely, or fall through to the default. 

build/audit/bin/post-integration-test.sh

@@ -1,10 +1,11 @@
 #!/usr/bin/env bash
 
 ###############################################################################
-# (C) Copyright IBM Corp. 2020
+# (C) Copyright IBM Corp. 2020, 2022
 #
 # SPDX-License-Identifier: Apache-2.0
 ###############################################################################
+set -ex
 
 # audit_post - executes for each AUDIT post integration steps
 audit_post(){

build/audit/bin/pre-integration-test.sh

@@ -1,10 +1,11 @@
 #!/usr/bin/env bash
 
 ###############################################################################
-# (C) Copyright IBM Corp. 2020
+# (C) Copyright IBM Corp. 2020, 2022
 #
 # SPDX-License-Identifier: Apache-2.0
 ###############################################################################
+set -ex
 
 # audit_pre - executes for each audit pre integration steps
 audit_pre(){

build/audit/bin/setup-prerequisites.sh

@@ -22,7 +22,7 @@ required_build(){
     mvn -B install --file fhir-parent -DskipTests -P include-fhir-igs,integration --no-transfer-progress
 
     # Build dockerfile
-    mvn -B dockerfile:build -f fhir-install --no-transfer-progress
+    mvn -B dockerfile:build -f fhir-install --no-transfer-progress -Ddocker.buildArg.VERBOSE=false
 }
 
 # audit_build - executes for each audit type.

build/audit/kafka/docker-compose.yml

@@ -49,7 +49,7 @@ services:
     ports:
       - 29092:29092
   fhir-server:
-    image: ibmcom/ibm-fhir-server:latest
+    image: linuxforhealth/fhir-server:latest
     depends_on:
       - kafka-1
       - kafka-2

build/audit/kafka/pre-integration-test.sh

@@ -5,10 +5,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 ###############################################################################
-
-set -o errexit
-set -o nounset
-set -o pipefail
+set -ex
 
 DIST="${WORKSPACE}/build/audit/kafka/workarea/volumes/dist"
 
@@ -37,7 +34,7 @@ config(){
     USERLIB="${DIST}/userlib"
     mkdir -p $USERLIB
     find ${WORKSPACE}/conformance -iname 'fhir-ig*.jar' -not -iname 'fhir*-tests.jar' -not -iname 'fhir*-test-*.jar' -exec cp -f {} ${USERLIB} \;
-    cp ${WORKSPACE}/fhir-operation-test/target/fhir-operation-*.jar ${USERLIB}
+    cp ${WORKSPACE}/operation/fhir-operation-test/target/fhir-operation-*.jar ${USERLIB}
     cp ${WORKSPACE}/term/operation/fhir-operation-term-cache/target/fhir-operation-*.jar ${USERLIB}
     echo "Finished copying fhir-server dependencies..."
 
@@ -76,7 +73,7 @@ bringup(){
     Docker container status:"
     docker ps -a
 
-    containerId=$(docker ps -a | grep kafka_fhir-server_1 | cut -d ' ' -f 1)
+    containerId=$(docker ps -a | grep fhir-server | cut -d ' ' -f 1)
     if [[ -z "${containerId}" ]]; then
         echo "Warning: Could not find the fhir container!!!"
     else

build/common/gather-logs.sh

@@ -29,7 +29,7 @@ package_logs(){
     echo $(date) > ${it_results}/runtime.txt
 
     # Look for the FHIR Server Container
-    containerId=$(docker ps -a | grep ibm-fhir-server | cut -d ' ' -f 1)
+    containerId=$(docker ps -a | grep fhir-server | cut -d ' ' -f 1)
     if [ -z "${containerId}" ]
     then
         echo "Warning: Could not find fhir container!!!"