Implemented Retry for OpenSearch Container (opensearch-project#304)

Signed-off-by: Theo Truong <theotr@amazon.com>

Signed-off-by: Theo Truong <theotr@amazon.com>

.ci/opensearch/Dockerfile.opensearch

@@ -5,4 +5,10 @@ ARG opensearch_path=/usr/share/opensearch
 ARG opensearch_yml=$opensearch_path/config/opensearch.yml
 
 ARG SECURE_INTEGRATION
+
+HEALTHCHECK --start-period=20s --interval=5s --retries=2 --timeout=1s \
+  CMD if [ "$SECURE_INTEGRATION" != "true" ]; \
+  then curl --fail localhost:9200/_cat/health; \
+  else curl --fail -k https:/localhost:9200/_cat/health -u admin:admin; fi
+
 RUN if [ "$SECURE_INTEGRATION" != "true" ] ; then $opensearch_path/bin/opensearch-plugin remove opensearch-security; fi

.ci/opensearch/docker-compose.yml

@@ -2,6 +2,7 @@ version: '3'
 
 services:
   opensearch:
+    restart: always
     build:
       context: .
       dockerfile: Dockerfile.opensearch
@@ -11,6 +12,17 @@ services:
     environment:
       - discovery.type=single-node
       - bootstrap.memory_lock=true
+      - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false}
     ports:
       - '9200:9200'
     user: opensearch
+  autoheal:
+    restart: always
+    image: willfarrell/autoheal
+    environment:
+      - AUTOHEAL_CONTAINER_LABEL=all
+      - AUTOHEAL_START_PERIOD=30
+      - AUTOHEAL_INTERVAL=5
+      - AUTOHEAL_DEFAULT_STOP_TIMEOUT=30
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock

.github/workflows/compatibility.yml

@@ -34,6 +34,7 @@ jobs:
           sudo sysctl -w vm.max_map_count=262144
 
       - name: Runs OpenSearch cluster
+        id: start_opensearch_cluster
         run: |
           export OPENSEARCH_VERSION=${{ matrix.entry.opensearch_version }}
           export SECURE_INTEGRATION=${{ matrix.secured }}
@@ -59,5 +60,6 @@ jobs:
           npm run test:integration:helpers-secure
 
       - name: Stop the OpenSearch cluster
+        if: ${{ steps.start_opensearch_cluster.outcome == 'success'}}
         run: |
           make cluster.opensearch.stop

Makefile

@@ -3,7 +3,7 @@ cluster.opensearch.build:
 
 cluster.opensearch.start:
 	docker-compose --project-directory .ci/opensearch up -d ;
-	sleep 20;
+	sleep 60;
 
 cluster.opensearch.stop:
 	docker-compose --project-directory .ci/opensearch down ;