Add retry attempts to runs and tests for retries (#65)

* Adding a test to make sure that the executable is re-launched if the exit code is not found
* Added a test that sets the success line for the time 2 minutes from the test to see if it re-runs until this occurs
* Print the options that are being added, print the command that is being run

---------

.github/workflows/test.yml

@@ -43,6 +43,7 @@ jobs:
             exit 1
           fi
           gcc executable-monitor/test.c -o executable-monitor/test.out
+          gcc -DEXIT_WITH_MINUTES executable-monitor/test.c -o executable-monitor/test_exit_current_minutes.out
           echo "::endgroup::"
           if [ "$?" = "0" ]; then
             readlink -f executable-monitor/test.out
@@ -52,7 +53,7 @@ jobs:
             exit 1
           fi
 
-      - name: Functional Test | Success Case | Executable Monitor | Retries, Success Line, and Exit Code | Pass on Success Line
+      - name: Functional Test | Success Case | Retries, Success Line, and Exit Code | Pass on Success Line
         id: test-executable-monitor-action-success-line
         uses: ./executable-monitor
         with:
@@ -62,7 +63,7 @@ jobs:
           success-exit-code: 0
           retry-attempts: 2
 
-      - name: Functional Test | Success Case | Executable Monitor | Retries, Success Line, and Exit Code | Pass on Exit Code
+      - name: Functional Test | Success Case | Retries, Success Line, and Exit Code | Pass on Exit Code
         id: test-executable-monitor-action-exit-code
         uses: ./executable-monitor
         with:
@@ -72,7 +73,7 @@ jobs:
           success-exit-code: 0
           retry-attempts: 2
 
-      - name: Functional Test | Success Case | Executable Monitor | Retries, Success Line, and Exit Code, With log file
+      - name: Functional Test | Success Case | Retries, Success Line, and Exit Code, With log file
         id: test-executable-monitor-action-log-file
         uses: ./executable-monitor
         with:
@@ -83,7 +84,7 @@ jobs:
           success-exit-code: 0
           retry-attempts: 2
 
-      - name: Functional Test | Success Case | Executable Monitor | Retries, Success Line, No Exit Code
+      - name: Functional Test | Success Case | Retries, Success Line, No Exit Code
         id: test-executable-monitor-action-no-exit-code
         uses: ./executable-monitor
         with:
@@ -92,7 +93,7 @@ jobs:
           success-line: "SLEEPING FOR 6 SECONDS"
           retry-attempts: 2
 
-      - name: Functional Test | Success Case | Executable Monitor | Retries, No Success Line, Exit Code
+      - name: Functional Test | Success Case | Retries, No Success Line, Exit Code
         id: test-executable-monitor-action-no-success-line
         uses: ./executable-monitor
         with:
@@ -101,39 +102,65 @@ jobs:
           success-exit-code: 0
           retry-attempts: 2
 
-      - name: Functional Test | Success Case | Executable Monitor | No Retries, Success Line, No Exit Code
+      - name: Functional Test | Success Case | No Retries, Success Line, No Exit Code
         id: test-executable-monitor-action-no-retry-attempts-success-line
         uses: ./executable-monitor
         with:
           exe-path: executable-monitor/test.out
           success-line: "SLEEPING FOR 6 SECONDS"
           timeout-seconds: 20
 
-      - name: Functional Test | Success Case | Executable Monitor | No Retries, No Success Line, Exit Code
+      - name: Functional Test | Success Case | No Retries, No Success Line, Exit Code
         id: test-executable-monitor-action-no-retry-attempts-exit-code
         uses: ./executable-monitor
         with:
           exe-path: executable-monitor/test.out
           success-exit-code: 0
           timeout-seconds: 75
 
-      - name: API Test | Success Case | Executable Monitor | Retries, Success Line, No Exit Code, Use Default Timeout
+      - name: API Test | Success Case | Retries, Success Line, No Exit Code, Use Default Timeout
         id: test-executable-monitor-action-no-exit-code-no-timeout
         uses: ./executable-monitor
         with:
           exe-path: executable-monitor/test.out
           success-line: "SLEEPING FOR 6 SECONDS"
           retry-attempts: 2
 
-      - name: API Test | Success Case | Executable Monitor | Retries, No Success Line, Exit Code, Use Default Timeout
+      - name: API Test | Success Case | Retries, No Success Line, Exit Code, Use Default Timeout
         id: test-executable-monitor-action-no-success-line-no-timeout
         uses: ./executable-monitor
         with:
           exe-path: executable-monitor/test.out
           success-exit-code: 0
           retry-attempts: 2
 
-      - name: Functional Test | Failure Case | Executable Monitor | Fail On Timeout
+      # Get future times from now, then look for that in the executable
+      - name: Get Future Times
+        id: future-time
+        run: |
+          # Get times for future demos
+          echo "time_success_line=$(date --date='2 minutes' +%H:%M)" >> "$GITHUB_ENV"
+          echo "time_exit_code=$(date --date='5 minutes' +%M)" >> "$GITHUB_ENV"
+
+      - name: Functional Test | Sucess Case | Test Retry Run occurs on Wrong exit code
+        id: test-executable-monitor-action-retry-success-line
+        uses: ./executable-monitor
+        with:
+          exe-path: executable-monitor/test.out
+          timeout-seconds: 20
+          retry-attempts: 10
+          success-line: ${{ env.time_success_line }}
+
+      - name: Functional Test | Success Case | Retry On Wrong Exit Code
+        id: test-executable-monitor-action-retry-exit-code
+        uses: ./executable-monitor
+        with:
+          exe-path: executable-monitor/test_exit_current_minutes.out
+          success-exit-code: ${{ env.time_exit_code }}
+          timeout-seconds: 60
+          retry-attempts: 10
+
+      - name: Functional Test | Failure Case | Fail On Timeout
         id: test-executable-monitor-action-fail-on-timeout
         continue-on-error: true
         uses: ./executable-monitor
@@ -144,7 +171,7 @@ jobs:
           success-exit-code: 0
           retry-attempts: 2
 
-      - name: API Test | Failure Case | Executable Monitor | Retries, No Success Line, No Exit Code
+      - name: API Test | Failure Case | Retries, No Success Metric
         continue-on-error: true
         id: test-executable-monitor-API-no-success-metric
         uses: ./executable-monitor
@@ -153,7 +180,7 @@ jobs:
           timeout-seconds: 20
           retry-attempts: 2
 
-      - name: API Test | Failure Case | Executable Monitor | No Exe Path
+      - name: API Test | Failure Case | No Exe Path
         continue-on-error: true
         id: test-executable-monitor-API-no-exe-path
         uses: ./executable-monitor
@@ -168,25 +195,25 @@ jobs:
         run: |
           # Check Last Step Failed
           echo "::group::Check Failure Test Cases"
-          if [ "${{ steps.test-executable-monitor-API-no-success-metric.outcome}}" = "failure" ]; then
-            echo -e "\033[32;3mtest-executable-monitor-API-no-success-metric had outcome '${{ steps.test-executable-monitor-API-no-success-metric.outcome}}' as intended\033[0m"
+          if [ "${{ steps.test-executable-monitor-action-fail-on-timeout.outcome}}" = "failure" ]; then
+            echo -e "\033[32;3mFunctional Test | Failure Case | Fail On Timeout had outcome '${{ steps.test-executable-monitor-action-fail-on-timeout.outcome}}' as intended\033[0m"
           else
             echo "::endgroup::"
-            echo -e "\033[32;31mtest-executable-monitor-API-no-success-metric had unexpected '${{ steps.test-executable-monitor-API-no-success-metric.outcome}}' exit condition\033[0m"
+            echo -e "\033[32;31mFunctional Test | Failure Case | Fail On Timeout had unexpected '${{ steps.test-executable-monitor-action-fail-on-timeout.outcome}}' exit condition\033[0m"
             exit 1
           fi
-          if [ "${{ steps.test-executable-monitor-API-no-exe-path.outcome}}" = "failure" ]; then
-          echo -e "\033[32;3mtest-executable-monitor-API-no-exe-path had outcome '${{ steps.test-executable-monitor-API-no-exe-path.outcome}}' as intended\033[0m"
+          if [ "${{ steps.test-executable-monitor-API-no-success-metric.outcome}}" = "failure" ]; then
+            echo -e "\033[32;3mAPI Test | Failure Case | Retries, No Success Metric had outcome '${{ steps.test-executable-monitor-API-no-success-metric.outcome}}' as intended\033[0m"
           else
             echo "::endgroup::"
-            echo -e "\033[32;31mtest-executable-monitor-API-no-exe-path had unexpected '${{ steps.test-executable-monitor-API-no-exe-path.outcome}}' exit condition\033[0m"
+            echo -e "\033[32;31mAPI Test | Failure Case | Retries, No Success Metric had unexpected '${{ steps.test-executable-monitor-API-no-success-metric.outcome}}' exit condition\033[0m"
             exit 1
           fi
-          if [ "${{ steps.test-executable-monitor-action-fail-on-timeout.outcome}}" = "failure" ]; then
-          echo -e "\033[32;3mtest-executable-monitor-action-fail-on-timeout had outcome '${{ steps.test-executable-monitor-action-fail-on-timeout.outcome}}' as intended\033[0m"
+          if [ "${{ steps.test-executable-monitor-API-no-exe-path.outcome}}" = "failure" ]; then
+            echo -e "\033[32;3mAPI Test | Failure Case | No Exe Path had outcome '${{ steps.test-executable-monitor-API-no-exe-path.outcome}}' as intended\033[0m"
           else
             echo "::endgroup::"
-            echo -e "\033[32;31mtest-executable-monitor-action-fail-on-timeout had unexpected '${{ steps.test-executable-monitor-action-fail-on-timeout.outcome}}' exit condition\033[0m"
+            echo -e "\033[32;31mAPI Test | Failure Case | No Exe Path had unexpected '${{ steps.test-executable-monitor-API-no-exe-path.outcome}}' exit condition\033[0m"
             exit 1
           fi
           echo "::endgroup::"

executable-monitor/action.yml

@@ -45,32 +45,60 @@ runs:
       - name: Run Executable with Monitoring
         run: |
           # Run Executable with Monitoring
-          echo "::group::Executable Output"
-          optArgs=" " 
+          echo "::group::Argument Parsing"
+          optArgs=" "
+          # Make sure we have an exit condition to look for
           if [ "${{ inputs.success-exit-code }}" = "" ] && [ "${{ inputs.success-line }}" = "" ]; then
             echo "::endgroup::"
             echo -e "\033[32;31mDid not supply an input of success-line or success-exit-code to search for\033[0m"
             exit 1
           fi
+
+          # Check if an exit code was provided
           if [ "${{ inputs.success-exit-code }}" != "" ]; then
-             optArgs="--success-exit-code=${{ inputs.success-exit-code }}$optArgs "
+            optArgs="--success-exit-code=${{ inputs.success-exit-code }} $optArgs"
+            echo "Adding --success-exit-code=${{ inputs.success-exit-code }} to the call"
           fi
-          
+
+          # Check for log directory/if a log file should be created
           if [ "${{ inputs.log-dir }}" != "" ]; then
             optArgs="--log-dir=${{ inputs.log-dir}} $optArgs"
+            echo "Adding --log-dir=${{ inputs.log-dir}} to the call"
+          fi
+
+          # Check for retry attempts
+          if [ "${{ inputs.retry-attempts }}" != "" ]; then
+            optArgs="--retry-attempts=${{ inputs.retry-attempts }} $optArgs"
+            echo "Adding --retry-attempts=${{ inputs.retry-attempts }} to the call"
           fi
-          echo "optArgs= $optArgs"
+
+          # Check if a success line was provided. Converting the github input to a bash variable
+          # Will cause it to tokenize on spaces when passed into python
+          # So just only add the success line if it exits.
+          # set +e so if the run fails we can capture that and print custom error message
+          set +e
           if [ "${{ inputs.success-line }}" != "" ]; then
+            successLineArgs="--success-line=\"${{ inputs.success-line}}\" $optArgs"
+            echo "Adding --success-line=${{ inputs.success-line}} to the call"
+            echo "::endgroup::"
+            echo -e "\033[1;33mRunning Command: python3 $GITHUB_ACTION_PATH/executable-monitor.py --exe-path=${{ inputs.exe-path }} --timeout-seconds=${{ inputs.timeout-seconds }} --success-line="${{ inputs.success-line }}" $optArgs  \033[0m"
+            echo "::group::Executable Output"
             python3 $GITHUB_ACTION_PATH/executable-monitor.py --exe-path=${{ inputs.exe-path }} --timeout-seconds=${{ inputs.timeout-seconds }} $optArgs --success-line="${{ inputs.success-line}}"
           else
+            echo "::endgroup::"
+            echo -e "\033[1;33mRunning Command: python3 $GITHUB_ACTION_PATH/executable-monitor.py --exe-path=${{ inputs.exe-path }} --timeout-seconds=${{ inputs.timeout-seconds }} $optArgs \033[0m"
+            echo "::group::Executable Output"
             python3 $GITHUB_ACTION_PATH/executable-monitor.py --exe-path=${{ inputs.exe-path }} --timeout-seconds=${{ inputs.timeout-seconds }} $optArgs
           fi
+          # Store status as the echo group will overwrite it
+          exitStatus=$?
           echo "::endgroup::"
-          if [ "$?" = "0" ]; then
+          if [ "$exitStatus" = "0" ]; then
             echo -e "\033[32;3mValid exit status found\033[0m"
             exit 0
           else
             echo -e "\033[32;31mDid not find a valid exit condition\033[0m"
+            set -e
             exit 1
           fi
         shell: bash

executable-monitor/executable-monitor.py

@@ -225,7 +225,7 @@ def runAndMonitor(args):
             exit_status = thread.exitcode
             logging.info(f"THREAD EXITED WITH EXITCODE {exit_status}")
 
-        if( ( attempts  < args.retry_attempts ) and exit_status == 1 ):
+        if( ( attempts  < args.retry_attempts ) and exit_status != 0 ):
             logging.warning(f"DID NOT RECEIVE SUCCESSFUL EXIT STATUS, TRYING RE-ATTEMPT {attempts+1} OF {args.retry_attempts}\n")
         else:
             break

executable-monitor/test.c

@@ -4,6 +4,8 @@
 #include <stdlib.h>
 #include <unistd.h>
 
+
+
 typedef struct DateAndTime
 {
     int year;
@@ -30,7 +32,7 @@ int main( int argc,
         printf( "This is a basic test application.\n" );
         printf( "It prints the date and time and then sleeps for loopCount * 3\n" );
         printf( "This program takes in two inputs, a loop count and an exit code\n" );
-        printf( "By default it will run 5 loops and exit with exit status 0\n" );
+        printf( "By default it will run %d loops and exit with exit status %d\n", totalLoops, exitCode );
     }
 
     if( argc == 2 )
@@ -75,7 +77,9 @@ int main( int argc,
                  );
         sleep( i * 3U );
     }
-
+#ifdef EXIT_WITH_MINUTES
+    exitCode = date_and_time.minutes;
+#endif
     printf( "EXITING TEST APPLICICATION WITH EXIT CODE = %d\n",exitCode );
     return exitCode;
 }