diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cee727ab..32429c3a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,6 +23,93 @@ jobs: path: coreMQTT exclude-files: lexicon.txt exclude-dirs: build,docs + test-executable-monitor: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Compile executable monitor test + id: compile-executable-monitor-test + shell: bash + run: | + # Compile executable monitor test + echo "::group::Compile executable monitor test" + sudo apt install build-essential + if [ "$?" = "0" ]; then + echo -e "\033[32;3mInstalled build-essential\033[0m" + exit 0 + else + echo "::endgroup::" + echo -e "\033[32;31mInstall build-essential failed...\033[0m" + exit 1 + fi + gcc executable-monitor/test.c -o executable-monitor/test.out + echo "::endgroup::" + if [ "$?" = "0" ]; then + echo -e "\033[32;3mCompiled executable-monitor/test.c\033[0m" + exit 0 + else + echo -e "\033[32;31mCompilation of executable-monitor/test.c failed...\033[0m" + exit 1 + fi + - name: Test Executable Monitor API - No success metric + continue-on-error: true + id: test-executable-monitor-API-no-success-metric + uses: ./executable-monitor + with: + exe-path: test.out + log-dir: demo_run_logs + timeout-seconds: 20 + retry-attempts: 2 + + - name: Check Last Step + if: success() || failure() && steps.test-executable-monitor-API-no-success-metric.outcome == 'failure' + run: | + # Check Last Step Failed + if [ "${{ steps.test-executable-monitor-API-no-success-metric.outcome}}" = "failure" ]; then + echo -e "\033[32;3mCheck Last Step had outcome '${{ steps.test-executable-monitor-API-no-success-metric.outcome}}' as intended\033[0m" + else + echo -e "\033[32;31mCheck Last Step had unexpected '${{ steps.test-executable-monitor-API-no-success-metric.outcome}}' exit condition\033[0m" + exit 1 + fi + + - name: Test Executable Monitor Action + id: test-executable-monitor-action + uses: ./executable-monitor + with: + exe-path: test.out + log-dir: demo_run_logs + timeout-seconds: 20 + success-line: "Sleep for 6 seconds" + success-exit-code: 0 + retry-attempts: 2 + + - name: Test Executable Monitor Action No Exit Code + id: test-executable-monitor-action-no-exit-code + uses: ./executable-monitor + with: + exe-path: test.out + log-dir: demo_run_logs + timeout-seconds: 20 + success-line: "Sleep for 6 seconds" + retry-attempts: 2 + + - name: Test Executable Monitor Action No Success Line + id: test-executable-monitor-action-no-success-line + uses: ./executable-monitor + with: + exe-path: test.out + log-dir: demo_run_logs + timeout-seconds: 20 + success-exit-code: 0 + retry-attempts: 2 + - name: Test Executable Monitor Action No Retry Attempts + id: test-executable-monitor-action-no-retry-attempts + uses: ./executable-monitor + with: + exe-path: test.out + log-dir: demo_run_logs + success-line: "Sleep for 6 seconds" + test-complexity-check: runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..6b2a67f3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.out +logDir/ diff --git a/executable-monitor/action.yml b/executable-monitor/action.yml index 01967dff..47556c84 100644 --- a/executable-monitor/action.yml +++ b/executable-monitor/action.yml @@ -10,7 +10,7 @@ inputs: success-line: description: 'Line of output from executable indicating success.' required: false - default: "Demo completed successfully." + default: "" timeout-seconds: description: 'Maximum amount of time to run the executable. Default is 600.' required: false @@ -45,7 +45,17 @@ runs: run: | # Run Executable with Monitoring echo "::group::Executable Output" - python3 $GITHUB_ACTION_PATH/executable-monitor.py --exe-path=${{ inputs.exe-path }} --timeout-seconds=${{ inputs.timeout-seconds }} --success-line="${{ inputs.success-line }}" --log-dir=${{ inputs.log-dir }} --retry-attempts=${{ inputs.retry-attempts }} --success-exit-status=${{ inputs.success-exit-status }} + if [ ( "${{ inputs.success-exit-status }}" = "" ) && ( "${{ inputs.success-line }}" = "" ) ]; then + echo "::endgroup::"" + echo -e "\033[32;31mDid not supply an input of success-line or success-exit-status to search for\033[0m" + exit 1 + elif [ "${{ inputs.success-exit-status }}" = "" ]; then + python3 $GITHUB_ACTION_PATH/executable-monitor.py --exe-path=${{ inputs.exe-path }} --timeout-seconds=${{ inputs.timeout-seconds }} --success-line="${{ inputs.success-line }}" --log-dir=${{ inputs.log-dir }} --retry-attempts=${{ inputs.retry-attempts }} + elif [ "${{ inputs.success-line }}" = "" ]; then + python3 $GITHUB_ACTION_PATH/executable-monitor.py --exe-path=${{ inputs.exe-path }} --timeout-seconds=${{ inputs.timeout-seconds }} --success-exit-status=${{ inputs.success-exit-status }} --log-dir=${{ inputs.log-dir }} --retry-attempts=${{ inputs.retry-attempts } + else + python3 $GITHUB_ACTION_PATH/executable-monitor.py --exe-path=${{ inputs.exe-path }} --timeout-seconds=${{ inputs.timeout-seconds }} --success-line="${{ inputs.success-line }}" --success-exit-status=${{ inputs.success-exit-status }} --log-dir=${{ inputs.log-dir }} --retry-attempts=${{ inputs.retry-attempts } + fi echo "::endgroup::" if [ "$?" = "0" ]; then echo -e "\033[32;3mValid exit status found\033[0m" diff --git a/executable-monitor/executable-monitor.py b/executable-monitor/executable-monitor.py index 568d1d15..86f81aa8 100755 --- a/executable-monitor/executable-monitor.py +++ b/executable-monitor/executable-monitor.py @@ -31,9 +31,6 @@ def runAndMonitor(args): file_logging_handler.setFormatter(file_logging_formatter) logging.getLogger().addHandler(file_logging_handler) - if args.success_exit_status is not None: - logging.info("Looking for exit status {0}".format(args.success_exit_status )) - # Initialize values exe_exit_status = None exe_exitted = False @@ -47,7 +44,7 @@ def runAndMonitor(args): cur_time_seconds = time.time() timeout_time_seconds = cur_time_seconds + args.timeout_seconds - logging.info("START OF DEVICE OUTPUT\n") + logging.info("START OF EXECUTABLE OUTPUT\n") # While a timeout hasn't happened, the executable is running, and an exit condition has not been met while ( not exit_condition_met ): @@ -56,18 +53,18 @@ def runAndMonitor(args): if (exe_exit_status is not None) and (exe_exitted is False): logging.info(f"EXECUTABLE CLOSED WITH STATUS: {exe_exit_status}") exe_exitted = True - if( args.success_line is None): - exit_condition_met = True + exit_condition_met = True exe_stdout_line = exe.stdout.readline() if(exe_stdout_line is not None) and (len(exe_stdout_line.strip()) > 1): # Check if the executable printed out its success line if ( args.success_line is not None ) and ( args.success_line in exe_stdout_line ) : logging.info(f"SUCCESS_LINE_FOUND: {exe_stdout_line}") + # Mark that we found the success line, kill the executable success_line_found = True - if( not wait_for_exit ): - exit_condition_met = True - exe.kill() + exit_condition_met = True + exe.kill() + time.sleep(.05) else: logging.info(exe_stdout_line) @@ -80,11 +77,9 @@ def runAndMonitor(args): # Sleep for a short duration between loops to not steal all system resources # time.sleep(.05) - if not exe_exitted: + if not exe_exitted and not success_line_found: logging.info(f"EXECUTABLE DID NOT EXIT, MANUALLY KILLING NOW") exe.kill() - - if not exit_condition_met: logging.info(f"PARSING REST OF LOG") # Capture remaining output and check for the successful line for exe_stdout_line in exe.stdout.readlines(): @@ -95,29 +90,32 @@ def runAndMonitor(args): logging.info("END OF DEVICE OUTPUT") logging.info("EXECUTABLE RUN SUMMARY:") - exit_status = 0 - - if args.success_line is not None: - if not success_line_found: - logging.error("Success Line: Success line not output.\n") - exit_status = 1 - if args.success_exit_status is not None: - if exe_exitted: - if exe_exit_status != args.success_exit_status: - exit_status = 1 - logging.info(f"Exit Status: {exe_exit_status}\n") - else: + exit_status = 1 + # Check if a success line was found if that is an option + if ( args.success_line is not None) and (not success_line_found ): + logging.error("Success Line: Success line not output.") + exit_status = 1 + elif( args.success_line is not None) and ( success_line_found ): + exit_status = 0 + logging.info(f"Success Line: Success line was output") + + # Check if a exit status was found if that was an option + if ( ( exit_status != 0 ) and ( args.success_exit_status is not None) ): + # If the executable had to be force killed mark it as a failure + if( not exe_exitted): logging.error("Exit Status: Executable did not exit by itself.\n") exit_status = 1 + # If the executable exited with a different status mark it as a failure + elif ( ( exe_exitted ) and ( exe_exit_status != args.success_exit_status ) ): + logging.error(f"Exit Status: {exe_exit_status} is not equal to requested exit status of {args.success_exit_status}\n") + exit_status = 1 + # If the executable exited with the same status as requested mark a success + elif ( ( exe_exitted ) and ( exe_exit_status == args.success_exit_status ) ): + logging.info(f"Exit Status: Executable exited with requested exit status") + exit_status = 0 - if( exit_status == 0 ): - logging.info(f"Run found a valid success metric\n") - - else: - logging.error("Run did not find a valid success metric.\n") - - logging.info(f"Runner thread exiiting with status {exit_status}") + logging.info(f"Runner thread exiting with status {exit_status}\n") exit(exit_status) if __name__ == '__main__': @@ -150,7 +148,7 @@ def runAndMonitor(args): required=False, help='Line that indicates executable completed successfully. Required if --success-exit-status is not used.') parser.add_argument('--success-exit-status', - type=str, + type=int, required=False, help='Exit status that indicates that the executable completed successfully. Required if --success-line is not used.') parser.add_argument('--retry-attempts', @@ -160,19 +158,16 @@ def runAndMonitor(args): args = parser.parse_args() - # GitHub action needs to be able to pass in a "blank" value. This means the parser needs to take in success exit status as a str - # Check if it was the blank value, if it was set it to None. If it was not convert it to an integer - if args.success_exit_status == "": - args.success_exit_status = None - else: - args.success_exit_status = int(args.success_exit_status) - if args.success_exit_status is None and args.success_line is None: - logging.info("Must specify at least one of the following: --success-line, --success-exit-status.") + logging.error("Must specify at least one of the following: --success-line, --success-exit-status.") sys.exit(1) + elif args.success_exit_status is not None and args.success_line is not None: + logging.warning("Received an option for success-line and success-exit-status.") + logging.warning("Be aware: This program will report SUCCESS on either of these conditions being met") + if not os.path.exists(args.exe_path): - logging.info(f'Input executable path \"{args.exe_path}\" does not exist.') + logging.error(f'Input executable path \"{args.exe_path}\" does not exist.') sys.exit(1) # Convert any relative path (like './') in passed argument to absolute path. @@ -192,19 +187,23 @@ def runAndMonitor(args): file_logging_handler.setFormatter(file_logging_formatter) logging.getLogger().addHandler(file_logging_handler) - if not args.retry_attempts: - retryAttempts = 0 - else: - retryAttempts = args.retry_attempts - logging.info(f"Running executable: {exe_abs_path} ") logging.info(f"Storing logs in: {log_dir}") logging.info(f"Timeout (seconds) per run: {args.timeout_seconds}") - logging.info(f"Searching for success line: {args.success_line}\n") - + + if not args.retry_attempts: + args.retry_attempts = 0 + else: + logging.info(f"Will relaunch the executable {args.retry_attempts} times to look for a valid success metric") + + if args.success_line is not None: + logging.info(f"Searching for success line: {args.success_line}") + if args.success_exit_status is not None: + logging.info(f"Searching for exit code: {args.success_exit_status}") + # Small increase on the timeout to allow the thread to try and timeout threadTimeout = ( args.timeout_seconds + 3 ) - for attempts in range(0,retryAttempts + 1): + for attempts in range(0,args.retry_attempts + 1): exit_status = 1 # Set the timeout for the thread thread = Process(target=runAndMonitor, args=(args,)) @@ -215,15 +214,15 @@ def runAndMonitor(args): # If the thread is still alive, the join() call timed out. if ( ( thread.exitcode is None ) and ( thread.is_alive() ) ): # Print the thread timeout they passed in to the log - logging.info(f"EXECUTABLE HAS HIT TIMEOUT OF {threadTimeout - 3} SECONDS: FORCE KILLING THREAD") + logging.warning(f"EXECUTABLE HAS HIT TIMEOUT OF {threadTimeout - 3} SECONDS: FORCE KILLING THREAD") thread.kill() exit_status = 1 else: exit_status = thread.exitcode logging.info(f"THREAD EXITED WITH EXITCODE {exit_status}") - if( ( attempts < retryAttempts ) and exit_status == 1 ): - logging.info(f"DID NOT RECEIVE SUCCESSFUL EXIT STATUS, TRYING RE-ATTEMPT {attempts+1} OF {retryAttempts}\n") + if( ( attempts < args.retry_attempts ) and exit_status == 1 ): + logging.warning(f"DID NOT RECEIVE SUCCESSFUL EXIT STATUS, TRYING RE-ATTEMPT {attempts+1} OF {args.retry_attempts}\n") else: break diff --git a/executable-monitor/test.c b/executable-monitor/test.c index dd18f472..03d5c524 100644 --- a/executable-monitor/test.c +++ b/executable-monitor/test.c @@ -4,7 +4,8 @@ #include #include -typedef struct DateAndTime { +typedef struct DateAndTime +{ int year; int month; int day; @@ -14,43 +15,67 @@ typedef struct DateAndTime { int msec; } DateAndTime; -int -main(void) +int main( int argc, + char ** argv ) { DateAndTime date_and_time; struct timeval tv; - struct tm *tm; - int i = 0; - //exit(0); - gettimeofday(&tv, NULL); + struct tm * tm; + int32_t loop = 0; + int32_t totalLoops = 5; + int32_t exitCode = 0; + + if( argc == 1 ) + { + printf( "This is a basic test application.\n" ); + printf( "It prints the date and time and then sleeps for loopCount * 3\n" ); + printf( "This program takes in two inputs, a loop count and an exit code\n" ); + printf( "By default it will run 5 loops and exit with exit status 0\n" ); + } + + if( argc == 2 ) + { + totalLoops = ( int32_t ) atoi( argv[ 2 ] ); + printf( "Will run for requested %d loops\n", totalLoops ); + } + + if( argc == 3 ) + { + exitCode = atoi( argv[ 3 ] ); + printf( "Will exit with supplied exit code %d\n", exitCode ); + } setvbuf( stdout, NULL, _IONBF, 0 ); - for(int i = 1; i < 20; i++){ - - tm = localtime(&tv.tv_sec); - printf("\r\n"); - // Add 1900 to get the right year value - // read the manual page for localtime() - // date_and_time.year = tm->tm_year + 1900; - // Months are 0 based in struct tm + + for(int i = 1; i < totalLoops; i++) + { + gettimeofday( &tv, NULL ); + tm = localtime( &tv.tv_sec ); + /* Add 1900 to get the right year value */ + /* read the manual page for localtime() */ + /* date_and_time.year = tm->tm_year + 1900; */ + /* Months are 0 based in struct tm */ + date_and_time.year = tm->tm_year + 1900; date_and_time.month = tm->tm_mon + 1; date_and_time.day = tm->tm_mday; date_and_time.hour = tm->tm_hour; date_and_time.minutes = tm->tm_min; date_and_time.seconds = tm->tm_sec; - date_and_time.msec = (int) (tv.tv_usec / 1000); - - fprintf(stdout, "%02d:%02d:%02d.%03d %02d-%02d-%04d Sleeping for %d seconds\n", - date_and_time.hour, - date_and_time.minutes, - date_and_time.seconds, - date_and_time.msec, - date_and_time.day, - date_and_time.month, - date_and_time.year, - i*i*i - ); - sleep(i*i*i); + date_and_time.msec = ( int ) ( tv.tv_usec / 1000 ); + + fprintf( stdout, "%02d:%02d:%02d.%03d %02d-%02d-%04d Sleeping for %d seconds\n", + date_and_time.hour, + date_and_time.minutes, + date_and_time.seconds, + date_and_time.msec, + date_and_time.day, + date_and_time.month, + date_and_time.year, + i * 3U + ); + sleep( i * 3U ); } - return 0; + + printf( "Exiting test application\n" ); + return exitCode; }