From 54e388a3752e81d6c8137f740097dc68540f5fad Mon Sep 17 00:00:00 2001 From: Eduardo Quezada Date: Wed, 22 Feb 2023 00:30:12 -0300 Subject: [PATCH 1/5] More detailed hydra test results --- test/test_runner.c | 4 +++- tools/mgba-rom-test-hydra/main.c | 29 +++++++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/test/test_runner.c b/test/test_runner.c index 0be163431703..c2d3f66d2502 100644 --- a/test/test_runner.c +++ b/test/test_runner.c @@ -194,8 +194,10 @@ void CB2_TestRunner(void) default: result = "UNKNOWN"; break; } - if (gTestRunnerState.expectedResult == gTestRunnerState.result) + if (gTestRunnerState.result == TEST_RESULT_PASS) MgbaPrintf_(":P%s%s\e[0m", color, result); + else if (gTestRunnerState.expectedResult == gTestRunnerState.result) + MgbaPrintf_(":K%s%s\e[0m", color, result); else MgbaPrintf_(":F%s%s\e[0m", color, result); } diff --git a/tools/mgba-rom-test-hydra/main.c b/tools/mgba-rom-test-hydra/main.c index 4ce9b09bd055..107ca339be7a 100644 --- a/tools/mgba-rom-test-hydra/main.c +++ b/tools/mgba-rom-test-hydra/main.c @@ -9,7 +9,11 @@ * COMMANDS * N: Sets the test name to the remainder of the line. * R: Sets the result to the remainder of the line, and flushes any - * output buffered since the previous R. */ + * output buffered since the previous R. + * + * //Missing P, F and K documentation, please tell me what to put here lol + * + */ #include #include #include @@ -39,6 +43,7 @@ struct Runner size_t output_buffer_capacity; char *output_buffer; int passes; + int knownFails; int results; }; @@ -76,8 +81,11 @@ static void handle_read(struct Runner *runner) case 'P': case 'F': + case 'K': if (soc[1] == 'P') runner->passes++; + else if (soc[1] == 'K') + runner->knownFails++; runner->results++; soc += 2; fprintf(stdout, "%s: ", runner->test_name); @@ -411,6 +419,7 @@ int main(int argc, char *argv[]) // Reap test runners and collate exit codes. int exit_code = 0; int passes = 0; + int knownFails = 0; int results = 0; for (int i = 0; i < nrunners; i++) { @@ -425,9 +434,25 @@ int main(int argc, char *argv[]) if (WIFEXITED(wstatus) && WEXITSTATUS(wstatus) > exit_code) exit_code = WEXITSTATUS(wstatus); passes += runners[i].passes; + knownFails += runners[i].knownFails; results += runners[i].results; } - fprintf(stdout, "%d/%d \e[32mPASS\e[0med\n", passes, results); + + if (results == 0) + { + fprintf(stdout, "\nNo tests found.\n"); + } + else + { + fprintf(stdout, "\n- Tests TOTAL: %d\n", results); + fprintf(stdout, "- Tests \e[32mPASSED: \e[0m %d\n", passes); + if (knownFails > 0) + fprintf(stdout, "- Tests \e[33mKNOWN_FAILING:\e[0m %d\n", knownFails); + if (passes + knownFails < results) + fprintf(stdout, "- Tests \e[31mFAILED: \e[0m %d\n", results - passes - knownFails); + } + fprintf(stdout, "\n"); + fflush(stdout); return exit_code; } From 5808e82434ab03d10396485028398f618858eac6 Mon Sep 17 00:00:00 2001 From: Eduardo Quezada Date: Wed, 22 Feb 2023 15:05:39 -0300 Subject: [PATCH 2/5] Added assumption fail total, made fails into their own counter and changed "SKIP" to "ASSUMPTION_FAIIL" in log --- test/test_runner.c | 7 ++++++- tools/mgba-rom-test-hydra/main.c | 31 ++++++++++++++++++++++--------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/test/test_runner.c b/test/test_runner.c index c2d3f66d2502..eb5d12c77893 100644 --- a/test/test_runner.c +++ b/test/test_runner.c @@ -187,7 +187,10 @@ void CB2_TestRunner(void) } break; case TEST_RESULT_PASS: result = "PASS"; break; - case TEST_RESULT_SKIP: result = "SKIP"; break; + case TEST_RESULT_SKIP: + result = "ASSUMPTION_FAIL"; + color = "\e[33m"; + break; case TEST_RESULT_INVALID: result = "INVALID"; break; case TEST_RESULT_ERROR: result = "ERROR"; break; case TEST_RESULT_TIMEOUT: result = "TIMEOUT"; break; @@ -196,6 +199,8 @@ void CB2_TestRunner(void) if (gTestRunnerState.result == TEST_RESULT_PASS) MgbaPrintf_(":P%s%s\e[0m", color, result); + else if (gTestRunnerState.result == TEST_RESULT_SKIP) + MgbaPrintf_(":A%s%s\e[0m", color, result); else if (gTestRunnerState.expectedResult == gTestRunnerState.result) MgbaPrintf_(":K%s%s\e[0m", color, result); else diff --git a/tools/mgba-rom-test-hydra/main.c b/tools/mgba-rom-test-hydra/main.c index 107ca339be7a..75c705f17758 100644 --- a/tools/mgba-rom-test-hydra/main.c +++ b/tools/mgba-rom-test-hydra/main.c @@ -44,6 +44,8 @@ struct Runner char *output_buffer; int passes; int knownFails; + int assumptionFails; + int fails; int results; }; @@ -80,12 +82,17 @@ static void handle_read(struct Runner *runner) break; case 'P': - case 'F': + runner->passes++; + goto add_to_results; case 'K': - if (soc[1] == 'P') - runner->passes++; - else if (soc[1] == 'K') - runner->knownFails++; + runner->knownFails++; + goto add_to_results; + case 'A': + runner->assumptionFails++; + goto add_to_results; + case 'F': + runner->fails++; +add_to_results: runner->results++; soc += 2; fprintf(stdout, "%s: ", runner->test_name); @@ -420,6 +427,8 @@ int main(int argc, char *argv[]) int exit_code = 0; int passes = 0; int knownFails = 0; + int assumptionFails = 0; + int fails = 0; int results = 0; for (int i = 0; i < nrunners; i++) { @@ -435,6 +444,8 @@ int main(int argc, char *argv[]) exit_code = WEXITSTATUS(wstatus); passes += runners[i].passes; knownFails += runners[i].knownFails; + assumptionFails += runners[i].assumptionFails; + fails += runners[i].fails; results += runners[i].results; } @@ -445,11 +456,13 @@ int main(int argc, char *argv[]) else { fprintf(stdout, "\n- Tests TOTAL: %d\n", results); - fprintf(stdout, "- Tests \e[32mPASSED: \e[0m %d\n", passes); + fprintf(stdout, "- Tests \e[32mPASSED\e[0m: %d\n", passes); if (knownFails > 0) - fprintf(stdout, "- Tests \e[33mKNOWN_FAILING:\e[0m %d\n", knownFails); - if (passes + knownFails < results) - fprintf(stdout, "- Tests \e[31mFAILED: \e[0m %d\n", results - passes - knownFails); + fprintf(stdout, "- Tests \e[33mKNOWN_FAILING\e[0m: %d\n", knownFails); + if (fails > 0) + fprintf(stdout, "- Tests \e[31mFAILED\e[0m : %d\n", fails); + if (assumptionFails > 0) + fprintf(stdout, "- \e[33mASSUMPTIONS_FAILED\e[0m: %d\n", assumptionFails); } fprintf(stdout, "\n"); From e3ed11475457999e828349eb499b3fa0ae25405a Mon Sep 17 00:00:00 2001 From: Eduardo Quezada Date: Wed, 22 Feb 2023 15:09:47 -0300 Subject: [PATCH 3/5] Updated commands comment --- tools/mgba-rom-test-hydra/main.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/mgba-rom-test-hydra/main.c b/tools/mgba-rom-test-hydra/main.c index 75c705f17758..f506d0428d31 100644 --- a/tools/mgba-rom-test-hydra/main.c +++ b/tools/mgba-rom-test-hydra/main.c @@ -10,9 +10,9 @@ * N: Sets the test name to the remainder of the line. * R: Sets the result to the remainder of the line, and flushes any * output buffered since the previous R. - * - * //Missing P, F and K documentation, please tell me what to put here lol - * + * P/K/F/A: Sets the result to the remaining of the line, flushes any + * output since the previous P/K/F/A and increment the number of + * passes/known fails/assumption fails/fails. */ #include #include From eb49e28e6cb9c57d03d5a5d952735a0d4c57fe7b Mon Sep 17 00:00:00 2001 From: Eduardo Quezada Date: Wed, 8 Mar 2023 08:26:33 -0300 Subject: [PATCH 4/5] Renamed TEST_RESULT_SKIP to TEST_RESULT_ASSUMPTION_FAIL --- test/test.h | 4 ++-- test/test_runner.c | 8 ++++---- test/test_runner_battle.c | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test.h b/test/test.h index c69a4fa925d1..27475bb48e40 100644 --- a/test/test.h +++ b/test/test.h @@ -9,7 +9,7 @@ enum TestResult { TEST_RESULT_FAIL, TEST_RESULT_PASS, - TEST_RESULT_SKIP, + TEST_RESULT_ASSUMPTION_FAIL, TEST_RESULT_INVALID, TEST_RESULT_ERROR, TEST_RESULT_TIMEOUT, @@ -77,7 +77,7 @@ s32 MgbaPrintf_(const char *fmt, ...); do \ { \ if (!(c)) \ - Test_ExitWithResult(TEST_RESULT_SKIP, "%s:%d: ASSUME failed", gTestRunnerState.test->filename, __LINE__); \ + Test_ExitWithResult(TEST_RESULT_ASSUMPTION_FAIL, "%s:%d: ASSUME failed", gTestRunnerState.test->filename, __LINE__); \ } while (0) #define EXPECT(c) \ diff --git a/test/test_runner.c b/test/test_runner.c index eb5d12c77893..9d33301f9d49 100644 --- a/test/test_runner.c +++ b/test/test_runner.c @@ -121,7 +121,7 @@ void CB2_TestRunner(void) // NOTE: Assumes that the compiler interns __FILE__. if (gTestRunnerState.skipFilename == gTestRunnerState.test->filename) { - gTestRunnerState.result = TEST_RESULT_SKIP; + gTestRunnerState.result = TEST_RESULT_ASSUMPTION_FAIL; } else { @@ -157,7 +157,7 @@ void CB2_TestRunner(void) color = "\e[32m"; MgbaPrintf_(":N%s", gTestRunnerState.test->name); } - else if (gTestRunnerState.result != TEST_RESULT_SKIP || gTestRunnerSkipIsFail) + else if (gTestRunnerState.result != TEST_RESULT_ASSUMPTION_FAIL || gTestRunnerSkipIsFail) { gTestRunnerState.exitCode = 1; color = "\e[31m"; @@ -187,7 +187,7 @@ void CB2_TestRunner(void) } break; case TEST_RESULT_PASS: result = "PASS"; break; - case TEST_RESULT_SKIP: + case TEST_RESULT_ASSUMPTION_FAIL: result = "ASSUMPTION_FAIL"; color = "\e[33m"; break; @@ -199,7 +199,7 @@ void CB2_TestRunner(void) if (gTestRunnerState.result == TEST_RESULT_PASS) MgbaPrintf_(":P%s%s\e[0m", color, result); - else if (gTestRunnerState.result == TEST_RESULT_SKIP) + else if (gTestRunnerState.result == TEST_RESULT_ASSUMPTION_FAIL) MgbaPrintf_(":A%s%s\e[0m", color, result); else if (gTestRunnerState.expectedResult == gTestRunnerState.result) MgbaPrintf_(":K%s%s\e[0m", color, result); diff --git a/test/test_runner_battle.c b/test/test_runner_battle.c index e40ed3e50b88..ec7ba275c2c2 100644 --- a/test/test_runner_battle.c +++ b/test/test_runner_battle.c @@ -720,7 +720,7 @@ static void CB2_BattleTest_NextTrial(void) case TEST_RESULT_PASS: STATE->observedPasses++; break; - case TEST_RESULT_SKIP: + case TEST_RESULT_ASSUMPTION_FAIL: STATE->skippedTrials++; if (STATE->skippedTrials > STATE->trials / 4) Test_ExitWithResult(TEST_RESULT_INVALID, "25%% of the trials were SKIPed"); From 24d8d3fbee471d51d119658765fa1ccbf9a35687 Mon Sep 17 00:00:00 2001 From: Eduardo Quezada Date: Wed, 8 Mar 2023 09:09:48 -0300 Subject: [PATCH 5/5] Updated switch format --- test/test_runner.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/test/test_runner.c b/test/test_runner.c index 9d33301f9d49..c8bd0f3a92de 100644 --- a/test/test_runner.c +++ b/test/test_runner.c @@ -186,15 +186,25 @@ void CB2_TestRunner(void) result = "FAIL"; } break; - case TEST_RESULT_PASS: result = "PASS"; break; + case TEST_RESULT_PASS: + result = "PASS"; + break; case TEST_RESULT_ASSUMPTION_FAIL: result = "ASSUMPTION_FAIL"; color = "\e[33m"; break; - case TEST_RESULT_INVALID: result = "INVALID"; break; - case TEST_RESULT_ERROR: result = "ERROR"; break; - case TEST_RESULT_TIMEOUT: result = "TIMEOUT"; break; - default: result = "UNKNOWN"; break; + case TEST_RESULT_INVALID: + result = "INVALID"; + break; + case TEST_RESULT_ERROR: + result = "ERROR"; + break; + case TEST_RESULT_TIMEOUT: + result = "TIMEOUT"; + break; + default: + result = "UNKNOWN"; + break; } if (gTestRunnerState.result == TEST_RESULT_PASS)