diff --git a/test/test.h b/test/test.h index c69a4fa925d1..27475bb48e40 100644 --- a/test/test.h +++ b/test/test.h @@ -9,7 +9,7 @@ enum TestResult { TEST_RESULT_FAIL, TEST_RESULT_PASS, - TEST_RESULT_SKIP, + TEST_RESULT_ASSUMPTION_FAIL, TEST_RESULT_INVALID, TEST_RESULT_ERROR, TEST_RESULT_TIMEOUT, @@ -77,7 +77,7 @@ s32 MgbaPrintf_(const char *fmt, ...); do \ { \ if (!(c)) \ - Test_ExitWithResult(TEST_RESULT_SKIP, "%s:%d: ASSUME failed", gTestRunnerState.test->filename, __LINE__); \ + Test_ExitWithResult(TEST_RESULT_ASSUMPTION_FAIL, "%s:%d: ASSUME failed", gTestRunnerState.test->filename, __LINE__); \ } while (0) #define EXPECT(c) \ diff --git a/test/test_runner.c b/test/test_runner.c index 0be163431703..c8bd0f3a92de 100644 --- a/test/test_runner.c +++ b/test/test_runner.c @@ -121,7 +121,7 @@ void CB2_TestRunner(void) // NOTE: Assumes that the compiler interns __FILE__. if (gTestRunnerState.skipFilename == gTestRunnerState.test->filename) { - gTestRunnerState.result = TEST_RESULT_SKIP; + gTestRunnerState.result = TEST_RESULT_ASSUMPTION_FAIL; } else { @@ -157,7 +157,7 @@ void CB2_TestRunner(void) color = "\e[32m"; MgbaPrintf_(":N%s", gTestRunnerState.test->name); } - else if (gTestRunnerState.result != TEST_RESULT_SKIP || gTestRunnerSkipIsFail) + else if (gTestRunnerState.result != TEST_RESULT_ASSUMPTION_FAIL || gTestRunnerSkipIsFail) { gTestRunnerState.exitCode = 1; color = "\e[31m"; @@ -186,16 +186,33 @@ void CB2_TestRunner(void) result = "FAIL"; } break; - case TEST_RESULT_PASS: result = "PASS"; break; - case TEST_RESULT_SKIP: result = "SKIP"; break; - case TEST_RESULT_INVALID: result = "INVALID"; break; - case TEST_RESULT_ERROR: result = "ERROR"; break; - case TEST_RESULT_TIMEOUT: result = "TIMEOUT"; break; - default: result = "UNKNOWN"; break; + case TEST_RESULT_PASS: + result = "PASS"; + break; + case TEST_RESULT_ASSUMPTION_FAIL: + result = "ASSUMPTION_FAIL"; + color = "\e[33m"; + break; + case TEST_RESULT_INVALID: + result = "INVALID"; + break; + case TEST_RESULT_ERROR: + result = "ERROR"; + break; + case TEST_RESULT_TIMEOUT: + result = "TIMEOUT"; + break; + default: + result = "UNKNOWN"; + break; } - if (gTestRunnerState.expectedResult == gTestRunnerState.result) + if (gTestRunnerState.result == TEST_RESULT_PASS) MgbaPrintf_(":P%s%s\e[0m", color, result); + else if (gTestRunnerState.result == TEST_RESULT_ASSUMPTION_FAIL) + MgbaPrintf_(":A%s%s\e[0m", color, result); + else if (gTestRunnerState.expectedResult == gTestRunnerState.result) + MgbaPrintf_(":K%s%s\e[0m", color, result); else MgbaPrintf_(":F%s%s\e[0m", color, result); } diff --git a/test/test_runner_battle.c b/test/test_runner_battle.c index e40ed3e50b88..ec7ba275c2c2 100644 --- a/test/test_runner_battle.c +++ b/test/test_runner_battle.c @@ -720,7 +720,7 @@ static void CB2_BattleTest_NextTrial(void) case TEST_RESULT_PASS: STATE->observedPasses++; break; - case TEST_RESULT_SKIP: + case TEST_RESULT_ASSUMPTION_FAIL: STATE->skippedTrials++; if (STATE->skippedTrials > STATE->trials / 4) Test_ExitWithResult(TEST_RESULT_INVALID, "25%% of the trials were SKIPed"); diff --git a/tools/mgba-rom-test-hydra/main.c b/tools/mgba-rom-test-hydra/main.c index 4ce9b09bd055..f506d0428d31 100644 --- a/tools/mgba-rom-test-hydra/main.c +++ b/tools/mgba-rom-test-hydra/main.c @@ -9,7 +9,11 @@ * COMMANDS * N: Sets the test name to the remainder of the line. * R: Sets the result to the remainder of the line, and flushes any - * output buffered since the previous R. */ + * output buffered since the previous R. + * P/K/F/A: Sets the result to the remaining of the line, flushes any + * output since the previous P/K/F/A and increment the number of + * passes/known fails/assumption fails/fails. + */ #include #include #include @@ -39,6 +43,9 @@ struct Runner size_t output_buffer_capacity; char *output_buffer; int passes; + int knownFails; + int assumptionFails; + int fails; int results; }; @@ -75,9 +82,17 @@ static void handle_read(struct Runner *runner) break; case 'P': + runner->passes++; + goto add_to_results; + case 'K': + runner->knownFails++; + goto add_to_results; + case 'A': + runner->assumptionFails++; + goto add_to_results; case 'F': - if (soc[1] == 'P') - runner->passes++; + runner->fails++; +add_to_results: runner->results++; soc += 2; fprintf(stdout, "%s: ", runner->test_name); @@ -411,6 +426,9 @@ int main(int argc, char *argv[]) // Reap test runners and collate exit codes. int exit_code = 0; int passes = 0; + int knownFails = 0; + int assumptionFails = 0; + int fails = 0; int results = 0; for (int i = 0; i < nrunners; i++) { @@ -425,9 +443,29 @@ int main(int argc, char *argv[]) if (WIFEXITED(wstatus) && WEXITSTATUS(wstatus) > exit_code) exit_code = WEXITSTATUS(wstatus); passes += runners[i].passes; + knownFails += runners[i].knownFails; + assumptionFails += runners[i].assumptionFails; + fails += runners[i].fails; results += runners[i].results; } - fprintf(stdout, "%d/%d \e[32mPASS\e[0med\n", passes, results); + + if (results == 0) + { + fprintf(stdout, "\nNo tests found.\n"); + } + else + { + fprintf(stdout, "\n- Tests TOTAL: %d\n", results); + fprintf(stdout, "- Tests \e[32mPASSED\e[0m: %d\n", passes); + if (knownFails > 0) + fprintf(stdout, "- Tests \e[33mKNOWN_FAILING\e[0m: %d\n", knownFails); + if (fails > 0) + fprintf(stdout, "- Tests \e[31mFAILED\e[0m : %d\n", fails); + if (assumptionFails > 0) + fprintf(stdout, "- \e[33mASSUMPTIONS_FAILED\e[0m: %d\n", assumptionFails); + } + fprintf(stdout, "\n"); + fflush(stdout); return exit_code; }