From f25bcd9c4a8dc297955b1190bc8ed6680e31ff5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Silveira?= Date: Sun, 3 Nov 2024 11:46:38 +0000 Subject: [PATCH] chore: improve tests --- .gitignore | 1 + CMakeLists.txt | 10 +- samples/error/{99.lang => 04.lang} | 0 samples/error/{999.lang => 05.lang} | 0 samples/test.py | 164 -------------------------- samples/valid/{1.lang => 01.lang} | 0 samples/valid/{2.lang => 02.lang} | 0 samples/valid/{3.lang => 03.lang} | 0 samples/valid/{4.lang => 04.lang} | 0 samples/valid/{5.lang => 05.lang} | 0 samples/valid/{6.lang => 06.lang} | 0 samples/valid/{7.lang => 07.lang} | 0 samples/valid/{8.lang => 08.lang} | 0 samples/valid/{9.lang => 09.lang} | 0 src/CMakeLists.txt | 7 -- src/main.cpp | 5 +- tests/main.py | 176 ++++++++++++++++++++++++++++ 17 files changed, 189 insertions(+), 174 deletions(-) rename samples/error/{99.lang => 04.lang} (100%) rename samples/error/{999.lang => 05.lang} (100%) delete mode 100644 samples/test.py rename samples/valid/{1.lang => 01.lang} (100%) rename samples/valid/{2.lang => 02.lang} (100%) rename samples/valid/{3.lang => 03.lang} (100%) rename samples/valid/{4.lang => 04.lang} (100%) rename samples/valid/{5.lang => 05.lang} (100%) rename samples/valid/{6.lang => 06.lang} (100%) rename samples/valid/{7.lang => 07.lang} (100%) rename samples/valid/{8.lang => 08.lang} (100%) rename samples/valid/{9.lang => 09.lang} (100%) delete mode 100644 src/CMakeLists.txt create mode 100644 tests/main.py diff --git a/.gitignore b/.gitignore index 714f3b4..156ba47 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ build/ .cache/ .vscode/ +__pycache__/ \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index e892f99..a764cfe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,10 +14,16 @@ set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti -fno-exceptions") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wno-unused-variable") include_directories(include ${LLVM_INCLUDE_DIRS}) add_definitions(${LLVM_DEFINITIONS}) -add_subdirectory(src) +file(GLOB_RECURSE SOURCES CONFIGURE_DEPENDS "src/*.cpp") + +add_executable(compiler ${SOURCES}) + +llvm_map_components_to_libnames(llvm_libs core support) + +target_link_libraries(compiler ${llvm_libs}) diff --git a/samples/error/99.lang b/samples/error/04.lang similarity index 100% rename from samples/error/99.lang rename to samples/error/04.lang diff --git a/samples/error/999.lang b/samples/error/05.lang similarity index 100% rename from samples/error/999.lang rename to samples/error/05.lang diff --git a/samples/test.py b/samples/test.py deleted file mode 100644 index b719146..0000000 --- a/samples/test.py +++ /dev/null @@ -1,164 +0,0 @@ -import sys -import subprocess - - -def test_compiler_valid(compiler: str, files: list[str]) -> None: - """ - Test the compiler with the given files. - - Args: - compiler (str): The compiler to test - files (list[str]): A list of file names - """ - - for file in files: - res = subprocess.run( - [compiler, file], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - - # Assert that the compiler returned successfully - if res.returncode != 0: - print(f"\nError: {file} did not return successfully") - print(res.stderr) - sys.exit(1) - - # Assert stderr is empty - if res.stderr: - print(f"\nError: {file} has non-empty stderr\n") - print(res.stderr) - sys.exit(1) - - # Assert that the compiler emits the correct output - print(f"- Test passed: {file}") - - -def test_compiler_error(compiler: str, files: list[tuple[str, str, str]]) -> None: - """ - Test the compiler's error handling by checking if the compiler emits the correct error - for the given files. - - Args: - compiler (str): The compiler to test - files (list[tuple[str, str, str]]): A list of tuples containing the file name, the expected error id and the location - """ - import json - - for file, error_id, location in files: - res = subprocess.run( - [compiler, "--error-format=json", file], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - - # Assert that the compiler returned an error - if res.returncode == 0: - print(f"\nError: {file} did not return an error") - sys.exit(1) - - # Assert stderr is not empty - if not res.stderr: - print(f"\nError: {file} has empty stderr") - sys.exit(1) - - # Assert stderr is a valid JSON - error_json = None - try: - error_json = json.loads(res.stderr) - except json.JSONDecodeError: - print(f"\nError: {file} has invalid JSON in stderr") - print(res.stderr) - sys.exit(1) - - # Check the length of the errors is 1 - if len(error_json) != 1: - print(f"\nError: {file} has more than one error") - print(res.stderr) - sys.exit(1) - - # Check the error id - if error_json[0]["id"] != error_id: - print( - f"\nError: {file} has incorrect error id: got {error_json[0]['id']}, expected {error_id}" - ) - sys.exit(1) - - # Check the error location - if error_json[0]["loc"] != location: - print( - f"\nError: {file} has incorrect error location: got {error_json[0]['loc']}, expected {location}" - ) - sys.exit(1) - - # Assert that the correct error is emitted - print(f"- Test passed: {file}") - - -def main(): - import argparse - - FILE_EXT = "lang" - VALID_PATH = "samples/valid" - ERROR_PATH = "samples/error" - - # Parse the command line arguments - - parser = argparse.ArgumentParser( - description="A testing script for compiler's error handling" - ) - - parser.add_argument("compiler", help="The compiler to test") - - # TODO: Actually implement this - parser.add_argument( - "-p", - "--pipeline", - help="The pipeline to test", - choices=["lex", "syn", "output", "all"], - default="all", - ) - - # Parse command line arguments - - args = parser.parse_args() - - # Test the compiler with the valid files - - valid_test_set = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"] - - valid_files = map( - lambda file: f"{VALID_PATH}/{file}.{FILE_EXT}", - valid_test_set, - ) - - print("Testing the compiler with the valid files:") - test_compiler_valid(args.compiler, valid_files) - - # Test the compiler with the error files - - error_test_set = [ - ("01", "lex-invalid-char", "1:1"), - ("02", "parse-unexpected-eof", "1:1"), - ("03", "parse-unexpected-token", "2:9"), - # ("1", "cfa-early-return-stmt", "2:5"), - # ("2", "cfa-invalid-break-stmt", "2:5"), - ] - - error_files = map( - lambda file: ( - f"{ERROR_PATH}/{file[0]}.{FILE_EXT}", - file[1], - f"{ERROR_PATH}/{file[0]}.{FILE_EXT}:{file[2]}", - ), - error_test_set, - ) - - print("\nTesting the compiler with the error files:") - test_compiler_error(args.compiler, error_files) - - -if __name__ == "__main__": - main() diff --git a/samples/valid/1.lang b/samples/valid/01.lang similarity index 100% rename from samples/valid/1.lang rename to samples/valid/01.lang diff --git a/samples/valid/2.lang b/samples/valid/02.lang similarity index 100% rename from samples/valid/2.lang rename to samples/valid/02.lang diff --git a/samples/valid/3.lang b/samples/valid/03.lang similarity index 100% rename from samples/valid/3.lang rename to samples/valid/03.lang diff --git a/samples/valid/4.lang b/samples/valid/04.lang similarity index 100% rename from samples/valid/4.lang rename to samples/valid/04.lang diff --git a/samples/valid/5.lang b/samples/valid/05.lang similarity index 100% rename from samples/valid/5.lang rename to samples/valid/05.lang diff --git a/samples/valid/6.lang b/samples/valid/06.lang similarity index 100% rename from samples/valid/6.lang rename to samples/valid/06.lang diff --git a/samples/valid/7.lang b/samples/valid/07.lang similarity index 100% rename from samples/valid/7.lang rename to samples/valid/07.lang diff --git a/samples/valid/8.lang b/samples/valid/08.lang similarity index 100% rename from samples/valid/8.lang rename to samples/valid/08.lang diff --git a/samples/valid/9.lang b/samples/valid/09.lang similarity index 100% rename from samples/valid/9.lang rename to samples/valid/09.lang diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt deleted file mode 100644 index 0e0ad43..0000000 --- a/src/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -file(GLOB_RECURSE SOURCES CONFIGURE_DEPENDS "*.cpp") - -add_executable(compiler ${SOURCES}) - -llvm_map_components_to_libnames(llvm_libs core support) - -target_link_libraries(compiler ${llvm_libs}) \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index 69e2f1e..4368d35 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -93,7 +93,7 @@ void reportErrors( } // namespace -int main(int argc, char **argv) { +int main(int argc, char **argv) try { llvm::cl::ParseCommandLineOptions(argc, argv, "Lang Compiler\n", nullptr, nullptr, true); @@ -244,4 +244,7 @@ int main(int argc, char **argv) { } return EXIT_SUCCESS; +} catch (const std::exception &e) { + llvm::errs() << "Error: " << e.what() << '\n'; + return EXIT_FAILURE; } diff --git a/tests/main.py b/tests/main.py new file mode 100644 index 0000000..eeedc02 --- /dev/null +++ b/tests/main.py @@ -0,0 +1,176 @@ +import json +from subprocess import CompletedProcess + + +def compile_program(file: str, opts: str = "") -> CompletedProcess[str]: + import subprocess + + """ + Compile the given program through the compiler executable and return the output. + + Args: + program (str): The program to compile. + + Returns: + Tuple[str, str]: A tuple containing the stdout and stderr of the compiler. + """ + # Open process with pipes for stdout and stderr + return subprocess.run( + ["./build/compiler", file, opts], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + +def test_valid_01() -> None: + res = compile_program("valid/01.lang") + + assert res.returncode == 0 + assert not res.stderr + + +def test_valid_02() -> None: + res = compile_program("valid/02.lang") + + assert res.returncode == 0 + assert not res.stderr + + +def test_valid_03() -> None: + res = compile_program("valid/03.lang") + + assert res.returncode == 0 + assert not res.stderr + + +def test_valid_04() -> None: + res = compile_program("valid/04.lang") + + assert res.returncode == 0 + assert not res.stderr + + +def test_valid_05() -> None: + res = compile_program("valid/05.lang") + + assert res.returncode == 0 + assert not res.stderr + + +def test_valid_06() -> None: + res = compile_program("valid/06.lang") + + assert res.returncode == 0 + assert not res.stderr + + +def test_valid_07() -> None: + res = compile_program("valid/07.lang") + + assert res.returncode == 0 + assert not res.stderr + + +def test_valid_08() -> None: + res = compile_program("valid/08.lang") + + assert res.returncode == 0 + assert not res.stderr + + +def test_valid_09() -> None: + res = compile_program("valid/09.lang") + + assert res.returncode == 0 + assert not res.stderr + + +def test_valid_10() -> None: + res = compile_program("valid/10.lang") + + assert res.returncode == 0 + assert not res.stderr + +def test_error_01() -> None: + res = compile_program("samples/error/01.lang", "--error-format=json") + + assert res.returncode != 0 + assert res.stderr + + try: + error = json.loads(res.stderr) + assert len(error) == 1 + assert error[0]["id"] == "lex-invalid-char" + assert error[0]["loc"] == "samples/error/01.lang:1:1" + except json.JSONDecodeError as e: + print(e) + assert False + +def test_error_02() -> None: + res = compile_program("samples/error/02.lang", "--error-format=json") + + assert res.returncode != 0 + assert res.stderr + + try: + error = json.loads(res.stderr) + assert len(error) == 1 + assert error[0]["id"] == "parse-unexpected-eof" + assert error[0]["loc"] == "samples/error/02.lang:1:1" + except json.JSONDecodeError as e: + print(e) + assert False + +def test_error_03() -> None: + res = compile_program("samples/error/03.lang", "--error-format=json") + + assert res.returncode != 0 + assert res.stderr + + print(res.stderr) + try: + error = json.loads(res.stderr) + assert len(error) == 1 + assert error[0]["id"] == "parse-unexpected-token" + assert error[0]["loc"] == "samples/error/03.lang:2:9" + except json.JSONDecodeError as e: + print(e) + assert False + +def test_error_04() -> None: + res = compile_program("samples/error/04.lang", "--error-format=json") + + assert res.returncode != 0 + assert res.stderr + + print(res.stderr) + try: + error = json.loads(res.stderr) + assert len(error) == 1 + assert error[0]["id"] == "cfa-early-return-stmt" + assert error[0]["loc"] == "samples/error/04.lang:2:5" + except json.JSONDecodeError as e: + print(e) + assert False + +def test_error_05() -> None: + res = compile_program("samples/error/05.lang", "--error-format=json") + + assert res.returncode != 0 + assert res.stderr + + print(res.stderr) + try: + error = json.loads(res.stderr) + assert len(error) == 1 + assert error[0]["id"] == "cfa-invalid-break-stmt" + assert error[0]["loc"] == "samples/error/05.lang:2:5" + except json.JSONDecodeError as e: + print(e) + assert False + +if __name__ == "__main__": + import pytest + + pytest.main([__file__])