From 495f07d1356f7fa0206d67195a97742a26085e97 Mon Sep 17 00:00:00 2001 From: fruffy Date: Sat, 27 Jan 2024 19:51:59 -0500 Subject: [PATCH] Add a P4Testgen library API. --- BUILD.bazel | 11 ++ backends/p4tools/BUILD.bazel | 35 +++- .../modules/testgen/lib/test_backend.cpp | 11 +- .../modules/testgen/lib/test_backend.h | 7 + .../modules/testgen/lib/test_framework.cpp | 14 ++ .../modules/testgen/lib/test_framework.h | 40 +++- backends/p4tools/modules/testgen/options.cpp | 2 - .../testgen/targets/bmv2/CMakeLists.txt | 1 + .../targets/bmv2/test/test_backend/ptf.cpp | 8 +- .../targets/bmv2/test/test_backend/stf.cpp | 8 +- .../bmv2/test/testgen_api/api_test.cpp | 102 ++++++++++ .../testgen/targets/bmv2/test_backend.cpp | 6 +- .../targets/bmv2/test_backend/metadata.cpp | 5 +- .../targets/bmv2/test_backend/metadata.h | 4 +- .../targets/bmv2/test_backend/protobuf.cpp | 58 ++++-- .../targets/bmv2/test_backend/protobuf.h | 43 +++-- .../targets/bmv2/test_backend/protobuf_ir.cpp | 24 ++- .../targets/bmv2/test_backend/protobuf_ir.h | 36 +++- .../testgen/targets/bmv2/test_backend/ptf.cpp | 7 +- .../testgen/targets/bmv2/test_backend/ptf.h | 4 +- .../testgen/targets/bmv2/test_backend/stf.cpp | 4 +- .../testgen/targets/bmv2/test_backend/stf.h | 4 +- .../testgen/targets/ebpf/backend/stf/stf.cpp | 4 +- .../testgen/targets/ebpf/backend/stf/stf.h | 4 +- .../targets/pna/backend/metadata/metadata.cpp | 5 +- .../targets/pna/backend/metadata/metadata.h | 4 +- .../testgen/targets/pna/backend/ptf/ptf.cpp | 5 +- .../testgen/targets/pna/backend/ptf/ptf.h | 4 +- .../testgen/targets/pna/test_backend.h | 1 + backends/p4tools/modules/testgen/testgen.cpp | 174 +++++++++++++----- backends/p4tools/modules/testgen/testgen.h | 7 +- test/gtest/helpers.cpp | 2 +- test/gtest/helpers.h | 3 +- 33 files changed, 509 insertions(+), 138 deletions(-) create mode 100644 backends/p4tools/modules/testgen/targets/bmv2/test/testgen_api/api_test.cpp diff --git a/BUILD.bazel b/BUILD.bazel index 6eedb12425a..88ef9302008 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -241,6 +241,17 @@ cc_library( ], ) +cc_library( + name = "test_helpers", + srcs = ["test/gtest/helpers.cpp"], + hdrs = ["test/gtest/helpers.h"], + data = ["test"], + deps = [ + ":ir_frontend_midend_control_plane", + "@com_google_googletest//:gtest", + ], +) + cc_library( name = "p4c_bmv2_common_lib", srcs = glob(["backends/bmv2/common/*.cpp"]), diff --git a/backends/p4tools/BUILD.bazel b/backends/p4tools/BUILD.bazel index 90090fcb80c..38138f35bab 100644 --- a/backends/p4tools/BUILD.bazel +++ b/backends/p4tools/BUILD.bazel @@ -125,27 +125,36 @@ filegroup( cc_library( name = "testgen_lib", srcs = glob([ - "modules/testgen/core/**/*.h", - "modules/testgen/lib/*.h", - "modules/testgen/*.h", - ]) + glob([ "modules/testgen/core/*.cpp", "modules/testgen/core/small_step/*.cpp", "modules/testgen/core/symbolic_executor/*.cpp", "modules/testgen/lib/*.cpp", + ]) + glob([ + "modules/testgen/core/*.h", + "modules/testgen/core/small_step/*.h", + "modules/testgen/core/symbolic_executor/*.h", + "modules/testgen/lib/*.h", ]) + [ - "common/options.h", "modules/testgen/options.cpp", "modules/testgen/testgen.cpp", ":register_testgen_targets", ":target_register_headers", ":testgen_targets_src", ], + # We onlu make top-level, library, and test backend header files visible to other targets. + hdrs = ["common/options.h"] + glob([ + "modules/testgen/lib/*.h", + "modules/testgen/*.h", + ]) + glob( + ["modules/testgen/targets/%s/test_backend/*.h" % target for target in TESTGEN_TARGETS], + ), copts = [ "-fexceptions", "-w", ], features = ["-use_header_modules"], + # Needs to be accessible by other targets. + visibility = ["//visibility:public"], deps = [ ":common", ":register_testgen_targets", @@ -182,3 +191,19 @@ build_test( ":p4testgen", ], ) + +cc_test( + name = "p4testgen_api_test", + srcs = [ + "modules/testgen/targets/bmv2/test/testgen_api/api_test.cpp", + ], + copts = [ + "-fexceptions", + "-w", + ], + deps = [ + ":testgen_lib", + "//:test_helpers", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/backends/p4tools/modules/testgen/lib/test_backend.cpp b/backends/p4tools/modules/testgen/lib/test_backend.cpp index 3624f7a7439..94d65d2436f 100644 --- a/backends/p4tools/modules/testgen/lib/test_backend.cpp +++ b/backends/p4tools/modules/testgen/lib/test_backend.cpp @@ -140,7 +140,16 @@ bool TestBackEnd::run(const FinalState &state) { // Output the test. Util::withTimer("backend", [this, &testSpec, &selectedBranches] { - testWriter->outputTest(testSpec, selectedBranches, testCount, coverage); + if (testWriter->isInFileMode()) { + testWriter->writeTestToFile(testSpec, selectedBranches, testCount, coverage); + } else { + auto testOpt = + testWriter->produceTest(testSpec, selectedBranches, testCount, coverage); + if (!testOpt.has_value()) { + BUG("Failed to produce test."); + } + tests.push_back(testOpt.value()); + } }); printTraces("============ End Test %1% ============\n", testCount); diff --git a/backends/p4tools/modules/testgen/lib/test_backend.h b/backends/p4tools/modules/testgen/lib/test_backend.h index f3138ab3890..74127150348 100644 --- a/backends/p4tools/modules/testgen/lib/test_backend.h +++ b/backends/p4tools/modules/testgen/lib/test_backend.h @@ -48,6 +48,9 @@ class TestBackEnd { /// The accumulated coverage of all finished test cases. Number in range [0, 1]. float coverage = 0; + /// The list of tests accumulated in the test back end. + AbstractTestList tests; + explicit TestBackEnd(const ProgramInfo &programInfo, const TestBackendConfiguration &testBackendConfiguration, SymbolicExecutor &symbex) @@ -133,6 +136,10 @@ class TestBackEnd { /// Returns the configuration options for the test back end. [[nodiscard]] const TestBackendConfiguration &getTestBackendConfiguration() const; + + /// Returns the list of tests accumulated in the test back end. + /// If the test write is in file mode this list will be empty. + [[nodiscard]] const AbstractTestList &getTests() const { return tests; } }; } // namespace P4Tools::P4Testgen diff --git a/backends/p4tools/modules/testgen/lib/test_framework.cpp b/backends/p4tools/modules/testgen/lib/test_framework.cpp index 9df7aa45dc3..ba9c2d30585 100644 --- a/backends/p4tools/modules/testgen/lib/test_framework.cpp +++ b/backends/p4tools/modules/testgen/lib/test_framework.cpp @@ -1,5 +1,7 @@ #include "backends/p4tools/modules/testgen/lib/test_framework.h" +#include "backends/p4tools/modules/testgen/lib/exceptions.h" + namespace P4Tools::P4Testgen { TestFramework::TestFramework(const TestBackendConfiguration &testBackendConfiguration) @@ -8,4 +10,16 @@ TestFramework::TestFramework(const TestBackendConfiguration &testBackendConfigur const TestBackendConfiguration &TestFramework::getTestBackendConfiguration() const { return testBackendConfiguration.get(); } + +bool TestFramework::isInFileMode() const { + return getTestBackendConfiguration().fileBasePath.has_value(); +} + +AbstractTestReferenceOrError TestFramework::produceTest(const TestSpec * /*spec*/, + cstring /*selectedBranches*/, + size_t /*testIdx*/, + float /*currentCoverage*/) { + TESTGEN_UNIMPLEMENTED("produceTest() not implemented for this test framework."); +} + } // namespace P4Tools::P4Testgen diff --git a/backends/p4tools/modules/testgen/lib/test_framework.h b/backends/p4tools/modules/testgen/lib/test_framework.h index fd544ea46cb..001b5c7a883 100644 --- a/backends/p4tools/modules/testgen/lib/test_framework.h +++ b/backends/p4tools/modules/testgen/lib/test_framework.h @@ -2,8 +2,11 @@ #define BACKENDS_P4TOOLS_MODULES_TESTGEN_LIB_TEST_FRAMEWORK_H_ #include +#include +#include #include #include +#include #include #include @@ -11,6 +14,7 @@ #include "backends/p4tools/common/lib/format_int.h" #include "backends/p4tools/common/lib/trace_event.h" +#include "lib/castable.h" #include "lib/cstring.h" #include "backends/p4tools/modules/testgen/lib/test_backend_configuration.h" @@ -19,6 +23,18 @@ namespace P4Tools::P4Testgen { +/// Type definitions for abstract tests. +struct AbstractTest : ICastable {}; +/// TODO: It would be nice if this were a reference to signal non-nullness. +/// Consider using an optional_ref implementation. +using AbstractTestReference = const AbstractTest *; +using AbstractTestReferenceOrError = std::optional; +using AbstractTestList = std::vector; + +/// An file path which may or may not be set. Can influence the execution behavior the test +/// framework. +using OptionalFilePath = std::optional; + /// THe default base class for the various test frameworks. Every test framework has a test /// name and a seed associated with it. Also contains a variety of common utility functions. class TestFramework { @@ -136,13 +152,25 @@ class TestFramework { /// The method used to output the test case to be implemented by /// all the test frameworks (eg. STF, PTF, etc.). - /// @param spec the testcase specification to be outputted - /// @param selectedBranches string describing branches selected for this testcase - /// @param testIdx testcase unique number identifier + /// @param spec the testcase specification to be outputted. + /// @param selectedBranches string describing branches selected for this testcase. + /// @param testIdx testcase unique number identifier. TODO: Make this a member? /// @param currentCoverage current coverage ratio (between 0.0 and 1.0) - // attaches arbitrary string data to the test preamble. - virtual void outputTest(const TestSpec *spec, cstring selectedBranches, size_t testIdx, - float currentCoverage) = 0; + /// TODO (https://github.com/p4lang/p4c/issues/4403): This should not return void but instead a + /// status. + virtual void writeTestToFile(const TestSpec *spec, cstring selectedBranches, size_t testIdx, + float currentCoverage) = 0; + + /// The method used to return the test case. This method is optional to each test framework. + /// @param spec the testcase specification to be outputted. + /// @param selectedBranches string describing branches selected for this testcase. + /// @param testIdx testcase unique number identifier. TODO: Make this a member? + /// @param currentCoverage current coverage ratio (between 0.0 and 1.0). + virtual AbstractTestReferenceOrError produceTest(const TestSpec *spec, cstring selectedBranches, + size_t testIdx, float currentCoverage); + + /// @Returns true if the test framework is configured to write to a file. + [[nodiscard]] bool isInFileMode() const; }; } // namespace P4Tools::P4Testgen diff --git a/backends/p4tools/modules/testgen/options.cpp b/backends/p4tools/modules/testgen/options.cpp index 30837b2ea5e..3789855d451 100644 --- a/backends/p4tools/modules/testgen/options.cpp +++ b/backends/p4tools/modules/testgen/options.cpp @@ -2,7 +2,6 @@ #include #include -#include #include #include #include @@ -12,7 +11,6 @@ #include "backends/p4tools/common/lib/util.h" #include "backends/p4tools/common/options.h" #include "lib/error.h" -#include "lib/exceptions.h" #include "backends/p4tools/modules/testgen/lib/logging.h" diff --git a/backends/p4tools/modules/testgen/targets/bmv2/CMakeLists.txt b/backends/p4tools/modules/testgen/targets/bmv2/CMakeLists.txt index e7f12f3ded8..286dd8a95c2 100644 --- a/backends/p4tools/modules/testgen/targets/bmv2/CMakeLists.txt +++ b/backends/p4tools/modules/testgen/targets/bmv2/CMakeLists.txt @@ -32,6 +32,7 @@ set(TESTGEN_SOURCES set(TESTGEN_GTEST_SOURCES ${TESTGEN_GTEST_SOURCES} + ${CMAKE_CURRENT_SOURCE_DIR}/test/testgen_api/api_test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/test_backend/ptf.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/test_backend/stf.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/small-step/binary.cpp diff --git a/backends/p4tools/modules/testgen/targets/bmv2/test/test_backend/ptf.cpp b/backends/p4tools/modules/testgen/targets/bmv2/test/test_backend/ptf.cpp index 283ccaaeedc..be30a46ad3d 100644 --- a/backends/p4tools/modules/testgen/targets/bmv2/test/test_backend/ptf.cpp +++ b/backends/p4tools/modules/testgen/targets/bmv2/test/test_backend/ptf.cpp @@ -83,7 +83,7 @@ TEST_F(PTFTest, Ptf01) { TestBackendConfiguration testBackendConfiguration{"test01", 1, "test01", 1}; auto testWriter = PTF(testBackendConfiguration); - testWriter.outputTest(&testSpec, "", 1, 0); + testWriter.writeTestToFile(&testSpec, "", 1, 0); } /// Create a test spec with two Exact matches and print an ptf test. @@ -114,7 +114,7 @@ TEST_F(PTFTest, Ptf02) { TestBackendConfiguration testBackendConfiguration{"test02", 1, "test02", 2}; auto testWriter = PTF(testBackendConfiguration); - testWriter.outputTest(&testSpec, "", 2, 0); + testWriter.writeTestToFile(&testSpec, "", 2, 0); } TableConfig PTFTest::gettest1TableConfig() { @@ -160,7 +160,7 @@ TEST_F(PTFTest, Ptf03) { TestBackendConfiguration testBackendConfiguration{"test03", 1, "test03", 3}; auto testWriter = PTF(testBackendConfiguration); try { - testWriter.outputTest(&testSpec, "", 3, 0); + testWriter.writeTestToFile(&testSpec, "", 3, 0); } catch (const Util::CompilerBug &e) { EXPECT_THAT(e.what(), HasSubstr("Unimplemented for Ternary FieldMatch")); } @@ -228,7 +228,7 @@ TEST_F(PTFTest, Ptf04) { TestBackendConfiguration testBackendConfiguration{"test04", 1, "test04", 4}; auto testWriter = PTF(testBackendConfiguration); try { - testWriter.outputTest(&testSpec, "", 4, 0); + testWriter.writeTestToFile(&testSpec, "", 4, 0); } catch (const Util::CompilerBug &e) { EXPECT_THAT(e.what(), HasSubstr("Unimplemented for Ternary FieldMatch")); } diff --git a/backends/p4tools/modules/testgen/targets/bmv2/test/test_backend/stf.cpp b/backends/p4tools/modules/testgen/targets/bmv2/test/test_backend/stf.cpp index 27e05c66d84..1ee4a8b0e54 100644 --- a/backends/p4tools/modules/testgen/targets/bmv2/test/test_backend/stf.cpp +++ b/backends/p4tools/modules/testgen/targets/bmv2/test/test_backend/stf.cpp @@ -83,7 +83,7 @@ TEST_F(STFTest, Stf01) { TestBackendConfiguration testBackendConfiguration{"test01", 1, "test01", 1}; auto testWriter = STF(testBackendConfiguration); - testWriter.outputTest(&testSpec, "", 1, 0); + testWriter.writeTestToFile(&testSpec, "", 1, 0); } /// Create a test spec with two Exact matches and print an stf test. @@ -114,7 +114,7 @@ TEST_F(STFTest, Stf02) { TestBackendConfiguration testBackendConfiguration{"test02", 1, "test02", 2}; auto testWriter = STF(testBackendConfiguration); - testWriter.outputTest(&testSpec, "", 2, 0); + testWriter.writeTestToFile(&testSpec, "", 2, 0); } TableConfig STFTest::gettest1TableConfig() { @@ -160,7 +160,7 @@ TEST_F(STFTest, Stf03) { TestBackendConfiguration testBackendConfiguration{"test03", 1, "test03", 3}; auto testWriter = STF(testBackendConfiguration); try { - testWriter.outputTest(&testSpec, "", 3, 0); + testWriter.writeTestToFile(&testSpec, "", 3, 0); } catch (const Util::CompilerBug &e) { EXPECT_THAT(e.what(), HasSubstr("Unimplemented for Ternary FieldMatch")); } @@ -228,7 +228,7 @@ TEST_F(STFTest, Stf04) { TestBackendConfiguration testBackendConfiguration{"test04", 1, "test04", 4}; auto testWriter = STF(testBackendConfiguration); try { - testWriter.outputTest(&testSpec, "", 4, 0); + testWriter.writeTestToFile(&testSpec, "", 4, 0); } catch (const Util::CompilerBug &e) { EXPECT_THAT(e.what(), HasSubstr("Unimplemented for Ternary FieldMatch")); } diff --git a/backends/p4tools/modules/testgen/targets/bmv2/test/testgen_api/api_test.cpp b/backends/p4tools/modules/testgen/targets/bmv2/test/testgen_api/api_test.cpp new file mode 100644 index 00000000000..d1ac85325ef --- /dev/null +++ b/backends/p4tools/modules/testgen/targets/bmv2/test/testgen_api/api_test.cpp @@ -0,0 +1,102 @@ +#include +#include + +#include "test/gtest/helpers.h" + +#include "backends/p4tools/modules/testgen/options.h" +#include "backends/p4tools/modules/testgen/targets/bmv2/test_backend/protobuf.h" +#include "backends/p4tools/modules/testgen/targets/bmv2/test_backend/protobuf_ir.h" +#include "backends/p4tools/modules/testgen/testgen.h" + +namespace Test { + +TEST(P4TestgenLibrary, GeneratesCorrectProtobufIrTest) { + std::stringstream streamTest; + streamTest << R"p4( +header ethernet_t { + bit<48> dst_addr; + bit<48> src_addr; + bit<16> ether_type; +} + +struct Headers { + ethernet_t eth_hdr; +} + +struct Metadata { } +parser parse(packet_in pkt, out Headers hdr, inout Metadata m, inout standard_metadata_t sm) { + state start { + pkt.extract(hdr.eth_hdr); + transition accept; + } +} +control ingress(inout Headers hdr, inout Metadata meta, inout standard_metadata_t sm) { + apply { + if (hdr.eth_hdr.dst_addr == 0xDEADDEADDEAD && hdr.eth_hdr.src_addr == 0xBEEFBEEFBEEF && hdr.eth_hdr.ether_type == 0xF00D) { + mark_to_drop(sm); + } + } +} +control egress(inout Headers hdr, inout Metadata meta, inout standard_metadata_t sm) { + apply {} +} +control deparse(packet_out pkt, in Headers hdr) { + apply { + pkt.emit(hdr.eth_hdr); + } +} +control verifyChecksum(inout Headers hdr, inout Metadata meta) { + apply {} +} +control computeChecksum(inout Headers hdr, inout Metadata meta) { + apply {} +} +V1Switch(parse(), verifyChecksum(), ingress(), egress(), computeChecksum(), deparse()) main; +)p4"; + + auto source = P4_SOURCE(P4Headers::V1MODEL, streamTest.str().c_str()); + auto compilerOptions = CompilerOptions(); + compilerOptions.target = "bmv2"; + compilerOptions.arch = "v1model"; + auto &testgenOptions = P4Tools::P4Testgen::TestgenOptions::get(); + testgenOptions.testBackend = "PROTOBUF_IR"; + testgenOptions.testBaseName = "dummy"; + // Create a bespoke packet for the Ethernet extract call. + testgenOptions.minPktSize = 112; + testgenOptions.maxPktSize = 112; + // Only generate one test. + testgenOptions.maxTests = 1; + + { + auto testListOpt = + P4Tools::P4Testgen::Testgen::generateTests(source, compilerOptions, testgenOptions); + + ASSERT_TRUE(testListOpt.has_value()); + auto testList = testListOpt.value(); + ASSERT_EQ(testList.size(), 1); + const auto *test = testList[0]; + const auto *protobufIrTest = test->to(); + ASSERT_TRUE(protobufIrTest != nullptr); + EXPECT_THAT(protobufIrTest->getFormattedTest(), ::testing::HasSubstr(R"(input_packet { + packet: "\xDE\xAD\xDE\xAD\xDE\xAD\xBE\xEF\xBE\xEF\xBE\xEF\xF0\x0D" + port: 0 +})")); + } + /// Now try running again with the test back end set to Protobuf. The result should be the same. + testgenOptions.testBackend = "PROTOBUF"; + + auto testListOpt = + P4Tools::P4Testgen::Testgen::generateTests(source, compilerOptions, testgenOptions); + + ASSERT_TRUE(testListOpt.has_value()); + auto testList = testListOpt.value(); + ASSERT_EQ(testList.size(), 1); + auto &test = testList[0]; + const auto *protobufTest = test->to(); + ASSERT_TRUE(protobufTest != nullptr); + EXPECT_THAT(protobufTest->getFormattedTest(), ::testing::HasSubstr(R"(input_packet { + packet: "\xDE\xAD\xDE\xAD\xDE\xAD\xBE\xEF\xBE\xEF\xBE\xEF\xF0\x0D" + port: 0 +})")); +} +} // namespace Test diff --git a/backends/p4tools/modules/testgen/targets/bmv2/test_backend.cpp b/backends/p4tools/modules/testgen/targets/bmv2/test_backend.cpp index 2ce382eb56d..3e48256d6bd 100644 --- a/backends/p4tools/modules/testgen/targets/bmv2/test_backend.cpp +++ b/backends/p4tools/modules/testgen/targets/bmv2/test_backend.cpp @@ -88,9 +88,6 @@ TestBackEnd::TestInfo Bmv2TestBackend::produceTestInfo( const TestSpec *Bmv2TestBackend::createTestSpec(const ExecutionState *executionState, const Model *finalModel, const TestInfo &testInfo) { - // Create a testSpec. - TestSpec *testSpec = nullptr; - const auto *ingressPayload = testInfo.inputPacket; const auto *ingressPayloadMask = IR::getConstant(IR::getBitType(1), 1); const auto ingressPacket = Packet(testInfo.inputPort, ingressPayload, ingressPayloadMask); @@ -99,7 +96,8 @@ const TestSpec *Bmv2TestBackend::createTestSpec(const ExecutionState *executionS if (!testInfo.packetIsDropped) { egressPacket = Packet(testInfo.outputPort, testInfo.outputPacket, testInfo.packetTaintMask); } - testSpec = new TestSpec(ingressPacket, egressPacket, testInfo.programTraces); + // Create a testSpec. + auto *testSpec = new TestSpec(ingressPacket, egressPacket, testInfo.programTraces); // If metadata mode is enabled, gather the user metadata variable form the parser. // Save the values of all the fields in it and return. diff --git a/backends/p4tools/modules/testgen/targets/bmv2/test_backend/metadata.cpp b/backends/p4tools/modules/testgen/targets/bmv2/test_backend/metadata.cpp index 27fb5a1ef74..c1aa5f68576 100644 --- a/backends/p4tools/modules/testgen/targets/bmv2/test_backend/metadata.cpp +++ b/backends/p4tools/modules/testgen/targets/bmv2/test_backend/metadata.cpp @@ -19,6 +19,7 @@ #include "nlohmann/json.hpp" #include "backends/p4tools/modules/testgen/lib/test_object.h" +#include "backends/p4tools/modules/testgen/options.h" #include "backends/p4tools/modules/testgen/targets/bmv2/test_spec.h" namespace P4Tools::P4Testgen::Bmv2 { @@ -134,8 +135,8 @@ void Metadata::emitTestcase(const TestSpec *testSpec, cstring selectedBranches, metadataFile.flush(); } -void Metadata::outputTest(const TestSpec *testSpec, cstring selectedBranches, size_t testId, - float currentCoverage) { +void Metadata::writeTestToFile(const TestSpec *testSpec, cstring selectedBranches, size_t testId, + float currentCoverage) { std::string testCase = getTestCaseTemplate(); emitTestcase(testSpec, selectedBranches, testId, testCase, currentCoverage); } diff --git a/backends/p4tools/modules/testgen/targets/bmv2/test_backend/metadata.h b/backends/p4tools/modules/testgen/targets/bmv2/test_backend/metadata.h index 7498e4d2620..8ec44f0ec53 100644 --- a/backends/p4tools/modules/testgen/targets/bmv2/test_backend/metadata.h +++ b/backends/p4tools/modules/testgen/targets/bmv2/test_backend/metadata.h @@ -22,8 +22,8 @@ class Metadata : public Bmv2TestFramework { explicit Metadata(const TestBackendConfiguration &testBackendConfiguration); /// Produce a Metadata test. - void outputTest(const TestSpec *spec, cstring selectedBranches, size_t testId, - float currentCoverage) override; + void writeTestToFile(const TestSpec *spec, cstring selectedBranches, size_t testId, + float currentCoverage) override; private: /// The output file. diff --git a/backends/p4tools/modules/testgen/targets/bmv2/test_backend/protobuf.cpp b/backends/p4tools/modules/testgen/targets/bmv2/test_backend/protobuf.cpp index ca146aa4be9..cb5fc574d05 100644 --- a/backends/p4tools/modules/testgen/targets/bmv2/test_backend/protobuf.cpp +++ b/backends/p4tools/modules/testgen/targets/bmv2/test_backend/protobuf.cpp @@ -23,6 +23,7 @@ #include "backends/p4tools/modules/testgen/lib/exceptions.h" #include "backends/p4tools/modules/testgen/lib/test_object.h" +#include "backends/p4tools/modules/testgen/options.h" #include "backends/p4tools/modules/testgen/targets/bmv2/test_spec.h" namespace P4Tools::P4Testgen::Bmv2 { @@ -158,14 +159,25 @@ inja::json Protobuf::getControlPlaneForTable(cstring tableName, cstring actionNa return rulesJson; } +inja::json Protobuf::getSend(const TestSpec *testSpec) const { + const auto *iPacket = testSpec->getIngressPacket(); + const auto *payload = iPacket->getEvaluatedPayload(); + inja::json sendJson; + sendJson["ig_port"] = iPacket->getPort(); + sendJson["pkt"] = formatHexExpressionWithSeparators(*payload); + sendJson["pkt_size"] = payload->type->width_bits(); + return sendJson; +} + inja::json Protobuf::getExpectedPacket(const TestSpec *testSpec) const { inja::json verifyData = inja::json::object(); - if (testSpec->getEgressPacket() != std::nullopt) { - const auto &packet = **testSpec->getEgressPacket(); - verifyData["eg_port"] = packet.getPort(); - const auto *payload = packet.getEvaluatedPayload(); - const auto *payloadMask = packet.getEvaluatedPayloadMask(); - verifyData["ignore_mask"] = formatHexExpressionWithSeparators(*payloadMask); + auto egressPacket = testSpec->getEgressPacket(); + if (egressPacket.has_value()) { + const auto *packet = egressPacket.value(); + verifyData["eg_port"] = packet->getPort(); + const auto *payload = packet->getEvaluatedPayload(); + const auto *mask = packet->getEvaluatedPayloadMask(); + verifyData["ignore_mask"] = formatHexExpressionWithSeparators(*mask); verifyData["exp_pkt"] = formatHexExpressionWithSeparators(*payload); } return verifyData; @@ -298,12 +310,13 @@ entities { return TEST_CASE; } -void Protobuf::emitTestcase(const TestSpec *testSpec, cstring selectedBranches, size_t testId, - const std::string &testCase, float currentCoverage) { +inja::json Protobuf::produceTestCase(const TestSpec *testSpec, cstring selectedBranches, + size_t testId, float currentCoverage) const { inja::json dataJson; if (selectedBranches != nullptr) { dataJson["selected_branches"] = selectedBranches.c_str(); } + auto optSeed = getTestBackendConfiguration().seed; if (optSeed.has_value()) { dataJson["seed"] = optSeed.value(); @@ -319,6 +332,22 @@ void Protobuf::emitTestcase(const TestSpec *testSpec, cstring selectedBranches, coverageStr << std::setprecision(2) << currentCoverage; dataJson["coverage"] = coverageStr.str(); + // Check whether this test has a clone configuration. + // These are special because they require additional instrumentation and produce two output + // packets. + auto cloneSpecs = testSpec->getTestObjectCategory("clone_specs"); + if (!cloneSpecs.empty()) { + dataJson["clone_specs"] = getClone(cloneSpecs); + } + auto meterValues = testSpec->getTestObjectCategory("meter_values"); + dataJson["meter_values"] = getMeter(meterValues); + + return dataJson; +} + +void Protobuf::writeTestToFile(const TestSpec *testSpec, cstring selectedBranches, size_t testId, + float currentCoverage) { + inja::json dataJson = produceTestCase(testSpec, selectedBranches, testId, currentCoverage); LOG5("Protobuf test back end: emitting testcase:" << std::setw(4) << dataJson); auto optBasePath = getTestBackendConfiguration().fileBasePath; @@ -327,14 +356,17 @@ void Protobuf::emitTestcase(const TestSpec *testSpec, cstring selectedBranches, incrementedbasePath.concat("_" + std::to_string(testId)); incrementedbasePath.replace_extension(".txtpb"); auto protobufFileStream = std::ofstream(incrementedbasePath); - inja::render_to(protobufFileStream, testCase, dataJson); + inja::render_to(protobufFileStream, getTestCaseTemplate(), dataJson); protobufFileStream.flush(); } -void Protobuf::outputTest(const TestSpec *testSpec, cstring selectedBranches, size_t testId, - float currentCoverage) { - std::string testCase = getTestCaseTemplate(); - emitTestcase(testSpec, selectedBranches, testId, testCase, currentCoverage); +AbstractTestReferenceOrError Protobuf::produceTest(const TestSpec *testSpec, + cstring selectedBranches, size_t testId, + float currentCoverage) { + inja::json dataJson = produceTestCase(testSpec, selectedBranches, testId, currentCoverage); + LOG5("ProtobufIR test back end: generated testcase:" << std::setw(4) << dataJson); + + return new ProtobufTest(inja::render(getTestCaseTemplate(), dataJson)); } } // namespace P4Tools::P4Testgen::Bmv2 diff --git a/backends/p4tools/modules/testgen/targets/bmv2/test_backend/protobuf.h b/backends/p4tools/modules/testgen/targets/bmv2/test_backend/protobuf.h index 0b47058d6b3..acdf5149a90 100644 --- a/backends/p4tools/modules/testgen/targets/bmv2/test_backend/protobuf.h +++ b/backends/p4tools/modules/testgen/targets/bmv2/test_backend/protobuf.h @@ -2,7 +2,6 @@ #define BACKENDS_P4TOOLS_MODULES_TESTGEN_TARGETS_BMV2_TEST_BACKEND_PROTOBUF_H_ #include -#include #include #include #include @@ -10,7 +9,6 @@ #include #include "backends/p4tools/common/control_plane/p4info_map.h" -#include "control-plane/p4RuntimeArchStandard.h" #include "control-plane/p4RuntimeSerializer.h" #include "ir/ir.h" #include "lib/cstring.h" @@ -21,7 +19,20 @@ namespace P4Tools::P4Testgen::Bmv2 { using P4::ControlPlaneAPI::p4rt_id_t; -using P4::ControlPlaneAPI::Standard::SymbolType; + +struct ProtobufTest : public AbstractTest { + private: + /// The formatted test. TODO: This should be a Protobuf object. + std::string formattedTest_; + + public: + explicit ProtobufTest(std::string formattedTest) : formattedTest_(std::move(formattedTest)) {} + + /// @return the formatted test. + [[nodiscard]] const std::string &getFormattedTest() const { return formattedTest_; } + + DECLARE_TYPEINFO(ProtobufTest); +}; /// Extracts information from the @testSpec to emit a Protobuf test case. class Protobuf : public Bmv2TestFramework { @@ -29,9 +40,11 @@ class Protobuf : public Bmv2TestFramework { explicit Protobuf(const TestBackendConfiguration &testBackendConfiguration, P4::P4RuntimeAPI p4RuntimeApi); - /// Produce a Protobuf test. - void outputTest(const TestSpec *spec, cstring selectedBranches, size_t testId, - float currentCoverage) override; + void writeTestToFile(const TestSpec *testSpec, cstring selectedBranches, size_t testId, + float currentCoverage) override; + + AbstractTestReferenceOrError produceTest(const TestSpec *testSpec, cstring selectedBranches, + size_t testIdx, float currentCoverage) override; private: /// The P4Runtime API generated by the compiler. This API can be used to look up the id of @@ -41,6 +54,12 @@ class Protobuf : public Bmv2TestFramework { /// A mapping from P4 control plane names to their mapped P4Runtime ids and vice versa. P4::ControlPlaneAPI::P4InfoMaps p4InfoMaps; + inja::json getControlPlane(const TestSpec *testSpec) const override; + + [[nodiscard]] inja::json getSend(const TestSpec *testSpec) const override; + + [[nodiscard]] inja::json getExpectedPacket(const TestSpec *testSpec) const override; + /// Wrapper helper function that automatically inserts separators for hex strings. static std::string formatHexExprWithSep(const IR::Expression *expr); @@ -52,21 +71,17 @@ class Protobuf : public Bmv2TestFramework { /// For the protobuf back end this is the "p4testgen.proto" file. void emitPreamble(const std::string &preamble); - /// Emits a test case. - /// @param testId specifies the test name. + /// Generates a test case. /// @param selectedBranches enumerates the choices the interpreter made for this path. + /// @param testId specifies the test name. /// @param currentCoverage contains statistics about the current coverage of this test and its /// preceding tests. - void emitTestcase(const TestSpec *testSpec, cstring selectedBranches, size_t testId, - const std::string &testCase, float currentCoverage); + inja::json produceTestCase(const TestSpec *testSpec, cstring selectedBranches, size_t testId, + float currentCoverage) const; /// @returns the inja test case template as a string. static std::string getTestCaseTemplate(); - inja::json getControlPlane(const TestSpec *testSpec) const override; - - inja::json getExpectedPacket(const TestSpec *testSpec) const override; - /// The Protobuf back end needs the parent table and action name to correctly identify the /// corresponding P4Runtme id. This is why we use a custom "getControlPlaneForTable" function /// here. diff --git a/backends/p4tools/modules/testgen/targets/bmv2/test_backend/protobuf_ir.cpp b/backends/p4tools/modules/testgen/targets/bmv2/test_backend/protobuf_ir.cpp index a6ece209bfa..e09a945d7de 100644 --- a/backends/p4tools/modules/testgen/targets/bmv2/test_backend/protobuf_ir.cpp +++ b/backends/p4tools/modules/testgen/targets/bmv2/test_backend/protobuf_ir.cpp @@ -14,6 +14,7 @@ #include "nlohmann/json.hpp" #include "backends/p4tools/modules/testgen/lib/exceptions.h" +#include "backends/p4tools/modules/testgen/lib/test_framework.h" #include "backends/p4tools/modules/testgen/targets/bmv2/test_spec.h" namespace P4Tools::P4Testgen::Bmv2 { @@ -307,8 +308,8 @@ inja::json ProtobufIr::getExpectedPacket(const TestSpec *testSpec) const { return verifyData; } -void ProtobufIr::emitTestcase(const TestSpec *testSpec, cstring selectedBranches, size_t testId, - const std::string &testCase, float currentCoverage) { +inja::json ProtobufIr::produceTestCase(const TestSpec *testSpec, cstring selectedBranches, + size_t testId, float currentCoverage) const { inja::json dataJson; if (selectedBranches != nullptr) { dataJson["selected_branches"] = selectedBranches.c_str(); @@ -339,6 +340,12 @@ void ProtobufIr::emitTestcase(const TestSpec *testSpec, cstring selectedBranches auto meterValues = testSpec->getTestObjectCategory("meter_values"); dataJson["meter_values"] = getMeter(meterValues); + return dataJson; +} + +void ProtobufIr::writeTestToFile(const TestSpec *testSpec, cstring selectedBranches, size_t testId, + float currentCoverage) { + inja::json dataJson = produceTestCase(testSpec, selectedBranches, testId, currentCoverage); LOG5("ProtobufIR test back end: emitting testcase:" << std::setw(4) << dataJson); auto optBasePath = getTestBackendConfiguration().fileBasePath; @@ -347,14 +354,17 @@ void ProtobufIr::emitTestcase(const TestSpec *testSpec, cstring selectedBranches incrementedbasePath.concat("_" + std::to_string(testId)); incrementedbasePath.replace_extension(".txtpb"); auto protobufFileStream = std::ofstream(incrementedbasePath); - inja::render_to(protobufFileStream, testCase, dataJson); + inja::render_to(protobufFileStream, getTestCaseTemplate(), dataJson); protobufFileStream.flush(); } -void ProtobufIr::outputTest(const TestSpec *testSpec, cstring selectedBranches, size_t testId, - float currentCoverage) { - std::string testCase = getTestCaseTemplate(); - emitTestcase(testSpec, selectedBranches, testId, testCase, currentCoverage); +AbstractTestReferenceOrError ProtobufIr::produceTest(const TestSpec *testSpec, + cstring selectedBranches, size_t testId, + float currentCoverage) { + inja::json dataJson = produceTestCase(testSpec, selectedBranches, testId, currentCoverage); + LOG5("ProtobufIR test back end: generated testcase:" << std::setw(4) << dataJson); + + return new ProtobufIrTest(inja::render(getTestCaseTemplate(), dataJson)); } } // namespace P4Tools::P4Testgen::Bmv2 diff --git a/backends/p4tools/modules/testgen/targets/bmv2/test_backend/protobuf_ir.h b/backends/p4tools/modules/testgen/targets/bmv2/test_backend/protobuf_ir.h index cc42d0f9dde..4c5b987af3d 100644 --- a/backends/p4tools/modules/testgen/targets/bmv2/test_backend/protobuf_ir.h +++ b/backends/p4tools/modules/testgen/targets/bmv2/test_backend/protobuf_ir.h @@ -2,9 +2,8 @@ #define BACKENDS_P4TOOLS_MODULES_TESTGEN_TARGETS_BMV2_TEST_BACKEND_PROTOBUF_IR_H_ #include -#include -#include #include +#include #include @@ -15,20 +14,38 @@ namespace P4Tools::P4Testgen::Bmv2 { +struct ProtobufIrTest : public AbstractTest { + private: + /// The formatted test. TODO: This should be a Protobuf object. + std::string formattedTest_; + + public: + explicit ProtobufIrTest(std::string formattedTest) : formattedTest_(std::move(formattedTest)) {} + + /// @return the formatted test. + [[nodiscard]] const std::string &getFormattedTest() const { return formattedTest_; } + + DECLARE_TYPEINFO(ProtobufIrTest); +}; + /// Extracts information from the @testSpec to emit a Protobuf IR test case. class ProtobufIr : public Bmv2TestFramework { public: explicit ProtobufIr(const TestBackendConfiguration &testBackendConfiguration); - virtual ~ProtobufIr() = default; + ~ProtobufIr() override = default; ProtobufIr(const ProtobufIr &) = default; ProtobufIr(ProtobufIr &&) = default; ProtobufIr &operator=(const ProtobufIr &) = default; ProtobufIr &operator=(ProtobufIr &&) = default; - void outputTest(const TestSpec *spec, cstring selectedBranches, size_t testId, - float currentCoverage) override; + void writeTestToFile(const TestSpec *testSpec, cstring selectedBranches, size_t testId, + float currentCoverage) override; + + AbstractTestReferenceOrError produceTest(const TestSpec *testSpec, cstring selectedBranches, + size_t testIdx, float currentCoverage) override; + private: [[nodiscard]] inja::json getControlPlaneForTable( const TableMatchMap &matches, const std::vector &args) const override; @@ -36,14 +53,13 @@ class ProtobufIr : public Bmv2TestFramework { [[nodiscard]] inja::json getExpectedPacket(const TestSpec *testSpec) const override; - private: - /// Emits a test case. - /// @param testId specifies the test name. + /// Generates a test case. /// @param selectedBranches enumerates the choices the interpreter made for this path. + /// @param testId specifies the test name. /// @param currentCoverage contains statistics about the current coverage of this test and its /// preceding tests. - void emitTestcase(const TestSpec *testSpec, cstring selectedBranches, size_t testId, - const std::string &testCase, float currentCoverage); + inja::json produceTestCase(const TestSpec *testSpec, cstring selectedBranches, size_t testId, + float currentCoverage) const; /// @returns the inja test case template as a string. static std::string getTestCaseTemplate(); diff --git a/backends/p4tools/modules/testgen/targets/bmv2/test_backend/ptf.cpp b/backends/p4tools/modules/testgen/targets/bmv2/test_backend/ptf.cpp index 291eced1ccc..c78334acf4a 100644 --- a/backends/p4tools/modules/testgen/targets/bmv2/test_backend/ptf.cpp +++ b/backends/p4tools/modules/testgen/targets/bmv2/test_backend/ptf.cpp @@ -12,9 +12,12 @@ #include "backends/p4tools/common/lib/format_int.h" #include "backends/p4tools/common/lib/util.h" #include "ir/ir.h" +#include "lib/exceptions.h" #include "lib/log.h" #include "nlohmann/json.hpp" +#include "backends/p4tools/modules/testgen/options.h" + namespace P4Tools::P4Testgen::Bmv2 { PTF::PTF(const TestBackendConfiguration &testBackendConfiguration) @@ -301,8 +304,8 @@ void PTF::emitTestcase(const TestSpec *testSpec, cstring selectedBranches, size_ ptfFileStream.flush(); } -void PTF::outputTest(const TestSpec *testSpec, cstring selectedBranches, size_t testId, - float currentCoverage) { +void PTF::writeTestToFile(const TestSpec *testSpec, cstring selectedBranches, size_t testId, + float currentCoverage) { std::string testCase = getTestCaseTemplate(); emitTestcase(testSpec, selectedBranches, testId, testCase, currentCoverage); } diff --git a/backends/p4tools/modules/testgen/targets/bmv2/test_backend/ptf.h b/backends/p4tools/modules/testgen/targets/bmv2/test_backend/ptf.h index 58b7d96dc83..00e9e10b878 100644 --- a/backends/p4tools/modules/testgen/targets/bmv2/test_backend/ptf.h +++ b/backends/p4tools/modules/testgen/targets/bmv2/test_backend/ptf.h @@ -27,8 +27,8 @@ class PTF : public Bmv2TestFramework { explicit PTF(const TestBackendConfiguration &testBackendConfiguration); /// Produce a PTF test. - void outputTest(const TestSpec *spec, cstring selectedBranches, size_t testId, - float currentCoverage) override; + void writeTestToFile(const TestSpec *spec, cstring selectedBranches, size_t testId, + float currentCoverage) override; private: /// Has the preamble been generated already? diff --git a/backends/p4tools/modules/testgen/targets/bmv2/test_backend/stf.cpp b/backends/p4tools/modules/testgen/targets/bmv2/test_backend/stf.cpp index 2e6c0c48205..874ac8dca6d 100644 --- a/backends/p4tools/modules/testgen/targets/bmv2/test_backend/stf.cpp +++ b/backends/p4tools/modules/testgen/targets/bmv2/test_backend/stf.cpp @@ -243,8 +243,8 @@ void STF::emitTestcase(const TestSpec *testSpec, cstring selectedBranches, size_ stfFileStream.flush(); } -void STF::outputTest(const TestSpec *testSpec, cstring selectedBranches, size_t testId, - float currentCoverage) { +void STF::writeTestToFile(const TestSpec *testSpec, cstring selectedBranches, size_t testId, + float currentCoverage) { std::string testCase = getTestCaseTemplate(); emitTestcase(testSpec, selectedBranches, testId, testCase, currentCoverage); } diff --git a/backends/p4tools/modules/testgen/targets/bmv2/test_backend/stf.h b/backends/p4tools/modules/testgen/targets/bmv2/test_backend/stf.h index 9cd751fa7ef..5a704b8306d 100644 --- a/backends/p4tools/modules/testgen/targets/bmv2/test_backend/stf.h +++ b/backends/p4tools/modules/testgen/targets/bmv2/test_backend/stf.h @@ -20,8 +20,8 @@ class STF : public Bmv2TestFramework { explicit STF(const TestBackendConfiguration &testBackendConfiguration); /// Produce an STF test. - void outputTest(const TestSpec *spec, cstring selectedBranches, size_t testId, - float currentCoverage) override; + void writeTestToFile(const TestSpec *spec, cstring selectedBranches, size_t testId, + float currentCoverage) override; private: /// Emits a test case. diff --git a/backends/p4tools/modules/testgen/targets/ebpf/backend/stf/stf.cpp b/backends/p4tools/modules/testgen/targets/ebpf/backend/stf/stf.cpp index de580707e5d..85ff79309d2 100644 --- a/backends/p4tools/modules/testgen/targets/ebpf/backend/stf/stf.cpp +++ b/backends/p4tools/modules/testgen/targets/ebpf/backend/stf/stf.cpp @@ -254,8 +254,8 @@ void STF::emitTestcase(const TestSpec *testSpec, cstring selectedBranches, size_ stfFileStream.flush(); } -void STF::outputTest(const TestSpec *testSpec, cstring selectedBranches, size_t testId, - float currentCoverage) { +void STF::writeTestToFile(const TestSpec *testSpec, cstring selectedBranches, size_t testId, + float currentCoverage) { std::string testCase = getTestCaseTemplate(); emitTestcase(testSpec, selectedBranches, testId, testCase, currentCoverage); } diff --git a/backends/p4tools/modules/testgen/targets/ebpf/backend/stf/stf.h b/backends/p4tools/modules/testgen/targets/ebpf/backend/stf/stf.h index 5237ce2b518..93a1c5409f7 100644 --- a/backends/p4tools/modules/testgen/targets/ebpf/backend/stf/stf.h +++ b/backends/p4tools/modules/testgen/targets/ebpf/backend/stf/stf.h @@ -26,8 +26,8 @@ class STF : public TestFramework { explicit STF(const TestBackendConfiguration &testBackendConfiguration); /// Produce an STF test. - void outputTest(const TestSpec *spec, cstring selectedBranches, size_t testId, - float currentCoverage) override; + void writeTestToFile(const TestSpec *spec, cstring selectedBranches, size_t testId, + float currentCoverage) override; private: /// Emits a test case. diff --git a/backends/p4tools/modules/testgen/targets/pna/backend/metadata/metadata.cpp b/backends/p4tools/modules/testgen/targets/pna/backend/metadata/metadata.cpp index a0234a92c2c..7f8b5cce2c4 100644 --- a/backends/p4tools/modules/testgen/targets/pna/backend/metadata/metadata.cpp +++ b/backends/p4tools/modules/testgen/targets/pna/backend/metadata/metadata.cpp @@ -18,6 +18,7 @@ #include "backends/p4tools/modules/testgen/lib/test_framework.h" #include "backends/p4tools/modules/testgen/lib/test_object.h" +#include "backends/p4tools/modules/testgen/options.h" #include "backends/p4tools/modules/testgen/targets/pna/test_spec.h" namespace P4Tools::P4Testgen::Pna { @@ -182,8 +183,8 @@ void Metadata::emitTestcase(const TestSpec *testSpec, cstring selectedBranches, metadataFile.flush(); } -void Metadata::outputTest(const TestSpec *testSpec, cstring selectedBranches, size_t testId, - float currentCoverage) { +void Metadata::writeTestToFile(const TestSpec *testSpec, cstring selectedBranches, size_t testId, + float currentCoverage) { std::string testCase = getTestCaseTemplate(); emitTestcase(testSpec, selectedBranches, testId, testCase, currentCoverage); } diff --git a/backends/p4tools/modules/testgen/targets/pna/backend/metadata/metadata.h b/backends/p4tools/modules/testgen/targets/pna/backend/metadata/metadata.h index 58df48f9fc0..f0149acd9d4 100644 --- a/backends/p4tools/modules/testgen/targets/pna/backend/metadata/metadata.h +++ b/backends/p4tools/modules/testgen/targets/pna/backend/metadata/metadata.h @@ -34,8 +34,8 @@ class Metadata : public TestFramework { explicit Metadata(const TestBackendConfiguration &testBackendConfiguration); /// Produce a Metadata test. - void outputTest(const TestSpec *spec, cstring selectedBranches, size_t testId, - float currentCoverage) override; + void writeTestToFile(const TestSpec *spec, cstring selectedBranches, size_t testId, + float currentCoverage) override; private: /// Emits the test preamble. This is only done once for all generated tests. diff --git a/backends/p4tools/modules/testgen/targets/pna/backend/ptf/ptf.cpp b/backends/p4tools/modules/testgen/targets/pna/backend/ptf/ptf.cpp index 054354c8f26..685da61bda4 100644 --- a/backends/p4tools/modules/testgen/targets/pna/backend/ptf/ptf.cpp +++ b/backends/p4tools/modules/testgen/targets/pna/backend/ptf/ptf.cpp @@ -20,6 +20,7 @@ #include "backends/p4tools/modules/testgen/lib/exceptions.h" #include "backends/p4tools/modules/testgen/lib/test_framework.h" +#include "backends/p4tools/modules/testgen/options.h" #include "backends/p4tools/modules/testgen/targets/pna/test_spec.h" namespace P4Tools::P4Testgen::Pna { @@ -409,8 +410,8 @@ void PTF::emitTestcase(const TestSpec *testSpec, cstring selectedBranches, size_ ptfFileStream.flush(); } -void PTF::outputTest(const TestSpec *testSpec, cstring selectedBranches, size_t testId, - float currentCoverage) { +void PTF::writeTestToFile(const TestSpec *testSpec, cstring selectedBranches, size_t testId, + float currentCoverage) { std::string testCase = getTestCaseTemplate(); emitTestcase(testSpec, selectedBranches, testId, testCase, currentCoverage); } diff --git a/backends/p4tools/modules/testgen/targets/pna/backend/ptf/ptf.h b/backends/p4tools/modules/testgen/targets/pna/backend/ptf/ptf.h index 95d01bb6e95..46e4eab6d23 100644 --- a/backends/p4tools/modules/testgen/targets/pna/backend/ptf/ptf.h +++ b/backends/p4tools/modules/testgen/targets/pna/backend/ptf/ptf.h @@ -39,8 +39,8 @@ class PTF : public TestFramework { explicit PTF(const TestBackendConfiguration &testBackendConfiguration); /// Produce a PTF test. - void outputTest(const TestSpec *spec, cstring selectedBranches, size_t testId, - float currentCoverage) override; + void writeTestToFile(const TestSpec *spec, cstring selectedBranches, size_t testId, + float currentCoverage) override; private: /// Emits the test preamble. This is only done once for all generated tests. diff --git a/backends/p4tools/modules/testgen/targets/pna/test_backend.h b/backends/p4tools/modules/testgen/targets/pna/test_backend.h index 72cc530980d..f50810b421a 100644 --- a/backends/p4tools/modules/testgen/targets/pna/test_backend.h +++ b/backends/p4tools/modules/testgen/targets/pna/test_backend.h @@ -15,6 +15,7 @@ #include "backends/p4tools/modules/testgen/core/program_info.h" #include "backends/p4tools/modules/testgen/core/symbolic_executor/symbolic_executor.h" +#include "backends/p4tools/modules/testgen/core/target.h" #include "backends/p4tools/modules/testgen/lib/execution_state.h" #include "backends/p4tools/modules/testgen/lib/test_backend.h" #include "backends/p4tools/modules/testgen/lib/test_spec.h" diff --git a/backends/p4tools/modules/testgen/testgen.cpp b/backends/p4tools/modules/testgen/testgen.cpp index 7e871d0f917..7c72976ded6 100644 --- a/backends/p4tools/modules/testgen/testgen.cpp +++ b/backends/p4tools/modules/testgen/testgen.cpp @@ -1,11 +1,13 @@ #include "backends/p4tools/modules/testgen/testgen.h" #include +#include #include #include #include #include +#include "backends/p4tools/common/compiler/context.h" #include "backends/p4tools/common/core/z3_solver.h" #include "backends/p4tools/common/lib/util.h" #include "frontends/common/parser_options.h" @@ -22,35 +24,83 @@ #include "backends/p4tools/modules/testgen/core/symbolic_executor/selected_branches.h" #include "backends/p4tools/modules/testgen/core/symbolic_executor/symbolic_executor.h" #include "backends/p4tools/modules/testgen/core/target.h" +#include "backends/p4tools/modules/testgen/lib/test_backend.h" +#include "backends/p4tools/modules/testgen/lib/test_framework.h" #include "backends/p4tools/modules/testgen/options.h" #include "backends/p4tools/modules/testgen/register.h" namespace P4Tools::P4Testgen { -void Testgen::registerTarget() { - // Register all available compiler targets. - // These are discovered by CMAKE, which fills out the register.h.in file. - registerCompilerTargets(); -} +namespace { +/// Pick the path selection algorithm for the symbolic executor. SymbolicExecutor *pickExecutionEngine(const TestgenOptions &testgenOptions, - const ProgramInfo *programInfo, AbstractSolver &solver) { + const ProgramInfo &programInfo, AbstractSolver &solver) { const auto &pathSelectionPolicy = testgenOptions.pathSelectionPolicy; if (pathSelectionPolicy == PathSelectionPolicy::GreedyStmtCoverage) { - return new GreedyNodeSelection(solver, *programInfo); + return new GreedyNodeSelection(solver, programInfo); } if (pathSelectionPolicy == PathSelectionPolicy::RandomBacktrack) { - return new RandomBacktrack(solver, *programInfo); + return new RandomBacktrack(solver, programInfo); } if (!testgenOptions.selectedBranches.empty()) { std::string selectedBranchesStr = testgenOptions.selectedBranches; - return new SelectedBranches(solver, *programInfo, selectedBranchesStr); + return new SelectedBranches(solver, programInfo, selectedBranchesStr); + } + return new DepthFirstSearch(solver, programInfo); +} + +/// Analyse the results of the symbolic execution and generate diagnostic messages. +int postProcess(const TestgenOptions &testgenOptions, const TestBackEnd &testBackend) { + // Do not print this warning if assertion mode is enabled. + if (testBackend.getTestCount() == 0 && !testgenOptions.assertionModeEnabled) { + ::warning( + "Unable to generate tests with given inputs. Double-check provided options and " + "parameters.\n"); + } + if (testBackend.getCoverage() < testgenOptions.minCoverage) { + ::error("The tests did not achieve requested coverage of %1%, the coverage is %2%.", + testgenOptions.minCoverage, testBackend.getCoverage()); + } + + return ::errorCount() == 0 ? EXIT_SUCCESS : EXIT_FAILURE; +} + +std::optional generateAndCollectAbstractTests( + const TestgenOptions &testgenOptions, const ProgramInfo &programInfo) { + if (!testgenOptions.testBaseName.has_value()) { + ::error( + "Test collection requires a test name. No name was provided as part of the " + "P4Testgen options."); + return std::nullopt; } - return new DepthFirstSearch(solver, *programInfo); + + // The test name is the stem of the output base path. + TestBackendConfiguration testBackendConfiguration{testgenOptions.testBaseName.value(), + testgenOptions.maxTests, std::nullopt, + testgenOptions.seed}; + // Need to declare the solver here to ensure its lifetime. + Z3Solver solver; + auto *symbolicExecutor = pickExecutionEngine(testgenOptions, programInfo, solver); + + // Each test back end has a different run function. + auto *testBackend = + TestgenTarget::getTestBackend(programInfo, testBackendConfiguration, *symbolicExecutor); + + // Define how to handle the final state for each test. This is target defined. + // We delegate execution to the symbolic executor. + symbolicExecutor->run([testBackend](auto &&finalState) { + return testBackend->run(std::forward(finalState)); + }); + auto result = postProcess(testgenOptions, *testBackend); + if (result != EXIT_SUCCESS) { + return std::nullopt; + } + return testBackend->getTests(); } -int generateAbstractTests(const TestgenOptions &testgenOptions, const ProgramInfo *programInfo, - SymbolicExecutor &symbex) { +int generateAndWriteAbstractTests(const TestgenOptions &testgenOptions, + const ProgramInfo &programInfo) { cstring inputFile = P4CContext::get().options().file; if (inputFile == nullptr) { ::error("No input file provided."); @@ -67,40 +117,77 @@ int generateAbstractTests(const TestgenOptions &testgenOptions, const ProgramInf cstring testDirStr = testgenOptions.outputDir; if (!testDirStr.isNullOrEmpty()) { auto testDir = std::filesystem::path(testDirStr.c_str()); - std::filesystem::create_directories(testDir); + try { + std::filesystem::create_directories(testDir); + } catch (const std::exception &err) { + ::error("Unable to create directory %1%: %2%", testDir.c_str(), err.what()); + return EXIT_FAILURE; + } testPath = testDir / testPath; } - // Each test back end has a different run function. // The test name is the stem of the output base path. TestBackendConfiguration testBackendConfiguration{testPath.c_str(), testgenOptions.maxTests, testPath, testgenOptions.seed}; + + // Need to declare the solver here to ensure its lifetime. + Z3Solver solver; + auto *symbolicExecutor = pickExecutionEngine(testgenOptions, programInfo, solver); + + // Each test back end has a different run function. auto *testBackend = - TestgenTarget::getTestBackend(*programInfo, testBackendConfiguration, symbex); + TestgenTarget::getTestBackend(programInfo, testBackendConfiguration, *symbolicExecutor); + // Define how to handle the final state for each test. This is target defined. // We delegate execution to the symbolic executor. - auto callBack = [testBackend](auto &&finalState) { + symbolicExecutor->run([testBackend](auto &&finalState) { return testBackend->run(std::forward(finalState)); - }; + }); + return postProcess(testgenOptions, *testBackend); +} - symbex.run(callBack); +std::optional generateTestsImpl(const std::string &program, + const CompilerOptions &compilerOptions, + const TestgenOptions &testgenOptions) { + // Register supported compiler targets. + registerCompilerTargets(); - // Do not print this warning if assertion mode is enabled. - if (testBackend->getTestCount() == 0 && !testgenOptions.assertionModeEnabled) { - ::warning( - "Unable to generate tests with given inputs. Double-check provided options and " - "parameters.\n"); + // Register supported Testgen targets. + registerTestgenTargets(); + + P4Tools::Target::init(compilerOptions.target.c_str(), compilerOptions.arch.c_str()); + + // Set up the compilation context. + auto *compileContext = new CompileContext(); + compileContext->options() = compilerOptions; + AutoCompileContext autoContext(compileContext); + // Run the compiler to get an IR and invoke the tool. + const auto compilerResultOpt = P4Tools::CompilerTarget::runCompiler(program); + if (!compilerResultOpt.has_value()) { + return std::nullopt; } - if (testBackend->getCoverage() < testgenOptions.minCoverage) { - ::error("The tests did not achieve requested coverage of %1%, the coverage is %2%.", - testgenOptions.minCoverage, testBackend->getCoverage()); + const auto *testgenCompilerResult = + compilerResultOpt.value().get().checkedTo(); + + const auto *programInfo = TestgenTarget::produceProgramInfo(*testgenCompilerResult); + if (programInfo == nullptr || ::errorCount() > 0) { + ::error("P4Testgen encountered errors during preprocessing."); + return std::nullopt; } - return ::errorCount() == 0 ? EXIT_SUCCESS : EXIT_FAILURE; + return generateAndCollectAbstractTests(testgenOptions, *programInfo); +} + +} // namespace + +void Testgen::registerTarget() { + // Register all available compiler targets. + // These are discovered by CMAKE, which fills out the register.h.in file. + registerCompilerTargets(); } int Testgen::mainImpl(const CompilerResult &compilerResult) { - // Register all available testgen targets. + // Register all available P4Testgen targets. // These are discovered by CMAKE, which fills out the register.h.in file. registerTestgenTargets(); @@ -108,27 +195,32 @@ int Testgen::mainImpl(const CompilerResult &compilerResult) { const auto *testgenCompilerResult = compilerResult.checkedTo(); const auto *programInfo = TestgenTarget::produceProgramInfo(*testgenCompilerResult); - if (programInfo == nullptr) { - ::error("Program not supported by target device and architecture."); - return EXIT_FAILURE; - } - if (::errorCount() > 0) { - ::error("Testgen: Encountered errors during preprocessing. Exiting"); + if (programInfo == nullptr || ::errorCount() > 0) { + ::error("P4Testgen encountered errors during preprocessing."); return EXIT_FAILURE; } - // Get the options and the seed. - const auto &testgenOptions = TestgenOptions::get(); auto seed = Utils::getCurrentSeed(); if (seed) { - printFeature("test_info", 4, "============ Program seed %1% =============\n", *seed); + printFeature("test_info", 4, "============ Prograam seed %1% =============\n", *seed); } - // Need to declare the solver here to ensure its lifetime. - Z3Solver solver; - auto *symbex = pickExecutionEngine(testgenOptions, programInfo, solver); + const auto &testgenOptions = TestgenOptions::get(); + return generateAndWriteAbstractTests(testgenOptions, *programInfo); +} - return generateAbstractTests(testgenOptions, programInfo, *symbex); +std::optional Testgen::generateTests(const std::string &program, + const CompilerOptions &compilerOptions, + const TestgenOptions &testgenOptions) { + try { + return generateTestsImpl(program, compilerOptions, testgenOptions); + } catch (const std::exception &e) { + std::cerr << "Internal error: " << e.what() << "\n"; + return std::nullopt; + } catch (...) { + return std::nullopt; + } + return std::nullopt; } } // namespace P4Tools::P4Testgen diff --git a/backends/p4tools/modules/testgen/testgen.h b/backends/p4tools/modules/testgen/testgen.h index bd6e985b150..59fbedaf307 100644 --- a/backends/p4tools/modules/testgen/testgen.h +++ b/backends/p4tools/modules/testgen/testgen.h @@ -2,8 +2,8 @@ #define BACKENDS_P4TOOLS_MODULES_TESTGEN_TESTGEN_H_ #include "backends/p4tools/common/p4ctool.h" -#include "ir/ir.h" +#include "backends/p4tools/modules/testgen/lib/test_framework.h" #include "backends/p4tools/modules/testgen/options.h" namespace P4Tools::P4Testgen { @@ -16,6 +16,11 @@ class Testgen : public AbstractP4cTool { int mainImpl(const CompilerResult &compilerResult) override; public: + /// + static std::optional generateTests(const std::string &program, + const CompilerOptions &options, + const TestgenOptions &testgenOptions); + virtual ~Testgen() = default; }; diff --git a/test/gtest/helpers.cpp b/test/gtest/helpers.cpp index 5270d04e02a..fd7a8afcf18 100644 --- a/test/gtest/helpers.cpp +++ b/test/gtest/helpers.cpp @@ -139,7 +139,7 @@ std::string P4CTestEnvironment::readHeader(const char *filename, bool preprocess P4CTestEnvironment::P4CTestEnvironment() { // Locate the headers based on the relative path of the file. std::filesystem::path srcFilePath{__FILE__}; - auto srcFileDir = srcFilePath.parent_path(); + auto srcFileDir = std::filesystem::absolute(srcFilePath.parent_path()); auto corePath = srcFileDir / "../../p4include/core.p4"; auto v1modelPath = srcFileDir / "../../p4include/v1model.p4"; auto psaPath = srcFileDir / "../../p4include/bmv2/psa.p4"; diff --git a/test/gtest/helpers.h b/test/gtest/helpers.h index 9522365ffa8..5c5d312d9ab 100644 --- a/test/gtest/helpers.h +++ b/test/gtest/helpers.h @@ -17,12 +17,13 @@ limitations under the License. #ifndef TEST_GTEST_HELPERS_H_ #define TEST_GTEST_HELPERS_H_ +#include + #include #include #include "frontends/common/options.h" #include "frontends/p4/parseAnnotations.h" -#include "gtest/gtest.h" namespace IR { class P4Program;