Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dont fail error suites #540

Merged
merged 4 commits into from
Dec 1, 2022
Merged

Dont fail error suites #540

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5f60a56
Dont't fail error suites
Jun 1, 2022
5384717
fix
Nov 10, 2022
5ba9426
fixes after rebase
Nov 16, 2022
cc4ee35
Delete unused code, suppress warnings, fix formatting
Nov 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions server/proto/testgen.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,13 +90,20 @@ message ProjectContext {
string buildDirRelativePath = 4;
}

enum ErrorMode {
FAILING = 0;
PASSING_IN_TARGET_ONLY = 1;
PASSING = 2;
}
Comment on lines +93 to +97
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you please add some docs about error mode? Probably, comments or doc at wiki?


message SettingsContext {
bool generateForStaticFunctions = 1;
bool verbose = 2;
int32 timeoutPerFunction = 3;
int32 timeoutPerTest = 4;
bool useDeterministicSearcher = 5;
bool useStubs = 6;
ErrorMode errorMode = 7;
}

message SnippetRequest {
Expand Down
6 changes: 3 additions & 3 deletions server/src/KleeGenerator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,8 @@ void KleeGenerator::parseKTestsToFinalCode(
const std::unordered_map<std::string, types::Type> &methodNameToReturnTypeMap,
const std::vector<MethodKtests> &kleeOutput,
const std::shared_ptr<LineInfo> &lineInfo,
bool verbose) {
bool verbose,
ErrorMode errorMode) {
for (const auto &batch : kleeOutput) {
bool filterByFlag = (lineInfo != nullptr && !lineInfo->forMethod && !lineInfo->forClass &&
!lineInfo->predicateInfo.has_value());
Expand All @@ -349,8 +350,7 @@ void KleeGenerator::parseKTestsToFinalCode(
}
auto predicate =
lineInfo ? lineInfo->predicateInfo : std::optional<LineInfo::PredicateInfo>{};

testsPrinter.genCode(methodDescription, predicate, verbose);
testsPrinter.genCode(methodDescription, predicate, verbose, errorMode);
}

printer::HeaderPrinter(Paths::getSourceLanguage(tests.sourceFilePath))
Expand Down
13 changes: 7 additions & 6 deletions server/src/KleeGenerator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,12 +112,13 @@ class KleeGenerator {
* @throws ExecutionProcessException if a Clang call returns non-zero code.
*/
void parseKTestsToFinalCode(
const utbot::ProjectContext &projectContext,
tests::Tests &tests,
const std::unordered_map<std::string, types::Type> &methodNameToReturnTypeMap,
const std::vector<MethodKtests> &kleeOutput,
const std::shared_ptr<LineInfo> &lineInfo = nullptr,
bool verbose = false);
const utbot::ProjectContext &projectContext,
tests::Tests &tests,
const std::unordered_map<std::string, types::Type> &methodNameToReturnTypeMap,
const std::vector<MethodKtests> &kleeOutput,
const std::shared_ptr<LineInfo> &lineInfo = nullptr,
bool verbose = false,
ErrorMode errorMode = ErrorMode::FAILING);

[[nodiscard]] fs::path getBitcodeFile(const fs::path &sourcePath) const;

Expand Down
2 changes: 1 addition & 1 deletion server/src/KleeRunner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ void KleeRunner::runKlee(const std::vector<tests::TestMethod> &testMethods,
}
auto kleeStats = writeKleeStats(Paths::kleeOutDirForFilePath(projectContext, filePath));
generator->parseKTestsToFinalCode(projectContext, tests, methodNameToReturnTypeMap, ktests,
lineInfo, settingsContext.verbose);
lineInfo, settingsContext.verbose, settingsContext.errorMode);
generationStats.addFileStats(kleeStats, tests);

sarif::sarifAddTestsToResults(projectContext, tests, sarifResults);
Expand Down
2 changes: 1 addition & 1 deletion server/src/Paths.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ namespace Paths {
return replaceExtension(path, StringUtils::stringFormat(".%s.err", suffix));
}

static bool errorFileExists(const fs::path &path, std::string const& suffix) {
bool errorFileExists(const fs::path &path, std::string const& suffix) {
return fs::exists(errorFile(path, suffix));
}

Expand Down
5 changes: 4 additions & 1 deletion server/src/Paths.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ namespace Paths {
const std::vector<fs::path> &dirNames,
const std::function<bool(const fs::path &path)> &filter);

bool errorFileExists(const fs::path &path, std::string const& suffix);

static inline void setOptPath(fs::path &path, const std::string &value) {
path = fs::path(value);
}
Expand Down Expand Up @@ -427,6 +429,7 @@ namespace Paths {
return "stubs" / relativeTestDirPath;
}

bool hasUncaughtException(const fs::path &path);
//endregion

//region utbot_report
Expand All @@ -447,6 +450,6 @@ namespace Paths {
//endregion

bool isHeadersEqual(const fs::path &srcPath, const fs::path &headerPath);
}
} // Paths

#endif //UNITTESTBOT_PATHS_H
9 changes: 6 additions & 3 deletions server/src/SettingsContext.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ namespace utbot {
int32_t timeoutPerFunction,
int32_t timeoutPerTest,
bool useDeterministicSearcher,
bool useStubs)
bool useStubs,
testsgen::ErrorMode errorMode)
: generateForStaticFunctions(generateForStaticFunctions),
verbose(verbose),
timeoutPerFunction(timeoutPerFunction > 0
Expand All @@ -17,14 +18,16 @@ namespace utbot {
timeoutPerTest(timeoutPerTest > 0
? std::make_optional(std::chrono::seconds{ timeoutPerTest })
: std::nullopt),
useDeterministicSearcher(useDeterministicSearcher), useStubs(useStubs) {
useDeterministicSearcher(useDeterministicSearcher), useStubs(useStubs),
errorMode(errorMode) {
}
SettingsContext::SettingsContext(const testsgen::SettingsContext &settingsContext)
: SettingsContext(settingsContext.generateforstaticfunctions(),
settingsContext.verbose(),
settingsContext.timeoutperfunction(),
settingsContext.timeoutpertest(),
settingsContext.usedeterministicsearcher(),
settingsContext.usestubs()) {
settingsContext.usestubs(),
settingsContext.errormode()) {
}
}
5 changes: 4 additions & 1 deletion server/src/SettingsContext.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

#include <chrono>
#include <optional>
#include <protobuf/testgen.grpc.pb.h>

namespace testsgen {
class SettingsContext;
Expand All @@ -18,13 +19,15 @@ namespace utbot {
int32_t timeoutPerFunction,
int32_t timeoutPerTest,
bool useDeterministicSearcher,
bool useStubs);
bool useStubs,
testsgen::ErrorMode errorMode);

const bool generateForStaticFunctions;
const bool verbose;
const std::optional<std::chrono::seconds> timeoutPerFunction, timeoutPerTest;
const bool useDeterministicSearcher;
const bool useStubs;
testsgen::ErrorMode errorMode;
};
}

Expand Down
2 changes: 2 additions & 0 deletions server/src/Tests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -744,6 +744,7 @@ void KTestObjectParser::parseTestCases(const UTBotKTestList &cases,

testCase.errorDescriptors = case_.errorDescriptors;

testCase.errorInfo = testCaseDescription.errorInfo;
if (filterByLineFlag) {
auto view = testCaseDescription.kleePathFlagSymbolicValue.view;
if (!view || view->getEntryValue(nullptr) != "1") {
Expand Down Expand Up @@ -824,6 +825,7 @@ Tests::TestCaseDescription KTestObjectParser::parseTestCaseParams(

Tests::TestCaseDescription testCaseDescription;
testCaseDescription.objects = ktest.objects;
testCaseDescription.errorInfo = ktest.errorInfo;

for (const auto &obj : testCaseDescription.objects) {
if (obj.name != LAZYNAME) {
Expand Down
7 changes: 6 additions & 1 deletion server/src/Tests.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include <klee/TestCase.h>
#include <tsl/ordered_map.h>
#include <tsl/ordered_set.h>
#include "Paths.h"

#include <cassert>
#include <climits>
Expand All @@ -25,6 +26,7 @@
#include <type_traits>
#include <utility>
#include <vector>
#include <utils/ErrorInfo.h>

using json = nlohmann::json;

Expand Down Expand Up @@ -78,6 +80,7 @@ namespace tests {
std::vector<UTBotKTestObject> objects;
Status status;
std::vector<std::string> errorDescriptors;
ErrorInfo errorInfo;

UTBotKTest(std::vector<UTBotKTestObject> objects,
const Status &status,
Expand Down Expand Up @@ -412,6 +415,7 @@ namespace tests {
std::optional <std::vector<TestCaseParamValue>> filesValues = std::nullopt;
std::optional<TestCaseParamValue> classPreValues;
std::optional<TestCaseParamValue> classPostValues;
ErrorInfo errorInfo;
};

struct MethodTestCase {
Expand Down Expand Up @@ -439,6 +443,7 @@ namespace tests {
std::optional<TestCaseParamValue> classPreValues;
std::optional<TestCaseParamValue> classPostValues;
std::vector<std::string> errorDescriptors;
ErrorInfo errorInfo;

[[nodiscard]] bool isError() const;
};
Expand Down Expand Up @@ -898,5 +903,5 @@ namespace tests {
const std::string &typeName,
size_t offsetInBits,
size_t lenInBits);
}
} // tests
#endif // UNITTESTBOT_TESTS_H
4 changes: 4 additions & 0 deletions server/src/commands/Commands.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,10 @@ bool Commands::SettingsContextOptionGroup::withStubs() const {
return !noStubs;
}

ErrorMode Commands::SettingsContextOptionGroup::getErrorMode() const {
return errorMode;
}

Commands::RunTestsCommands::RunTestsCommands(Commands::MainCommands &commands) {
runCommand = commands.getRunTestsCommand();

Expand Down
5 changes: 4 additions & 1 deletion server/src/commands/Commands.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
#include <CLI11.hpp>
#include <string>


using namespace ::testsgen;

namespace Commands {
extern uint32_t threadsPerUser;
Expand Down Expand Up @@ -243,6 +243,8 @@ namespace Commands {

[[nodiscard]] bool withStubs() const;

[[nodiscard]] ErrorMode getErrorMode() const;

private:
CLI::Option_group *settingsContextOptions;
bool generateForStaticFunctions = true;
Expand All @@ -251,6 +253,7 @@ namespace Commands {
int32_t timeoutPerTest = 30;
bool noDeterministicSearcher = false;
bool noStubs = false;
ErrorMode errorMode = ErrorMode::FAILING;
};
};

Expand Down
10 changes: 4 additions & 6 deletions server/src/coverage/TestRunner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,18 +64,16 @@ std::vector<UnitTest> TestRunner::getTestsFromMakefile(const fs::path &makefile,
StringUtils::trim(s);
}
std::string testSuite;
std::vector<std::string> testsList;
for (const std::string &s : gtestListTestsOutput) {
std::vector<UnitTest> testList;
for (const std::string &s: gtestListTestsOutput) {
if (s.back() == '.') {
testSuite = s;
testSuite.pop_back();
} else {
testsList.push_back(s);
testList.push_back({testFilePath, testSuite, s});
}
}
return CollectionUtils::transform(testsList, [&testFilePath, &testSuite](std::string const &name) {
return UnitTest{ testFilePath, testSuite, name };
});
return testList;
}

std::vector<UnitTest> TestRunner::getTestsToLaunch() {
Expand Down
Loading