Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

router check tool: add flag for only printing failed tests #8160

Merged
merged 22 commits into from
Sep 11, 2019
Merged

router check tool: add flag for only printing failed tests #8160

Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
acaf2b6
Add line on installing XCode
Sep 3, 2019
0a6aec7
Try again with sign-off
Sep 4, 2019
99d685b
Merge remote-tracking branch 'upstream/master'
Sep 5, 2019
b08e9e7
Add flag for printing failed tests only
Sep 5, 2019
cfc566f
Add unit tests, fix formatting
Sep 5, 2019
7272326
newline
Sep 5, 2019
f58805c
Remove no-output test
Sep 5, 2019
0609bdb
Update docs and version history
Sep 6, 2019
d3d80ee
Merge branch 'master' into master
Sep 6, 2019
474eee3
Remove shorthand
Sep 6, 2019
6b40624
Merge branch 'master' of github.com:LisaLudique/envoy
Sep 6, 2019
a642ba5
Change method name and remove call from deprecated method
Sep 6, 2019
0f0fc9e
Restructure test result output
Sep 9, 2019
15d58bd
Lint
Sep 9, 2019
9e58cfc
Use vector instead of map for non-unique test names
Sep 9, 2019
b74ea99
Format
Sep 9, 2019
e5d1b36
Restore proto details test
Sep 10, 2019
b6d8f3d
Remove comments
Sep 10, 2019
ed0b7f8
Change expected test output to account for CI tests
Sep 10, 2019
a474b4c
Refactor output printing, add comment
Sep 10, 2019
8c9012f
Formatting
Sep 10, 2019
cc362e3
Fix error in test script and add test for multiple errors in test
Sep 11, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions docs/root/install/tools/route_table_check_tool.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ Usage
-d, --details
Show detailed test execution results. The first line indicates the test name.

--only-show-failures
Displays test results for failed tests. Omits test names for passing tests if the details flag is set.

-p, --useproto
Use Proto test file schema

Expand Down
1 change: 1 addition & 0 deletions docs/root/intro/version_history.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ Version history
* router check tool: add coverage reporting & enforcement.
* router check tool: add comprehensive coverage reporting.
* router check tool: add deprecated field check.
* router check tool: add flag for only printing results of failed tests.
* thrift_proxy: fix crashing bug on invalid transport/protocol framing
* tls: added verification of IP address SAN fields in certificates against configured SANs in the
* tracing: added support to the Zipkin reporter for sending list of spans as Zipkin JSON v2 and protobuf message over HTTP.
Expand Down
139 changes: 83 additions & 56 deletions test/tools/router_check/router.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ bool RouterCheckTool::compareEntriesInJson(const std::string& expected_route_jso
}
Json::ObjectSharedPtr validate = check_config->getObject("validate");

using checkerFunc = std::function<bool(ToolConfig&, const std::string&)>;
using checkerFunc = std::function<bool(ToolConfig&, const std::string&, const std::string&)>;
const std::unordered_map<std::string, checkerFunc> checkers = {
{"cluster_name",
[this](auto&... params) -> bool { return this->compareCluster(params...); }},
Expand All @@ -134,9 +134,9 @@ bool RouterCheckTool::compareEntriesInJson(const std::string& expected_route_jso
if (validate->hasObject(test.first)) {
const std::string& expected = validate->getString(test.first);
if (tool_config.route_ == nullptr) {
compareResults("", expected, test.first);
compareResults("", expected, test.first, test_name);
} else {
if (!test.second(tool_config, expected)) {
if (!test.second(tool_config, expected, test_name)) {
no_failures = false;
}
}
Expand All @@ -146,7 +146,7 @@ bool RouterCheckTool::compareEntriesInJson(const std::string& expected_route_jso
if (validate->hasObject("header_fields")) {
for (const Json::ObjectSharedPtr& header_field : validate->getObjectArray("header_fields")) {
if (!compareHeaderField(tool_config, header_field->getString("field"),
header_field->getString("value"))) {
header_field->getString("value"), test_name)) {
no_failures = false;
}
}
Expand All @@ -156,7 +156,7 @@ bool RouterCheckTool::compareEntriesInJson(const std::string& expected_route_jso
for (const Json::ObjectSharedPtr& header_field :
validate->getObjectArray("custom_header_fields")) {
if (!compareCustomHeaderField(tool_config, header_field->getString("field"),
header_field->getString("value"))) {
header_field->getString("value"), test_name)) {
no_failures = false;
}
}
Expand All @@ -183,13 +183,11 @@ bool RouterCheckTool::compareEntries(const std::string& expected_routes) {
tool_config.route_ = config_->route(*tool_config.headers_, tool_config.random_value_);

const std::string& test_name = check_config.test_name();
if (details_) {
std::cout << test_name << std::endl;
}
tests_.insert({test_name, {}});
LisaLudique marked this conversation as resolved.
Show resolved Hide resolved
const envoy::RouterCheckToolSchema::ValidationAssert& validate = check_config.validate();

using checkerFunc =
std::function<bool(ToolConfig&, const envoy::RouterCheckToolSchema::ValidationAssert&)>;
using checkerFunc = std::function<bool(
ToolConfig&, const envoy::RouterCheckToolSchema::ValidationAssert&, const std::string&)>;
checkerFunc checkers[] = {
[this](auto&... params) -> bool { return this->compareCluster(params...); },
[this](auto&... params) -> bool { return this->compareVirtualCluster(params...); },
Expand All @@ -203,40 +201,55 @@ bool RouterCheckTool::compareEntries(const std::string& expected_routes) {

// Call appropriate function for each match case.
for (const auto& test : checkers) {
if (!test(tool_config, validate)) {
if (!test(tool_config, validate, test_name)) {
no_failures = false;
}
}
}
// Output failure details to stdout if details_ flag is set to true
LisaLudique marked this conversation as resolved.
Show resolved Hide resolved
for (const auto& test_result : tests_) {
// All test names are printed if the details_ flag is true unless only_show_failures_ is also
// true.
if ((details_ && !only_show_failures_) ||
(only_show_failures_ && !test_result.second.empty())) {
std::cout << test_result.first << std::endl;
for (const auto& failure : test_result.second) {
std::cerr << failure << std::endl;
}
}
}

return no_failures;
}

bool RouterCheckTool::compareCluster(ToolConfig& tool_config, const std::string& expected) {
bool RouterCheckTool::compareCluster(ToolConfig& tool_config, const std::string& expected,
const std::string& test_name) {
std::string actual = "";

if (tool_config.route_->routeEntry() != nullptr) {
actual = tool_config.route_->routeEntry()->clusterName();
}
const bool matches = compareResults(actual, expected, "cluster_name");
const bool matches = compareResults(actual, expected, "cluster_name", test_name);
if (matches && tool_config.route_->routeEntry() != nullptr) {
coverage_.markClusterCovered(*tool_config.route_->routeEntry());
}
return matches;
}

bool RouterCheckTool::compareCluster(
ToolConfig& tool_config, const envoy::RouterCheckToolSchema::ValidationAssert& expected) {
bool RouterCheckTool::compareCluster(ToolConfig& tool_config,
const envoy::RouterCheckToolSchema::ValidationAssert& expected,
const std::string& test_name) {
if (!expected.has_cluster_name()) {
return true;
}
if (tool_config.route_ == nullptr) {
return compareResults("", expected.cluster_name().value(), "cluster_name");
return compareResults("", expected.cluster_name().value(), "cluster_name", test_name);
}
return compareCluster(tool_config, expected.cluster_name().value());
return compareCluster(tool_config, expected.cluster_name().value(), test_name);
}

bool RouterCheckTool::compareVirtualCluster(ToolConfig& tool_config, const std::string& expected) {
bool RouterCheckTool::compareVirtualCluster(ToolConfig& tool_config, const std::string& expected,
const std::string& test_name) {
std::string actual = "";

if (tool_config.route_->routeEntry() != nullptr &&
Expand All @@ -245,49 +258,54 @@ bool RouterCheckTool::compareVirtualCluster(ToolConfig& tool_config, const std::
tool_config.route_->routeEntry()->virtualCluster(*tool_config.headers_)->statName();
actual = tool_config.symbolTable().toString(stat_name);
}
const bool matches = compareResults(actual, expected, "virtual_cluster_name");
const bool matches = compareResults(actual, expected, "virtual_cluster_name", test_name);
if (matches && tool_config.route_->routeEntry() != nullptr) {
coverage_.markVirtualClusterCovered(*tool_config.route_->routeEntry());
}
return matches;
}

bool RouterCheckTool::compareVirtualCluster(
ToolConfig& tool_config, const envoy::RouterCheckToolSchema::ValidationAssert& expected) {
ToolConfig& tool_config, const envoy::RouterCheckToolSchema::ValidationAssert& expected,
const std::string& test_name) {
if (!expected.has_virtual_cluster_name()) {
return true;
}
if (tool_config.route_ == nullptr) {
return compareResults("", expected.virtual_cluster_name().value(), "virtual_cluster_name");
return compareResults("", expected.virtual_cluster_name().value(), "virtual_cluster_name",
test_name);
}
return compareVirtualCluster(tool_config, expected.virtual_cluster_name().value());
return compareVirtualCluster(tool_config, expected.virtual_cluster_name().value(), test_name);
}

bool RouterCheckTool::compareVirtualHost(ToolConfig& tool_config, const std::string& expected) {
bool RouterCheckTool::compareVirtualHost(ToolConfig& tool_config, const std::string& expected,
const std::string& test_name) {
std::string actual = "";
if (tool_config.route_->routeEntry() != nullptr) {
Stats::StatName stat_name = tool_config.route_->routeEntry()->virtualHost().statName();
actual = tool_config.symbolTable().toString(stat_name);
}
const bool matches = compareResults(actual, expected, "virtual_host_name");
const bool matches = compareResults(actual, expected, "virtual_host_name", test_name);
if (matches && tool_config.route_->routeEntry() != nullptr) {
coverage_.markVirtualHostCovered(*tool_config.route_->routeEntry());
}
return matches;
}

bool RouterCheckTool::compareVirtualHost(
ToolConfig& tool_config, const envoy::RouterCheckToolSchema::ValidationAssert& expected) {
ToolConfig& tool_config, const envoy::RouterCheckToolSchema::ValidationAssert& expected,
const std::string& test_name) {
if (!expected.has_virtual_host_name()) {
return true;
}
if (tool_config.route_ == nullptr) {
return compareResults("", expected.virtual_host_name().value(), "virtual_host_name");
return compareResults("", expected.virtual_host_name().value(), "virtual_host_name", test_name);
}
return compareVirtualHost(tool_config, expected.virtual_host_name().value());
return compareVirtualHost(tool_config, expected.virtual_host_name().value(), test_name);
}

bool RouterCheckTool::compareRewritePath(ToolConfig& tool_config, const std::string& expected) {
bool RouterCheckTool::compareRewritePath(ToolConfig& tool_config, const std::string& expected,
const std::string& test_name) {
std::string actual = "";
Envoy::StreamInfo::StreamInfoImpl stream_info(Envoy::Http::Protocol::Http11,
factory_context_->dispatcher().timeSource());
Expand All @@ -300,25 +318,27 @@ bool RouterCheckTool::compareRewritePath(ToolConfig& tool_config, const std::str

actual = tool_config.headers_->get_(Http::Headers::get().Path);
}
const bool matches = compareResults(actual, expected, "path_rewrite");
const bool matches = compareResults(actual, expected, "path_rewrite", test_name);
if (matches && tool_config.route_->routeEntry() != nullptr) {
coverage_.markPathRewriteCovered(*tool_config.route_->routeEntry());
}
return matches;
}

bool RouterCheckTool::compareRewritePath(
ToolConfig& tool_config, const envoy::RouterCheckToolSchema::ValidationAssert& expected) {
ToolConfig& tool_config, const envoy::RouterCheckToolSchema::ValidationAssert& expected,
const std::string& test_name) {
if (!expected.has_path_rewrite()) {
return true;
}
if (tool_config.route_ == nullptr) {
return compareResults("", expected.path_rewrite().value(), "path_rewrite");
return compareResults("", expected.path_rewrite().value(), "path_rewrite", test_name);
}
return compareRewritePath(tool_config, expected.path_rewrite().value());
return compareRewritePath(tool_config, expected.path_rewrite().value(), test_name);
}

bool RouterCheckTool::compareRewriteHost(ToolConfig& tool_config, const std::string& expected) {
bool RouterCheckTool::compareRewriteHost(ToolConfig& tool_config, const std::string& expected,
const std::string& test_name) {
std::string actual = "";
Envoy::StreamInfo::StreamInfoImpl stream_info(Envoy::Http::Protocol::Http11,
factory_context_->dispatcher().timeSource());
Expand All @@ -331,54 +351,58 @@ bool RouterCheckTool::compareRewriteHost(ToolConfig& tool_config, const std::str

actual = tool_config.headers_->get_(Http::Headers::get().Host);
}
const bool matches = compareResults(actual, expected, "host_rewrite");
const bool matches = compareResults(actual, expected, "host_rewrite", test_name);
if (matches && tool_config.route_->routeEntry() != nullptr) {
coverage_.markHostRewriteCovered(*tool_config.route_->routeEntry());
}
return matches;
}

bool RouterCheckTool::compareRewriteHost(
ToolConfig& tool_config, const envoy::RouterCheckToolSchema::ValidationAssert& expected) {
ToolConfig& tool_config, const envoy::RouterCheckToolSchema::ValidationAssert& expected,
const std::string& test_name) {
if (!expected.has_host_rewrite()) {
return true;
}
if (tool_config.route_ == nullptr) {
return compareResults("", expected.host_rewrite().value(), "host_rewrite");
return compareResults("", expected.host_rewrite().value(), "host_rewrite", test_name);
}
return compareRewriteHost(tool_config, expected.host_rewrite().value());
return compareRewriteHost(tool_config, expected.host_rewrite().value(), test_name);
}

bool RouterCheckTool::compareRedirectPath(ToolConfig& tool_config, const std::string& expected) {
bool RouterCheckTool::compareRedirectPath(ToolConfig& tool_config, const std::string& expected,
const std::string& test_name) {
std::string actual = "";
if (tool_config.route_->directResponseEntry() != nullptr) {
actual = tool_config.route_->directResponseEntry()->newPath(*tool_config.headers_);
}

const bool matches = compareResults(actual, expected, "path_redirect");
const bool matches = compareResults(actual, expected, "path_redirect", test_name);
if (matches && tool_config.route_->routeEntry() != nullptr) {
coverage_.markRedirectPathCovered(*tool_config.route_->routeEntry());
}
return matches;
}

bool RouterCheckTool::compareRedirectPath(
ToolConfig& tool_config, const envoy::RouterCheckToolSchema::ValidationAssert& expected) {
ToolConfig& tool_config, const envoy::RouterCheckToolSchema::ValidationAssert& expected,
const std::string& test_name) {
if (!expected.has_path_redirect()) {
return true;
}
if (tool_config.route_ == nullptr) {
return compareResults("", expected.path_redirect().value(), "path_redirect");
return compareResults("", expected.path_redirect().value(), "path_redirect", test_name);
}
return compareRedirectPath(tool_config, expected.path_redirect().value());
return compareRedirectPath(tool_config, expected.path_redirect().value(), test_name);
}

bool RouterCheckTool::compareHeaderField(
ToolConfig& tool_config, const envoy::RouterCheckToolSchema::ValidationAssert& expected) {
ToolConfig& tool_config, const envoy::RouterCheckToolSchema::ValidationAssert& expected,
const std::string& test_name) {
bool no_failures = true;
if (expected.header_fields().data()) {
for (const envoy::api::v2::core::HeaderValue& header : expected.header_fields()) {
if (!compareHeaderField(tool_config, header.key(), header.value())) {
if (!compareHeaderField(tool_config, header.key(), header.value(), test_name)) {
no_failures = false;
}
}
Expand All @@ -387,13 +411,15 @@ bool RouterCheckTool::compareHeaderField(
}

bool RouterCheckTool::compareHeaderField(ToolConfig& tool_config, const std::string& field,
const std::string& expected) {
const std::string& expected,
const std::string& test_name) {
std::string actual = tool_config.headers_->get_(field);
return compareResults(actual, expected, "check_header");
return compareResults(actual, expected, "check_header", test_name);
}

bool RouterCheckTool::compareCustomHeaderField(ToolConfig& tool_config, const std::string& field,
const std::string& expected) {
const std::string& expected,
const std::string& test_name) {
std::string actual = "";
Envoy::StreamInfo::StreamInfoImpl stream_info(Envoy::Http::Protocol::Http11,
factory_context_->dispatcher().timeSource());
Expand All @@ -403,15 +429,16 @@ bool RouterCheckTool::compareCustomHeaderField(ToolConfig& tool_config, const st
true);
actual = tool_config.headers_->get_(field);
}
return compareResults(actual, expected, "custom_header");
return compareResults(actual, expected, "custom_header", test_name);
}

bool RouterCheckTool::compareCustomHeaderField(
ToolConfig& tool_config, const envoy::RouterCheckToolSchema::ValidationAssert& expected) {
ToolConfig& tool_config, const envoy::RouterCheckToolSchema::ValidationAssert& expected,
const std::string& test_name) {
bool no_failures = true;
if (expected.custom_header_fields().data()) {
for (const envoy::api::v2::core::HeaderValue& header : expected.custom_header_fields()) {
if (!compareCustomHeaderField(tool_config, header.key(), header.value())) {
if (!compareCustomHeaderField(tool_config, header.key(), header.value(), test_name)) {
no_failures = false;
}
}
Expand All @@ -420,16 +447,13 @@ bool RouterCheckTool::compareCustomHeaderField(
}

bool RouterCheckTool::compareResults(const std::string& actual, const std::string& expected,
const std::string& test_type) {
const std::string& test_type, const std::string& test_name) {
if (expected == actual) {
return true;
}

// Output failure details to stdout if details_ flag is set to true
if (details_) {
std::cerr << "expected: [" << expected << "], actual: [" << actual
<< "], test type: " << test_type << std::endl;
}
tests_[test_name].push_back("expected: [" + expected + "], actual: [" + actual +
"], test type: " + test_type);
return false;
}

Expand All @@ -446,6 +470,8 @@ Options::Options(int argc, char** argv) {
TCLAP::CmdLine cmd("router_check_tool", ' ', "none", true);
TCLAP::SwitchArg is_proto("p", "useproto", "Use Proto test file schema", cmd, false);
TCLAP::SwitchArg is_detailed("d", "details", "Show detailed test execution results", cmd, false);
TCLAP::SwitchArg only_show_failures("", "only-show-failures", "Only display failing tests", cmd,
false);
TCLAP::SwitchArg disable_deprecation_check("", "disable-deprecation-check",
"Disable deprecated fields check", cmd, false);
TCLAP::ValueArg<double> fail_under("f", "fail-under",
Expand All @@ -468,6 +494,7 @@ Options::Options(int argc, char** argv) {

is_proto_ = is_proto.getValue();
is_detailed_ = is_detailed.getValue();
only_show_failures_ = only_show_failures.getValue();
fail_under_ = fail_under.getValue();
comprehensive_coverage_ = comprehensive_coverage.getValue();
disable_deprecation_check_ = disable_deprecation_check.getValue();
Expand Down
Loading