diff --git a/engine/atf.cpp b/engine/atf.cpp index eb63be20..b1cb1a95 100644 --- a/engine/atf.cpp +++ b/engine/atf.cpp @@ -39,6 +39,7 @@ extern "C" { #include "engine/atf_list.hpp" #include "engine/atf_result.hpp" #include "engine/exceptions.hpp" +#include "engine/execenv/execenv.hpp" #include "model/test_case.hpp" #include "model/test_program.hpp" #include "model/test_result.hpp" @@ -190,7 +191,9 @@ engine::atf_interface::exec_test(const model::test_program& test_program, args.push_back(F("-r%s") % (control_directory / result_name)); args.push_back(test_case_name); - process::exec(test_program.absolute_path(), args); + + engine::execenv::init(test_program, test_case_name); + engine::execenv::exec(test_program, test_case_name, args); } @@ -219,7 +222,8 @@ engine::atf_interface::exec_cleanup( } args.push_back(F("%s:cleanup") % test_case_name); - process::exec(test_program.absolute_path(), args); + + engine::execenv::exec(test_program, test_case_name, args); } diff --git a/engine/atf_list.cpp b/engine/atf_list.cpp index a16b889c..1ce32f2d 100644 --- a/engine/atf_list.cpp +++ b/engine/atf_list.cpp @@ -121,6 +121,10 @@ engine::parse_atf_metadata(const model::properties_map& props) mdbuilder.set_string("has_cleanup", value); } else if (name == "require.arch") { mdbuilder.set_string("allowed_architectures", value); + } else if (name == "execenv") { + mdbuilder.set_string("execenv", value); + } else if (name == "execenv.jail") { + mdbuilder.set_string("execenv_jail", value); } else if (name == "require.config") { mdbuilder.set_string("required_configs", value); } else if (name == "require.files") { diff --git a/engine/execenv/execenv.cpp b/engine/execenv/execenv.cpp new file mode 100644 index 00000000..42e72b25 --- /dev/null +++ b/engine/execenv/execenv.cpp @@ -0,0 +1,106 @@ +// Copyright (c) 2023 Igor Ostapenko +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/execenv/execenv.hpp" + +#include "engine/execenv/jail.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/test_program.hpp" +#include "utils/fs/path.hpp" +#include "utils/process/operations.hpp" + +namespace execenv = engine::execenv; +namespace process = utils::process; + +using utils::process::args_vector; + + +/// Initialize execution environment. +/// +/// It's expected to be called inside a fork which runs interface::exec_test(), +/// so we can fail a test fast if its execution environment setup fails, and +/// test execution could use the configured proc environment, if expected. +/// +/// \param program The test program binary absolute path. +/// \param test_case_name Name of the test case. +void +execenv::init(const model::test_program& test_program, + const std::string& test_case_name) +{ + const model::test_case& test_case = test_program.find(test_case_name); + + if (test_case.get_metadata().is_execenv_jail()) { + return execenv::jail::init(test_program, test_case_name); + } else { + // host environment by default + return; + } +} + + +/// Execute within an execution environment. +/// +/// It's expected to be called inside a fork which runs interface::exec_test(). +/// +/// \param program The test program binary absolute path. +/// \param test_case_name Name of the test case. +void +execenv::exec(const model::test_program& test_program, + const std::string& test_case_name, + const args_vector& args) throw() +{ + const model::test_case& test_case = test_program.find(test_case_name); + + if (test_case.get_metadata().is_execenv_jail()) { + execenv::jail::exec(test_program, test_case_name, args); + } else { + // host environment by default + process::exec(test_program.absolute_path(), args); + } +} + + +/// Cleanup execution environment. +/// +/// It's expected to be called inside a fork for execenv cleanup. +/// +/// \param program The test program binary absolute path. +/// \param test_case_name Name of the test case. +void +execenv::cleanup(const model::test_program& test_program, + const std::string& test_case_name) +{ + const model::test_case& test_case = test_program.find(test_case_name); + + if (test_case.get_metadata().is_execenv_jail()) { + return execenv::jail::cleanup(test_program, test_case_name); + } else { + // cleanup is not expected to be called for host environment + std::exit(EXIT_SUCCESS); + } +} diff --git a/engine/execenv/execenv.hpp b/engine/execenv/execenv.hpp new file mode 100644 index 00000000..2c7719bf --- /dev/null +++ b/engine/execenv/execenv.hpp @@ -0,0 +1,59 @@ +// Copyright (c) 2023 Igor Ostapenko +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/execenv/execenv.hpp +/// Execution environment multiplexer. +/// +/// A test case may ask for a specific execution environment like running in +/// a jail, what needs initialization before the test run and cleanup after. +/// +/// By default, there is no specific execution environment, so called host +/// environment, and no additional initialization or cleanup is done. + +#if !defined(ENGINE_EXECENV_EXECENV_HPP) +#define ENGINE_EXECENV_EXECENV_HPP + +#include "model/test_program.hpp" +#include "utils/defs.hpp" +#include "utils/process/operations_fwd.hpp" + +namespace engine { +namespace execenv { + + +void init(const model::test_program&, const std::string&); + +void exec(const model::test_program&, const std::string&, + const utils::process::args_vector&) throw() UTILS_NORETURN; + +void cleanup(const model::test_program&, const std::string&); + + +} // namespace execenv +} // namespace engine + +#endif // !defined(ENGINE_EXECENV_EXECENV_HPP) diff --git a/engine/execenv/jail.cpp b/engine/execenv/jail.cpp new file mode 100644 index 00000000..d30ebcd6 --- /dev/null +++ b/engine/execenv/jail.cpp @@ -0,0 +1,138 @@ +// Copyright (c) 2023 Igor Ostapenko +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/execenv/jail.hpp" + +extern "C" { +// FreeBSD Jail +#include +} + +#include + +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/test_program.hpp" +#include "utils/fs/path.hpp" +#include "utils/process/jail.hpp" +#include "utils/process/operations.hpp" + +namespace execenv = engine::execenv; +namespace process = utils::process; +namespace fs = utils::fs; + +using utils::process::args_vector; + + +namespace { + + +static const int jail_name_max_len = MAXHOSTNAMELEN - 1; +static const char* jail_name_prefix = "kyua"; + +/// Constructs a jail name based on program and test case. +/// +/// The formula is "kyua" + + "_" + . +/// All non-alphanumeric chars are replaced with "_". +/// +/// If a resulting string exceeds maximum allowed length of a jail name, +/// then it's shortened from the left side keeping the "kyua" prefix. +/// +/// \param program The test program. +/// \param test_case_name Name of the test case. +/// +/// \return A jail name string. +static std::string +make_jail_name(const fs::path& program, const std::string& test_case_name) +{ + std::string name = std::regex_replace( + program.str() + "_" + test_case_name, + std::regex(R"([^A-Za-z0-9_])"), + "_"); + + const std::string::size_type limit = + jail_name_max_len - strlen(jail_name_prefix); + if (name.length() > limit) + name.erase(0, name.length() - limit); + + return jail_name_prefix + name; +} + + +} // anonymous namespace + + +/// Initialize execution environment. +/// +/// It's expected to be called inside a fork which runs interface::exec_test(), +/// so we can fail a test fast if its execution environment setup fails. +/// +/// \param program The test program. +/// \param test_case_name Name of the test case. +void +execenv::jail::init(const model::test_program& test_program, + const std::string& test_case_name) +{ + const model::test_case& test_case = test_program.find(test_case_name); + + process::jail::create( + make_jail_name(test_program.absolute_path(), test_case_name), + test_case.get_metadata().execenv_jail()); +} + + +/// Execute within an execution environment. +/// +/// It's expected to be called inside a fork which runs interface::exec_test(). +/// +/// \param program The test program. +/// \param test_case_name Name of the test case. +/// \param args The arguments to pass to the binary, without the program name. +void +execenv::jail::exec(const model::test_program& test_program, + const std::string& test_case_name, + const args_vector& args) throw() +{ + process::jail::exec( + make_jail_name(test_program.absolute_path(), test_case_name), + test_program.absolute_path(), args); +} + + +/// Cleanup execution environment. +/// +/// It's expected to be called inside a fork for execenv cleanup. +/// +/// \param program The test program. +/// \param test_case_name Name of the test case. +void +execenv::jail::cleanup(const model::test_program& test_program, + const std::string& test_case_name) +{ + process::jail::remove( + make_jail_name(test_program.absolute_path(), test_case_name)); +} diff --git a/engine/execenv/jail.hpp b/engine/execenv/jail.hpp new file mode 100644 index 00000000..30a0c0a6 --- /dev/null +++ b/engine/execenv/jail.hpp @@ -0,0 +1,55 @@ +// Copyright (c) 2023 Igor Ostapenko +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/execenv/jail.hpp +/// FreeBSD jail execution environment. + +#if !defined(ENGINE_EXECENV_JAIL_HPP) +#define ENGINE_EXECENV_JAIL_HPP + +#include "model/test_program.hpp" +#include "utils/defs.hpp" +#include "utils/process/operations_fwd.hpp" + +namespace engine { +namespace execenv { +namespace jail { + + +void init(const model::test_program&, const std::string&); + +void exec(const model::test_program&, const std::string&, + const utils::process::args_vector&) throw() UTILS_NORETURN; + +void cleanup(const model::test_program&, const std::string&); + + +} // namespace jail +} // namespace execenv +} // namespace engine + +#endif // !defined(ENGINE_EXECENV_JAIL_HPP) diff --git a/engine/plain.cpp b/engine/plain.cpp index 8346e50b..7d35541d 100644 --- a/engine/plain.cpp +++ b/engine/plain.cpp @@ -34,6 +34,7 @@ extern "C" { #include +#include "engine/execenv/execenv.hpp" #include "model/test_case.hpp" #include "model/test_program.hpp" #include "model/test_result.hpp" @@ -104,7 +105,9 @@ engine::plain_interface::exec_test( } process::args_vector args; - process::exec(test_program.absolute_path(), args); + + engine::execenv::init(test_program, test_case_name); + engine::execenv::exec(test_program, test_case_name, args); } diff --git a/engine/scheduler.cpp b/engine/scheduler.cpp index e7b51d23..b6f4198a 100644 --- a/engine/scheduler.cpp +++ b/engine/scheduler.cpp @@ -40,6 +40,7 @@ extern "C" { #include "engine/config.hpp" #include "engine/exceptions.hpp" +#include "engine/execenv/execenv.hpp" #include "engine/requirements.hpp" #include "model/context.hpp" #include "model/metadata.hpp" @@ -87,6 +88,10 @@ using utils::optional; datetime::delta scheduler::cleanup_timeout(60, 0); +/// Timeout for the test case execenv cleanup operation. +datetime::delta scheduler::execenv_cleanup_timeout(60, 0); + + /// Timeout for the test case listing operation. /// /// TODO(jmmv): This is here only for testing purposes. Maybe we should expose @@ -206,6 +211,18 @@ struct test_exec_data : public exec_data { /// denote that no further attempts shall be made at cleaning this up. bool needs_cleanup; + /// Whether this test case still needs to have its execenv cleanup executed. + /// + /// This is set externally when the cleanup routine is actually invoked to + /// denote that no further attempts shall be made at cleaning this up. + bool needs_execenv_cleanup; + + /// Original PID of the test case subprocess. + /// + /// This is used for the cleanup upon termination by a signal, to reap the + /// leftovers and form missing exit_handle. + int pid; + /// The exit_handle for this test once it has completed. /// /// This is set externally when the test case has finished, as we need this @@ -222,12 +239,14 @@ struct test_exec_data : public exec_data { test_exec_data(const model::test_program_ptr test_program_, const std::string& test_case_name_, const std::shared_ptr< scheduler::interface > interface_, - const config::tree& user_config_) : + const config::tree& user_config_, + const int pid_) : exec_data(test_program_, test_case_name_), - interface(interface_), user_config(user_config_) + interface(interface_), user_config(user_config_), pid(pid_) { const model::test_case& test_case = test_program->find(test_case_name); needs_cleanup = test_case.get_metadata().has_cleanup(); + needs_execenv_cleanup = test_case.get_metadata().has_execenv(); } }; @@ -266,6 +285,40 @@ struct cleanup_exec_data : public exec_data { }; +/// Maintenance data held while a test execenv cleanup is being executed. +/// +/// Instances of this object are related to a previous test_exec_data, as +/// cleanup routines can only exist once the test has been run. +struct execenv_exec_data : public exec_data { + /// The exit handle of the test. This is necessary so that we can return + /// the correct exit_handle to the user of the scheduler. + executor::exit_handle body_exit_handle; + + /// The final result of the test's body. This is necessary to compute the + /// right return value for a test with a cleanup routine: the body result is + /// respected if it is a "bad" result; else the result of the cleanup + /// routine is used if it has failed. + model::test_result body_result; + + /// Constructor. + /// + /// \param test_program_ Test program data for this test case. + /// \param test_case_name_ Name of the test case. + /// \param body_exit_handle_ If not none, exit handle of the body + /// corresponding to the cleanup routine represented by this exec_data. + /// \param body_result_ If not none, result of the body corresponding to the + /// cleanup routine represented by this exec_data. + execenv_exec_data(const model::test_program_ptr test_program_, + const std::string& test_case_name_, + const executor::exit_handle& body_exit_handle_, + const model::test_result& body_result_) : + exec_data(test_program_, test_case_name_), + body_exit_handle(body_exit_handle_), body_result(body_result_) + { + } +}; + + /// Shared pointer to exec_data. /// /// We require this because we want exec_data to not be copyable, and thus we @@ -492,6 +545,39 @@ class run_test_cleanup { }; +/// Functor to execute a test execenv cleanup in a child process. +class run_execenv_cleanup { + /// Test program to execute. + const model::test_program _test_program; + + /// Name of the test case to execute. + const std::string& _test_case_name; + +public: + /// Constructor. + /// + /// \param test_program Test program to execute. + /// \param test_case_name Name of the test case to execute. + run_execenv_cleanup( + const model::test_program_ptr test_program, + const std::string& test_case_name) : + _test_program(force_absolute_paths(*test_program)), + _test_case_name(test_case_name) + { + } + + /// Body of the subprocess. + /// + /// \param control_directory The testcase directory where cleanup will be + /// run from. + void + operator()(const fs::path& /* control_directory */) + { + engine::execenv::cleanup(_test_program, _test_case_name); + } +}; + + /// Obtains the right scheduler interface for a given test program. /// /// \param name The name of the interface of the test program. @@ -835,6 +921,22 @@ struct engine::scheduler::scheduler_handle::impl : utils::noncopyable { % test_data->test_case_name); } } + + const test_exec_data_vector td = tests_needing_execenv_cleanup(); + + for (test_exec_data_vector::const_iterator iter = td.begin(); + iter != td.end(); ++iter) { + const test_exec_data* test_data = *iter; + + try { + sync_execenv_cleanup(test_data); + } catch (const std::runtime_error& e) { + LW(F("Failed to run execenv cleanup routine for %s:%s on abrupt " + "termination") + % test_data->test_program->relative_path() + % test_data->test_case_name); + } + } } /// Finds any pending exec_datas that correspond to tests needing cleanup. @@ -856,6 +958,8 @@ struct engine::scheduler::scheduler_handle::impl : utils::noncopyable { if (test_data->needs_cleanup) { tests_data.push_back(test_data); test_data->needs_cleanup = false; + if (!test_data->exit_handle) + test_data->exit_handle = generic.reap(test_data->pid); } } catch (const std::bad_cast& e) { // Do nothing for cleanup_exec_data objects. @@ -865,6 +969,37 @@ struct engine::scheduler::scheduler_handle::impl : utils::noncopyable { return tests_data; } + /// Finds any pending exec_datas that correspond to tests needing execenv + /// cleanup. + /// + /// \return The collection of test_exec_data objects that have their + /// specific execenv property set. + test_exec_data_vector + tests_needing_execenv_cleanup(void) + { + test_exec_data_vector tests_data; + + for (exec_data_map::const_iterator iter = all_exec_data.begin(); + iter != all_exec_data.end(); ++iter) { + const exec_data_ptr data = (*iter).second; + + try { + test_exec_data* test_data = &dynamic_cast< test_exec_data& >( + *data.get()); + if (test_data->needs_execenv_cleanup) { + tests_data.push_back(test_data); + test_data->needs_execenv_cleanup = false; + if (!test_data->exit_handle) + test_data->exit_handle = generic.reap(test_data->pid); + } + } catch (const std::bad_cast& e) { + // Do nothing for other objects. + } + } + + return tests_data; + } + /// Cleans up a single test case synchronously. /// /// \param test_data The data of the previously executed test case to be @@ -926,6 +1061,61 @@ struct engine::scheduler::scheduler_handle::impl : utils::noncopyable { return handle; } + + /// Cleans up a single test case execenv synchronously. + /// + /// \param test_data The data of the previously executed test case to be + /// cleaned up. + void + sync_execenv_cleanup(const test_exec_data* test_data) + { + // The message in this result should never be seen by the user, but use + // something reasonable just in case it leaks and we need to pinpoint + // the call site. + model::test_result result(model::test_result_broken, + "Test case died abruptly"); + + const executor::exec_handle cleanup_handle = spawn_execenv_cleanup( + test_data->test_program, test_data->test_case_name, + test_data->exit_handle.get(), result); + generic.wait(cleanup_handle); + } + + /// Forks and executes a test case execenv cleanup asynchronously. + /// + /// \param test_program The container test program. + /// \param test_case_name The name of the test case to run. + /// \param body_handle The exit handle of the test case's corresponding + /// body. The cleanup will be executed in the same context. + /// \param body_result The result of the test case's corresponding body. + /// + /// \return A handle for the background operation. Used to match the result + /// of the execution returned by wait_any() with this invocation. + executor::exec_handle + spawn_execenv_cleanup(const model::test_program_ptr test_program, + const std::string& test_case_name, + const executor::exit_handle& body_handle, + const model::test_result& body_result) + { + generic.check_interrupt(); + + LI(F("Spawning %s:%s (execenv cleanup)") + % test_program->absolute_path() % test_case_name); + + const executor::exec_handle handle = generic.spawn_followup( + run_execenv_cleanup(test_program, test_case_name), + body_handle, execenv_cleanup_timeout); + + const exec_data_ptr data(new execenv_exec_data( + test_program, test_case_name, body_handle, body_result)); + LD(F("Inserting %s into all_exec_data (execenv cleanup)") % handle.pid()); + INV_MSG(all_exec_data.find(handle.pid()) == all_exec_data.end(), + F("PID %s already in all_exec_data; not properly cleaned " + "up or reused too fast") % handle.pid());; + all_exec_data.insert(exec_data_map::value_type(handle.pid(), data)); + + return handle; + } }; @@ -1115,7 +1305,7 @@ scheduler::scheduler_handle::spawn_test( unprivileged_user); const exec_data_ptr data(new test_exec_data( - test_program, test_case_name, interface, user_config)); + test_program, test_case_name, interface, user_config, handle.pid())); LD(F("Inserting %s into all_exec_data") % handle.pid()); INV_MSG( _pimpl->all_exec_data.find(handle.pid()) == _pimpl->all_exec_data.end(), @@ -1150,6 +1340,8 @@ scheduler::scheduler_handle::wait_any(void) _pimpl->generic, handle); optional< model::test_result > result; + + // test itself try { test_exec_data* test_data = &dynamic_cast< test_exec_data& >( *data.get()); @@ -1185,6 +1377,7 @@ scheduler::scheduler_handle::wait_any(void) // if the test's body reports a skip (because actions could have // already been taken). test_data->needs_cleanup = false; + test_data->needs_execenv_cleanup = false; } } if (!result) { @@ -1209,7 +1402,6 @@ scheduler::scheduler_handle::wait_any(void) _pimpl->spawn_cleanup(test_data->test_program, test_data->test_case_name, test_data->user_config, handle, result.get()); - test_data->needs_cleanup = false; // TODO(jmmv): Chaining this call is ugly. We'd be better off by // looping over terminated processes until we got a result suitable @@ -1218,7 +1410,21 @@ scheduler::scheduler_handle::wait_any(void) // of test cases do not have cleanup routines. return wait_any(); } + + if (test_data->needs_execenv_cleanup) { + INV(test_case.get_metadata().has_execenv()); + _pimpl->spawn_execenv_cleanup(test_data->test_program, + test_data->test_case_name, + handle, result.get()); + test_data->needs_execenv_cleanup = false; + return wait_any(); + } } catch (const std::bad_cast& e) { + // ok, let's check for another type + } + + // test cleanup + try { const cleanup_exec_data* cleanup_data = &dynamic_cast< const cleanup_exec_data& >(*data.get()); LD(F("Got %s from all_exec_data (cleanup)") % handle.original_pid()); @@ -1257,7 +1463,65 @@ scheduler::scheduler_handle::wait_any(void) _pimpl->all_exec_data.erase(handle.original_pid()); handle = cleanup_data->body_exit_handle; + + const exec_data_map::iterator it = _pimpl->all_exec_data.find( + handle.original_pid()); + if (it != _pimpl->all_exec_data.end()) { + exec_data_ptr d = (*it).second; + test_exec_data* test_data = &dynamic_cast< test_exec_data& >( + *d.get()); + const model::test_case& test_case = + cleanup_data->test_program->find(cleanup_data->test_case_name); + test_data->needs_cleanup = false; + + if (test_data->needs_execenv_cleanup) { + INV(test_case.get_metadata().has_execenv()); + _pimpl->spawn_execenv_cleanup(cleanup_data->test_program, + cleanup_data->test_case_name, + handle, result.get()); + test_data->needs_execenv_cleanup = false; + return wait_any(); + } + } + } catch (const std::bad_cast& e) { + // ok, let's check for another type } + + // execenv cleanup + try { + const execenv_exec_data* execenv_data = + &dynamic_cast< const execenv_exec_data& >(*data.get()); + LD(F("Got %s from all_exec_data (execenv cleanup)") % handle.original_pid()); + + const model::test_result& body_result = execenv_data->body_result; + if (body_result.good()) { + if (!handle.status()) { + result = model::test_result(model::test_result_broken, + "Test case execenv cleanup timed out"); + } else { + if (!handle.status().get().exited() || + handle.status().get().exitstatus() != EXIT_SUCCESS) { + result = model::test_result( + model::test_result_broken, + "Test case execenv cleanup did not terminate successfully"); // ? + } else { + result = body_result; + } + } + } else { + result = body_result; + } + + LD(F("Removing %s from all_exec_data (execenv cleanup) in favor of %s") + % handle.original_pid() + % execenv_data->body_exit_handle.original_pid()); + _pimpl->all_exec_data.erase(handle.original_pid()); + + handle = execenv_data->body_exit_handle; + } catch (const std::bad_cast& e) { + // ok, it was one of the types above + } + INV(result); std::shared_ptr< result_handle::bimpl > result_handle_bimpl( diff --git a/engine/scheduler.hpp b/engine/scheduler.hpp index 24ff0b5a..ee01c83b 100644 --- a/engine/scheduler.hpp +++ b/engine/scheduler.hpp @@ -262,6 +262,7 @@ class scheduler_handle { extern utils::datetime::delta cleanup_timeout; +extern utils::datetime::delta execenv_cleanup_timeout; extern utils::datetime::delta list_timeout; diff --git a/engine/tap.cpp b/engine/tap.cpp index 85e23857..cc8c961c 100644 --- a/engine/tap.cpp +++ b/engine/tap.cpp @@ -35,6 +35,7 @@ extern "C" { #include #include "engine/exceptions.hpp" +#include "engine/execenv/execenv.hpp" #include "engine/tap_parser.hpp" #include "model/test_case.hpp" #include "model/test_program.hpp" @@ -151,7 +152,9 @@ engine::tap_interface::exec_test( } process::args_vector args; - process::exec(test_program.absolute_path(), args); + + engine::execenv::init(test_program, test_case_name); + engine::execenv::exec(test_program, test_case_name, args); } diff --git a/model/metadata.cpp b/model/metadata.cpp index d27e3237..58129f09 100644 --- a/model/metadata.cpp +++ b/model/metadata.cpp @@ -249,6 +249,8 @@ init_tree(config::tree& tree) tree.define< config::string_node >("description"); tree.define< config::bool_node >("has_cleanup"); tree.define< config::bool_node >("is_exclusive"); + tree.define< config::string_node >("execenv"); + tree.define< config::string_node >("execenv_jail"); tree.define< config::strings_set_node >("required_configs"); tree.define< bytes_node >("required_disk_space"); tree.define< paths_set_node >("required_files"); @@ -272,6 +274,8 @@ set_defaults(config::tree& tree) tree.set< config::string_node >("description", ""); tree.set< config::bool_node >("has_cleanup", false); tree.set< config::bool_node >("is_exclusive", false); + tree.set< config::string_node >("execenv", ""); + tree.set< config::string_node >("execenv_jail", ""); tree.set< config::strings_set_node >("required_configs", model::strings_set()); tree.set< bytes_node >("required_disk_space", units::bytes(0)); @@ -486,13 +490,64 @@ bool model::metadata::is_exclusive(void) const { if (_pimpl->props.is_set("is_exclusive")) { - return _pimpl->props.lookup< config::bool_node >("is_exclusive"); + const bool is_excl = + _pimpl->props.lookup< config::bool_node >("is_exclusive"); + return is_excl && !is_execenv_jail(); } else { return get_defaults().lookup< config::bool_node >("is_exclusive"); } } +/// Returns execution environment name. +/// +/// \return Name of configured execution environment. +const std::string& +model::metadata::execenv(void) const +{ + if (_pimpl->props.is_set("execenv")) { + return _pimpl->props.lookup< config::string_node >("execenv"); + } else { + return get_defaults().lookup< config::string_node >("execenv"); + } +} + + +/// Returns whether the test has any specific execenv apart from "host" one. +/// +/// \return True if there is a non-host execenv configured; false otherwise. +bool +model::metadata::has_execenv(void) const +{ + const std::string& name = execenv(); + return !name.empty() && name != "host"; +} + + +/// Returns execenv jail parameters string to run a test with. +/// +/// \return String of jail parameters. +const std::string& +model::metadata::execenv_jail(void) const +{ + if (_pimpl->props.is_set("execenv_jail")) { + return _pimpl->props.lookup< config::string_node >("execenv_jail"); + } else { + return get_defaults().lookup< config::string_node >("execenv_jail"); + } +} + + +/// Returns whether the test is configured for jail execenv. +/// +/// \return True if there is a jail execenv is set; false otherwise. +bool +model::metadata::is_execenv_jail(void) const +{ + return execenv() == "jail"; +} + + /// Returns the list of configuration variables needed by the test. /// /// \return Set of configuration variables. @@ -920,6 +975,36 @@ model::metadata_builder::set_is_exclusive(const bool exclusive) } +/// Sets execution environment name. +/// +/// \param name Execution environment name. +/// +/// \return A reference to this builder. +/// +/// \throw model::error If the value is invalid. +model::metadata_builder& +model::metadata_builder::set_execenv(const std::string& name) +{ + set< config::string_node >(_pimpl->props, "execenv", name); + return *this; +} + + +/// Sets execenv jail parameters string to run the test with. +/// +/// \param params String of jail parameters. +/// +/// \return A reference to this builder. +/// +/// \throw model::error If the value is invalid. +model::metadata_builder& +model::metadata_builder::set_execenv_jail(const std::string& params) +{ + set< config::string_node >(_pimpl->props, "execenv_jail", params); + return *this; +} + + /// Sets the list of configuration variables needed by the test. /// /// \param vars Set of configuration variables. diff --git a/model/metadata.hpp b/model/metadata.hpp index c7dd4519..c3cbc0cf 100644 --- a/model/metadata.hpp +++ b/model/metadata.hpp @@ -69,6 +69,10 @@ class metadata { const std::string& description(void) const; bool has_cleanup(void) const; bool is_exclusive(void) const; + const std::string& execenv(void) const; + bool has_execenv(void) const; + const std::string& execenv_jail(void) const; + bool is_execenv_jail(void) const; const strings_set& required_configs(void) const; const utils::units::bytes& required_disk_space(void) const; const paths_set& required_files(void) const; @@ -112,6 +116,8 @@ class metadata_builder : utils::noncopyable { metadata_builder& set_description(const std::string&); metadata_builder& set_has_cleanup(const bool); metadata_builder& set_is_exclusive(const bool); + metadata_builder& set_execenv(const std::string&); + metadata_builder& set_execenv_jail(const std::string&); metadata_builder& set_required_configs(const strings_set&); metadata_builder& set_required_disk_space(const utils::units::bytes&); metadata_builder& set_required_files(const paths_set&); diff --git a/utils/process/executor.cpp b/utils/process/executor.cpp index a0063261..7e18837a 100644 --- a/utils/process/executor.cpp +++ b/utils/process/executor.cpp @@ -689,6 +689,34 @@ struct utils::process::executor::executor_handle::impl : utils::noncopyable { data._pimpl->state_owners, all_exec_handles))); } + + executor::exit_handle + reap(const int original_pid) + { + const exec_handles_map::iterator iter = all_exec_handles.find( + original_pid); + exec_handle& data = (*iter).second; + data._pimpl->timer.unprogram(); + + if (!fs::exists(data.stdout_file())) { + std::ofstream new_stdout(data.stdout_file().c_str()); + } + if (!fs::exists(data.stderr_file())) { + std::ofstream new_stderr(data.stderr_file().c_str()); + } + + return exit_handle(std::shared_ptr< exit_handle::impl >( + new exit_handle::impl( + data.pid(), + none, + data._pimpl->unprivileged_user, + data._pimpl->start_time, datetime::timestamp::now(), + data.control_directory(), + data.stdout_file(), + data.stderr_file(), + data._pimpl->state_owners, + all_exec_handles))); + } }; @@ -879,6 +907,20 @@ executor::executor_handle::wait_any(void) } +/// Forms exit_handle for the given PID subprocess. +/// +/// Can be used in the cases when we want to do cleanup(s) of a killed test +/// subprocess, but we do not have exit handle as we usually do after normal +/// wait mechanism. +/// +/// \return A pointer to an object describing the subprocess. +executor::exit_handle +executor::executor_handle::reap(const int pid) +{ + return _pimpl->reap(pid); +} + + /// Checks if an interrupt has fired. /// /// Calls to this function should be sprinkled in strategic places through the diff --git a/utils/process/executor.hpp b/utils/process/executor.hpp index 858ad9c8..1b5faf51 100644 --- a/utils/process/executor.hpp +++ b/utils/process/executor.hpp @@ -215,6 +215,7 @@ class executor_handle { exit_handle wait(const exec_handle); exit_handle wait_any(void); + exit_handle reap(const int); void check_interrupt(void) const; }; diff --git a/utils/process/jail.cpp b/utils/process/jail.cpp new file mode 100644 index 00000000..bca5f7e2 --- /dev/null +++ b/utils/process/jail.cpp @@ -0,0 +1,283 @@ +// Copyright (c) 2023 Igor Ostapenko +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "utils/process/jail.hpp" + +extern "C" { +#include +#include +#include + +// FreeBSD sysctl facility +#include + +// FreeBSD Jail syscalls +#include +#include + +// FreeBSD Jail library +#include +} + +#include +#include + +#include "utils/fs/path.hpp" +#include "utils/process/child.ipp" +#include "utils/format/macros.hpp" +#include "utils/process/operations.hpp" +#include "utils/process/status.hpp" + +namespace fs = utils::fs; +namespace process = utils::process; + +using utils::process::args_vector; +using utils::process::child; + + +namespace { + + +static std::vector< std::string > +parse_jail_params_string(const std::string& str) +{ + std::vector< std::string > params; + std::string p; + char quote = 0; + + for (const char& c : str) { + // whitespace delimited parameter + if (quote == 0) { + if (std::isspace(c)) { + if (p.empty()) + continue; + params.push_back(p); + p = ""; + } + else if (c == '"' || c == '\'') { + if (!p.empty()) + params.push_back(p); + p = ""; + quote = c; + } + else + p += c; + } + + // quoted parameter + else { + if (c == quote) { + if (!p.empty()) + params.push_back(p); + p = ""; + quote = 0; + } + else + p += c; + } + } + + // leftovers + if (!p.empty()) + params.push_back(p); + + return params; +} + + +/// Functor to run a program. +class run { + /// Program binary absolute path. + const utils::fs::path& _program; + + /// Program arguments. + const args_vector& _args; + +public: + /// Constructor. + /// + /// \param program Program binary absolute path. + /// \param args Program arguments. + run( + const utils::fs::path& program, + const args_vector& args) : + _program(program), + _args(args) + { + } + + /// Body of the subprocess. + void + operator()(void) + { + process::exec(_program, _args); + } +}; + + +} // anonymous namespace + + +/// Create a jail with a given name and params string. +/// +/// A new jail will always be 'persist', thus the caller is expected to remove +/// the jail eventually via jail::remove(). +/// +/// It's expected to be run in a subprocess. +/// +/// \param jail_name Name of a new jail. +/// \param jail_params String of jail parameters. +void +process::jail::create(const std::string& jail_name, + const std::string& jail_params) +{ + args_vector av; + + // creation flag + av.push_back("-qc"); + + // jail name + av.push_back("name=" + jail_name); + + // determine maximum allowed children.max + int max; + size_t len = sizeof(max); + if (sysctlbyname("security.jail.children.max", &max, &len, NULL, 0) != 0) { + std::cerr << "sysctlbyname(security.jail.children.max) errors: " + << strerror(errno) << ".\n"; + std::exit(EXIT_FAILURE); + } + if (len < sizeof(max)) { + std::cerr << "sysctlbyname(security.jail.children.max) provides less " + "data (" << len << ") than expected (" << sizeof(max) << ").\n"; + std::exit(EXIT_FAILURE); + } + if (max < 0) { + std::cerr << "sysctlbyname(security.jail.children.max) yields " + "abnormal " << max << ".\n"; + std::exit(EXIT_FAILURE); + } + if (max > 0) + max--; // a child jail must have less than parent's children.max + av.push_back("children.max=" + std::to_string(max)); + + // test defined jail params + const std::vector< std::string > params = parse_jail_params_string(jail_params); + for (const std::string& p : params) + av.push_back(p); + + // it must be persist + av.push_back("persist"); + + // invoke jail + std::auto_ptr< process::child > child = child::fork_capture( + run(fs::path("/usr/sbin/jail"), av)); + process::status status = child->wait(); + + // expect success + if (status.exited() && status.exitstatus() == EXIT_SUCCESS) + return; + + // otherwise, let us know what jail thinks and fail fast + std::cerr << child->output().rdbuf(); + std::exit(EXIT_FAILURE); +} + + +/// Executes an external binary in a jail and replaces the current process. +/// +/// \param jail_name Name of the jail to run within. +/// \param program The test program binary absolute path. +/// \param args The arguments to pass to the binary, without the program name. +void +process::jail::exec(const std::string& jail_name, + const fs::path& program, + const args_vector& args) throw() +{ + // get work dir prepared by kyua + char cwd[PATH_MAX]; + if (getcwd(cwd, sizeof(cwd)) == NULL) { + std::cerr << "process::jail::exec: getcwd() errors: " + << strerror(errno) << ".\n"; + std::exit(EXIT_FAILURE); + } + + // get jail id by its name + int jid = jail_getid(jail_name.c_str()); + if (jid < 0) { + std::cerr << "process::jail::exec: jail_getid() errors: " + << strerror(errno) << ": " << jail_errmsg << ".\n"; + std::exit(EXIT_FAILURE); + } + + // attach to the jail + if (jail_attach(jid) == -1) { + std::cerr << "process::jail::exec: jail_attach() errors: " + << strerror(errno) << ".\n"; + std::exit(EXIT_FAILURE); + } + + // set back the expected work dir + if (chdir(cwd) == -1) { + std::cerr << "process::jail::exec: chdir() errors: " + << strerror(errno) << ".\n"; + std::exit(EXIT_FAILURE); + } + + process::exec(program, args); +} + + +/// Removes a jail with a given name. +/// +/// It's expected to be run in a subprocess. +/// +/// \param jail_name Name of a jail to remove. +void +process::jail::remove(const std::string& jail_name) +{ + args_vector av; + + // removal flag + av.push_back("-r"); + + // jail name + av.push_back(jail_name); + + // invoke jail + std::auto_ptr< process::child > child = child::fork_capture( + run(fs::path("/usr/sbin/jail"), av)); + process::status status = child->wait(); + + // expect success + if (status.exited() && status.exitstatus() == EXIT_SUCCESS) + std::exit(EXIT_SUCCESS); + + // otherwise, let us know what jail thinks and fail fast + std::cerr << child->output().rdbuf(); + std::exit(EXIT_FAILURE); +} diff --git a/utils/process/jail.hpp b/utils/process/jail.hpp new file mode 100644 index 00000000..2643e213 --- /dev/null +++ b/utils/process/jail.hpp @@ -0,0 +1,55 @@ +// Copyright (c) 2023 Igor Ostapenko +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file utils/process/jail.hpp +/// Collection of utilities for FreeBSD jail. + +#if !defined(UTILS_PROCESS_JAIL_HPP) +#define UTILS_PROCESS_JAIL_HPP + +#include "utils/defs.hpp" +#include "utils/fs/path_fwd.hpp" +#include "utils/process/operations_fwd.hpp" + +namespace utils { +namespace process { +namespace jail { + + +void create(const std::string&, const std::string&); + +void exec(const std::string&, const utils::fs::path&, const args_vector&) + throw() UTILS_NORETURN; + +void remove(const std::string&); + + +} // namespace jail +} // namespace process +} // namespace utils + +#endif // !defined(UTILS_PROCESS_JAIL_HPP)