From d6f19b475e57e4e2842e681a6e5f36f63bf2c339 Mon Sep 17 00:00:00 2001 From: Wouter De Borger Date: Wed, 28 Mar 2018 11:29:38 +0200 Subject: [PATCH 01/14] temp --- src/inmanta/protocol.py | 346 ++++++++++++++++++++++++---------------- 1 file changed, 211 insertions(+), 135 deletions(-) diff --git a/src/inmanta/protocol.py b/src/inmanta/protocol.py index e9b3d214b5..c22ff75bfd 100644 --- a/src/inmanta/protocol.py +++ b/src/inmanta/protocol.py @@ -16,6 +16,15 @@ Contact: code@inmanta.com """ + +""" + +RestServer => manages tornado/handlers, marshalling, dispatching, and endpoints + +ServerEndpoint + +ServerEndpoint.server [1] -- RestServer.endpoints [1:] +""" import logging import socket import time @@ -126,86 +135,6 @@ def create(cls, transport_class, endpoint=None): """ return transport_class(endpoint) - def __init__(self, endpoint=None): - self.__end_point = endpoint - self.daemon = True - self._connected = False - - endpoint = property(lambda x: x.__end_point) - - def get_id(self): - """ - Returns a unique id for a transport on an endpoint - """ - return "%s_%s_transport" % (self.__end_point.name, self.__class__.__transport_name__) - - id = property(get_id) - - def start_endpoint(self): - """ - Start the transport as endpoint - """ - self.start() - - def stop_endpoint(self): - """ - Stop the transport as endpoint - """ - self.stop() - - def start_client(self): - """ - Start this transport as client - """ - self.start() - - def stop_client(self): - """ - Stop this transport as client - """ - self.stop() - - def start(self): - """ - Start the transport as a new thread - """ - - def stop(self): - """ - Stop the transport - """ - self._connected = False - - def call(self, method, destination=None, **kwargs): - """ - Perform a method call - """ - raise NotImplementedError() - - def _decode(self, body): - """ - Decode a response body - """ - if body is not None and len(body) > 0: - body = json.loads(tornado.escape.to_basestring(body)) - else: - body = None - - return body - - def set_connected(self): - """ - Mark this transport as connected - """ - LOGGER.debug("Transport %s is connected", self.get_id()) - self._connected = True - - def is_connected(self): - """ - Is this transport connected - """ - return self._connected - def custom_json_encoder(o): """ @@ -489,20 +418,7 @@ def authorize_request(auth_data, metadata, message, config): return -class RESTTransport(Transport): - """" - A REST (json body over http) transport. Only methods that operate on resource can use all - HTTP verbs. For other methods the POST verb is used. - """ - __transport_name__ = "rest" - - def __init__(self, endpoint, connection_timout=120): - super().__init__(endpoint) - self.set_connected() - self._handlers = [] - self.token = inmanta_config.Config.get(self.id, "token", None) - self.connection_timout = connection_timout - self.headers = set() +class RESTBase: def _create_base_url(self, properties, msg=None, versioned=True): """ @@ -522,6 +438,17 @@ def _create_base_url(self, properties, msg=None, versioned=True): return url + def _decode(self, body): + """ + Decode a response body + """ + if body is not None and len(body) > 0: + body = json.loads(tornado.escape.to_basestring(body)) + else: + body = None + + return body + def create_op_mapping(self): """ Build a mapping between urls, ops and methods @@ -550,26 +477,6 @@ def create_op_mapping(self): self.headers = headers return url_map - def match_call(self, url, method): - """ - Get the method call for the given url and http method - """ - url_map = self.create_op_mapping() - for url_re, handlers in url_map.items(): - if not url_re.endswith("$"): - url_re += "$" - match = re.match(url_re, url) - if match and method in handlers: - return match.groupdict(), handlers[method] - - return None, None - - def return_error_msg(self, status=500, msg="", headers={}): - body = {"message": msg} - headers["Content-Type"] = "application/json" - LOGGER.debug("Signaling error to client: %d, %s, %s", status, body, headers) - return body, headers, status - @gen.coroutine def _execute_call(self, kwargs, http_method, config, message, request_headers, auth=None): if "api_version" in config[0] and config[0]["api_version"] is None: @@ -719,34 +626,77 @@ def _execute_call(self, kwargs, http_method, config, message, request_headers, a LOGGER.exception("An exception occured during the request.") return self.return_error_msg(500, "An exception occured: " + str(e.args), headers) - def add_static_handler(self, location, path, default_filename=None, start=False): + +class ServerSlice(object): + + def __init__(self, io_loop, name): + self._name = name + self._io_loop = io_loop + self.create_endpoint_metadata() + + def add_future(self, future): """ - Configure a static handler to serve data from the specified path. + Add a future to the ioloop to be handled, but do not require the result. """ - if location[0] != "/": - location = "/" + location + def handle_result(f): + try: + f.result() + except Exception as e: + LOGGER.exception("An exception occurred while handling a future: %s", str(e)) - if location[-1] != "/": - location = location + "/" + self._io_loop.add_future(future, handle_result) - options = {"path": path} - if default_filename is None: - options["default_filename"] = "index.html" + def create_endpoint_metadata(self): + total_dict = {method_name: getattr(self, method_name) + for method_name in dir(self) if callable(getattr(self, method_name))} - self._handlers.append((r"%s(.*)" % location, tornado.web.StaticFileHandler, options)) - self._handlers.append((r"%s" % location[:-1], tornado.web.RedirectHandler, {"url": location})) + methods = self.__methods__ + for name, attr in total_dict.items(): + if name[0:2] != "__" and hasattr(attr, "__protocol_method__"): + if attr.__protocol_method__ in methods: + raise Exception("Unable to register multiple handlers for the same method.") - if start: - self._handlers.append((r"/", tornado.web.RedirectHandler, {"url": location})) + methods[attr.__protocol_method__] = (name, attr) - def add_static_content(self, path, content, content_type="application/javascript"): - self._handlers.append((r"%s(.*)" % path, StaticContentHandler, {"transport": self, "content": content, - "content_type": content_type})) + self.__methods__ = methods + + def start(self): + pass + + def stop(self): + pass + + +class RESTServer(RESTBase): + + def __init__(self, connection_timout=120): + self.__end_points = [] + self._handlers = [] + self.token = inmanta_config.Config.get(self.id, "token", None) + self.connection_timout = connection_timout + self.headers = set() - def start_endpoint(self): + def add_endpoint(self, endpoint: ServerSlice): + self.endpoints.extend(endpoint) + + def get_id(self): + """ + Returns a unique id for a transport on an endpoint + """ + return "%s_%s_transport" % (self.__end_point.name, "rest") + + id = property(get_id) + + endpoint = property(lambda x: x.__end_point) + + def start(self): """ Start the transport """ + + for endpoint in self.__end_points: + endpoint.start() + url_map = self.create_op_mapping() for url, configs in url_map.items(): @@ -778,12 +728,120 @@ def start_endpoint(self): self.http_server.listen(port) LOGGER.debug("Start REST transport") - super().start() def stop_endpoint(self): - super().stop() + self._connected = False self.http_server.stop() + def add_static_handler(self, location, path, default_filename=None, start=False): + """ + Configure a static handler to serve data from the specified path. + """ + if location[0] != "/": + location = "/" + location + + if location[-1] != "/": + location = location + "/" + + options = {"path": path} + if default_filename is None: + options["default_filename"] = "index.html" + + self._handlers.append((r"%s(.*)" % location, tornado.web.StaticFileHandler, options)) + self._handlers.append((r"%s" % location[:-1], tornado.web.RedirectHandler, {"url": location})) + + if start: + self._handlers.append((r"/", tornado.web.RedirectHandler, {"url": location})) + + def add_static_content(self, path, content, content_type="application/javascript"): + self._handlers.append((r"%s(.*)" % path, StaticContentHandler, {"transport": self, "content": content, + "content_type": content_type})) + + def return_error_msg(self, status=500, msg="", headers={}): + body = {"message": msg} + headers["Content-Type"] = "application/json" + LOGGER.debug("Signaling error to client: %d, %s, %s", status, body, headers) + return body, headers, status + + +class RESTTransport(Transport, RESTBase): + """" + A REST (json body over http) transport. Only methods that operate on resource can use all + HTTP verbs. For other methods the POST verb is used. + """ + __transport_name__ = "rest" + + def __init__(self, endpoint, connection_timout=120): + self.__end_point = endpoint + self.daemon = True + self._connected = False + self.set_connected() + self._handlers = [] + self.token = inmanta_config.Config.get(self.id, "token", None) + self.connection_timout = connection_timout + self.headers = set() + + endpoint = property(lambda x: x.__end_point) + + def get_id(self): + """ + Returns a unique id for a transport on an endpoint + """ + return "%s_%s_transport" % (self.__end_point.name, self.__class__.__transport_name__) + + id = property(get_id) + + def start_client(self): + """ + Start this transport as client + """ + self.start() + + def stop_client(self): + """ + Stop this transport as client + """ + self.stop() + + def start(self): + """ + Start the transport as a new thread + """ + pass + + def stop(self): + """ + Stop the transport + """ + self._connected = False + + def set_connected(self): + """ + Mark this transport as connected + """ + LOGGER.debug("Transport %s is connected", self.get_id()) + self._connected = True + + def is_connected(self): + """ + Is this transport connected + """ + return self._connected + + def match_call(self, url, method): + """ + Get the method call for the given url and http method + """ + url_map = self.create_op_mapping() + for url_re, handlers in url_map.items(): + if not url_re.endswith("$"): + url_re += "$" + match = re.match(url_re, url) + if match and method in handlers: + return match.groupdict(), handlers[method] + + return None, None + def _get_client_config(self): """ Load the configuration for the client @@ -1148,7 +1206,7 @@ def get_client(self): return self.client -class ServerEndpoint(Endpoint, metaclass=EndpointMeta): +class ServerEndpoint(Endpoint): """ A service that receives method calls over one or more transports """ @@ -1156,9 +1214,10 @@ class ServerEndpoint(Endpoint, metaclass=EndpointMeta): def __init__(self, name, io_loop, transport=RESTTransport, interval=60, hangtime=None): super().__init__(io_loop, name) + self.__methods__ = {} self._transport = transport - self._transport_instance = Transport.create(self._transport, self) + self._transport_instance = RESTServer(self) self._sched = Scheduler(self._io_loop) self._heartbeat_cb = None @@ -1169,6 +1228,22 @@ def __init__(self, name, io_loop, transport=RESTTransport, interval=60, hangtime hangtime = interval * 3 / 4 self.hangtime = hangtime + self.add_endpoint(self) + + def add_endpoint(self, endpoint): + total_dict = {method_name: getattr(endpoint, method_name) + for method_name in dir(endpoint) if callable(getattr(endpoint, method_name))} + + methods = self.__methods__ + for name, attr in total_dict.items(): + if name[0:2] != "__" and hasattr(attr, "__protocol_method__"): + if attr.__protocol_method__ in methods: + raise Exception("Unable to register multiple handlers for the same method.") + + methods[attr.__protocol_method__] = (name, attr) + + self.__methods__ = methods + def schedule(self, call, interval=60): self._sched.add_action(call, interval) @@ -1428,6 +1503,7 @@ class SyncClient(object): """ A synchronous client that communicates with end-point based on its configuration """ + def __init__(self, name, timeout=120): self.name = name self.timeout = timeout From 122118b565f5bf9949563c4fb8bdaac1aeb4ccfc Mon Sep 17 00:00:00 2001 From: Wouter De Borger Date: Thu, 29 Mar 2018 11:37:33 +0200 Subject: [PATCH 02/14] first stage factoring out of server --- src/inmanta/app.py | 5 +- src/inmanta/deploy.py | 6 +- src/inmanta/protocol.py | 160 ++++++++++++++++++++--------------- src/inmanta/server/server.py | 2 +- tests/conftest.py | 9 +- tests/server_test.py | 6 +- tests/test_2way_protocol.py | 16 ++-- tests/test_agent.py | 4 +- tests/test_server.py | 95 ++++++++++++--------- tests/test_server_agent.py | 56 ++++++------ 10 files changed, 208 insertions(+), 151 deletions(-) diff --git a/src/inmanta/app.py b/src/inmanta/app.py index 4dc9876f66..5a679abd84 100755 --- a/src/inmanta/app.py +++ b/src/inmanta/app.py @@ -34,6 +34,7 @@ from inmanta.export import cfg_env, ModelExporter from inmanta.ast import CompilerException import yaml +from inmanta.protocol import RESTServer LOGGER = logging.getLogger() @@ -43,8 +44,10 @@ def start_server(options): from inmanta import server io_loop = IOLoop.current() + rs = RESTServer() s = server.Server(io_loop) - s.start() + rs.add_endpoint(s) + rs.start() try: io_loop.start() diff --git a/src/inmanta/deploy.py b/src/inmanta/deploy.py index 3d0b1ce494..361d3f43b6 100644 --- a/src/inmanta/deploy.py +++ b/src/inmanta/deploy.py @@ -24,6 +24,7 @@ from mongobox import mongobox from tornado import gen, process from inmanta import module, config, server, agent, protocol, const, data +from inmanta.protocol import RESTServer LOGGER = logging.getLogger(__name__) @@ -91,9 +92,12 @@ def _setup_server(self, no_agent_log): config.Config.set("cmdline_rest_transport", "port", str(self._server_port)) # start the server + rs = RESTServer() self._server = server.Server(database_host="localhost", database_port=self._mongoport, io_loop=self._io_loop, agent_no_log=no_agent_log) - self._server.start() + rs.add_endpoint(self._server) + rs.start() + LOGGER.debug("Started server on port %d", self._server_port) return True diff --git a/src/inmanta/protocol.py b/src/inmanta/protocol.py index c22ff75bfd..317e4e8c4e 100644 --- a/src/inmanta/protocol.py +++ b/src/inmanta/protocol.py @@ -254,7 +254,7 @@ class RESTHandler(tornado.web.RequestHandler): A generic class use by the transport """ - def initialize(self, transport: Transport, config): + def initialize(self, transport: "RESTServer", config): self._transport = transport self._config = config @@ -449,34 +449,6 @@ def _decode(self, body): return body - def create_op_mapping(self): - """ - Build a mapping between urls, ops and methods - """ - url_map = defaultdict(dict) - headers = set() - for method, method_handlers in self.endpoint.__methods__.items(): - properties = method.__protocol_properties__ - call = (self.endpoint, method_handlers[0]) - - if "arg_options" in properties: - for opts in properties["arg_options"].values(): - if "header" in opts: - headers.add(opts["header"]) - - url = self._create_base_url(properties) - properties["api_version"] = "1" - url_map[url][properties["operation"]] = (properties, call, method.__wrapped__) - - url = self._create_base_url(properties, versioned=False) - properties = properties.copy() - properties["api_version"] = None - url_map[url][properties["operation"]] = (properties, call, method.__wrapped__) - - headers.add("Authorization") - self.headers = headers - return url_map - @gen.coroutine def _execute_call(self, kwargs, http_method, config, message, request_headers, auth=None): if "api_version" in config[0] and config[0]["api_version"] is None: @@ -633,6 +605,19 @@ def __init__(self, io_loop, name): self._name = name self._io_loop = io_loop self.create_endpoint_metadata() + self._end_point_names = [] + + name = property(lambda self: self._name) + + def get_end_point_names(self): + return self._end_point_names + + def add_end_point_name(self, name): + """ + Add an additional name to this endpoint to which it reacts and sends out in heartbeats + """ + LOGGER.debug("Adding '%s' as endpoint", name) + self._end_point_names.append(name) def add_future(self, future): """ @@ -650,11 +635,11 @@ def create_endpoint_metadata(self): total_dict = {method_name: getattr(self, method_name) for method_name in dir(self) if callable(getattr(self, method_name))} - methods = self.__methods__ + methods = {} for name, attr in total_dict.items(): if name[0:2] != "__" and hasattr(attr, "__protocol_method__"): if attr.__protocol_method__ in methods: - raise Exception("Unable to register multiple handlers for the same method.") + raise Exception("Unable to register multiple handlers for the same method. %s" % attr.__protocol_method__) methods[attr.__protocol_method__] = (name, attr) @@ -671,32 +656,67 @@ class RESTServer(RESTBase): def __init__(self, connection_timout=120): self.__end_points = [] + self.__endpoint_dict = {} self._handlers = [] self.token = inmanta_config.Config.get(self.id, "token", None) self.connection_timout = connection_timout self.headers = set() def add_endpoint(self, endpoint: ServerSlice): - self.endpoints.extend(endpoint) + self.__end_points.append(endpoint) + self.__endpoint_dict[endpoint.name] = endpoint + + def get_endpoint(self, name): + return self.__endpoint_dict[name] def get_id(self): """ Returns a unique id for a transport on an endpoint """ - return "%s_%s_transport" % (self.__end_point.name, "rest") + return "server_rest_transport" id = property(get_id) - endpoint = property(lambda x: x.__end_point) + def create_op_mapping(self): + """ + Build a mapping between urls, ops and methods + """ + url_map = defaultdict(dict) + + # TODO: avoid colliding handlers + + for endpoint in self.__end_points: + for method, method_handlers in endpoint.__methods__.items(): + properties = method.__protocol_properties__ + call = (endpoint, method_handlers[0]) + + if "arg_options" in properties: + for opts in properties["arg_options"].values(): + if "header" in opts: + self.headers.add(opts["header"]) + + url = self._create_base_url(properties) + properties["api_version"] = "1" + url_map[url][properties["operation"]] = (properties, call, method.__wrapped__) + print(url) + + url = self._create_base_url(properties, versioned=False) + properties = properties.copy() + properties["api_version"] = None + url_map[url][properties["operation"]] = (properties, call, method.__wrapped__) + print(url) + + return url_map def start(self): """ Start the transport """ - + LOGGER.debug("Starting Server Rest Endpoint") + for endpoint in self.__end_points: endpoint.start() - + url_map = self.create_op_mapping() for url, configs in url_map.items(): @@ -729,8 +749,10 @@ def start(self): LOGGER.debug("Start REST transport") - def stop_endpoint(self): - self._connected = False + def stop(self): + LOGGER.debug("Stoppin Server Rest Endpoint") + for endpoint in self.__end_points: + endpoint.stop() self.http_server.stop() def add_static_handler(self, location, path, default_filename=None, start=False): @@ -828,6 +850,34 @@ def is_connected(self): """ return self._connected + def create_op_mapping(self): + """ + Build a mapping between urls, ops and methods + """ + url_map = defaultdict(dict) + headers = set() + for method, method_handlers in self.endpoint.__methods__.items(): + properties = method.__protocol_properties__ + call = (self.endpoint, method_handlers[0]) + + if "arg_options" in properties: + for opts in properties["arg_options"].values(): + if "header" in opts: + headers.add(opts["header"]) + + url = self._create_base_url(properties) + properties["api_version"] = "1" + url_map[url][properties["operation"]] = (properties, call, method.__wrapped__) + + url = self._create_base_url(properties, versioned=False) + properties = properties.copy() + properties["api_version"] = None + url_map[url][properties["operation"]] = (properties, call, method.__wrapped__) + + headers.add("Authorization") + self.headers = headers + return url_map + def match_call(self, url, method): """ Get the method call for the given url and http method @@ -1206,18 +1256,17 @@ def get_client(self): return self.client -class ServerEndpoint(Endpoint): +class ServerEndpoint(ServerSlice): """ A service that receives method calls over one or more transports """ __methods__ = {} - def __init__(self, name, io_loop, transport=RESTTransport, interval=60, hangtime=None): + def __init__(self, name, io_loop, interval=60, hangtime=None): super().__init__(io_loop, name) - self.__methods__ = {} - self._transport = transport self._transport_instance = RESTServer(self) + self._transport_instance.add_endpoint(self) self._sched = Scheduler(self._io_loop) self._heartbeat_cb = None @@ -1228,40 +1277,13 @@ def __init__(self, name, io_loop, transport=RESTTransport, interval=60, hangtime hangtime = interval * 3 / 4 self.hangtime = hangtime - self.add_endpoint(self) - - def add_endpoint(self, endpoint): - total_dict = {method_name: getattr(endpoint, method_name) - for method_name in dir(endpoint) if callable(getattr(endpoint, method_name))} - - methods = self.__methods__ - for name, attr in total_dict.items(): - if name[0:2] != "__" and hasattr(attr, "__protocol_method__"): - if attr.__protocol_method__ in methods: - raise Exception("Unable to register multiple handlers for the same method.") - - methods[attr.__protocol_method__] = (name, attr) - - self.__methods__ = methods - def schedule(self, call, interval=60): self._sched.add_action(call, interval) - def start(self): - """ - Start this end-point using the central configuration - """ - LOGGER.debug("Starting transport for endpoint %s", self.name) - if self._transport_instance is not None: - self._transport_instance.start_endpoint() - def stop(self): """ Stop the end-point and all of its transports """ - if self._transport_instance is not None: - self._transport_instance.stop_endpoint() - LOGGER.debug("Stopped %s", self._transport_instance) # terminate all sessions cleanly for session in self._sessions.copy().values(): session.expire(0) diff --git a/src/inmanta/server/server.py b/src/inmanta/server/server.py index 62db5a0a42..dc9329abf2 100644 --- a/src/inmanta/server/server.py +++ b/src/inmanta/server/server.py @@ -80,7 +80,7 @@ def __init__(self, io_loop, database_host=None, database_port=None, agent_no_log self._fact_expire = opt.server_fact_expire.get() self._fact_renew = opt.server_fact_renew.get() - self.add_end_point_name(self.node_name) + #self.add_end_point_name(self.node_name) self.schedule(self.renew_expired_facts, self._fact_renew) self.schedule(self._purge_versions, opt.server_purge_version_interval.get()) diff --git a/tests/conftest.py b/tests/conftest.py index 17a069b78a..50815683e8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,6 +41,7 @@ from tornado import gen import re from tornado.ioloop import IOLoop +from inmanta.protocol import RESTServer DEFAULT_PORT_ENVVAR = 'MONGOBOX_PORT' @@ -156,13 +157,15 @@ def server(inmanta_config, io_loop, mongo_db, mongo_client, motor): data.use_motor(motor) + rs = RESTServer() server = Server(database_host="localhost", database_port=int(mongo_db.port), io_loop=io_loop) - server.start() + rs.add_endpoint(server) + rs.start() - yield server + yield rs del IOLoop._instance - server.stop() + rs.stop() shutil.rmtree(state_dir) diff --git a/tests/server_test.py b/tests/server_test.py index af7a8e4e4d..e9713f6759 100644 --- a/tests/server_test.py +++ b/tests/server_test.py @@ -27,6 +27,7 @@ from inmanta import config from inmanta.server import Server from tornado.testing import AsyncTestCase +from inmanta.protocol import RESTServer LOGGER = logging.getLogger(__name__) PORT = "45678" @@ -66,10 +67,13 @@ def setUp(self): if mongo_port is None: raise Exception("MONGOBOX_PORT env variable not available. Make sure test are executed with --with-mongobox") + self.rs = RESTServer() self.server = Server(database_host="localhost", database_port=int(mongo_port), io_loop=self.io_loop) + self.rs.add_endpoint(self.server) + self.rs.start() def tearDown(self): - self.server.stop() + self.rs.stop() # does not work with current pymongo for db_name in self.mongo_client.database_names(): self.mongo_client.drop_database(db_name) diff --git a/tests/test_2way_protocol.py b/tests/test_2way_protocol.py index f041cce40a..2fad215437 100644 --- a/tests/test_2way_protocol.py +++ b/tests/test_2way_protocol.py @@ -27,6 +27,7 @@ from tornado.gen import sleep from utils import retry_limited from tornado.ioloop import IOLoop +from inmanta.protocol import RESTServer LOGGER = logging.getLogger(__name__) @@ -119,8 +120,10 @@ def test_2way_protocol(free_port, logs=False): Config.set("cmdline_rest_transport", "port", free_port) io_loop = IOLoop.current() + rs = RESTServer() server = Server("server", io_loop) - server.start() + rs.add_endpoint(server) + rs.start() agent = Agent("agent", io_loop) agent.add_end_point_name("agent") @@ -144,7 +147,7 @@ def do_call(): io_loop.start() except KeyboardInterrupt: io_loop.stop() - server.stop() + rs.stop() agent.stop() @@ -171,9 +174,12 @@ def test_timeout(free_port): Config.set("compiler_rest_transport", "port", free_port) Config.set("client_rest_transport", "port", free_port) Config.set("cmdline_rest_transport", "port", free_port) + + rs = RESTServer() server = Server("server", io_loop, interval=2) - server.start() - + rs.add_endpoint(server) + rs.start() + env = uuid.uuid4() # agent 1 @@ -225,5 +231,5 @@ def do_call(): io_loop.start() except KeyboardInterrupt: io_loop.stop() - server.stop() + rs.stop() agent.stop() diff --git a/tests/test_agent.py b/tests/test_agent.py index aaffcc81cc..5383e44c32 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -29,8 +29,8 @@ def test_agent_get_status(io_loop, server, environment): myagent.add_end_point_name("agent1") myagent.start() - yield retry_limited(lambda: len(server._sessions) == 1, 0.5) - clients = server._sessions.values() + yield retry_limited(lambda: len(server.get_endpoint("server")._sessions) == 1, 0.5) + clients = server.get_endpoint("server")._sessions.values() assert len(clients) == 1 clients = [x for x in clients] client = clients[0].get_client() diff --git a/tests/test_server.py b/tests/test_server.py index 2bf9230dc5..c719fe4a5b 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -25,7 +25,7 @@ from inmanta.agent.agent import Agent from inmanta import data, protocol from inmanta import const -from inmanta.server import config as opt +from inmanta.server import config as opt, agentmanager from datetime import datetime from uuid import UUID from inmanta.export import upload_code @@ -43,35 +43,39 @@ def test_autostart(server, client, environment): env = yield data.Environment.get_by_id(uuid.UUID(environment)) yield env.set(data.AUTOSTART_AGENT_MAP, {"iaas_agent": "", "iaas_agentx": ""}) - yield server.agentmanager.ensure_agent_registered(env, "iaas_agent") - yield server.agentmanager.ensure_agent_registered(env, "iaas_agentx") + agentmanager = server.get_endpoint("server").agentmanager + serverendpoint = server.get_endpoint("server") - res = yield server.agentmanager._ensure_agents(env, ["iaas_agent"]) + + yield agentmanager.ensure_agent_registered(env, "iaas_agent") + yield agentmanager.ensure_agent_registered(env, "iaas_agentx") + + res = yield agentmanager._ensure_agents(env, ["iaas_agent"]) assert res - yield retry_limited(lambda: len(server._sessions) == 1, 20) - assert len(server._sessions) == 1 - res = yield server.agentmanager._ensure_agents(env, ["iaas_agent"]) + yield retry_limited(lambda: len(serverendpoint._sessions) == 1, 20) + assert len(serverendpoint._sessions) == 1 + res = yield agentmanager._ensure_agents(env, ["iaas_agent"]) assert not res - assert len(server._sessions) == 1 + assert len(serverendpoint._sessions) == 1 LOGGER.warning("Killing agent") - server.agentmanager._agent_procs[env.id].terminate() - yield retry_limited(lambda: len(server._sessions) == 0, 20) - res = yield server.agentmanager._ensure_agents(env, ["iaas_agent"]) + agentmanager._agent_procs[env.id].terminate() + yield retry_limited(lambda: len(serverendpoint._sessions) == 0, 20) + res = yield agentmanager._ensure_agents(env, ["iaas_agent"]) assert res - yield retry_limited(lambda: len(server._sessions) == 1, 3) - assert len(server._sessions) == 1 + yield retry_limited(lambda: len(serverendpoint._sessions) == 1, 3) + assert len(serverendpoint._sessions) == 1 # second agent for same env - res = yield server.agentmanager._ensure_agents(env, ["iaas_agentx"]) + res = yield agentmanager._ensure_agents(env, ["iaas_agentx"]) assert res - yield retry_limited(lambda: len(server._sessions) == 1, 20) - assert len(server._sessions) == 1 + yield retry_limited(lambda: len(serverendpoint._sessions) == 1, 20) + assert len(serverendpoint._sessions) == 1 # Test stopping all agents - yield server.agentmanager.stop_agents(env) - assert len(server._sessions) == 0 - assert len(server.agentmanager._agent_procs) == 0 + yield agentmanager.stop_agents(env) + assert len(serverendpoint._sessions) == 0 + assert len(agentmanager._agent_procs) == 0 @pytest.mark.gen_test(timeout=60) @@ -80,6 +84,10 @@ def test_autostart_dual_env(client, server): """ Test auto start of agent """ + + agentmanager = server.get_endpoint("server").agentmanager + serverendpoint = server.get_endpoint("server") + result = yield client.create_project("env-test") assert result.code == 200 project_id = result.result["project"]["id"] @@ -96,18 +104,18 @@ def test_autostart_dual_env(client, server): env2 = yield data.Environment.get_by_id(uuid.UUID(env_id2)) yield env2.set(data.AUTOSTART_AGENT_MAP, {"iaas_agent": ""}) - yield server.agentmanager.ensure_agent_registered(env, "iaas_agent") - yield server.agentmanager.ensure_agent_registered(env2, "iaas_agent") + yield agentmanager.ensure_agent_registered(env, "iaas_agent") + yield agentmanager.ensure_agent_registered(env2, "iaas_agent") - res = yield server.agentmanager._ensure_agents(env, ["iaas_agent"]) + res = yield agentmanager._ensure_agents(env, ["iaas_agent"]) assert res - yield retry_limited(lambda: len(server._sessions) == 1, 20) - assert len(server._sessions) == 1 + yield retry_limited(lambda: len(serverendpoint._sessions) == 1, 20) + assert len(serverendpoint._sessions) == 1 - res = yield server.agentmanager._ensure_agents(env2, ["iaas_agent"]) + res = yield agentmanager._ensure_agents(env2, ["iaas_agent"]) assert res - yield retry_limited(lambda: len(server._sessions) == 2, 20) - assert len(server._sessions) == 2 + yield retry_limited(lambda: len(serverendpoint._sessions) == 2, 20) + assert len(serverendpoint._sessions) == 2 @pytest.mark.gen_test(timeout=60) @@ -119,28 +127,31 @@ def test_autostart_batched(client, server, environment): env = yield data.Environment.get_by_id(uuid.UUID(environment)) yield env.set(data.AUTOSTART_AGENT_MAP, {"iaas_agent": "", "iaas_agentx": ""}) - yield server.agentmanager.ensure_agent_registered(env, "iaas_agent") - yield server.agentmanager.ensure_agent_registered(env, "iaas_agentx") + agentmanager = server.get_endpoint("server").agentmanager + serverendpoint = server.get_endpoint("server") + + yield agentmanager.ensure_agent_registered(env, "iaas_agent") + yield agentmanager.ensure_agent_registered(env, "iaas_agentx") - res = yield server.agentmanager._ensure_agents(env, ["iaas_agent", "iaas_agentx"]) + res = yield agentmanager._ensure_agents(env, ["iaas_agent", "iaas_agentx"]) assert res - yield retry_limited(lambda: len(server._sessions) == 1, 20) - assert len(server._sessions) == 1 - res = yield server.agentmanager._ensure_agents(env, ["iaas_agent"]) + yield retry_limited(lambda: len(serverendpoint._sessions) == 1, 20) + assert len(serverendpoint._sessions) == 1 + res = yield agentmanager._ensure_agents(env, ["iaas_agent"]) assert not res - assert len(server._sessions) == 1 + assert len(serverendpoint._sessions) == 1 - res = yield server.agentmanager._ensure_agents(env, ["iaas_agent", "iaas_agentx"]) + res = yield agentmanager._ensure_agents(env, ["iaas_agent", "iaas_agentx"]) assert not res - assert len(server._sessions) == 1 + assert len(serverendpoint._sessions) == 1 LOGGER.warning("Killing agent") - server.agentmanager._agent_procs[env.id].terminate() - yield retry_limited(lambda: len(server._sessions) == 0, 20) - res = yield server.agentmanager._ensure_agents(env, ["iaas_agent", "iaas_agentx"]) + agentmanager._agent_procs[env.id].terminate() + yield retry_limited(lambda: len(serverendpoint._sessions) == 0, 20) + res = yield agentmanager._ensure_agents(env, ["iaas_agent", "iaas_agentx"]) assert res - yield retry_limited(lambda: len(server._sessions) == 1, 3) - assert len(server._sessions) == 1 + yield retry_limited(lambda: len(serverendpoint._sessions) == 1, 3) + assert len(serverendpoint._sessions) == 1 @pytest.mark.gen_test(timeout=10) @@ -160,7 +171,7 @@ def test_version_removal(client, server): for _i in range(20): version += 1 - yield server._purge_versions() + yield server.get_endpoint("server")._purge_versions() res = yield client.put_version(tid=env_id, version=version, resources=[], unknowns=[], version_info={}) assert res.code == 200 result = yield client.get_project(id=project_id) diff --git a/tests/test_server_agent.py b/tests/test_server_agent.py index 0443a3ed2c..c6b5c24a6e 100644 --- a/tests/test_server_agent.py +++ b/tests/test_server_agent.py @@ -38,6 +38,7 @@ from inmanta.config import Config from inmanta.server.server import Server from inmanta.ast import CompilerException +from inmanta.protocol import RESTServer logger = logging.getLogger("inmanta.test.server_agent") @@ -380,16 +381,19 @@ def test_server_restart(resource_container, io_loop, server, mongo_db, client): code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server.agentmanager.sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 1, 10) resource_container.Provider.set("agent1", "key2", "incorrect_value") resource_container.Provider.set("agent1", "key3", "value") server.stop() + rs = RESTServer() server = Server(database_host="localhost", database_port=int(mongo_db.port), io_loop=io_loop) - server.start() - yield retry_limited(lambda: len(server.agentmanager.sessions) == 1, 10) + rs.add_endpoint(server) + rs.start() + + yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 1, 10) version = int(time.time()) @@ -482,7 +486,7 @@ def test_server_restart(resource_container, io_loop, server, mongo_db, client): assert not resource_container.Provider.isset("agent1", "key3") agent.stop() - server.stop() + rs.stop() @pytest.mark.gen_test(timeout=30) @@ -504,7 +508,7 @@ def test_spontaneous_deploy(resource_container, io_loop, server, client): code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server.agentmanager.sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 1, 10) resource_container.Provider.set("agent1", "key2", "incorrect_value") resource_container.Provider.set("agent1", "key3", "value") @@ -583,7 +587,7 @@ def test_dual_agent(resource_container, io_loop, server, client, environment): myagent.add_end_point_name("agent1") myagent.add_end_point_name("agent2") myagent.start() - yield retry_limited(lambda: len(server._sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("server")._sessions) == 1, 10) resource_container.Provider.set("agent1", "key1", "incorrect_value") resource_container.Provider.set("agent2", "key1", "incorrect_value") @@ -682,7 +686,7 @@ def test_snapshot_restore(resource_container, client, server, io_loop): code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server._sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("server")._sessions) == 1, 10) resource_container.Provider.set("agent1", "key", "value") @@ -789,8 +793,8 @@ def test_server_agent_api(resource_container, client, server, io_loop): code_loader=False) agent.start() - yield retry_limited(lambda: len(server.agentmanager.sessions) == 2, 10) - assert len(server.agentmanager.sessions) == 2 + yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 2, 10) + assert len(server.get_endpoint("server").agentmanager.sessions) == 2 result = yield client.list_agent_processes(env_id) assert result.code == 200 @@ -881,7 +885,7 @@ def test_get_facts(resource_container, client, server, io_loop): code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server._sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("server")._sessions) == 1, 10) resource_container.Provider.set("agent1", "key", "value") @@ -929,7 +933,7 @@ def test_purged_facts(resource_container, client, server, io_loop, environment): code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server._sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("server")._sessions) == 1, 10) resource_container.Provider.set("agent1", "key", "value") @@ -1026,7 +1030,7 @@ def test_unkown_parameters(resource_container, client, server, io_loop): code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server._sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("server")._sessions) == 1, 10) resource_container.Provider.set("agent1", "key", "value") @@ -1054,7 +1058,7 @@ def test_unkown_parameters(resource_container, client, server, io_loop): result = yield client.release_version(env_id, version, True) assert result.code == 200 - yield server.renew_expired_facts() + yield server.get_endpoint("server").renew_expired_facts() env_id = uuid.UUID(env_id) params = yield data.Parameter.get_list(environment=env_id, resource_id=resource_id_wov) @@ -1082,7 +1086,7 @@ def test_fail(resource_container, client, server, io_loop): code_loader=False, poolsize=10) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server._sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("server")._sessions) == 1, 10) resource_container.Provider.set("agent1", "key", "value") @@ -1192,7 +1196,7 @@ def test_wait(resource_container, client, server, io_loop): agent.start() # wait for agent - yield retry_limited(lambda: len(server._sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("server")._sessions) == 1, 10) # set the deploy environment resource_container.Provider.set("agent1", "key", "value") @@ -1343,7 +1347,7 @@ def test_multi_instance(resource_container, client, server, io_loop): agent.start() # wait for agent - yield retry_limited(lambda: len(server._sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("server")._sessions) == 1, 10) # set the deploy environment resource_container.Provider.set("agent1", "key", "value") @@ -1467,13 +1471,13 @@ def test_cross_agent_deps(resource_container, io_loop, server, client): code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server.agentmanager.sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 1, 10) agent2 = Agent(io_loop, hostname="node2", environment=env_id, agent_map={"agent2": "localhost"}, code_loader=False) agent2.add_end_point_name("agent2") agent2.start() - yield retry_limited(lambda: len(server.agentmanager.sessions) == 2, 10) + yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 2, 10) resource_container.Provider.set("agent1", "key2", "incorrect_value") resource_container.Provider.set("agent1", "key3", "value") @@ -1570,7 +1574,7 @@ def test_dryrun_scale(resource_container, io_loop, server, client): code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server.agentmanager.sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 1, 10) version = int(time.time()) @@ -1622,7 +1626,7 @@ def test_send_events(resource_container, io_loop, environment, server, client): code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server.agentmanager.sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 1, 10) version = int(time.time()) @@ -1685,13 +1689,13 @@ def test_send_events_cross_agent(resource_container, io_loop, environment, serve code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server.agentmanager.sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 1, 10) agent2 = Agent(io_loop, hostname="node2", environment=environment, agent_map={"agent2": "localhost"}, code_loader=False) agent2.add_end_point_name("agent2") agent2.start() - yield retry_limited(lambda: len(server.agentmanager.sessions) == 2, 10) + yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 2, 10) version = int(time.time()) @@ -1758,7 +1762,7 @@ def test_send_events_cross_agent_restart(resource_container, io_loop, environmen code_loader=False) agent2.add_end_point_name("agent2") agent2.start() - yield retry_limited(lambda: len(server.agentmanager.sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 1, 10) version = int(time.time()) @@ -1808,7 +1812,7 @@ def test_send_events_cross_agent_restart(resource_container, io_loop, environmen code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server.agentmanager.sessions) == 2, 10) + yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 2, 10) while (result.result["model"]["total"] - result.result["model"]["done"]) > 0: result = yield client.get_version(environment, version) @@ -1839,7 +1843,7 @@ def test_auto_deploy(io_loop, server, client, resource_container, environment): code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server.agentmanager.sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 1, 10) resource_container.Provider.set("agent1", "key2", "incorrect_value") resource_container.Provider.set("agent1", "key3", "value") @@ -2174,7 +2178,7 @@ def wait_for_version(cnt): return versions.result - project_dir = os.path.join(server._server_storage["environments"], str(environment)) + project_dir = os.path.join(server.get_endpoint("server")._server_storage["environments"], str(environment)) project_source = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data", "project") shutil.copytree(project_source, project_dir) From ce511475a569494e066d05298f099dd4f437f8bc Mon Sep 17 00:00:00 2001 From: Wouter De Borger Date: Mon, 2 Apr 2018 12:23:50 +0200 Subject: [PATCH 03/14] temp commit --- src/inmanta/protocol.py | 53 ++++++++++++++++++++---------------- src/inmanta/server/server.py | 4 +-- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/src/inmanta/protocol.py b/src/inmanta/protocol.py index 317e4e8c4e..f063a4c99e 100644 --- a/src/inmanta/protocol.py +++ b/src/inmanta/protocol.py @@ -606,9 +606,13 @@ def __init__(self, io_loop, name): self._io_loop = io_loop self.create_endpoint_metadata() self._end_point_names = [] + self._handlers = [] name = property(lambda self: self._name) + def get_handlers(self): + return self._handlers + def get_end_point_names(self): return self._end_point_names @@ -651,6 +655,30 @@ def start(self): def stop(self): pass + def add_static_handler(self, location, path, default_filename=None, start=False): + """ + Configure a static handler to serve data from the specified path. + """ + if location[0] != "/": + location = "/" + location + + if location[-1] != "/": + location = location + "/" + + options = {"path": path} + if default_filename is None: + options["default_filename"] = "index.html" + + self._handlers.append((r"%s(.*)" % location, tornado.web.StaticFileHandler, options)) + self._handlers.append((r"%s" % location[:-1], tornado.web.RedirectHandler, {"url": location})) + + if start: + self._handlers.append((r"/", tornado.web.RedirectHandler, {"url": location})) + + def add_static_content(self, path, content, content_type="application/javascript"): + self._handlers.append((r"%s(.*)" % path, StaticContentHandler, {"transport": self, "content": content, + "content_type": content_type})) + class RESTServer(RESTBase): @@ -716,6 +744,7 @@ def start(self): for endpoint in self.__end_points: endpoint.start() + self.handlers.extend(endpoint.get_handlers()) url_map = self.create_op_mapping() @@ -755,30 +784,6 @@ def stop(self): endpoint.stop() self.http_server.stop() - def add_static_handler(self, location, path, default_filename=None, start=False): - """ - Configure a static handler to serve data from the specified path. - """ - if location[0] != "/": - location = "/" + location - - if location[-1] != "/": - location = location + "/" - - options = {"path": path} - if default_filename is None: - options["default_filename"] = "index.html" - - self._handlers.append((r"%s(.*)" % location, tornado.web.StaticFileHandler, options)) - self._handlers.append((r"%s" % location[:-1], tornado.web.RedirectHandler, {"url": location})) - - if start: - self._handlers.append((r"/", tornado.web.RedirectHandler, {"url": location})) - - def add_static_content(self, path, content, content_type="application/javascript"): - self._handlers.append((r"%s(.*)" % path, StaticContentHandler, {"transport": self, "content": content, - "content_type": content_type})) - def return_error_msg(self, status=500, msg="", headers={}): body = {"message": msg} headers["Content-Type"] = "application/json" diff --git a/src/inmanta/server/server.py b/src/inmanta/server/server.py index dc9329abf2..e15a93f10f 100644 --- a/src/inmanta/server/server.py +++ b/src/inmanta/server/server.py @@ -144,8 +144,8 @@ def setup_dashboard(self): 'backend': window.location.origin+'/'%s }); """ % auth - self._transport_instance.add_static_content("/dashboard/config.js", content=content) - self._transport_instance.add_static_handler("/dashboard", dashboard_path, start=True) + self.add_static_content("/dashboard/config.js", content=content) + self.add_static_handler("/dashboard", dashboard_path, start=True) @gen.coroutine def _purge_versions(self): From 17ceebb8f604ff27a8686ce91fb1c0c53ce67d70 Mon Sep 17 00:00:00 2001 From: Wouter De Borger Date: Wed, 4 Apr 2018 20:35:15 +0200 Subject: [PATCH 04/14] temp --- src/inmanta/protocol.py | 11 ++++++++--- tests/test_server_agent.py | 10 +++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/inmanta/protocol.py b/src/inmanta/protocol.py index f063a4c99e..e9fff36c7d 100644 --- a/src/inmanta/protocol.py +++ b/src/inmanta/protocol.py @@ -21,9 +21,13 @@ RestServer => manages tornado/handlers, marshalling, dispatching, and endpoints -ServerEndpoint +ServerSlice => contributes handlers and methods + +ServerSlice.server [1] -- RestServer.endpoints [1:] + + + -ServerEndpoint.server [1] -- RestServer.endpoints [1:] """ import logging import socket @@ -121,6 +125,7 @@ def callback(self, fnc): self._callback = fnc +#todo: has to go class Transport(object): """ This class implements a transport for the Inmanta protocol. @@ -744,7 +749,7 @@ def start(self): for endpoint in self.__end_points: endpoint.start() - self.handlers.extend(endpoint.get_handlers()) + self._handlers.extend(endpoint.get_handlers()) url_map = self.create_op_mapping() diff --git a/tests/test_server_agent.py b/tests/test_server_agent.py index c6b5c24a6e..1aaf1e8e56 100644 --- a/tests/test_server_agent.py +++ b/tests/test_server_agent.py @@ -388,10 +388,10 @@ def test_server_restart(resource_container, io_loop, server, mongo_db, client): server.stop() - rs = RESTServer() - server = Server(database_host="localhost", database_port=int(mongo_db.port), io_loop=io_loop) - rs.add_endpoint(server) - rs.start() + server = RESTServer() + xserver = Server(database_host="localhost", database_port=int(mongo_db.port), io_loop=io_loop) + server.add_endpoint(xserver) + server.start() yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 1, 10) @@ -486,7 +486,7 @@ def test_server_restart(resource_container, io_loop, server, mongo_db, client): assert not resource_container.Provider.isset("agent1", "key3") agent.stop() - rs.stop() + server.stop() @pytest.mark.gen_test(timeout=30) From a2df6bf467a01faca010fed13cb3e4a7b0f13c90 Mon Sep 17 00:00:00 2001 From: Wouter De Borger Date: Tue, 17 Apr 2018 15:12:26 +0200 Subject: [PATCH 05/14] fix testcases --- src/inmanta/protocol.py | 204 +++++++++++++++++++------------------ tests/conftest.py | 12 ++- tests/test_server_agent.py | 2 +- 3 files changed, 113 insertions(+), 105 deletions(-) diff --git a/src/inmanta/protocol.py b/src/inmanta/protocol.py index e9fff36c7d..f01a192338 100644 --- a/src/inmanta/protocol.py +++ b/src/inmanta/protocol.py @@ -14,20 +14,6 @@ limitations under the License. Contact: code@inmanta.com -""" - - -""" - -RestServer => manages tornado/handlers, marshalling, dispatching, and endpoints - -ServerSlice => contributes handlers and methods - -ServerSlice.server [1] -- RestServer.endpoints [1:] - - - - """ import logging import socket @@ -57,90 +43,18 @@ LOGGER = logging.getLogger(__name__) INMANTA_MT_HEADER = "X-Inmanta-tid" +""" -class Result(object): - """ - A result of a method call - """ - - def __init__(self, multiple=False, code=0, result=None): - self._multiple = multiple - if multiple: - self._result = [] - if result is not None: - self._result.append(result) - else: - self._result = result - self.code = code - self._callback = None - - def add_result(self, result): - """ - Add a new result to an instance - - :param result: The result to store - """ - assert(self._multiple) - self._result.append(result) - if self._callback: - self._callback(self) - - def get_result(self): - """ - Only when the result is marked as available the result can be returned - """ - if self.available(): - return self._result - raise Exception("The result is not yet available") - - def set_result(self, value): - if not self.available(): - assert(not self._multiple) - self._result = value - if self._callback: - self._callback(self) - - def available(self): - if self._multiple: - return len(self._result) > 0 is not None or self.code > 0 - else: - return self._result is not None or self.code > 0 - - def wait(self, timeout=60): - """ - Wait for the result to become available - """ - count = 0 - while count < timeout: - time.sleep(0.1) - count += 0.1 - - result = property(get_result, set_result) - - def callback(self, fnc): - """ - Set a callback function that is to be called when the result is ready. When multiple - results are expected, the callback is called for each result. - """ - self._callback = fnc +RestServer => manages tornado/handlers, marshalling, dispatching, and endpoints +ServerSlice => contributes handlers and methods -#todo: has to go -class Transport(object): - """ - This class implements a transport for the Inmanta protocol. +ServerSlice.server [1] -- RestServer.endpoints [1:] - :param end_point_name: The name of the endpoint to which this transport belongs. This is used - for logging and configuration purposes - """ - @classmethod - def create(cls, transport_class, endpoint=None): - """ - Create an instance of the transport class - """ - return transport_class(endpoint) +""" +# Util functions def custom_json_encoder(o): """ A custom json encoder that knows how to encode other types commonly used by Inmanta @@ -187,8 +101,10 @@ def gzipped_json(value): return True, gzip_value.getvalue() -class UnauhorizedError(Exception): - pass +def sh(msg, max_len=10): + if len(msg) < max_len: + return msg + return msg[0:max_len - 3] + "..." def encode_token(client_types, environment=None, idempotent=False, expire=None): @@ -254,6 +170,93 @@ def decode_token(token): return payload +class UnauhorizedError(Exception): + pass + + +class Result(object): + """ + A result of a method call + """ + + def __init__(self, multiple=False, code=0, result=None): + self._multiple = multiple + if multiple: + self._result = [] + if result is not None: + self._result.append(result) + else: + self._result = result + self.code = code + self._callback = None + + def add_result(self, result): + """ + Add a new result to an instance + + :param result: The result to store + """ + assert(self._multiple) + self._result.append(result) + if self._callback: + self._callback(self) + + def get_result(self): + """ + Only when the result is marked as available the result can be returned + """ + if self.available(): + return self._result + raise Exception("The result is not yet available") + + def set_result(self, value): + if not self.available(): + assert(not self._multiple) + self._result = value + if self._callback: + self._callback(self) + + def available(self): + if self._multiple: + return len(self._result) > 0 is not None or self.code > 0 + else: + return self._result is not None or self.code > 0 + + def wait(self, timeout=60): + """ + Wait for the result to become available + """ + count = 0 + while count < timeout: + time.sleep(0.1) + count += 0.1 + + result = property(get_result, set_result) + + def callback(self, fnc): + """ + Set a callback function that is to be called when the result is ready. When multiple + results are expected, the callback is called for each result. + """ + self._callback = fnc + + +# todo: has to go +class Transport(object): + """ + This class implements a transport for the Inmanta protocol. + + :param end_point_name: The name of the endpoint to which this transport belongs. This is used + for logging and configuration purposes + """ + @classmethod + def create(cls, transport_class, endpoint=None): + """ + Create an instance of the transport class + """ + return transport_class(endpoint) + + class RESTHandler(tornado.web.RequestHandler): """ A generic class use by the transport @@ -387,12 +390,6 @@ def get(self, *args, **kwargs): self.set_status(200) -def sh(msg, max_len=10): - if len(msg) < max_len: - return msg - return msg[0:max_len - 3] + "..." - - def authorize_request(auth_data, metadata, message, config): """ Authorize a request based on the given data @@ -605,10 +602,14 @@ def _execute_call(self, kwargs, http_method, config, message, request_headers, a class ServerSlice(object): + """ + An API serving part of the server. + """ def __init__(self, io_loop, name): self._name = name self._io_loop = io_loop + self.create_endpoint_metadata() self._end_point_names = [] self._handlers = [] @@ -619,6 +620,7 @@ def get_handlers(self): return self._handlers def get_end_point_names(self): + # TODO: why? return self._end_point_names def add_end_point_name(self, name): @@ -785,9 +787,9 @@ def start(self): def stop(self): LOGGER.debug("Stoppin Server Rest Endpoint") + self.http_server.stop() for endpoint in self.__end_points: endpoint.stop() - self.http_server.stop() def return_error_msg(self, status=500, msg="", headers={}): body = {"message": msg} diff --git a/tests/conftest.py b/tests/conftest.py index 50815683e8..f537b40029 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -211,12 +211,18 @@ def server_multi(inmanta_config, io_loop, mongo_db, mongo_client, request): config.Config.set("config", "executable", os.path.abspath(os.path.join(__file__, "../../src/inmanta/app.py"))) config.Config.set("server", "agent-timeout", "2") + rs = RESTServer() server = Server(database_host="localhost", database_port=int(mongo_db.port), io_loop=io_loop) - server.start() + rs.add_endpoint(server) + rs.start() - yield server + yield rs - server.stop() + try: + del IOLoop._instance + except Exception: + pass + rs.stop() shutil.rmtree(state_dir) diff --git a/tests/test_server_agent.py b/tests/test_server_agent.py index 1aaf1e8e56..186d3576e1 100644 --- a/tests/test_server_agent.py +++ b/tests/test_server_agent.py @@ -241,7 +241,7 @@ def test_dryrun_and_deploy(io_loop, server_multi, client_multi, resource_contain agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server_multi.agentmanager.sessions) == 1, 10) + yield retry_limited(lambda: len(server_multi.get_endpoint("server").agentmanager.sessions) == 1, 10) resource_container.Provider.set("agent1", "key2", "incorrect_value") resource_container.Provider.set("agent1", "key3", "value") From 430c4644225973fc2f3c9fbb663e4bd13e95fbc0 Mon Sep 17 00:00:00 2001 From: Wouter De Borger Date: Tue, 17 Apr 2018 15:41:11 +0200 Subject: [PATCH 06/14] added bootloader --- src/inmanta/app.py | 9 ++++---- src/inmanta/protocol.py | 6 +++--- src/inmanta/server/bootloader.py | 37 ++++++++++++++++++++++++++++++++ tests/conftest.py | 33 ++++++++++++++++------------ tests/test_server.py | 2 +- 5 files changed, 64 insertions(+), 23 deletions(-) create mode 100644 src/inmanta/server/bootloader.py diff --git a/src/inmanta/app.py b/src/inmanta/app.py index 5a679abd84..cfed12acea 100755 --- a/src/inmanta/app.py +++ b/src/inmanta/app.py @@ -35,6 +35,7 @@ from inmanta.ast import CompilerException import yaml from inmanta.protocol import RESTServer +from inmanta.server.bootloader import InmantaBootloader LOGGER = logging.getLogger() @@ -44,16 +45,14 @@ def start_server(options): from inmanta import server io_loop = IOLoop.current() - rs = RESTServer() - s = server.Server(io_loop) - rs.add_endpoint(s) - rs.start() + ibl = InmantaBootloader() + ibl.start() try: io_loop.start() except KeyboardInterrupt: IOLoop.current().stop() - s.stop() + ibl.stop() @command("agent", help_msg="Start the inmanta agent") diff --git a/src/inmanta/protocol.py b/src/inmanta/protocol.py index f01a192338..5b1829f46e 100644 --- a/src/inmanta/protocol.py +++ b/src/inmanta/protocol.py @@ -420,7 +420,7 @@ def authorize_request(auth_data, metadata, message, config): return -class RESTBase: +class RESTBase(object): def _create_base_url(self, properties, msg=None, versioned=True): """ @@ -602,8 +602,8 @@ def _execute_call(self, kwargs, http_method, config, message, request_headers, a class ServerSlice(object): - """ - An API serving part of the server. + """ + An API serving part of the server. """ def __init__(self, io_loop, name): diff --git a/src/inmanta/server/bootloader.py b/src/inmanta/server/bootloader.py new file mode 100644 index 0000000000..b82f599ae3 --- /dev/null +++ b/src/inmanta/server/bootloader.py @@ -0,0 +1,37 @@ +""" + Copyright 2018 Inmanta + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + Contact: code@inmanta.com +""" +from tornado.ioloop import IOLoop +from inmanta import server +from inmanta.protocol import RESTServer + + +class InmantaBootloader: + + def __init__(self): + self.restserver = RESTServer() + + def get_server_slice(self): + io_loop = IOLoop.current() + return server.Server(io_loop) + + def get_server_slices(self): + return [self.get_server_slice()] + + def start(self): + for mypart in self.get_server_slices(): + self.restserver.add_endpoint(mypart) + self.restserver.start() + + def stop(self): + self.restserver.stop() diff --git a/tests/conftest.py b/tests/conftest.py index f537b40029..2a1e6738e7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,6 +42,7 @@ import re from tornado.ioloop import IOLoop from inmanta.protocol import RESTServer +from inmanta.server.bootloader import InmantaBootloader DEFAULT_PORT_ENVVAR = 'MONGOBOX_PORT' @@ -144,7 +145,9 @@ def server(inmanta_config, io_loop, mongo_db, mongo_client, motor): state_dir = tempfile.mkdtemp() port = get_free_tcp_port() - config.Config.get("database", "name", "inmanta-" + ''.join(random.choice(string.ascii_letters) for _ in range(10))) + config.Config.set("database", "name", "inmanta-" + ''.join(random.choice(string.ascii_letters) for _ in range(10))) + config.Config.set("database", "host", "localhost") + config.Config.set("database", "port", str(mongo_db.port)) config.Config.set("config", "state-dir", state_dir) config.Config.set("config", "log-dir", os.path.join(state_dir, "logs")) config.Config.set("server_rest_transport", "port", port) @@ -157,15 +160,13 @@ def server(inmanta_config, io_loop, mongo_db, mongo_client, motor): data.use_motor(motor) - rs = RESTServer() - server = Server(database_host="localhost", database_port=int(mongo_db.port), io_loop=io_loop) - rs.add_endpoint(server) - rs.start() + ibl = InmantaBootloader() + ibl.start() - yield rs + yield ibl.restserver + ibl.stop() del IOLoop._instance - rs.stop() shutil.rmtree(state_dir) @@ -173,6 +174,8 @@ def server(inmanta_config, io_loop, mongo_db, mongo_client, motor): params=[(True, True), (True, False), (False, True), (False, False)], ids=["SSL and Auth", "SSL", "Auth", "Normal"]) def server_multi(inmanta_config, io_loop, mongo_db, mongo_client, request): + IOLoop._instance = io_loop + from inmanta.server import Server state_dir = tempfile.mkdtemp() @@ -200,7 +203,9 @@ def server_multi(inmanta_config, io_loop, mongo_db, mongo_client, request): config.Config.set(x, "token", token) port = get_free_tcp_port() - config.Config.get("database", "name", "inmanta-" + ''.join(random.choice(string.ascii_letters) for _ in range(10))) + config.Config.set("database", "name", "inmanta-" + ''.join(random.choice(string.ascii_letters) for _ in range(10))) + config.Config.set("database", "host", "localhost") + config.Config.set("database", "port", str(mongo_db.port)) config.Config.set("config", "state-dir", state_dir) config.Config.set("config", "log-dir", os.path.join(state_dir, "logs")) config.Config.set("server_rest_transport", "port", port) @@ -211,18 +216,18 @@ def server_multi(inmanta_config, io_loop, mongo_db, mongo_client, request): config.Config.set("config", "executable", os.path.abspath(os.path.join(__file__, "../../src/inmanta/app.py"))) config.Config.set("server", "agent-timeout", "2") - rs = RESTServer() - server = Server(database_host="localhost", database_port=int(mongo_db.port), io_loop=io_loop) - rs.add_endpoint(server) - rs.start() + ibl = InmantaBootloader() + ibl.start() - yield rs + yield ibl.restserver + + ibl.stop() try: del IOLoop._instance except Exception: pass - rs.stop() + shutil.rmtree(state_dir) diff --git a/tests/test_server.py b/tests/test_server.py index c719fe4a5b..ae8700f18b 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -25,7 +25,7 @@ from inmanta.agent.agent import Agent from inmanta import data, protocol from inmanta import const -from inmanta.server import config as opt, agentmanager +from inmanta.server import config as opt from datetime import datetime from uuid import UUID from inmanta.export import upload_code From ce7f6a847b68d0df7acabf292999090a3f50aafd Mon Sep 17 00:00:00 2001 From: Wouter De Borger Date: Tue, 17 Apr 2018 16:35:07 +0200 Subject: [PATCH 07/14] split off session management --- src/inmanta/protocol.py | 52 +++++++++++-- src/inmanta/server/__init__.py | 2 +- src/inmanta/server/agentmanager.py | 120 +++++++++++++++++++---------- src/inmanta/server/bootloader.py | 8 +- src/inmanta/server/server.py | 53 ++----------- tests/conftest.py | 4 +- tests/test_2way_protocol.py | 6 +- tests/test_server.py | 48 +++++++----- 8 files changed, 166 insertions(+), 127 deletions(-) diff --git a/src/inmanta/protocol.py b/src/inmanta/protocol.py index 5b1829f46e..ef820f6053 100644 --- a/src/inmanta/protocol.py +++ b/src/inmanta/protocol.py @@ -40,6 +40,9 @@ import ssl import jwt +from inmanta.server import config as opt + + LOGGER = logging.getLogger(__name__) INMANTA_MT_HEADER = "X-Inmanta-tid" @@ -613,6 +616,7 @@ def __init__(self, io_loop, name): self.create_endpoint_metadata() self._end_point_names = [] self._handlers = [] + self._sched = Scheduler(self._io_loop) name = property(lambda self: self._name) @@ -641,6 +645,9 @@ def handle_result(f): LOGGER.exception("An exception occurred while handling a future: %s", str(e)) self._io_loop.add_future(future, handle_result) + + def schedule(self, call, interval=60): + self._sched.add_action(call, interval) def create_endpoint_metadata(self): total_dict = {method_name: getattr(self, method_name) @@ -656,6 +663,9 @@ def create_endpoint_metadata(self): self.__methods__ = methods + def prestart(self, server): + pass + def start(self): pass @@ -696,6 +706,8 @@ def __init__(self, connection_timout=120): self.token = inmanta_config.Config.get(self.id, "token", None) self.connection_timout = connection_timout self.headers = set() + self.sessions_handler = SessionEndpoint(IOLoop.current()) + self.add_endpoint(self.sessions_handler) def add_endpoint(self, endpoint: ServerSlice): self.__end_points.append(endpoint) @@ -704,6 +716,9 @@ def add_endpoint(self, endpoint: ServerSlice): def get_endpoint(self, name): return self.__endpoint_dict[name] + def validate_sid(self, sid): + return self.sessions_handler.validate_sid(sid) + def get_id(self): """ Returns a unique id for a transport on an endpoint @@ -749,6 +764,9 @@ def start(self): """ LOGGER.debug("Starting Server Rest Endpoint") + for endpoint in self.__end_points: + endpoint.prestart(self) + for endpoint in self.__end_points: endpoint.start() self._handlers.extend(endpoint.get_handlers()) @@ -1268,18 +1286,29 @@ def get_client(self): return self.client -class ServerEndpoint(ServerSlice): +class SessionListener: + + def new_session(self, session: Session): + pass + + def expire(self, session: Session, timeout): + pass + + def seen(self, session: Session, endpoint_names: list): + pass + + +class SessionEndpoint(ServerSlice): """ A service that receives method calls over one or more transports """ __methods__ = {} - def __init__(self, name, io_loop, interval=60, hangtime=None): - super().__init__(io_loop, name) + def __init__(self, io_loop): + super().__init__(io_loop, "session") - self._transport_instance = RESTServer(self) - self._transport_instance.add_endpoint(self) - self._sched = Scheduler(self._io_loop) + interval = opt.agent_timeout.get() + hangtime = opt.agent_hangtime.get() self._heartbeat_cb = None self.agent_handles = {} @@ -1288,9 +1317,10 @@ def __init__(self, name, io_loop, interval=60, hangtime=None): if hangtime is None: hangtime = interval * 3 / 4 self.hangtime = hangtime + self.listeners = [] - def schedule(self, call, interval=60): - self._sched.add_action(call, interval) + def add_listener(self, listener): + self.listeners.append(listener) def stop(self): """ @@ -1312,9 +1342,13 @@ def get_or_create_session(self, sid, tid, endpoint_names, nodename): if sid not in self._sessions: session = self.new_session(sid, tid, endpoint_names, nodename) self._sessions[sid] = session + for listener in self.listeners: + listener.new_session(session) else: session = self._sessions[sid] self.seen(session, endpoint_names) + for listener in self.listeners: + listener.seen(session, endpoint_names) return session @@ -1324,6 +1358,8 @@ def new_session(self, sid, tid, endpoint_names, nodename): def expire(self, session: Session, timeout): LOGGER.debug("Expired session with id %s, last seen %d seconds ago" % (session.get_id(), timeout)) + for listener in self.listeners: + listener.expire(session, timeout) del self._sessions[session.id] def seen(self, session: Session, endpoint_names: list): diff --git a/src/inmanta/server/__init__.py b/src/inmanta/server/__init__.py index 69c49ab020..5b33054deb 100644 --- a/src/inmanta/server/__init__.py +++ b/src/inmanta/server/__init__.py @@ -14,4 +14,4 @@ # flake8: noqa: F401 -from inmanta.server.server import Server \ No newline at end of file +#from inmanta.server.server import Server \ No newline at end of file diff --git a/src/inmanta/server/agentmanager.py b/src/inmanta/server/agentmanager.py index e1a70db6e7..b28c1eb9b0 100644 --- a/src/inmanta/server/agentmanager.py +++ b/src/inmanta/server/agentmanager.py @@ -21,7 +21,7 @@ from tornado import locks from inmanta.config import Config -from inmanta import data +from inmanta import data, methods from inmanta import protocol from inmanta.asyncutil import retry_limited from . import config as server_config @@ -33,6 +33,9 @@ import sys import subprocess import uuid +from inmanta.protocol import ServerSlice +from inmanta.server import config as opt +from tornado.ioloop import IOLoop LOGGER = logging.getLogger(__name__) @@ -62,27 +65,36 @@ | | +---------------+ + +get_resources_for_agent + +resource_action_update + +dryrun_update + +set_parameters """ -class AgentManager(object): +class AgentManager(ServerSlice): ''' This class contains all server functionality related to the management of agents ''' - def __init__(self, server, closesessionsonstart=True, fact_back_off=60): - self._server = server + def __init__(self, restserver, closesessionsonstart=True, fact_back_off=None): + super(AgentManager, self).__init__(IOLoop.current(), "agentmanager") + self.restserver = restserver + + if fact_back_off is None: + fact_back_off = opt.server_fact_resource_block.get() self._agent_procs = {} # env uuid -> subprocess.Popen - server.add_future(self.start_agents()) # back-off timer for fact requests self._fact_resource_block = fact_back_off # per resource time of last fact request self._fact_resource_block_set = {} - self._server_storage = server._server_storage - # session lock self.session_lock = locks.Lock() # all sessions @@ -92,13 +104,26 @@ def __init__(self, server, closesessionsonstart=True, fact_back_off=60): self.closesessionsonstart = closesessionsonstart - # From server - def new_session(self, session: protocol.Session): + def prestart(self, server): + ServerSlice.prestart(self, server) + self._server = server.get_endpoint("server") + self._server_storage = self._server._server_storage + server.get_endpoint("session").add_listener(self) + + def new_session(self, session): self.add_future(self.register_session(session, datetime.now())) - def expire(self, session: protocol.Session): + def expire(self, session, timeout): self.add_future(self.expire_session(session, datetime.now())) + def seen(self, session, endpoint_names): + if set(session.endpoint_names) != set(endpoint_names): + LOGGER.warning("Agent endpoint set changed, this should not occur, update ignored (was %s is %s)" % + (set(session.endpoint_names), set(endpoint_names))) + # start async, let it run free + self.add_future(self.flush_agent_presence(session, datetime.now())) + + # From server def get_agent_client(self, tid: uuid.UUID, endpoint): if isinstance(tid, str): tid = uuid.UUID(tid) @@ -107,24 +132,14 @@ def get_agent_client(self, tid: uuid.UUID, endpoint): return self.tid_endpoint_to_session[(tid, endpoint)].get_client() return None - def seen(self, session, endpoint_names): - if set(session.endpoint_names) != set(endpoint_names): - LOGGER.warning("Agent endpoint set changed, this should not occur, update ignored (was %s is %s)" % - (set(session.endpoint_names), set(endpoint_names))) - # start async, let it run free - self.add_future(self.flush_agent_presence(session, datetime.now())) - def start(self): + self.add_future(self.start_agents()) if self.closesessionsonstart: self.add_future(self.clean_db()) def stop(self): self.terminate_agents() - # To Server - def add_future(self, future): - self._server.add_future(future) - # Agent Management @gen.coroutine @@ -320,9 +335,25 @@ def _fork_inmanta(self, args, outfile, errfile, cwd=None): errhandle.close() # External APIS + @protocol.handle(methods.NodeMethod.get_agent_process, agent_id="id") + @gen.coroutine + def get_agent_process(self, agent_id): + return (yield self.agentmanager.get_agent_process_report(agent_id)) + @protocol.handle(methods.ServerAgentApiMethod.trigger_agent, agent_id="id", env="tid") @gen.coroutine - def list_agent_processes(self, tid, expired): + def trigger_agent(self, env, agent_id): + yield self.agentmanager.trigger_agent(env.id, agent_id) + + @protocol.handle(methods.NodeMethod.list_agent_processes) + @gen.coroutine + def list_agent_processes(self, environment, expired): + if environment is not None: + env = yield data.Environment.get_by_id(environment) + if env is None: + return 404, {"message": "The given environment id does not exist!"} + + tid = environment if tid is not None: if expired: aps = yield data.AgentProcess.get_by_env(tid) @@ -347,6 +378,30 @@ def list_agent_processes(self, tid, expired): return 200, {"processes": processes} + @protocol.handle(methods.ServerAgentApiMethod.list_agents, env="tid") + @gen.coroutine + def list_agents(self, env): + tid = env.id + if tid is not None: + ags = yield data.Agent.get_list(environment=tid) + else: + ags = yield data.Agent.get_list() + + return 200, {"agents": [a.to_dict() for a in ags], "servertime": datetime.now().isoformat()} + + @protocol.handle(methods.AgentRecovery.get_state, env="tid") + @gen.coroutine + def get_state(self, env: uuid.UUID, sid: uuid.UUID, agent: str): + tid = env.id + if isinstance(tid, str): + tid = uuid.UUID(tid) + key = (tid, agent) + if key in self.tid_endpoint_to_session: + session = self.tid_endpoint_to_session[(tid, agent)] + if session.id == sid: + return 200, {"enabled": True} + return 200, {"enabled": False} + @gen.coroutine def get_agent_process_report(self, apid: uuid.UUID): ap = yield data.AgentProcess.get_by_id(apid) @@ -359,15 +414,6 @@ def get_agent_process_report(self, apid: uuid.UUID): result = yield client.get_status() return result.code, result.get_result() - @gen.coroutine - def list_agents(self, tid): - if tid is not None: - ags = yield data.Agent.get_list(environment=tid) - else: - ags = yield data.Agent.get_list() - - return 200, {"agents": [a.to_dict() for a in ags], "servertime": datetime.now().isoformat()} - # Start/stop agents @gen.coroutine def _ensure_agents(self, env: data.Environment, agents: list, restart: bool=False): @@ -534,17 +580,7 @@ def _request_parameter(self, env_id: uuid.UUID, resource_id): else: return 404, {"message": "resource_id parameter is required."} - @gen.coroutine - def get_state(self, tid: uuid.UUID, sid: uuid.UUID, agent: str): - if isinstance(tid, str): - tid = uuid.UUID(tid) - key = (tid, agent) - if key in self.tid_endpoint_to_session: - session = self.tid_endpoint_to_session[(tid, agent)] - if session.id == sid: - return 200, {"enabled": True} - return 200, {"enabled": False} - + @gen.coroutine def start_agents(self): """ diff --git a/src/inmanta/server/bootloader.py b/src/inmanta/server/bootloader.py index b82f599ae3..39cd9b3ffb 100644 --- a/src/inmanta/server/bootloader.py +++ b/src/inmanta/server/bootloader.py @@ -12,8 +12,9 @@ Contact: code@inmanta.com """ from tornado.ioloop import IOLoop -from inmanta import server +from inmanta.server import server from inmanta.protocol import RESTServer +from inmanta.server.agentmanager import AgentManager class InmantaBootloader: @@ -25,8 +26,11 @@ def get_server_slice(self): io_loop = IOLoop.current() return server.Server(io_loop) + def get_agent_manager_slice(self): + return AgentManager(self.restserver) + def get_server_slices(self): - return [self.get_server_slice()] + return [self.get_server_slice(), self.get_agent_manager_slice()] def start(self): for mypart in self.get_server_slices(): diff --git a/src/inmanta/server/server.py b/src/inmanta/server/server.py index e15a93f10f..1536affecf 100644 --- a/src/inmanta/server/server.py +++ b/src/inmanta/server/server.py @@ -53,14 +53,14 @@ DBLIMIT = 100000 -class Server(protocol.ServerEndpoint): +class Server(protocol.ServerSlice): """ The central Inmanta server that communicates with clients and agents and persists configuration information """ def __init__(self, io_loop, database_host=None, database_port=None, agent_no_log=False): - super().__init__("server", io_loop=io_loop, interval=opt.agent_timeout.get(), hangtime=opt.agent_hangtime.get()) + super().__init__(io_loop=io_loop, name="server") LOGGER.info("Starting server endpoint") self._server_storage = self.check_storage() self._agent_no_log = agent_no_log @@ -89,31 +89,18 @@ def __init__(self, io_loop, database_host=None, database_port=None, agent_no_log self._recompiles = defaultdict(lambda: None) - self.agentmanager = AgentManager(self, fact_back_off=opt.server_fact_resource_block.get()) - self.setup_dashboard() self.dryrun_lock = locks.Lock() - def new_session(self, sid, tid, endpoint_names, nodename): - session = protocol.ServerEndpoint.new_session(self, sid, tid, endpoint_names, nodename) - self.agentmanager.new_session(session) - return session - - def expire(self, session, timeout): - self.agentmanager.expire(session) - protocol.ServerEndpoint.expire(self, session, timeout) - - def seen(self, session, endpoint_names): - self.agentmanager.seen(session, endpoint_names) - protocol.ServerEndpoint.seen(self, session, endpoint_names) + def prestart(self, server): + self.agentmanager = server.get_endpoint("agentmanager") + protocol.SessionEndpoint.prestart(self, server) def start(self): super().start() - self.agentmanager.start() def stop(self): super().stop() - self.agentmanager.stop() def get_agent_client(self, tid: UUID, endpoint): return self.agentmanager.get_agent_client(tid, endpoint) @@ -644,36 +631,6 @@ def file_diff(self, a, b): return 200, {"diff": list(diff)} - @protocol.handle(methods.NodeMethod.get_agent_process, agent_id="id") - @gen.coroutine - def get_agent_process(self, agent_id): - return (yield self.agentmanager.get_agent_process_report(agent_id)) - - @protocol.handle(methods.ServerAgentApiMethod.trigger_agent, agent_id="id", env="tid") - @gen.coroutine - def trigger_agent(self, env, agent_id): - yield self.agentmanager.trigger_agent(env.id, agent_id) - - @protocol.handle(methods.NodeMethod.list_agent_processes) - @gen.coroutine - def list_agent_processes(self, environment, expired): - if environment is not None: - env = yield data.Environment.get_by_id(environment) - if env is None: - return 404, {"message": "The given environment id does not exist!"} - - return (yield self.agentmanager.list_agent_processes(environment, expired)) - - @protocol.handle(methods.ServerAgentApiMethod.list_agents, env="tid") - @gen.coroutine - def list_agents(self, env): - return (yield self.agentmanager.list_agents(env.id)) - - @protocol.handle(methods.AgentRecovery.get_state, env="tid") - @gen.coroutine - def get_state(self, env: uuid.UUID, sid: uuid.UUID, agent: str): - return (yield self.agentmanager.get_state(env.id, sid, agent)) - @protocol.handle(methods.ResourceMethod.get_resource, resource_id="id", env="tid") @gen.coroutine def get_resource(self, env, resource_id, logs, status, log_action, log_limit): diff --git a/tests/conftest.py b/tests/conftest.py index 2a1e6738e7..d491bc71d5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -141,7 +141,7 @@ def server(inmanta_config, io_loop, mongo_db, mongo_client, motor): # causes handler failure IOLoop._instance = io_loop - from inmanta.server import Server + from inmanta.server.server import Server state_dir = tempfile.mkdtemp() port = get_free_tcp_port() @@ -176,7 +176,7 @@ def server(inmanta_config, io_loop, mongo_db, mongo_client, motor): def server_multi(inmanta_config, io_loop, mongo_db, mongo_client, request): IOLoop._instance = io_loop - from inmanta.server import Server + from inmanta.server.server import Server state_dir = tempfile.mkdtemp() ssl, auth = request.param diff --git a/tests/test_2way_protocol.py b/tests/test_2way_protocol.py index 2fad215437..e5410fad4e 100644 --- a/tests/test_2way_protocol.py +++ b/tests/test_2way_protocol.py @@ -48,10 +48,10 @@ def get_agent_status(self, id): from inmanta import protocol # NOQA -class Server(protocol.ServerEndpoint): +class Server(protocol.SessionEndpoint): def __init__(self, name, io_loop, interval=60): - protocol.ServerEndpoint.__init__(self, name, io_loop, interval=interval) + protocol.SessionEndpoint.__init__(self, name, io_loop, interval=interval) self.expires = 0 @protocol.handle(StatusMethod.get_status_x) @@ -67,7 +67,7 @@ def get_status_x(self, tid): return 200, {"agents": status_list} def expire(self, session, timeout): - protocol.ServerEndpoint.expire(self, session, timeout) + protocol.SessionEndpoint.expire(self, session, timeout) print(session._sid) self.expires += 1 diff --git a/tests/test_server.py b/tests/test_server.py index ae8700f18b..10c71d1408 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -43,8 +43,10 @@ def test_autostart(server, client, environment): env = yield data.Environment.get_by_id(uuid.UUID(environment)) yield env.set(data.AUTOSTART_AGENT_MAP, {"iaas_agent": "", "iaas_agentx": ""}) - agentmanager = server.get_endpoint("server").agentmanager + agentmanager = server.get_endpoint("agentmanager") serverendpoint = server.get_endpoint("server") + sessionendpoint = server.get_endpoint("session") + yield agentmanager.ensure_agent_registered(env, "iaas_agent") @@ -52,29 +54,29 @@ def test_autostart(server, client, environment): res = yield agentmanager._ensure_agents(env, ["iaas_agent"]) assert res - yield retry_limited(lambda: len(serverendpoint._sessions) == 1, 20) - assert len(serverendpoint._sessions) == 1 + yield retry_limited(lambda: len(sessionendpoint._sessions) == 1, 20) + assert len(sessionendpoint._sessions) == 1 res = yield agentmanager._ensure_agents(env, ["iaas_agent"]) assert not res - assert len(serverendpoint._sessions) == 1 + assert len(sessionendpoint._sessions) == 1 LOGGER.warning("Killing agent") agentmanager._agent_procs[env.id].terminate() - yield retry_limited(lambda: len(serverendpoint._sessions) == 0, 20) + yield retry_limited(lambda: len(sessionendpoint._sessions) == 0, 20) res = yield agentmanager._ensure_agents(env, ["iaas_agent"]) assert res - yield retry_limited(lambda: len(serverendpoint._sessions) == 1, 3) - assert len(serverendpoint._sessions) == 1 + yield retry_limited(lambda: len(sessionendpoint._sessions) == 1, 3) + assert len(sessionendpoint._sessions) == 1 # second agent for same env res = yield agentmanager._ensure_agents(env, ["iaas_agentx"]) assert res - yield retry_limited(lambda: len(serverendpoint._sessions) == 1, 20) - assert len(serverendpoint._sessions) == 1 + yield retry_limited(lambda: len(sessionendpoint._sessions) == 1, 20) + assert len(sessionendpoint._sessions) == 1 # Test stopping all agents yield agentmanager.stop_agents(env) - assert len(serverendpoint._sessions) == 0 + assert len(sessionendpoint._sessions) == 0 assert len(agentmanager._agent_procs) == 0 @@ -87,6 +89,8 @@ def test_autostart_dual_env(client, server): agentmanager = server.get_endpoint("server").agentmanager serverendpoint = server.get_endpoint("server") + sessionendpoint = server.get_endpoint("session") + result = yield client.create_project("env-test") assert result.code == 200 @@ -109,13 +113,13 @@ def test_autostart_dual_env(client, server): res = yield agentmanager._ensure_agents(env, ["iaas_agent"]) assert res - yield retry_limited(lambda: len(serverendpoint._sessions) == 1, 20) - assert len(serverendpoint._sessions) == 1 + yield retry_limited(lambda: len(sessionendpoint._sessions) == 1, 20) + assert len(sessionendpoint._sessions) == 1 res = yield agentmanager._ensure_agents(env2, ["iaas_agent"]) assert res - yield retry_limited(lambda: len(serverendpoint._sessions) == 2, 20) - assert len(serverendpoint._sessions) == 2 + yield retry_limited(lambda: len(sessionendpoint._sessions) == 2, 20) + assert len(sessionendpoint._sessions) == 2 @pytest.mark.gen_test(timeout=60) @@ -124,6 +128,8 @@ def test_autostart_batched(client, server, environment): """ Test auto start of agent """ + sessionendpoint = server.get_endpoint("session") + env = yield data.Environment.get_by_id(uuid.UUID(environment)) yield env.set(data.AUTOSTART_AGENT_MAP, {"iaas_agent": "", "iaas_agentx": ""}) @@ -135,23 +141,23 @@ def test_autostart_batched(client, server, environment): res = yield agentmanager._ensure_agents(env, ["iaas_agent", "iaas_agentx"]) assert res - yield retry_limited(lambda: len(serverendpoint._sessions) == 1, 20) - assert len(serverendpoint._sessions) == 1 + yield retry_limited(lambda: len(sessionendpoint._sessions) == 1, 20) + assert len(sessionendpoint._sessions) == 1 res = yield agentmanager._ensure_agents(env, ["iaas_agent"]) assert not res - assert len(serverendpoint._sessions) == 1 + assert len(sessionendpoint._sessions) == 1 res = yield agentmanager._ensure_agents(env, ["iaas_agent", "iaas_agentx"]) assert not res - assert len(serverendpoint._sessions) == 1 + assert len(sessionendpoint._sessions) == 1 LOGGER.warning("Killing agent") agentmanager._agent_procs[env.id].terminate() - yield retry_limited(lambda: len(serverendpoint._sessions) == 0, 20) + yield retry_limited(lambda: len(sessionendpoint._sessions) == 0, 20) res = yield agentmanager._ensure_agents(env, ["iaas_agent", "iaas_agentx"]) assert res - yield retry_limited(lambda: len(serverendpoint._sessions) == 1, 3) - assert len(serverendpoint._sessions) == 1 + yield retry_limited(lambda: len(sessionendpoint._sessions) == 1, 3) + assert len(sessionendpoint._sessions) == 1 @pytest.mark.gen_test(timeout=10) From 18c6d877decc959bae1e420f21ef6734a9aee596 Mon Sep 17 00:00:00 2001 From: Wouter De Borger Date: Tue, 17 Apr 2018 16:45:08 +0200 Subject: [PATCH 08/14] fixed more tests --- src/inmanta/server/agentmanager.py | 4 ++-- src/inmanta/server/server.py | 1 - tests/test_server_agent.py | 27 ++++++++++++++------------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/inmanta/server/agentmanager.py b/src/inmanta/server/agentmanager.py index b28c1eb9b0..ca5b2424d0 100644 --- a/src/inmanta/server/agentmanager.py +++ b/src/inmanta/server/agentmanager.py @@ -338,12 +338,12 @@ def _fork_inmanta(self, args, outfile, errfile, cwd=None): @protocol.handle(methods.NodeMethod.get_agent_process, agent_id="id") @gen.coroutine def get_agent_process(self, agent_id): - return (yield self.agentmanager.get_agent_process_report(agent_id)) + return (yield self.get_agent_process_report(agent_id)) @protocol.handle(methods.ServerAgentApiMethod.trigger_agent, agent_id="id", env="tid") @gen.coroutine def trigger_agent(self, env, agent_id): - yield self.agentmanager.trigger_agent(env.id, agent_id) + raise NotImplemented() @protocol.handle(methods.NodeMethod.list_agent_processes) @gen.coroutine diff --git a/src/inmanta/server/server.py b/src/inmanta/server/server.py index 1536affecf..8dace733c9 100644 --- a/src/inmanta/server/server.py +++ b/src/inmanta/server/server.py @@ -94,7 +94,6 @@ def __init__(self, io_loop, database_host=None, database_port=None, agent_no_log def prestart(self, server): self.agentmanager = server.get_endpoint("agentmanager") - protocol.SessionEndpoint.prestart(self, server) def start(self): super().start() diff --git a/tests/test_server_agent.py b/tests/test_server_agent.py index 186d3576e1..6c79a7d232 100644 --- a/tests/test_server_agent.py +++ b/tests/test_server_agent.py @@ -39,6 +39,7 @@ from inmanta.server.server import Server from inmanta.ast import CompilerException from inmanta.protocol import RESTServer +from inmanta.server.bootloader import InmantaBootloader logger = logging.getLogger("inmanta.test.server_agent") @@ -388,10 +389,10 @@ def test_server_restart(resource_container, io_loop, server, mongo_db, client): server.stop() - server = RESTServer() - xserver = Server(database_host="localhost", database_port=int(mongo_db.port), io_loop=io_loop) - server.add_endpoint(xserver) - server.start() + ibl = InmantaBootloader() + server = ibl.restserver + + ibl.start() yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 1, 10) @@ -486,7 +487,7 @@ def test_server_restart(resource_container, io_loop, server, mongo_db, client): assert not resource_container.Provider.isset("agent1", "key3") agent.stop() - server.stop() + ibl.stop() @pytest.mark.gen_test(timeout=30) @@ -587,7 +588,7 @@ def test_dual_agent(resource_container, io_loop, server, client, environment): myagent.add_end_point_name("agent1") myagent.add_end_point_name("agent2") myagent.start() - yield retry_limited(lambda: len(server.get_endpoint("server")._sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("session")._sessions) == 1, 10) resource_container.Provider.set("agent1", "key1", "incorrect_value") resource_container.Provider.set("agent2", "key1", "incorrect_value") @@ -686,7 +687,7 @@ def test_snapshot_restore(resource_container, client, server, io_loop): code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server.get_endpoint("server")._sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("session")._sessions) == 1, 10) resource_container.Provider.set("agent1", "key", "value") @@ -885,7 +886,7 @@ def test_get_facts(resource_container, client, server, io_loop): code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server.get_endpoint("server")._sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("session")._sessions) == 1, 10) resource_container.Provider.set("agent1", "key", "value") @@ -933,7 +934,7 @@ def test_purged_facts(resource_container, client, server, io_loop, environment): code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server.get_endpoint("server")._sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("session")._sessions) == 1, 10) resource_container.Provider.set("agent1", "key", "value") @@ -1030,7 +1031,7 @@ def test_unkown_parameters(resource_container, client, server, io_loop): code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server.get_endpoint("server")._sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("session")._sessions) == 1, 10) resource_container.Provider.set("agent1", "key", "value") @@ -1086,7 +1087,7 @@ def test_fail(resource_container, client, server, io_loop): code_loader=False, poolsize=10) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server.get_endpoint("server")._sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("session")._sessions) == 1, 10) resource_container.Provider.set("agent1", "key", "value") @@ -1196,7 +1197,7 @@ def test_wait(resource_container, client, server, io_loop): agent.start() # wait for agent - yield retry_limited(lambda: len(server.get_endpoint("server")._sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("session")._sessions) == 1, 10) # set the deploy environment resource_container.Provider.set("agent1", "key", "value") @@ -1347,7 +1348,7 @@ def test_multi_instance(resource_container, client, server, io_loop): agent.start() # wait for agent - yield retry_limited(lambda: len(server.get_endpoint("server")._sessions) == 1, 10) + yield retry_limited(lambda: len(server.get_endpoint("session")._sessions) == 1, 10) # set the deploy environment resource_container.Provider.set("agent1", "key", "value") From 29e4a98c5961e96208b84e7ae53716ef209787fb Mon Sep 17 00:00:00 2001 From: Wouter De Borger Date: Tue, 24 Apr 2018 12:34:40 +0200 Subject: [PATCH 09/14] cleanup of code --- src/inmanta/app.py | 2 - src/inmanta/deploy.py | 11 +- src/inmanta/protocol.py | 696 ++--------------------------- src/inmanta/server/__init__.py | 4 +- src/inmanta/server/agentmanager.py | 13 +- src/inmanta/server/bootloader.py | 11 +- src/inmanta/server/protocol.py | 602 +++++++++++++++++++++++++ src/inmanta/server/server.py | 13 +- src/inmanta/util.py | 45 ++ tests/conftest.py | 3 - tests/server_test.py | 84 ---- tests/test_2way_protocol.py | 156 ++++--- tests/test_server.py | 17 +- tests/test_server_agent.py | 55 ++- 14 files changed, 845 insertions(+), 867 deletions(-) create mode 100644 src/inmanta/server/protocol.py delete mode 100644 tests/server_test.py diff --git a/src/inmanta/app.py b/src/inmanta/app.py index cfed12acea..e577447d2e 100755 --- a/src/inmanta/app.py +++ b/src/inmanta/app.py @@ -34,7 +34,6 @@ from inmanta.export import cfg_env, ModelExporter from inmanta.ast import CompilerException import yaml -from inmanta.protocol import RESTServer from inmanta.server.bootloader import InmantaBootloader LOGGER = logging.getLogger() @@ -42,7 +41,6 @@ @command("server", help_msg="Start the inmanta server") def start_server(options): - from inmanta import server io_loop = IOLoop.current() ibl = InmantaBootloader() diff --git a/src/inmanta/deploy.py b/src/inmanta/deploy.py index 361d3f43b6..60c43d7fd4 100644 --- a/src/inmanta/deploy.py +++ b/src/inmanta/deploy.py @@ -23,8 +23,8 @@ from mongobox import mongobox from tornado import gen, process -from inmanta import module, config, server, agent, protocol, const, data -from inmanta.protocol import RESTServer +from inmanta import module, config, agent, protocol, const, data +from inmanta.server.bootloader import InmantaBootloader LOGGER = logging.getLogger(__name__) @@ -90,12 +90,11 @@ def _setup_server(self, no_agent_log): config.Config.set("compiler_rest_transport", "port", str(self._server_port)) config.Config.set("client_rest_transport", "port", str(self._server_port)) config.Config.set("cmdline_rest_transport", "port", str(self._server_port)) + config.Config.set("database", "host", "localhost") + config.Config.set("database", "port", str(self._mongoport)) # start the server - rs = RESTServer() - self._server = server.Server(database_host="localhost", database_port=self._mongoport, io_loop=self._io_loop, - agent_no_log=no_agent_log) - rs.add_endpoint(self._server) + rs = InmantaBootloader(agent_no_log=no_agent_log) rs.start() LOGGER.debug("Started server on port %d", self._server_port) diff --git a/src/inmanta/protocol.py b/src/inmanta/protocol.py index ef820f6053..8086e2fe7f 100644 --- a/src/inmanta/protocol.py +++ b/src/inmanta/protocol.py @@ -30,17 +30,15 @@ import io import gzip -import tornado.web -from tornado import gen, queues, web +import tornado +from tornado import gen, web from inmanta import methods, const, execute from inmanta import config as inmanta_config -from tornado.httpserver import HTTPServer from tornado.httpclient import HTTPRequest, AsyncHTTPClient, HTTPError from tornado.ioloop import IOLoop -import ssl import jwt -from inmanta.server import config as opt +from inmanta.util import Scheduler LOGGER = logging.getLogger(__name__) @@ -173,6 +171,37 @@ def decode_token(token): return payload +def authorize_request(auth_data, metadata, message, config): + """ + Authorize a request based on the given data + """ + if auth_data is None: + return + + # Enforce environment restrictions + env_key = const.INMANTA_URN + "env" + if env_key in auth_data: + if env_key not in metadata: + raise UnauhorizedError("The authorization token is scoped to a specific environment.") + + if metadata[env_key] != "all" and auth_data[env_key] != metadata[env_key]: + raise UnauhorizedError("The authorization token is not valid for the requested environment.") + + # Enforce client_types restrictions + ok = False + ct_key = const.INMANTA_URN + "ct" + for ct in auth_data[ct_key]: + if ct in config[0]["client_types"]: + ok = True + + if not ok: + raise UnauhorizedError("The authorization token does not have a valid client type for this call." + + " (%s provided, %s expected" % (auth_data[ct_key], config[0]["client_types"])) + + return + + +# API class UnauhorizedError(Exception): pass @@ -244,185 +273,10 @@ def callback(self, fnc): self._callback = fnc -# todo: has to go -class Transport(object): - """ - This class implements a transport for the Inmanta protocol. - - :param end_point_name: The name of the endpoint to which this transport belongs. This is used - for logging and configuration purposes - """ - @classmethod - def create(cls, transport_class, endpoint=None): - """ - Create an instance of the transport class - """ - return transport_class(endpoint) - - -class RESTHandler(tornado.web.RequestHandler): - """ - A generic class use by the transport - """ - - def initialize(self, transport: "RESTServer", config): - self._transport = transport - self._config = config - - def _get_config(self, http_method): - if http_method.upper() not in self._config: - allowed = ", ".join(self._config.keys()) - self.set_header("Allow", allowed) - self._transport.return_error_msg(405, "%s is not supported for this url. Supported methods: %s" % - (http_method, allowed)) - return - - return self._config[http_method] - - def get_auth_token(self, headers: dict): - """ - Get the auth token provided by the caller. The token is provided as a bearer token. - """ - if "Authorization" not in headers: - return None - - parts = headers["Authorization"].split(" ") - if len(parts) == 0 or parts[0].lower() != "bearer" or len(parts) > 2 or len(parts) == 1: - LOGGER.warning("Invalid authentication header, Inmanta expects a bearer token. (%s was provided)", - headers["Authorization"]) - return None - - return decode_token(parts[1]) - - def respond(self, body, headers, status): - if body is not None: - self.write(json_encode(body)) - - for header, value in headers.items(): - self.set_header(header, value) - - self.set_status(status) - - @gen.coroutine - def _call(self, kwargs, http_method, call_config): - """ - An rpc like call - """ - if call_config is None: - body, headers, status = self._transport.return_error_msg(404, "This method does not exist.") - self.respond(body, headers, status) - return - - self.set_header("Access-Control-Allow-Origin", "*") - try: - message = self._transport._decode(self.request.body) - if message is None: - message = {} - - for key, value in self.request.query_arguments.items(): - if len(value) == 1: - message[key] = value[0].decode("latin-1") - else: - message[key] = [v.decode("latin-1") for v in value] - - request_headers = self.request.headers - - try: - auth_token = self.get_auth_token(request_headers) - except UnauhorizedError as e: - self.respond(*self._transport.return_error_msg(403, "Access denied: " + e.args[0])) - return - - auth_enabled = inmanta_config.Config.get("server", "auth", False) - if not auth_enabled or auth_token is not None: - result = yield self._transport._execute_call(kwargs, http_method, call_config, - message, request_headers, auth_token) - self.respond(*result) - else: - self.respond(*self._transport.return_error_msg(401, "Access to this resource is unauthorized.")) - except ValueError: - LOGGER.exception("An exception occured") - self.respond(*self._transport.return_error_msg(500, "Unable to decode request body")) - - @gen.coroutine - def head(self, *args, **kwargs): - yield self._call(http_method="HEAD", call_config=self._get_config("HEAD"), kwargs=kwargs) - - @gen.coroutine - def get(self, *args, **kwargs): - yield self._call(http_method="GET", call_config=self._get_config("GET"), kwargs=kwargs) - - @gen.coroutine - def post(self, *args, **kwargs): - yield self._call(http_method="POST", call_config=self._get_config("POST"), kwargs=kwargs) - - @gen.coroutine - def delete(self, *args, **kwargs): - yield self._call(http_method="DELETE", call_config=self._get_config("DELETE"), kwargs=kwargs) - - @gen.coroutine - def patch(self, *args, **kwargs): - yield self._call(http_method="PATCH", call_config=self._get_config("PATCH"), kwargs=kwargs) - - @gen.coroutine - def put(self, *args, **kwargs): - yield self._call(http_method="PUT", call_config=self._get_config("PUT"), kwargs=kwargs) - - @gen.coroutine - def options(self, *args, **kwargs): - allow_headers = "Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token" - if len(self._transport.headers): - allow_headers += ", " + ", ".join(self._transport.headers) - - self.set_header("Access-Control-Allow-Origin", "*") - self.set_header("Access-Control-Allow-Methods", "HEAD, GET, POST, PUT, OPTIONS, DELETE, PATCH") - self.set_header("Access-Control-Allow-Headers", allow_headers) - - self.set_status(200) - - -class StaticContentHandler(tornado.web.RequestHandler): - def initialize(self, transport: Transport, content, content_type): - self._transport = transport - self._content = content - self._content_type = content_type - - def get(self, *args, **kwargs): - self.set_header("Content-Type", self._content_type) - self.write(self._content) - self.set_status(200) - - -def authorize_request(auth_data, metadata, message, config): - """ - Authorize a request based on the given data - """ - if auth_data is None: - return - - # Enforce environment restrictions - env_key = const.INMANTA_URN + "env" - if env_key in auth_data: - if env_key not in metadata: - raise UnauhorizedError("The authorization token is scoped to a specific environment.") - - if metadata[env_key] != "all" and auth_data[env_key] != metadata[env_key]: - raise UnauhorizedError("The authorization token is not valid for the requested environment.") - - # Enforce client_types restrictions - ok = False - ct_key = const.INMANTA_URN + "ct" - for ct in auth_data[ct_key]: - if ct in config[0]["client_types"]: - ok = True - - if not ok: - raise UnauhorizedError("The authorization token does not have a valid client type for this call." + - " (%s provided, %s expected" % (auth_data[ct_key], config[0]["client_types"])) - - return +# Tornado Interface +# Shared class RESTBase(object): def _create_base_url(self, properties, msg=None, versioned=True): @@ -604,219 +458,8 @@ def _execute_call(self, kwargs, http_method, config, message, request_headers, a return self.return_error_msg(500, "An exception occured: " + str(e.args), headers) -class ServerSlice(object): - """ - An API serving part of the server. - """ - - def __init__(self, io_loop, name): - self._name = name - self._io_loop = io_loop - - self.create_endpoint_metadata() - self._end_point_names = [] - self._handlers = [] - self._sched = Scheduler(self._io_loop) - - name = property(lambda self: self._name) - - def get_handlers(self): - return self._handlers - - def get_end_point_names(self): - # TODO: why? - return self._end_point_names - - def add_end_point_name(self, name): - """ - Add an additional name to this endpoint to which it reacts and sends out in heartbeats - """ - LOGGER.debug("Adding '%s' as endpoint", name) - self._end_point_names.append(name) - - def add_future(self, future): - """ - Add a future to the ioloop to be handled, but do not require the result. - """ - def handle_result(f): - try: - f.result() - except Exception as e: - LOGGER.exception("An exception occurred while handling a future: %s", str(e)) - - self._io_loop.add_future(future, handle_result) - - def schedule(self, call, interval=60): - self._sched.add_action(call, interval) - - def create_endpoint_metadata(self): - total_dict = {method_name: getattr(self, method_name) - for method_name in dir(self) if callable(getattr(self, method_name))} - - methods = {} - for name, attr in total_dict.items(): - if name[0:2] != "__" and hasattr(attr, "__protocol_method__"): - if attr.__protocol_method__ in methods: - raise Exception("Unable to register multiple handlers for the same method. %s" % attr.__protocol_method__) - - methods[attr.__protocol_method__] = (name, attr) - - self.__methods__ = methods - - def prestart(self, server): - pass - - def start(self): - pass - - def stop(self): - pass - - def add_static_handler(self, location, path, default_filename=None, start=False): - """ - Configure a static handler to serve data from the specified path. - """ - if location[0] != "/": - location = "/" + location - - if location[-1] != "/": - location = location + "/" - - options = {"path": path} - if default_filename is None: - options["default_filename"] = "index.html" - - self._handlers.append((r"%s(.*)" % location, tornado.web.StaticFileHandler, options)) - self._handlers.append((r"%s" % location[:-1], tornado.web.RedirectHandler, {"url": location})) - - if start: - self._handlers.append((r"/", tornado.web.RedirectHandler, {"url": location})) - - def add_static_content(self, path, content, content_type="application/javascript"): - self._handlers.append((r"%s(.*)" % path, StaticContentHandler, {"transport": self, "content": content, - "content_type": content_type})) - - -class RESTServer(RESTBase): - - def __init__(self, connection_timout=120): - self.__end_points = [] - self.__endpoint_dict = {} - self._handlers = [] - self.token = inmanta_config.Config.get(self.id, "token", None) - self.connection_timout = connection_timout - self.headers = set() - self.sessions_handler = SessionEndpoint(IOLoop.current()) - self.add_endpoint(self.sessions_handler) - - def add_endpoint(self, endpoint: ServerSlice): - self.__end_points.append(endpoint) - self.__endpoint_dict[endpoint.name] = endpoint - - def get_endpoint(self, name): - return self.__endpoint_dict[name] - - def validate_sid(self, sid): - return self.sessions_handler.validate_sid(sid) - - def get_id(self): - """ - Returns a unique id for a transport on an endpoint - """ - return "server_rest_transport" - - id = property(get_id) - - def create_op_mapping(self): - """ - Build a mapping between urls, ops and methods - """ - url_map = defaultdict(dict) - - # TODO: avoid colliding handlers - - for endpoint in self.__end_points: - for method, method_handlers in endpoint.__methods__.items(): - properties = method.__protocol_properties__ - call = (endpoint, method_handlers[0]) - - if "arg_options" in properties: - for opts in properties["arg_options"].values(): - if "header" in opts: - self.headers.add(opts["header"]) - - url = self._create_base_url(properties) - properties["api_version"] = "1" - url_map[url][properties["operation"]] = (properties, call, method.__wrapped__) - print(url) - - url = self._create_base_url(properties, versioned=False) - properties = properties.copy() - properties["api_version"] = None - url_map[url][properties["operation"]] = (properties, call, method.__wrapped__) - print(url) - - return url_map - - def start(self): - """ - Start the transport - """ - LOGGER.debug("Starting Server Rest Endpoint") - - for endpoint in self.__end_points: - endpoint.prestart(self) - - for endpoint in self.__end_points: - endpoint.start() - self._handlers.extend(endpoint.get_handlers()) - - url_map = self.create_op_mapping() - - for url, configs in url_map.items(): - handler_config = {} - for op, cfg in configs.items(): - handler_config[op] = cfg - - self._handlers.append((url, RESTHandler, {"transport": self, "config": handler_config})) - LOGGER.debug("Registering handler(s) for url %s and methods %s" % (url, ", ".join(handler_config.keys()))) - - port = 8888 - if self.id in inmanta_config.Config.get() and "port" in inmanta_config.Config.get()[self.id]: - port = inmanta_config.Config.get()[self.id]["port"] - - application = tornado.web.Application(self._handlers, compress_response=True) - - crt = inmanta_config.Config.get("server", "ssl_cert_file", None) - key = inmanta_config.Config.get("server", "ssl_key_file", None) - - if(crt is not None and key is not None): - ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - ssl_ctx.load_cert_chain(crt, key) - - self.http_server = HTTPServer(application, decompress_request=True, ssl_options=ssl_ctx) - LOGGER.debug("Created REST transport with SSL") - else: - self.http_server = HTTPServer(application, decompress_request=True) - - self.http_server.listen(port) - - LOGGER.debug("Start REST transport") - - def stop(self): - LOGGER.debug("Stoppin Server Rest Endpoint") - self.http_server.stop() - for endpoint in self.__end_points: - endpoint.stop() - - def return_error_msg(self, status=500, msg="", headers={}): - body = {"message": msg} - headers["Content-Type"] = "application/json" - LOGGER.debug("Signaling error to client: %d, %s, %s", status, body, headers) - return body, headers, status - - -class RESTTransport(Transport, RESTBase): +# Client side +class RESTTransport(RESTBase): """" A REST (json body over http) transport. Only methods that operate on resource can use all HTTP verbs. For other methods the POST verb is used. @@ -1079,51 +722,6 @@ def __new__(cls, class_name, bases, dct): return type.__new__(cls, class_name, bases, dct) -class Scheduler(object): - """ - An event scheduler class - """ - - def __init__(self, io_loop): - self._scheduled = set() - self._io_loop = io_loop - - def add_action(self, action, interval, initial_delay=None): - """ - Add a new action - - :param action A function to call periodically - :param interval The interval between execution of actions - :param initial_delay Delay to the first execution, default to interval - """ - - if initial_delay is None: - initial_delay = interval - - LOGGER.debug("Scheduling action %s every %d seconds with initial delay %d", action, interval, initial_delay) - - def action_function(): - LOGGER.info("Calling %s" % action) - if action in self._scheduled: - try: - action() - except Exception: - LOGGER.exception("Uncaught exception while executing scheduled action") - - finally: - self._io_loop.call_later(interval, action_function) - - self._io_loop.call_later(initial_delay, action_function) - self._scheduled.add(action) - - def remove(self, action): - """ - Remove a scheduled action - """ - if action in self._scheduled: - self._scheduled.remove(action) - - class Endpoint(object): """ An end-point in the rpc framework @@ -1178,222 +776,6 @@ def get_node_name(self): node_name = property(get_node_name) -class Session(object): - """ - An environment that segments agents connected to the server - """ - - def __init__(self, sessionstore, io_loop, sid, hang_interval, timout, tid, endpoint_names, nodename): - self._sid = sid - self._interval = hang_interval - self._timeout = timout - self._sessionstore = sessionstore - self._seen = time.time() - self._callhandle = None - self.expired = False - - self.tid = tid - self.endpoint_names = endpoint_names - self.nodename = nodename - - self._io_loop = io_loop - - self._replies = {} - self.check_expire() - self._queue = queues.Queue() - - self.client = ReturnClient(str(sid), self) - - def check_expire(self): - if self.expired: - LOGGER.exception("Tried to expire session already expired") - ttw = self._timeout + self._seen - time.time() - if ttw < 0: - self.expire(self._seen - time.time()) - else: - self._callhandle = self._io_loop.call_later(ttw, self.check_expire) - - def get_id(self): - return self._sid - - id = property(get_id) - - def expire(self, timeout): - self.expired = True - if self._callhandle is not None: - self._io_loop.remove_timeout(self._callhandle) - self._sessionstore.expire(self, timeout) - - def seen(self): - self._seen = time.time() - - def _set_timeout(self, future, timeout, log_message): - def on_timeout(): - if not self.expired: - LOGGER.warning(log_message) - future.set_exception(gen.TimeoutError()) - - timeout_handle = self._io_loop.add_timeout(self._io_loop.time() + timeout, on_timeout) - future.add_done_callback(lambda _: self._io_loop.remove_timeout(timeout_handle)) - - def put_call(self, call_spec, timeout=10): - future = tornado.concurrent.Future() - - reply_id = uuid.uuid4() - - LOGGER.debug("Putting call %s: %s %s for agent %s in queue", reply_id, call_spec["method"], call_spec["url"], self._sid) - - q = self._queue - call_spec["reply_id"] = reply_id - q.put(call_spec) - self._set_timeout(future, timeout, "Call %s: %s %s for agent %s timed out." % - (reply_id, call_spec["method"], call_spec["url"], self._sid)) - self._replies[call_spec["reply_id"]] = future - - return future - - @gen.coroutine - def get_calls(self): - """ - Get all calls queued for a node. If no work is available, wait until timeout. This method returns none if a call - fails. - """ - try: - q = self._queue - call_list = [] - call = yield q.get(timeout=self._io_loop.time() + self._interval) - call_list.append(call) - while q.qsize() > 0: - call = yield q.get() - call_list.append(call) - - return call_list - - except gen.TimeoutError: - return None - - def set_reply(self, reply_id, data): - LOGGER.log(3, "Received Reply: %s", reply_id) - if reply_id in self._replies: - future = self._replies[reply_id] - del self._replies[reply_id] - if not future.done(): - future.set_result(data) - else: - LOGGER.debug("Received Reply that is unknown: %s", reply_id) - - def get_client(self): - return self.client - - -class SessionListener: - - def new_session(self, session: Session): - pass - - def expire(self, session: Session, timeout): - pass - - def seen(self, session: Session, endpoint_names: list): - pass - - -class SessionEndpoint(ServerSlice): - """ - A service that receives method calls over one or more transports - """ - __methods__ = {} - - def __init__(self, io_loop): - super().__init__(io_loop, "session") - - interval = opt.agent_timeout.get() - hangtime = opt.agent_hangtime.get() - - self._heartbeat_cb = None - self.agent_handles = {} - self._sessions = {} - self.interval = interval - if hangtime is None: - hangtime = interval * 3 / 4 - self.hangtime = hangtime - self.listeners = [] - - def add_listener(self, listener): - self.listeners.append(listener) - - def stop(self): - """ - Stop the end-point and all of its transports - """ - # terminate all sessions cleanly - for session in self._sessions.copy().values(): - session.expire(0) - - def validate_sid(self, sid): - if isinstance(sid, str): - sid = uuid.UUID(sid) - return sid in self._sessions - - def get_or_create_session(self, sid, tid, endpoint_names, nodename): - if isinstance(sid, str): - sid = uuid.UUID(sid) - - if sid not in self._sessions: - session = self.new_session(sid, tid, endpoint_names, nodename) - self._sessions[sid] = session - for listener in self.listeners: - listener.new_session(session) - else: - session = self._sessions[sid] - self.seen(session, endpoint_names) - for listener in self.listeners: - listener.seen(session, endpoint_names) - - return session - - def new_session(self, sid, tid, endpoint_names, nodename): - LOGGER.debug("New session with id %s on node %s for env %s with endpoints %s" % (sid, nodename, tid, endpoint_names)) - return Session(self, self._io_loop, sid, self.hangtime, self.interval, tid, endpoint_names, nodename) - - def expire(self, session: Session, timeout): - LOGGER.debug("Expired session with id %s, last seen %d seconds ago" % (session.get_id(), timeout)) - for listener in self.listeners: - listener.expire(session, timeout) - del self._sessions[session.id] - - def seen(self, session: Session, endpoint_names: list): - LOGGER.debug("Seen session with id %s" % (session.get_id())) - session.seen() - - @handle(methods.HeartBeatMethod.heartbeat, env="tid") - @gen.coroutine - def heartbeat(self, sid, env, endpoint_names, nodename): - LOGGER.debug("Received heartbeat from %s for agents %s in %s", nodename, ",".join(endpoint_names), env.id) - - session = self.get_or_create_session(sid, env.id, endpoint_names, nodename) - - LOGGER.debug("Let node %s wait for method calls to become available. (long poll)", nodename) - call_list = yield session.get_calls() - if call_list is not None: - LOGGER.debug("Pushing %d method calls to node %s", len(call_list), nodename) - return 200, {"method_calls": call_list} - else: - LOGGER.debug("Heartbeat wait expired for %s, returning. (long poll)", nodename) - - return 200 - - @handle(methods.HeartBeatMethod.heartbeat_reply) - @gen.coroutine - def heartbeat_reply(self, sid, reply_id, data): - try: - env = self._sessions[sid] - env.set_reply(reply_id, data) - return 200 - except Exception: - LOGGER.warning("could not deliver agent reply with sid=%s and reply_id=%s" % (sid, reply_id), exc_info=True) - - class AgentEndPoint(Endpoint, metaclass=EndpointMeta): """ An endpoint for clients that make calls to a server and that receive calls back from the server using long-poll @@ -1556,7 +938,7 @@ def __init__(self, name, ioloop=None, transport=RESTTransport): self._transport_instance = None LOGGER.debug("Start transport for client %s", self.name) - tr = Transport.create(self._transport, self) + tr = self._transport(self) self._transport_instance = tr @gen.coroutine diff --git a/src/inmanta/server/__init__.py b/src/inmanta/server/__init__.py index 5b33054deb..0f0df24707 100644 --- a/src/inmanta/server/__init__.py +++ b/src/inmanta/server/__init__.py @@ -14,4 +14,6 @@ # flake8: noqa: F401 -#from inmanta.server.server import Server \ No newline at end of file +SLICE_SERVER = "server" +SLICE_AGENT_MANAGER = "agentmanager" +SLICE_SESSION_MANAGER = "session" diff --git a/src/inmanta/server/agentmanager.py b/src/inmanta/server/agentmanager.py index ca5b2424d0..3c35842e15 100644 --- a/src/inmanta/server/agentmanager.py +++ b/src/inmanta/server/agentmanager.py @@ -22,7 +22,7 @@ from inmanta.config import Config from inmanta import data, methods -from inmanta import protocol +from inmanta.server import protocol, SLICE_AGENT_MANAGER from inmanta.asyncutil import retry_limited from . import config as server_config @@ -33,12 +33,15 @@ import sys import subprocess import uuid -from inmanta.protocol import ServerSlice +from inmanta.server.protocol import ServerSlice from inmanta.server import config as opt from tornado.ioloop import IOLoop +from inmanta.protocol import encode_token LOGGER = logging.getLogger(__name__) + + agent_lock = locks.Lock() @@ -82,7 +85,7 @@ class AgentManager(ServerSlice): ''' def __init__(self, restserver, closesessionsonstart=True, fact_back_off=None): - super(AgentManager, self).__init__(IOLoop.current(), "agentmanager") + super(AgentManager, self).__init__(IOLoop.current(), SLICE_AGENT_MANAGER) self.restserver = restserver if fact_back_off is None: @@ -141,7 +144,6 @@ def stop(self): self.terminate_agents() # Agent Management - @gen.coroutine def ensure_agent_registered(self, env: data.Environment, nodename: str): """ @@ -520,7 +522,7 @@ def _make_agent_config(self, env: data.Environment, agent_names: list, agent_map "statedir": privatestatedir, "agent_splay": agent_splay, "agent_interval": agent_interval} if server_config.server_enable_auth.get(): - token = protocol.encode_token(["agent"], environment_id) + token = encode_token(["agent"], environment_id) config += """ token=%s """ % (token) @@ -580,7 +582,6 @@ def _request_parameter(self, env_id: uuid.UUID, resource_id): else: return 404, {"message": "resource_id parameter is required."} - @gen.coroutine def start_agents(self): """ diff --git a/src/inmanta/server/bootloader.py b/src/inmanta/server/bootloader.py index 39cd9b3ffb..36ec28aafe 100644 --- a/src/inmanta/server/bootloader.py +++ b/src/inmanta/server/bootloader.py @@ -13,24 +13,25 @@ """ from tornado.ioloop import IOLoop from inmanta.server import server -from inmanta.protocol import RESTServer +from inmanta.server.protocol import RESTServer from inmanta.server.agentmanager import AgentManager -class InmantaBootloader: +class InmantaBootloader(object): - def __init__(self): + def __init__(self, agent_no_log=False): self.restserver = RESTServer() + self.agent_no_log = agent_no_log def get_server_slice(self): io_loop = IOLoop.current() - return server.Server(io_loop) + return server.Server(io_loop, agent_no_log=self.agent_no_log) def get_agent_manager_slice(self): return AgentManager(self.restserver) def get_server_slices(self): - return [self.get_server_slice(), self.get_agent_manager_slice()] + return [self.get_server_slice(), self.get_agent_manager_slice()] def start(self): for mypart in self.get_server_slices(): diff --git a/src/inmanta/server/protocol.py b/src/inmanta/server/protocol.py new file mode 100644 index 0000000000..ad30b2cb91 --- /dev/null +++ b/src/inmanta/server/protocol.py @@ -0,0 +1,602 @@ +""" + Copyright 2018 Inmanta + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + Contact: code@inmanta.com +""" +from inmanta.util import Scheduler +from inmanta.protocol import RESTBase, decode_token, json_encode, UnauhorizedError, ReturnClient, handle + +from inmanta import config as inmanta_config, methods +from inmanta.server import config as opt, SLICE_SESSION_MANAGER + +import tornado.web +from tornado import gen, queues +from tornado.ioloop import IOLoop +from tornado.httpserver import HTTPServer + +import logging +import ssl +import time +import uuid +from _collections import defaultdict + + +LOGGER = logging.getLogger(__name__) + + +# Server Side +class RESTServer(RESTBase): + + def __init__(self, connection_timout=120): + self.__end_points = [] + self.__endpoint_dict = {} + self._handlers = [] + self.token = inmanta_config.Config.get(self.id, "token", None) + self.connection_timout = connection_timout + self.headers = set() + self.sessions_handler = SessionManager(IOLoop.current()) + self.add_endpoint(self.sessions_handler) + + def add_endpoint(self, endpoint: "ServerSlice"): + self.__end_points.append(endpoint) + self.__endpoint_dict[endpoint.name] = endpoint + + def get_endpoint(self, name): + return self.__endpoint_dict[name] + + def validate_sid(self, sid): + return self.sessions_handler.validate_sid(sid) + + def get_id(self): + """ + Returns a unique id for a transport on an endpoint + """ + return "server_rest_transport" + + id = property(get_id) + + def create_op_mapping(self): + """ + Build a mapping between urls, ops and methods + """ + url_map = defaultdict(dict) + + # TODO: avoid colliding handlers + + for endpoint in self.__end_points: + for method, method_handlers in endpoint.__methods__.items(): + properties = method.__protocol_properties__ + call = (endpoint, method_handlers[0]) + + if "arg_options" in properties: + for opts in properties["arg_options"].values(): + if "header" in opts: + self.headers.add(opts["header"]) + + url = self._create_base_url(properties) + properties["api_version"] = "1" + url_map[url][properties["operation"]] = (properties, call, method.__wrapped__) + print(url) + + url = self._create_base_url(properties, versioned=False) + properties = properties.copy() + properties["api_version"] = None + url_map[url][properties["operation"]] = (properties, call, method.__wrapped__) + print(url) + + return url_map + + def start(self): + """ + Start the transport + """ + LOGGER.debug("Starting Server Rest Endpoint") + + for endpoint in self.__end_points: + endpoint.prestart(self) + + for endpoint in self.__end_points: + endpoint.start() + self._handlers.extend(endpoint.get_handlers()) + + url_map = self.create_op_mapping() + + for url, configs in url_map.items(): + handler_config = {} + for op, cfg in configs.items(): + handler_config[op] = cfg + + self._handlers.append((url, RESTHandler, {"transport": self, "config": handler_config})) + LOGGER.debug("Registering handler(s) for url %s and methods %s" % (url, ", ".join(handler_config.keys()))) + + port = 8888 + if self.id in inmanta_config.Config.get() and "port" in inmanta_config.Config.get()[self.id]: + port = inmanta_config.Config.get()[self.id]["port"] + + application = tornado.web.Application(self._handlers, compress_response=True) + + crt = inmanta_config.Config.get("server", "ssl_cert_file", None) + key = inmanta_config.Config.get("server", "ssl_key_file", None) + + if(crt is not None and key is not None): + ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + ssl_ctx.load_cert_chain(crt, key) + + self.http_server = HTTPServer(application, decompress_request=True, ssl_options=ssl_ctx) + LOGGER.debug("Created REST transport with SSL") + else: + self.http_server = HTTPServer(application, decompress_request=True) + + self.http_server.listen(port) + + LOGGER.debug("Start REST transport") + + def stop(self): + LOGGER.debug("Stoppin Server Rest Endpoint") + self.http_server.stop() + for endpoint in self.__end_points: + endpoint.stop() + + def return_error_msg(self, status=500, msg="", headers={}): + body = {"message": msg} + headers["Content-Type"] = "application/json" + LOGGER.debug("Signaling error to client: %d, %s, %s", status, body, headers) + return body, headers, status + + +class ServerSlice(object): + """ + An API serving part of the server. + """ + + def __init__(self, io_loop, name): + self._name = name + self._io_loop = io_loop + + self.create_endpoint_metadata() + self._end_point_names = [] + self._handlers = [] + self._sched = Scheduler(self._io_loop) + + def prestart(self, server: RESTServer): + """Called by the RestServer host prior to start, can be used to collect references to other server slices""" + pass + + def start(self): + pass + + def stop(self): + pass + + name = property(lambda self: self._name) + + def get_handlers(self): + return self._handlers + + def get_end_point_names(self): + # TODO: why? + return self._end_point_names + + def add_end_point_name(self, name): + """ + Add an additional name to this endpoint to which it reacts and sends out in heartbeats + """ + LOGGER.debug("Adding '%s' as endpoint", name) + self._end_point_names.append(name) + + def add_future(self, future): + """ + Add a future to the ioloop to be handled, but do not require the result. + """ + def handle_result(f): + try: + f.result() + except Exception as e: + LOGGER.exception("An exception occurred while handling a future: %s", str(e)) + + self._io_loop.add_future(future, handle_result) + + def schedule(self, call, interval=60): + self._sched.add_action(call, interval) + + def create_endpoint_metadata(self): + total_dict = {method_name: getattr(self, method_name) + for method_name in dir(self) if callable(getattr(self, method_name))} + + methods = {} + for name, attr in total_dict.items(): + if name[0:2] != "__" and hasattr(attr, "__protocol_method__"): + if attr.__protocol_method__ in methods: + raise Exception("Unable to register multiple handlers for the same method. %s" % attr.__protocol_method__) + + methods[attr.__protocol_method__] = (name, attr) + + self.__methods__ = methods + + def add_static_handler(self, location, path, default_filename=None, start=False): + """ + Configure a static handler to serve data from the specified path. + """ + if location[0] != "/": + location = "/" + location + + if location[-1] != "/": + location = location + "/" + + options = {"path": path} + if default_filename is None: + options["default_filename"] = "index.html" + + self._handlers.append((r"%s(.*)" % location, tornado.web.StaticFileHandler, options)) + self._handlers.append((r"%s" % location[:-1], tornado.web.RedirectHandler, {"url": location})) + + if start: + self._handlers.append((r"/", tornado.web.RedirectHandler, {"url": location})) + + def add_static_content(self, path, content, content_type="application/javascript"): + self._handlers.append((r"%s(.*)" % path, StaticContentHandler, {"transport": self, "content": content, + "content_type": content_type})) + + +class Session(object): + """ + An environment that segments agents connected to the server + """ + + def __init__(self, sessionstore, io_loop, sid, hang_interval, timout, tid, endpoint_names, nodename): + self._sid = sid + self._interval = hang_interval + self._timeout = timout + self._sessionstore = sessionstore + self._seen = time.time() + self._callhandle = None + self.expired = False + + self.tid = tid + self.endpoint_names = endpoint_names + self.nodename = nodename + + self._io_loop = io_loop + + self._replies = {} + self.check_expire() + self._queue = queues.Queue() + + self.client = ReturnClient(str(sid), self) + + def check_expire(self): + if self.expired: + LOGGER.exception("Tried to expire session already expired") + ttw = self._timeout + self._seen - time.time() + if ttw < 0: + self.expire(self._seen - time.time()) + else: + self._callhandle = self._io_loop.call_later(ttw, self.check_expire) + + def get_id(self): + return self._sid + + id = property(get_id) + + def expire(self, timeout): + self.expired = True + if self._callhandle is not None: + self._io_loop.remove_timeout(self._callhandle) + self._sessionstore.expire(self, timeout) + + def seen(self): + self._seen = time.time() + + def _set_timeout(self, future, timeout, log_message): + def on_timeout(): + if not self.expired: + LOGGER.warning(log_message) + future.set_exception(gen.TimeoutError()) + + timeout_handle = self._io_loop.add_timeout(self._io_loop.time() + timeout, on_timeout) + future.add_done_callback(lambda _: self._io_loop.remove_timeout(timeout_handle)) + + def put_call(self, call_spec, timeout=10): + future = tornado.concurrent.Future() + + reply_id = uuid.uuid4() + + LOGGER.debug("Putting call %s: %s %s for agent %s in queue", reply_id, call_spec["method"], call_spec["url"], self._sid) + + q = self._queue + call_spec["reply_id"] = reply_id + q.put(call_spec) + self._set_timeout(future, timeout, "Call %s: %s %s for agent %s timed out." % + (reply_id, call_spec["method"], call_spec["url"], self._sid)) + self._replies[call_spec["reply_id"]] = future + + return future + + @gen.coroutine + def get_calls(self): + """ + Get all calls queued for a node. If no work is available, wait until timeout. This method returns none if a call + fails. + """ + try: + q = self._queue + call_list = [] + call = yield q.get(timeout=self._io_loop.time() + self._interval) + call_list.append(call) + while q.qsize() > 0: + call = yield q.get() + call_list.append(call) + + return call_list + + except gen.TimeoutError: + return None + + def set_reply(self, reply_id, data): + LOGGER.log(3, "Received Reply: %s", reply_id) + if reply_id in self._replies: + future = self._replies[reply_id] + del self._replies[reply_id] + if not future.done(): + future.set_result(data) + else: + LOGGER.debug("Received Reply that is unknown: %s", reply_id) + + def get_client(self): + return self.client + + +class SessionListener(object): + + def new_session(self, session: Session): + pass + + def expire(self, session: Session, timeout): + pass + + def seen(self, session: Session, endpoint_names: list): + pass + + +# Internals +class SessionManager(ServerSlice): + """ + A service that receives method calls over one or more transports + """ + __methods__ = {} + + def __init__(self, io_loop): + super().__init__(io_loop, SLICE_SESSION_MANAGER) + + # Config + interval = opt.agent_timeout.get() + hangtime = opt.agent_hangtime.get() + + if hangtime is None: + hangtime = interval * 3 / 4 + + self.hangtime = hangtime + self.interval = interval + + # Session management + self._heartbeat_cb = None + self.agent_handles = {} + self._sessions = {} + + # Listeners + self.listeners = [] + + def add_listener(self, listener): + self.listeners.append(listener) + + def stop(self): + """ + Stop the end-point and all of its transports + """ + # terminate all sessions cleanly + for session in self._sessions.copy().values(): + session.expire(0) + + def validate_sid(self, sid): + if isinstance(sid, str): + sid = uuid.UUID(sid) + return sid in self._sessions + + def get_or_create_session(self, sid, tid, endpoint_names, nodename): + if isinstance(sid, str): + sid = uuid.UUID(sid) + + if sid not in self._sessions: + session = self.new_session(sid, tid, endpoint_names, nodename) + self._sessions[sid] = session + for listener in self.listeners: + listener.new_session(session) + else: + session = self._sessions[sid] + self.seen(session, endpoint_names) + for listener in self.listeners: + listener.seen(session, endpoint_names) + + return session + + def new_session(self, sid, tid, endpoint_names, nodename): + LOGGER.debug("New session with id %s on node %s for env %s with endpoints %s" % (sid, nodename, tid, endpoint_names)) + return Session(self, self._io_loop, sid, self.hangtime, self.interval, tid, endpoint_names, nodename) + + def expire(self, session: Session, timeout): + LOGGER.debug("Expired session with id %s, last seen %d seconds ago" % (session.get_id(), timeout)) + for listener in self.listeners: + listener.expire(session, timeout) + del self._sessions[session.id] + + def seen(self, session: Session, endpoint_names: list): + LOGGER.debug("Seen session with id %s" % (session.get_id())) + session.seen() + + @handle(methods.HeartBeatMethod.heartbeat, env="tid") + @gen.coroutine + def heartbeat(self, sid, env, endpoint_names, nodename): + LOGGER.debug("Received heartbeat from %s for agents %s in %s", nodename, ",".join(endpoint_names), env.id) + + session = self.get_or_create_session(sid, env.id, endpoint_names, nodename) + + LOGGER.debug("Let node %s wait for method calls to become available. (long poll)", nodename) + call_list = yield session.get_calls() + if call_list is not None: + LOGGER.debug("Pushing %d method calls to node %s", len(call_list), nodename) + return 200, {"method_calls": call_list} + else: + LOGGER.debug("Heartbeat wait expired for %s, returning. (long poll)", nodename) + + return 200 + + @handle(methods.HeartBeatMethod.heartbeat_reply) + @gen.coroutine + def heartbeat_reply(self, sid, reply_id, data): + try: + env = self._sessions[sid] + env.set_reply(reply_id, data) + return 200 + except Exception: + LOGGER.warning("could not deliver agent reply with sid=%s and reply_id=%s" % (sid, reply_id), exc_info=True) + + +class RESTHandler(tornado.web.RequestHandler): + """ + A generic class use by the transport + """ + + def initialize(self, transport: "RESTServer", config): + self._transport = transport + self._config = config + + def _get_config(self, http_method): + if http_method.upper() not in self._config: + allowed = ", ".join(self._config.keys()) + self.set_header("Allow", allowed) + self._transport.return_error_msg(405, "%s is not supported for this url. Supported methods: %s" % + (http_method, allowed)) + return + + return self._config[http_method] + + def get_auth_token(self, headers: dict): + """ + Get the auth token provided by the caller. The token is provided as a bearer token. + """ + if "Authorization" not in headers: + return None + + parts = headers["Authorization"].split(" ") + if len(parts) == 0 or parts[0].lower() != "bearer" or len(parts) > 2 or len(parts) == 1: + LOGGER.warning("Invalid authentication header, Inmanta expects a bearer token. (%s was provided)", + headers["Authorization"]) + return None + + return decode_token(parts[1]) + + def respond(self, body, headers, status): + if body is not None: + self.write(json_encode(body)) + + for header, value in headers.items(): + self.set_header(header, value) + + self.set_status(status) + + @gen.coroutine + def _call(self, kwargs, http_method, call_config): + """ + An rpc like call + """ + if call_config is None: + body, headers, status = self._transport.return_error_msg(404, "This method does not exist.") + self.respond(body, headers, status) + return + + self.set_header("Access-Control-Allow-Origin", "*") + try: + message = self._transport._decode(self.request.body) + if message is None: + message = {} + + for key, value in self.request.query_arguments.items(): + if len(value) == 1: + message[key] = value[0].decode("latin-1") + else: + message[key] = [v.decode("latin-1") for v in value] + + request_headers = self.request.headers + + try: + auth_token = self.get_auth_token(request_headers) + except UnauhorizedError as e: + self.respond(*self._transport.return_error_msg(403, "Access denied: " + e.args[0])) + return + + auth_enabled = inmanta_config.Config.get("server", "auth", False) + if not auth_enabled or auth_token is not None: + result = yield self._transport._execute_call(kwargs, http_method, call_config, + message, request_headers, auth_token) + self.respond(*result) + else: + self.respond(*self._transport.return_error_msg(401, "Access to this resource is unauthorized.")) + except ValueError: + LOGGER.exception("An exception occured") + self.respond(*self._transport.return_error_msg(500, "Unable to decode request body")) + + @gen.coroutine + def head(self, *args, **kwargs): + yield self._call(http_method="HEAD", call_config=self._get_config("HEAD"), kwargs=kwargs) + + @gen.coroutine + def get(self, *args, **kwargs): + yield self._call(http_method="GET", call_config=self._get_config("GET"), kwargs=kwargs) + + @gen.coroutine + def post(self, *args, **kwargs): + yield self._call(http_method="POST", call_config=self._get_config("POST"), kwargs=kwargs) + + @gen.coroutine + def delete(self, *args, **kwargs): + yield self._call(http_method="DELETE", call_config=self._get_config("DELETE"), kwargs=kwargs) + + @gen.coroutine + def patch(self, *args, **kwargs): + yield self._call(http_method="PATCH", call_config=self._get_config("PATCH"), kwargs=kwargs) + + @gen.coroutine + def put(self, *args, **kwargs): + yield self._call(http_method="PUT", call_config=self._get_config("PUT"), kwargs=kwargs) + + @gen.coroutine + def options(self, *args, **kwargs): + allow_headers = "Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token" + if len(self._transport.headers): + allow_headers += ", " + ", ".join(self._transport.headers) + + self.set_header("Access-Control-Allow-Origin", "*") + self.set_header("Access-Control-Allow-Methods", "HEAD, GET, POST, PUT, OPTIONS, DELETE, PATCH") + self.set_header("Access-Control-Allow-Headers", allow_headers) + + self.set_status(200) + + +class StaticContentHandler(tornado.web.RequestHandler): + def initialize(self, transport: "RESTServer", content, content_type): + self._transport = transport + self._content = content + self._content_type = content_type + + def get(self, *args, **kwargs): + self.set_header("Content-Type", self._content_type) + self.write(self._content) + self.set_status(200) diff --git a/src/inmanta/server/server.py b/src/inmanta/server/server.py index 8dace733c9..aa9d19c5df 100644 --- a/src/inmanta/server/server.py +++ b/src/inmanta/server/server.py @@ -39,13 +39,13 @@ from inmanta import const from inmanta import data, config from inmanta import methods -from inmanta import protocol +from inmanta.server import protocol, SLICE_SERVER from inmanta.ast import type from inmanta.resources import Id from inmanta.server import config as opt -from inmanta.server.agentmanager import AgentManager import json from inmanta.util import hash_file +from inmanta.protocol import encode_token LOGGER = logging.getLogger(__name__) agent_lock = locks.Lock() @@ -60,8 +60,9 @@ class Server(protocol.ServerSlice): """ def __init__(self, io_loop, database_host=None, database_port=None, agent_no_log=False): - super().__init__(io_loop=io_loop, name="server") + super().__init__(io_loop=io_loop, name=SLICE_SERVER) LOGGER.info("Starting server endpoint") + self._server_storage = self.check_storage() self._agent_no_log = agent_no_log @@ -80,8 +81,6 @@ def __init__(self, io_loop, database_host=None, database_port=None, agent_no_log self._fact_expire = opt.server_fact_expire.get() self._fact_renew = opt.server_fact_renew.get() - #self.add_end_point_name(self.node_name) - self.schedule(self.renew_expired_facts, self._fact_renew) self.schedule(self._purge_versions, opt.server_purge_version_interval.get()) @@ -1503,7 +1502,7 @@ def _recompile_environment(self, environment_id, update_repo=False, wait=0, meta cmd = inmanta_path + ["-vvv", "export", "-e", str(environment_id), "--server_address", server_address, "--server_port", opt.transport_port.get(), "--metadata", json.dumps(metadata)] if config.Config.get("server", "auth", False): - token = protocol.encode_token(["compiler", "api"], str(environment_id)) + token = encode_token(["compiler", "api"], str(environment_id)) cmd.append("--token") cmd.append(token) @@ -1773,4 +1772,4 @@ def create_token(self, env, client_types, idempotent): """ Create a new auth token for this environment """ - return 200, {"token": protocol.encode_token(client_types, str(env.id), idempotent)} + return 200, {"token": encode_token(client_types, str(env.id), idempotent)} diff --git a/src/inmanta/util.py b/src/inmanta/util.py index f24d85f700..b459532ac9 100644 --- a/src/inmanta/util.py +++ b/src/inmanta/util.py @@ -64,3 +64,48 @@ def hash_file(content): sha1sum.update(content) return sha1sum.hexdigest() + + +class Scheduler(object): + """ + An event scheduler class + """ + + def __init__(self, io_loop): + self._scheduled = set() + self._io_loop = io_loop + + def add_action(self, action, interval, initial_delay=None): + """ + Add a new action + + :param action A function to call periodically + :param interval The interval between execution of actions + :param initial_delay Delay to the first execution, default to interval + """ + + if initial_delay is None: + initial_delay = interval + + LOGGER.debug("Scheduling action %s every %d seconds with initial delay %d", action, interval, initial_delay) + + def action_function(): + LOGGER.info("Calling %s" % action) + if action in self._scheduled: + try: + action() + except Exception: + LOGGER.exception("Uncaught exception while executing scheduled action") + + finally: + self._io_loop.call_later(interval, action_function) + + self._io_loop.call_later(initial_delay, action_function) + self._scheduled.add(action) + + def remove(self, action): + """ + Remove a scheduled action + """ + if action in self._scheduled: + self._scheduled.remove(action) diff --git a/tests/conftest.py b/tests/conftest.py index d491bc71d5..9fb68ead26 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,7 +41,6 @@ from tornado import gen import re from tornado.ioloop import IOLoop -from inmanta.protocol import RESTServer from inmanta.server.bootloader import InmantaBootloader @@ -141,7 +140,6 @@ def server(inmanta_config, io_loop, mongo_db, mongo_client, motor): # causes handler failure IOLoop._instance = io_loop - from inmanta.server.server import Server state_dir = tempfile.mkdtemp() port = get_free_tcp_port() @@ -176,7 +174,6 @@ def server(inmanta_config, io_loop, mongo_db, mongo_client, motor): def server_multi(inmanta_config, io_loop, mongo_db, mongo_client, request): IOLoop._instance = io_loop - from inmanta.server.server import Server state_dir = tempfile.mkdtemp() ssl, auth = request.param diff --git a/tests/server_test.py b/tests/server_test.py deleted file mode 100644 index e9713f6759..0000000000 --- a/tests/server_test.py +++ /dev/null @@ -1,84 +0,0 @@ -""" - Copyright 2016 Inmanta - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - Contact: code@inmanta.com -""" - -import os -import tempfile -import shutil -import logging -import string -import random - -from mongobox.unittest import MongoTestCase -from inmanta import config -from inmanta.server import Server -from tornado.testing import AsyncTestCase -from inmanta.protocol import RESTServer - -LOGGER = logging.getLogger(__name__) -PORT = "45678" - - -class ServerTest(MongoTestCase, AsyncTestCase): - - def __init__(self, methodName='runTest'): # noqa: H803 - MongoTestCase.__init__(self, methodName) - AsyncTestCase.__init__(self, methodName) - - self.state_dir = None - self.server = None - - def setUp(self): - MongoTestCase.setUp(self) - AsyncTestCase.setUp(self) - - self.state_dir = tempfile.mkdtemp() - cfg = os.path.join(self.state_dir, "inmanta.cfg") - with open(cfg, "w"): - pass - config.Config.load_config(cfg) - config.Config.get("database", "name", "inmanta-" + ''.join(random.choice(string.ascii_letters) for _ in range(10))) - config.Config.set("config", "state-dir", self.state_dir) - config.Config.set("config", "log-dir", os.path.join(self.state_dir, "logs")) - config.Config.set("server_rest_transport", "port", PORT) - config.Config.set("agent_rest_transport", "port", PORT) - config.Config.set("compiler_rest_transport", "port", PORT) - config.Config.set("client_rest_transport", "port", PORT) - config.Config.set("cmdline_rest_transport", "port", PORT) - config.Config.set("config", "executable", os.path.abspath(os.path.join(__file__, "../../src/inmanta/app.py"))) - config.Config.set("server", "agent-timeout", "2") - - LOGGER.info("Starting server") - mongo_port = os.getenv('MONGOBOX_PORT') - if mongo_port is None: - raise Exception("MONGOBOX_PORT env variable not available. Make sure test are executed with --with-mongobox") - - self.rs = RESTServer() - self.server = Server(database_host="localhost", database_port=int(mongo_port), io_loop=self.io_loop) - self.rs.add_endpoint(self.server) - self.rs.start() - - def tearDown(self): - self.rs.stop() - # does not work with current pymongo - for db_name in self.mongo_client.database_names(): - self.mongo_client.drop_database(db_name) - # end fix - shutil.rmtree(self.state_dir) - - AsyncTestCase.tearDown(self) - MongoTestCase.tearDown(self) diff --git a/tests/test_2way_protocol.py b/tests/test_2way_protocol.py index e5410fad4e..ed7a9018e8 100644 --- a/tests/test_2way_protocol.py +++ b/tests/test_2way_protocol.py @@ -21,13 +21,16 @@ import uuid import colorlog -from inmanta import methods +from inmanta import methods, data from tornado import gen import pytest from tornado.gen import sleep from utils import retry_limited from tornado.ioloop import IOLoop -from inmanta.protocol import RESTServer +from inmanta.server.protocol import RESTServer, SessionListener, ServerSlice +from inmanta.server import SLICE_SESSION_MANAGER, server +from inmanta.methods import ENV_ARG +import importlib LOGGER = logging.getLogger(__name__) @@ -40,7 +43,7 @@ def get_status_x(self, tid: uuid.UUID): pass @methods.protocol(operation="GET", id=True, server_agent=True, timeout=10) - def get_agent_status(self, id): + def get_agent_status_x(self, id): pass @@ -48,38 +51,55 @@ def get_agent_status(self, id): from inmanta import protocol # NOQA -class Server(protocol.SessionEndpoint): +class SessionSpy(SessionListener, ServerSlice): - def __init__(self, name, io_loop, interval=60): - protocol.SessionEndpoint.__init__(self, name, io_loop, interval=interval) + def __init__(self): + ServerSlice.__init__(self, IOLoop.current(), "sessionspy") self.expires = 0 + self.__sessions = [] + + def new_session(self, session): + self.__sessions.append(session) @protocol.handle(StatusMethod.get_status_x) @gen.coroutine def get_status_x(self, tid): status_list = [] - for session in self._sessions.values(): + for session in self.__sessions: client = session.get_client() - status = yield client.get_agent_status("x") + status = yield client.get_agent_status_x("x") if status is not None and status.code == 200: status_list.append(status.result) return 200, {"agents": status_list} def expire(self, session, timeout): - protocol.SessionEndpoint.expire(self, session, timeout) + self.__sessions.remove(session) print(session._sid) self.expires += 1 + def get_sessions(self): + return self.__sessions + class Agent(protocol.AgentEndPoint): - @protocol.handle(StatusMethod.get_agent_status) + @protocol.handle(StatusMethod.get_agent_status_x) @gen.coroutine - def get_agent_status(self, id): + def get_agent_status_x(self, id): return 200, {"status": "ok", "agents": self.end_point_names} +importlib.reload(protocol) +importlib.reload(server.protocol) + + +@gen.coroutine +def get_environment(env: uuid.UUID, metadata: dict): + return data.Environment(from_mongo=True, _id=env, name="test", project=env, repo_url="xx", repo_branch="xx") + + +@pytest.mark.gen_test(timeout=30) def test_2way_protocol(free_port, logs=False): from inmanta.config import Config @@ -119,19 +139,26 @@ def test_2way_protocol(free_port, logs=False): Config.set("client_rest_transport", "port", free_port) Config.set("cmdline_rest_transport", "port", free_port) - io_loop = IOLoop.current() - rs = RESTServer() - server = Server("server", io_loop) - rs.add_endpoint(server) - rs.start() + # Disable validation of envs + old_get_env = ENV_ARG["getter"] + ENV_ARG["getter"] = get_environment + + try: + io_loop = IOLoop.current() + rs = RESTServer() + server = SessionSpy() + rs.get_endpoint(SLICE_SESSION_MANAGER).add_listener(server) + rs.add_endpoint(server) + rs.start() - agent = Agent("agent", io_loop) - agent.add_end_point_name("agent") - agent.set_environment(uuid.uuid4()) - agent.start() + agent = Agent("agent", io_loop) + agent.add_end_point_name("agent") + agent.set_environment(uuid.uuid4()) + agent.start() + + yield retry_limited(lambda: len(server.get_sessions()) == 1, 0.1) + assert len(server.get_sessions()) == 1 - @gen.coroutine - def do_call(): client = protocol.Client("client") status = yield client.get_status_x(str(agent.environment)) assert status.code == 200 @@ -141,24 +168,21 @@ def do_call(): server.stop() io_loop.stop() - io_loop.add_callback(do_call) - io_loop.add_timeout(io_loop.time() + 2, lambda: io_loop.stop()) - try: - io_loop.start() - except KeyboardInterrupt: - io_loop.stop() - rs.stop() - agent.stop() + rs.stop() + agent.stop() + finally: + ENV_ARG["getter"] = old_get_env @gen.coroutine def check_sessions(sessions): for s in sessions: - a = yield s.client.get_agent_status("X") + a = yield s.client.get_agent_status_x("X") assert a.get_result()['status'] == 'ok' @pytest.mark.slowtest +@pytest.mark.gen_test(timeout=30) def test_timeout(free_port): from inmanta.config import Config @@ -174,25 +198,31 @@ def test_timeout(free_port): Config.set("compiler_rest_transport", "port", free_port) Config.set("client_rest_transport", "port", free_port) Config.set("cmdline_rest_transport", "port", free_port) - - rs = RESTServer() - server = Server("server", io_loop, interval=2) - rs.add_endpoint(server) - rs.start() - - env = uuid.uuid4() - - # agent 1 - agent = Agent("agent", io_loop) - agent.add_end_point_name("agent") - agent.set_environment(env) - agent.start() + Config.set("server", "agent-timeout", "1") + + # Disable validation of envs + old_get_env = ENV_ARG["getter"] + ENV_ARG["getter"] = get_environment + + try: + + rs = RESTServer() + server = SessionSpy() + rs.get_endpoint(SLICE_SESSION_MANAGER).add_listener(server) + rs.add_endpoint(server) + rs.start() + + env = uuid.uuid4() + + # agent 1 + agent = Agent("agent", io_loop) + agent.add_end_point_name("agent") + agent.set_environment(env) + agent.start() - @gen.coroutine - def do_call(): # wait till up - yield retry_limited(lambda: len(server._sessions) == 1, 0.1) - assert len(server._sessions) == 1 + yield retry_limited(lambda: len(server.get_sessions()) == 1, 0.1) + assert len(server.get_sessions()) == 1 # agent 2 agent2 = Agent("agent", io_loop) @@ -201,14 +231,14 @@ def do_call(): agent2.start() # wait till up - yield retry_limited(lambda: len(server._sessions) == 2, 0.1) - assert len(server._sessions) == 2 + yield retry_limited(lambda: len(server.get_sessions()) == 2, 0.1) + assert len(server.get_sessions()) == 2 # see if it stays up - yield(check_sessions(server._sessions.values())) + yield(check_sessions(server.get_sessions())) yield sleep(2) - assert len(server._sessions) == 2 - yield(check_sessions(server._sessions.values())) + assert len(server.get_sessions()) == 2 + yield(check_sessions(server.get_sessions())) # take it down agent2.stop() @@ -216,20 +246,14 @@ def do_call(): # timout yield sleep(2) # check if down - assert len(server._sessions) == 1 - print(server._sessions) - yield(check_sessions(server._sessions.values())) + assert len(server.get_sessions()) == 1 + print(server.get_sessions()) + yield(check_sessions(server.get_sessions())) assert server.expires == 1 agent.stop() server.stop() - io_loop.stop() - - io_loop.add_callback(do_call) - io_loop.add_timeout(io_loop.time() + 2, lambda: io_loop.stop()) - try: - io_loop.start() - except KeyboardInterrupt: - io_loop.stop() - rs.stop() - agent.stop() + rs.stop() + agent.stop() + finally: + ENV_ARG["getter"] = old_get_env diff --git a/tests/test_server.py b/tests/test_server.py index 10c71d1408..33a564f99a 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -25,7 +25,7 @@ from inmanta.agent.agent import Agent from inmanta import data, protocol from inmanta import const -from inmanta.server import config as opt +from inmanta.server import config as opt, SLICE_AGENT_MANAGER, SLICE_SESSION_MANAGER from datetime import datetime from uuid import UUID from inmanta.export import upload_code @@ -43,11 +43,8 @@ def test_autostart(server, client, environment): env = yield data.Environment.get_by_id(uuid.UUID(environment)) yield env.set(data.AUTOSTART_AGENT_MAP, {"iaas_agent": "", "iaas_agentx": ""}) - agentmanager = server.get_endpoint("agentmanager") - serverendpoint = server.get_endpoint("server") - sessionendpoint = server.get_endpoint("session") - - + agentmanager = server.get_endpoint(SLICE_AGENT_MANAGER) + sessionendpoint = server.get_endpoint(SLICE_SESSION_MANAGER) yield agentmanager.ensure_agent_registered(env, "iaas_agent") yield agentmanager.ensure_agent_registered(env, "iaas_agentx") @@ -88,10 +85,8 @@ def test_autostart_dual_env(client, server): """ agentmanager = server.get_endpoint("server").agentmanager - serverendpoint = server.get_endpoint("server") sessionendpoint = server.get_endpoint("session") - result = yield client.create_project("env-test") assert result.code == 200 project_id = result.result["project"]["id"] @@ -128,13 +123,11 @@ def test_autostart_batched(client, server, environment): """ Test auto start of agent """ - sessionendpoint = server.get_endpoint("session") - env = yield data.Environment.get_by_id(uuid.UUID(environment)) yield env.set(data.AUTOSTART_AGENT_MAP, {"iaas_agent": "", "iaas_agentx": ""}) - agentmanager = server.get_endpoint("server").agentmanager - serverendpoint = server.get_endpoint("server") + agentmanager = server.get_endpoint(SLICE_AGENT_MANAGER) + sessionendpoint = server.get_endpoint(SLICE_SESSION_MANAGER) yield agentmanager.ensure_agent_registered(env, "iaas_agent") yield agentmanager.ensure_agent_registered(env, "iaas_agentx") diff --git a/tests/test_server_agent.py b/tests/test_server_agent.py index 6c79a7d232..cdcbfbba28 100644 --- a/tests/test_server_agent.py +++ b/tests/test_server_agent.py @@ -36,10 +36,9 @@ from inmanta.agent.agent import Agent from utils import retry_limited, assert_equal_ish, UNKWN from inmanta.config import Config -from inmanta.server.server import Server from inmanta.ast import CompilerException -from inmanta.protocol import RESTServer from inmanta.server.bootloader import InmantaBootloader +from inmanta.server import SLICE_AGENT_MANAGER logger = logging.getLogger("inmanta.test.server_agent") @@ -230,6 +229,9 @@ def test_dryrun_and_deploy(io_loop, server_multi, client_multi, resource_contain There is a second agent with an undefined resource. The server will shortcut the dryrun and deploy for this resource without an agent being present. """ + + agentmanager = server_multi.get_endpoint(SLICE_AGENT_MANAGER) + resource_container.Provider.reset() result = yield client_multi.create_project("env-test") project_id = result.result["project"]["id"] @@ -242,7 +244,7 @@ def test_dryrun_and_deploy(io_loop, server_multi, client_multi, resource_contain agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server_multi.get_endpoint("server").agentmanager.sessions) == 1, 10) + yield retry_limited(lambda: len(agentmanager.sessions) == 1, 10) resource_container.Provider.set("agent1", "key2", "incorrect_value") resource_container.Provider.set("agent1", "key3", "value") @@ -371,6 +373,8 @@ def test_server_restart(resource_container, io_loop, server, mongo_db, client): """ dryrun and deploy a configuration model """ + agentmanager = server.get_endpoint(SLICE_AGENT_MANAGER) + resource_container.Provider.reset() result = yield client.create_project("env-test") project_id = result.result["project"]["id"] @@ -382,7 +386,7 @@ def test_server_restart(resource_container, io_loop, server, mongo_db, client): code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 1, 10) + yield retry_limited(lambda: len(agentmanager.sessions) == 1, 10) resource_container.Provider.set("agent1", "key2", "incorrect_value") resource_container.Provider.set("agent1", "key3", "value") @@ -391,10 +395,9 @@ def test_server_restart(resource_container, io_loop, server, mongo_db, client): ibl = InmantaBootloader() server = ibl.restserver - ibl.start() - yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 1, 10) + yield retry_limited(lambda: len(agentmanager.sessions) == 1, 10) version = int(time.time()) @@ -495,6 +498,8 @@ def test_spontaneous_deploy(resource_container, io_loop, server, client): """ dryrun and deploy a configuration model """ + agentmanager = server.get_endpoint(SLICE_AGENT_MANAGER) + resource_container.Provider.reset() result = yield client.create_project("env-test") project_id = result.result["project"]["id"] @@ -509,7 +514,7 @@ def test_spontaneous_deploy(resource_container, io_loop, server, client): code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 1, 10) + yield retry_limited(lambda: len(agentmanager.sessions) == 1, 10) resource_container.Provider.set("agent1", "key2", "incorrect_value") resource_container.Provider.set("agent1", "key3", "value") @@ -781,6 +786,8 @@ def test_snapshot_restore(resource_container, client, server, io_loop): @pytest.mark.gen_test def test_server_agent_api(resource_container, client, server, io_loop): + agentmanager = server.get_endpoint(SLICE_AGENT_MANAGER) + result = yield client.create_project("env-test") project_id = result.result["project"]["id"] @@ -794,8 +801,8 @@ def test_server_agent_api(resource_container, client, server, io_loop): code_loader=False) agent.start() - yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 2, 10) - assert len(server.get_endpoint("server").agentmanager.sessions) == 2 + yield retry_limited(lambda: len(agentmanager.sessions) == 2, 10) + assert len(agentmanager.sessions) == 2 result = yield client.list_agent_processes(env_id) assert result.code == 200 @@ -1459,6 +1466,8 @@ def test_cross_agent_deps(resource_container, io_loop, server, client): """ deploy a configuration model with cross host dependency """ + agentmanager = server.get_endpoint(SLICE_AGENT_MANAGER) + resource_container.Provider.reset() # config for recovery mechanism Config.set("config", "agent-interval", "10") @@ -1472,13 +1481,13 @@ def test_cross_agent_deps(resource_container, io_loop, server, client): code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 1, 10) + yield retry_limited(lambda: len(agentmanager.sessions) == 1, 10) agent2 = Agent(io_loop, hostname="node2", environment=env_id, agent_map={"agent2": "localhost"}, code_loader=False) agent2.add_end_point_name("agent2") agent2.start() - yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 2, 10) + yield retry_limited(lambda: len(agentmanager.sessions) == 2, 10) resource_container.Provider.set("agent1", "key2", "incorrect_value") resource_container.Provider.set("agent1", "key3", "value") @@ -1564,6 +1573,8 @@ def test_dryrun_scale(resource_container, io_loop, server, client): """ test dryrun scaling """ + agentmanager = server.get_endpoint(SLICE_AGENT_MANAGER) + resource_container.Provider.reset() result = yield client.create_project("env-test") project_id = result.result["project"]["id"] @@ -1575,7 +1586,7 @@ def test_dryrun_scale(resource_container, io_loop, server, client): code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 1, 10) + yield retry_limited(lambda: len(agentmanager.sessions) == 1, 10) version = int(time.time()) @@ -1622,12 +1633,14 @@ def test_send_events(resource_container, io_loop, environment, server, client): """ Send and receive events within one agent """ + agentmanager = server.get_endpoint(SLICE_AGENT_MANAGER) + resource_container.Provider.reset() agent = Agent(io_loop, hostname="node1", environment=environment, agent_map={"agent1": "localhost"}, code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 1, 10) + yield retry_limited(lambda: len(agentmanager.sessions) == 1, 10) version = int(time.time()) @@ -1685,18 +1698,20 @@ def test_send_events_cross_agent(resource_container, io_loop, environment, serve """ Send and receive events over agents """ + agentmanager = server.get_endpoint(SLICE_AGENT_MANAGER) + resource_container.Provider.reset() agent = Agent(io_loop, hostname="node1", environment=environment, agent_map={"agent1": "localhost"}, code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 1, 10) + yield retry_limited(lambda: len(agentmanager.sessions) == 1, 10) agent2 = Agent(io_loop, hostname="node2", environment=environment, agent_map={"agent2": "localhost"}, code_loader=False) agent2.add_end_point_name("agent2") agent2.start() - yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 2, 10) + yield retry_limited(lambda: len(agentmanager.sessions) == 2, 10) version = int(time.time()) @@ -1758,12 +1773,14 @@ def test_send_events_cross_agent_restart(resource_container, io_loop, environmen """ Send and receive events over agents with agents starting after deploy """ + agentmanager = server.get_endpoint(SLICE_AGENT_MANAGER) + resource_container.Provider.reset() agent2 = Agent(io_loop, hostname="node2", environment=environment, agent_map={"agent2": "localhost"}, code_loader=False) agent2.add_end_point_name("agent2") agent2.start() - yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 1, 10) + yield retry_limited(lambda: len(agentmanager.sessions) == 1, 10) version = int(time.time()) @@ -1813,7 +1830,7 @@ def test_send_events_cross_agent_restart(resource_container, io_loop, environmen code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 2, 10) + yield retry_limited(lambda: len(agentmanager.sessions) == 2, 10) while (result.result["model"]["total"] - result.result["model"]["done"]) > 0: result = yield client.get_version(environment, version) @@ -1839,12 +1856,14 @@ def test_auto_deploy(io_loop, server, client, resource_container, environment): """ dryrun and deploy a configuration model automatically """ + agentmanager = server.get_endpoint(SLICE_AGENT_MANAGER) + resource_container.Provider.reset() agent = Agent(io_loop, hostname="node1", environment=environment, agent_map={"agent1": "localhost"}, code_loader=False) agent.add_end_point_name("agent1") agent.start() - yield retry_limited(lambda: len(server.get_endpoint("server").agentmanager.sessions) == 1, 10) + yield retry_limited(lambda: len(agentmanager.sessions) == 1, 10) resource_container.Provider.set("agent1", "key2", "incorrect_value") resource_container.Provider.set("agent1", "key3", "value") From c2c3bd098c2530e5f7f36268778df8069d300395 Mon Sep 17 00:00:00 2001 From: Wouter De Borger Date: Tue, 24 Apr 2018 13:22:49 +0200 Subject: [PATCH 10/14] fixed agentmanager test --- src/inmanta/server/agentmanager.py | 4 ++-- tests/test_agent_manager.py | 17 +++++++++++------ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/inmanta/server/agentmanager.py b/src/inmanta/server/agentmanager.py index 3c35842e15..2c8decd52a 100644 --- a/src/inmanta/server/agentmanager.py +++ b/src/inmanta/server/agentmanager.py @@ -383,8 +383,8 @@ def list_agent_processes(self, environment, expired): @protocol.handle(methods.ServerAgentApiMethod.list_agents, env="tid") @gen.coroutine def list_agents(self, env): - tid = env.id - if tid is not None: + if env is not None: + tid = env.id ags = yield data.Agent.get_list(environment=tid) else: ags = yield data.Agent.get_list() diff --git a/tests/test_agent_manager.py b/tests/test_agent_manager.py index 6710afd120..a0637a77b4 100644 --- a/tests/test_agent_manager.py +++ b/tests/test_agent_manager.py @@ -24,6 +24,7 @@ from tornado import gen from inmanta.protocol import Result from utils import assert_equal_ish, UNKWN +from collections import namedtuple class Collector(object): @@ -84,6 +85,7 @@ def test_primary_selection(motor): futures = Collector() server.add_future.side_effect = futures am = AgentManager(server, False) + am.add_future = futures @gen.coroutine def assert_agent(name: str, state: str, sid: UUID): @@ -134,7 +136,7 @@ def assert_agents(s1, s2, s3, sid1=None, sid2=None, sid3=None): yield assert_agents("paused", "up", "up", sid2=ts1.id, sid3=ts2.id) # expire first - am.expire(ts1) + am.expire(ts1, 100) yield futures.proccess() assert len(am.sessions) == 1 ts2.get_client().set_state.assert_called_with("agent2", True) @@ -142,7 +144,7 @@ def assert_agents(s1, s2, s3, sid1=None, sid2=None, sid3=None): yield assert_agents("paused", "up", "up", sid2=ts2.id, sid3=ts2.id) # expire second - am.expire(ts2) + am.expire(ts2, 100) yield futures.proccess() assert len(am.sessions) == 0 yield assert_agents("paused", "down", "down") @@ -165,6 +167,7 @@ def test_api(motor): futures = Collector() server.add_future.side_effect = futures am = AgentManager(server, False) + am.add_future = futures # one session ts1 = MockSession(uuid4(), env.id, ["agent1", "agent2"], "ts1") @@ -247,7 +250,7 @@ def dummy_status(): 'environment': env2.id, "state": "down"}]} assert_equal_ish(shouldbe, all_agents, ['name']) - code, all_agents = yield am.list_agents(env2.id) + code, all_agents = yield am.list_agents(env2) assert code == 200 shouldbe = { 'agents': [{'name': 'agent4', 'paused': False, 'last_failover': '', 'primary': '', @@ -269,6 +272,7 @@ def test_db_clean(motor): futures = Collector() server.add_future.side_effect = futures am = AgentManager(server, False) + am.add_future = futures @gen.coroutine def assert_agent(name: str, state: str, sid: UUID): @@ -318,7 +322,7 @@ def assert_agents(s1, s2, s3, sid1=None, sid2=None, sid3=None): yield assert_agents("paused", "up", "up", sid2=ts1.id, sid3=ts2.id) # expire first - am.expire(ts1) + am.expire(ts1, 100) yield futures.proccess() assert len(am.sessions) == 1 ts2.get_client().set_state.assert_called_with("agent2", True) @@ -327,6 +331,7 @@ def assert_agents(s1, s2, s3, sid1=None, sid2=None, sid3=None): # failover am = AgentManager(server, False) + am.add_future = futures yield am.clean_db() # one session @@ -354,7 +359,7 @@ def assert_agents(s1, s2, s3, sid1=None, sid2=None, sid3=None): yield assert_agents("paused", "up", "up", sid2=ts1.id, sid3=ts2.id) # expire first - am.expire(ts1) + am.expire(ts1, 100) yield futures.proccess() assert len(am.sessions) == 1 ts2.get_client().set_state.assert_called_with("agent2", True) @@ -362,7 +367,7 @@ def assert_agents(s1, s2, s3, sid1=None, sid2=None, sid3=None): yield assert_agents("paused", "up", "up", sid2=ts2.id, sid3=ts2.id) # expire second - am.expire(ts2) + am.expire(ts2, 100) yield futures.proccess() assert len(am.sessions) == 0 yield assert_agents("paused", "down", "down") From 8c58e23a8cfe023d6c4c5074af3200b61d0ffe5f Mon Sep 17 00:00:00 2001 From: Wouter De Borger Date: Tue, 24 Apr 2018 13:27:26 +0200 Subject: [PATCH 11/14] test case cleanup --- tests/test_agent.py | 5 +++-- tests/test_agent_manager.py | 1 - tests/test_server_agent.py | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_agent.py b/tests/test_agent.py index 5383e44c32..9f662e8dde 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -19,6 +19,7 @@ import pytest from utils import retry_limited from inmanta.agent import reporting +from inmanta.server import SLICE_SESSION_MANAGER @pytest.mark.slowtest @@ -29,8 +30,8 @@ def test_agent_get_status(io_loop, server, environment): myagent.add_end_point_name("agent1") myagent.start() - yield retry_limited(lambda: len(server.get_endpoint("server")._sessions) == 1, 0.5) - clients = server.get_endpoint("server")._sessions.values() + yield retry_limited(lambda: len(server.get_endpoint(SLICE_SESSION_MANAGER)._sessions) == 1, 0.5) + clients = server.get_endpoint(SLICE_SESSION_MANAGER)._sessions.values() assert len(clients) == 1 clients = [x for x in clients] client = clients[0].get_client() diff --git a/tests/test_agent_manager.py b/tests/test_agent_manager.py index a0637a77b4..c36c724008 100644 --- a/tests/test_agent_manager.py +++ b/tests/test_agent_manager.py @@ -24,7 +24,6 @@ from tornado import gen from inmanta.protocol import Result from utils import assert_equal_ish, UNKWN -from collections import namedtuple class Collector(object): diff --git a/tests/test_server_agent.py b/tests/test_server_agent.py index cdcbfbba28..2ebfa3d538 100644 --- a/tests/test_server_agent.py +++ b/tests/test_server_agent.py @@ -396,6 +396,7 @@ def test_server_restart(resource_container, io_loop, server, mongo_db, client): ibl = InmantaBootloader() server = ibl.restserver ibl.start() + agentmanager = server.get_endpoint(SLICE_AGENT_MANAGER) yield retry_limited(lambda: len(agentmanager.sessions) == 1, 10) From 7560a48f08a8f1e7bd119949db2de37e6e5906b8 Mon Sep 17 00:00:00 2001 From: Wouter De Borger Date: Tue, 24 Apr 2018 13:37:25 +0200 Subject: [PATCH 12/14] cleanup print statements --- src/inmanta/server/protocol.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/inmanta/server/protocol.py b/src/inmanta/server/protocol.py index ad30b2cb91..27a7fe671b 100644 --- a/src/inmanta/server/protocol.py +++ b/src/inmanta/server/protocol.py @@ -84,14 +84,10 @@ def create_op_mapping(self): url = self._create_base_url(properties) properties["api_version"] = "1" url_map[url][properties["operation"]] = (properties, call, method.__wrapped__) - print(url) - url = self._create_base_url(properties, versioned=False) properties = properties.copy() properties["api_version"] = None url_map[url][properties["operation"]] = (properties, call, method.__wrapped__) - print(url) - return url_map def start(self): From 41fefa203aabf8732d81e433761bc7e65d515e7c Mon Sep 17 00:00:00 2001 From: Bart Vanbrabant Date: Fri, 27 Apr 2018 10:08:56 +0200 Subject: [PATCH 13/14] Header fixes --- src/inmanta/protocol.py | 2 +- src/inmanta/server/__init__.py | 6 +++++- src/inmanta/server/agentmanager.py | 2 +- src/inmanta/server/bootloader.py | 4 ++++ src/inmanta/server/config.py | 2 +- src/inmanta/server/protocol.py | 4 ++++ src/inmanta/server/server.py | 2 +- 7 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/inmanta/protocol.py b/src/inmanta/protocol.py index 8086e2fe7f..b81664c81f 100644 --- a/src/inmanta/protocol.py +++ b/src/inmanta/protocol.py @@ -1,5 +1,5 @@ """ - Copyright 2017 Inmanta + Copyright 2018 Inmanta Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/inmanta/server/__init__.py b/src/inmanta/server/__init__.py index 0f0df24707..3de52f0d73 100644 --- a/src/inmanta/server/__init__.py +++ b/src/inmanta/server/__init__.py @@ -1,14 +1,18 @@ """ - Copyright 2017 Inmanta + Copyright 2018 Inmanta + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + Contact: code@inmanta.com """ diff --git a/src/inmanta/server/agentmanager.py b/src/inmanta/server/agentmanager.py index 51d119e004..2f70b3b901 100644 --- a/src/inmanta/server/agentmanager.py +++ b/src/inmanta/server/agentmanager.py @@ -1,5 +1,5 @@ """ - Copyright 2017 Inmanta + Copyright 2018 Inmanta Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/inmanta/server/bootloader.py b/src/inmanta/server/bootloader.py index 36ec28aafe..4b8bf28525 100644 --- a/src/inmanta/server/bootloader.py +++ b/src/inmanta/server/bootloader.py @@ -1,14 +1,18 @@ """ Copyright 2018 Inmanta + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + Contact: code@inmanta.com """ from tornado.ioloop import IOLoop diff --git a/src/inmanta/server/config.py b/src/inmanta/server/config.py index 6738f9cf33..0eb543794e 100644 --- a/src/inmanta/server/config.py +++ b/src/inmanta/server/config.py @@ -1,5 +1,5 @@ """ - Copyright 2017 Inmanta + Copyright 2018 Inmanta Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/inmanta/server/protocol.py b/src/inmanta/server/protocol.py index 27a7fe671b..e7b3bf27f9 100644 --- a/src/inmanta/server/protocol.py +++ b/src/inmanta/server/protocol.py @@ -1,14 +1,18 @@ """ Copyright 2018 Inmanta + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + Contact: code@inmanta.com """ from inmanta.util import Scheduler diff --git a/src/inmanta/server/server.py b/src/inmanta/server/server.py index 8f0fb4d3fd..6f8e5905b5 100644 --- a/src/inmanta/server/server.py +++ b/src/inmanta/server/server.py @@ -1,5 +1,5 @@ """ - Copyright 2017 Inmanta + Copyright 2018 Inmanta Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From eff15a903a0a35462e9add2deec3d773cfcafcbc Mon Sep 17 00:00:00 2001 From: Bart Vanbrabant Date: Fri, 27 Apr 2018 10:30:33 +0200 Subject: [PATCH 14/14] Remove unused imports --- src/inmanta/deploy.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/inmanta/deploy.py b/src/inmanta/deploy.py index 2a9cc42c70..1f9fb7da72 100644 --- a/src/inmanta/deploy.py +++ b/src/inmanta/deploy.py @@ -24,8 +24,7 @@ import socket from mongobox import mongobox -from tornado import gen, process -from inmanta import module, config, agent, protocol, const, data +from inmanta import module, config, protocol, const, data LOGGER = logging.getLogger(__name__)