From 455b1977e58a055f4178172b28f97bc1e559e0d9 Mon Sep 17 00:00:00 2001 From: goedzo Date: Wed, 19 Jul 2017 17:36:06 +0200 Subject: [PATCH 1/7] This eventhandler update writes also to a json file in web folder With this change, the interesting events are also written to a json file in web folder. This allows the web-ui to display the event messages on screen. With this the web-ui can now see in detail what the bot is actually doing by refreshing the json file. With self.MaxEventlog = 50 you can limit the amount of lines to save on disk. This to avoid the json file gets way to large to deal with! --- pokecli.py | 2 +- pokemongo_bot/__init__.py | 8 +-- pokemongo_bot/event_manager.py | 127 ++++++++++++++++++++++++++++++++- 3 files changed, 130 insertions(+), 7 deletions(-) diff --git a/pokecli.py b/pokecli.py index 23d726d1c5..d46a28d327 100644 --- a/pokecli.py +++ b/pokecli.py @@ -144,7 +144,7 @@ def setup_logging(config): handler.setFormatter(formatter) def start_bot(bot, config): - bot.start() + bot.start(bot) initialize_task(bot, config) bot.metrics.capture_stats() bot.health_record = BotEvent(config) diff --git a/pokemongo_bot/__init__.py b/pokemongo_bot/__init__.py index 769b61f47d..a451840039 100644 --- a/pokemongo_bot/__init__.py +++ b/pokemongo_bot/__init__.py @@ -158,8 +158,8 @@ def __init__(self, db, config): saved_info[key] = self.config.client_id saved_info.close() - def start(self): - self._setup_event_system() + def start(self, bot): + self._setup_event_system(bot) self.sleep_schedule = SleepSchedule(self, self.config.sleep_schedule) if self.config.sleep_schedule else None if self.sleep_schedule: self.sleep_schedule.work() @@ -174,7 +174,7 @@ def start(self): random.seed() - def _setup_event_system(self): + def _setup_event_system(self, bot): handlers = [] color = self.config.logging and 'color' in self.config.logging and self.config.logging['color'] @@ -199,7 +199,7 @@ def _setup_event_system(self): remote_control = WebsocketRemoteControl(self).start() # @var EventManager - self.event_manager = EventManager(self.config.walker_limit_output, *handlers) + self.event_manager = EventManager(bot, self.config.walker_limit_output, *handlers) self._register_events() if self.config.show_events: self.event_manager.event_report() diff --git a/pokemongo_bot/event_manager.py b/pokemongo_bot/event_manager.py index 2be9f02f66..1983085b75 100644 --- a/pokemongo_bot/event_manager.py +++ b/pokemongo_bot/event_manager.py @@ -3,6 +3,18 @@ from __future__ import print_function from sys import stdout +from datetime import datetime, timedelta +import time +import json +import os + + +from pokemongo_bot.base_dir import _base_dir + + +class FileIOException(Exception): + pass + class EventNotRegisteredException(Exception): pass @@ -20,15 +32,122 @@ def __init__(self): def handle_event(self, event, kwargs): raise NotImplementedError("Please implement") - + +class Event(object): + """ + Representation of an event. + """ + def __init__(self, event, sender=None, level='info', formatted='', data={}): + """ + Representation of an Event + :return: An Event + :rtype: Event + """ + t = datetime.today() + self.timestamp = t.strftime('%Y-%m-%d %H:%M:%S') + self.event = str(event).encode('ascii', 'xmlcharrefreplace') + if sender==None: + self.sender = sender + else: + self.sender = str(sender).encode('ascii', 'xmlcharrefreplace') + + self.level = str(level).encode('ascii', 'xmlcharrefreplace') + self.formatted = str(formatted).encode('ascii', 'xmlcharrefreplace') + self.data = str(data).encode('ascii', 'xmlcharrefreplace') + self.friendly_msg = "" + + if not formatted: + self.friendly_msg = self.data + else: + #Format the data + self.friendly_msg = formatted.format(**data) + + def __str__(self): + return self.timestamp + ": [" + str(self.event) + "] " + str(self.data) + + +class Events(object): + def __init__(self, bot): + self._events = [] + self.MaxEventlog = 50 + self.bot = bot + self._username = self.bot.config.username + #Hardcoded to avoid the file is being flooded on disk + self._write_debug = False + + def retrieve_data(self): + return self._events + + def get(self, number): + return self._events[number] + + def all(self): + return list(self._data.values()) + + def remove_event_by_num(self, number): + del self._events[number] + + + def add_event(self, event): + if event.level=="debug" and self._write_debug==False: + return + else: + self._events.append(event) + #Check if we exceed the max log entries + if len(self._events) > self.MaxEventlog: + self.remove_event_by_num(0) + #Write file to disk + self.update_web_event() + + def init_event_outfile(self): + web_event = os.path.join(_base_dir, "web", "events-%s.json" % self._username) + + if not os.path.exists(web_event): + self.bot.logger.info('No events file %s found. Creating a new one' % web_event) + + json_events = [] + + with open(web_event, "w") as outfile: + json.dump(json_events, outfile) + + def update_web_event(self): + web_event = os.path.join(_base_dir, "web", "events-%s.json" % self._username) + + if not os.path.exists(web_event): + self.init_event_outfile() + + json_events = self.jsonify_events() + #self.bot.logger.info('####### Writing %s' % json_events) + + try: + with open(web_event, "w") as outfile: + json.dump(json_events, outfile) + except (IOError, ValueError) as e: + self.bot.logger.info('[x] Error while opening events file for write: %s' % e, 'red') + except: + raise FileIOException("Unexpected error writing to {}".web_event) + + def jsonify_events(self): + json_events = [] + for event in self._events: + json_events.append({"event": {"timestamp": event.timestamp, "friendly_msg": event.friendly_msg, "event": event.event, "level": event.level, "formatted": event.formatted, "data": event.data}}) + return json_events + + class EventManager(object): - def __init__(self, limit_output=False, *handlers): + def __init__(self, bot ,limit_output=False, *handlers): + + self._registered_events = dict() self._handlers = list(handlers) or [] self._last_event = None self._limit_output = limit_output + self.bot = bot + self._EventLog = Events(self.bot) + + def event_report(self): for event, parameters in self._registered_events.iteritems(): print('-'*80) @@ -75,3 +194,7 @@ def emit(self, event, sender=None, level='info', formatted='', data={}): # send off to the handlers for handler in self._handlers: handler.handle_event(event, sender, level, formatted_msg, data) + + #Log the event in the event_log + l_event = Event(event, sender, level, formatted, data) + self._EventLog.add_event(l_event) From 3f99f72a485c4ff5713674986248f222c5a9958d Mon Sep 17 00:00:00 2001 From: goedzo Date: Wed, 19 Jul 2017 17:57:14 +0200 Subject: [PATCH 2/7] Fixed automatic build script test Fixed test --- tests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index 17f4615145..6c5d38777a 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -34,7 +34,7 @@ class FakeBot(PokemonGoBot): def __init__(self): self.config = MagicMock(websocket_server_url=False, show_events=False) self.api = FakeApi() - self.event_manager = EventManager() + self.event_manager = EventManager(PokemonGoBot) self._setup_event_system() def updateConfig(self, conf): From d2cf3f0e860eb4fa5253d1655bea7bb4aeebcf93 Mon Sep 17 00:00:00 2001 From: goedzo Date: Wed, 19 Jul 2017 18:07:02 +0200 Subject: [PATCH 3/7] Fixing other auto test errors --- pokemongo_bot/test/log_interval_test.py | 2 +- tests/__init__.py | 4 ++-- tests/location_parser_test.py | 2 +- tests/update_live_stats_test.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pokemongo_bot/test/log_interval_test.py b/pokemongo_bot/test/log_interval_test.py index 9b74dae373..604e363630 100644 --- a/pokemongo_bot/test/log_interval_test.py +++ b/pokemongo_bot/test/log_interval_test.py @@ -18,7 +18,7 @@ class LogDelayTestCase(unittest.TestCase): } def setUp(self): - self.bot = FakeBot() + self.bot = FakeBot(self.bot) self.bot.event_manager = MagicMock() self.worker = MoveToFort(self.bot, self.config) diff --git a/tests/__init__.py b/tests/__init__.py index 6c5d38777a..96af041791 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -31,10 +31,10 @@ def create_request(self, return_value='mock return'): return request class FakeBot(PokemonGoBot): - def __init__(self): + def __init__(self,bot): self.config = MagicMock(websocket_server_url=False, show_events=False) self.api = FakeApi() - self.event_manager = EventManager(PokemonGoBot) + self.event_manager = EventManager(bot) self._setup_event_system() def updateConfig(self, conf): diff --git a/tests/location_parser_test.py b/tests/location_parser_test.py index d59218bf57..c07ea6a9c0 100644 --- a/tests/location_parser_test.py +++ b/tests/location_parser_test.py @@ -9,7 +9,7 @@ class TestLocationParser(unittest.TestCase): def setUp(self): - self.bot = FakeBot() + self.bot = FakeBot(self.bot) config = dict( test=False, location='Paris', diff --git a/tests/update_live_stats_test.py b/tests/update_live_stats_test.py index f9e518e43e..31e0bb036b 100644 --- a/tests/update_live_stats_test.py +++ b/tests/update_live_stats_test.py @@ -26,7 +26,7 @@ class UpdateLiveStatsTestCase(unittest.TestCase): } def setUp(self): - self.bot = FakeBot() + self.bot = FakeBot(self.bot) self.bot._player = {'username': 'Username'} self.bot.config.username = 'Login' self.worker = UpdateLiveStats(self.bot, self.config) From b4baed8cbfddb43bcfa65e1dad70409ccec77282 Mon Sep 17 00:00:00 2001 From: goedzo Date: Wed, 19 Jul 2017 18:14:08 +0200 Subject: [PATCH 4/7] Fixing build test --- pokemongo_bot/event_manager.py | 5 ++++- pokemongo_bot/test/log_interval_test.py | 2 +- tests/__init__.py | 5 +++-- tests/location_parser_test.py | 2 +- tests/update_live_stats_test.py | 2 +- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pokemongo_bot/event_manager.py b/pokemongo_bot/event_manager.py index 1983085b75..462905fb22 100644 --- a/pokemongo_bot/event_manager.py +++ b/pokemongo_bot/event_manager.py @@ -71,7 +71,10 @@ def __init__(self, bot): self._events = [] self.MaxEventlog = 50 self.bot = bot - self._username = self.bot.config.username + if bot==None: + self._username = "TESTBUILD" + else: + self._username = self.bot.config.username #Hardcoded to avoid the file is being flooded on disk self._write_debug = False diff --git a/pokemongo_bot/test/log_interval_test.py b/pokemongo_bot/test/log_interval_test.py index 604e363630..9b74dae373 100644 --- a/pokemongo_bot/test/log_interval_test.py +++ b/pokemongo_bot/test/log_interval_test.py @@ -18,7 +18,7 @@ class LogDelayTestCase(unittest.TestCase): } def setUp(self): - self.bot = FakeBot(self.bot) + self.bot = FakeBot() self.bot.event_manager = MagicMock() self.worker = MoveToFort(self.bot, self.config) diff --git a/tests/__init__.py b/tests/__init__.py index 96af041791..fbafecf697 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -31,10 +31,11 @@ def create_request(self, return_value='mock return'): return request class FakeBot(PokemonGoBot): - def __init__(self,bot): + def __init__(self): self.config = MagicMock(websocket_server_url=False, show_events=False) self.api = FakeApi() - self.event_manager = EventManager(bot) + self.bot = bot + self.event_manager = EventManager(None) self._setup_event_system() def updateConfig(self, conf): diff --git a/tests/location_parser_test.py b/tests/location_parser_test.py index c07ea6a9c0..d59218bf57 100644 --- a/tests/location_parser_test.py +++ b/tests/location_parser_test.py @@ -9,7 +9,7 @@ class TestLocationParser(unittest.TestCase): def setUp(self): - self.bot = FakeBot(self.bot) + self.bot = FakeBot() config = dict( test=False, location='Paris', diff --git a/tests/update_live_stats_test.py b/tests/update_live_stats_test.py index 31e0bb036b..f9e518e43e 100644 --- a/tests/update_live_stats_test.py +++ b/tests/update_live_stats_test.py @@ -26,7 +26,7 @@ class UpdateLiveStatsTestCase(unittest.TestCase): } def setUp(self): - self.bot = FakeBot(self.bot) + self.bot = FakeBot() self.bot._player = {'username': 'Username'} self.bot.config.username = 'Login' self.worker = UpdateLiveStats(self.bot, self.config) From 5111f16d8a3cb0a7b1bf9a64cd3d6176bc59233e Mon Sep 17 00:00:00 2001 From: goedzo Date: Wed, 19 Jul 2017 18:17:48 +0200 Subject: [PATCH 5/7] Another build test issue --- tests/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index fbafecf697..23ffccc370 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -34,7 +34,6 @@ class FakeBot(PokemonGoBot): def __init__(self): self.config = MagicMock(websocket_server_url=False, show_events=False) self.api = FakeApi() - self.bot = bot self.event_manager = EventManager(None) self._setup_event_system() From 7c709251d8d21afd4722ed002c3a2a7dab299936 Mon Sep 17 00:00:00 2001 From: goedzo Date: Wed, 19 Jul 2017 18:22:31 +0200 Subject: [PATCH 6/7] Trying again --- tests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index 23ffccc370..83fc28695e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -35,7 +35,7 @@ def __init__(self): self.config = MagicMock(websocket_server_url=False, show_events=False) self.api = FakeApi() self.event_manager = EventManager(None) - self._setup_event_system() + self._setup_event_system(None) def updateConfig(self, conf): self.config.__dict__.update(conf) From 47d976c9a796bf14daec186a9ab42f1e65004d89 Mon Sep 17 00:00:00 2001 From: goedzo Date: Wed, 19 Jul 2017 18:32:09 +0200 Subject: [PATCH 7/7] Don't save file on disk when in testbuild --- pokemongo_bot/event_manager.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pokemongo_bot/event_manager.py b/pokemongo_bot/event_manager.py index 462905fb22..1902a5790f 100644 --- a/pokemongo_bot/event_manager.py +++ b/pokemongo_bot/event_manager.py @@ -92,6 +92,10 @@ def remove_event_by_num(self, number): def add_event(self, event): + #Do not log anything on disk when in Jenkins test build + if self._username == "TESTBUILD": + return + if event.level=="debug" and self._write_debug==False: return else: