From 0fd5dfe3977046c2b47378bed55cee4e0318fb04 Mon Sep 17 00:00:00 2001 From: Gunther Cox Date: Sat, 27 Jan 2018 13:04:08 -0500 Subject: [PATCH] Convert benchmark tests into unit tests --- tests/benchmarks.py | 93 ----------------------- tests/test_benchmarks.py | 158 +++++++++++++++++++++++++++++++++++++++ tox.ini | 6 -- 3 files changed, 158 insertions(+), 99 deletions(-) delete mode 100644 tests/benchmarks.py create mode 100644 tests/test_benchmarks.py diff --git a/tests/benchmarks.py b/tests/benchmarks.py deleted file mode 100644 index 4e759d301..000000000 --- a/tests/benchmarks.py +++ /dev/null @@ -1,93 +0,0 @@ -""" -Run this file to generate benchmark data for various -chat bot configurations. -""" - -from chatterbot import ChatBot -from chatterbot import utils -from pymongo.errors import ServerSelectionTimeoutError -from pymongo import MongoClient - - -BASE_CONFIGURATION = { - 'trainer': 'chatterbot.trainers.ListTrainer' -} - -CONFIGURATIONS = [ - { - 'description': 'Test the levenshtein distance comparison algorithm and file storage', - 'logic_adapters': [ - { - 'import_path': 'chatterbot.logic.BestMatch', - 'statement_comparison_function': 'chatterbot.comparisons.levenshtein_distance', - 'response_selection_method': 'chatterbot.response_selection.get_first_response' - } - ], - 'storage_adapter': { - 'import_path': 'chatterbot.storage.SQLStorageAdapter' - }, - }, - { - 'description': 'Test the synset distance comparison algorithm and file storage', - 'logic_adapters': [ - { - 'import_path': 'chatterbot.logic.BestMatch', - 'statement_comparison_function': 'chatterbot.comparisons.synset_distance', - 'response_selection_method': 'chatterbot.response_selection.get_first_response' - } - ], - 'storage_adapter': { - 'import_path': 'chatterbot.storage.SQLStorageAdapter' - }, - } -] - -# Skip these tests if a mongo client is not running -try: - client = MongoClient( - serverSelectionTimeoutMS=0.01 - ) - client.server_info() - - CONFIGURATIONS.extend([ - { - 'description': 'Test the levenshtein distance comparison algorithm and Mongo DB storage', - 'logic_adapters': [ - { - 'import_path': 'chatterbot.logic.BestMatch', - 'statement_comparison_function': 'chatterbot.comparisons.levenshtein_distance', - 'response_selection_method': 'chatterbot.response_selection.get_first_response' - } - ], - 'storage_adapter': 'chatterbot.storage.MongoDatabaseAdapter' - }, - { - 'description': 'Test the synset distance comparison algorithm and Mongo DB storage', - 'logic_adapters': [ - { - 'import_path': 'chatterbot.logic.BestMatch', - 'statement_comparison_function': 'chatterbot.comparisons.synset_distance', - 'response_selection_method': 'chatterbot.response_selection.get_first_response' - } - ], - 'storage_adapter': 'chatterbot.storage.MongoDatabaseAdapter' - } - ]) - -except ServerSelectionTimeoutError: - print('Not running Mongo DB benchmarking') - - -STATEMENT_LIST = utils.generate_strings(10) - -for config in CONFIGURATIONS: - configuration = BASE_CONFIGURATION.copy() - configuration.update(config) - - chatbot = ChatBot('Benchmark', **configuration) - chatbot.train(STATEMENT_LIST) - - durration = utils.get_response_time(chatbot) - - print(configuration['description']) - print('Durration was {} seconds'.format(durration)) diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py new file mode 100644 index 000000000..5d05d18e2 --- /dev/null +++ b/tests/test_benchmarks.py @@ -0,0 +1,158 @@ +""" +These tests are designed to test execution time for +various chat bot configurations to help prevent +performance based regressions when changes are made. +""" + +from .base_case import ChatBotSQLTestCase, ChatBotMongoTestCase +from chatterbot import ChatBot +from chatterbot import utils +import sys +import logging + + +logging.basicConfig( + stream=sys.stdout, + level=logging.INFO +) + +STATEMENT_LIST = utils.generate_strings(10) + + +class BenchmarkingMixin(object): + + def setUp(self): + super(BenchmarkingMixin, self).setUp() + + self.logger = logging.getLogger(__name__) + + def get_kwargs(self): + kwargs = super(BenchmarkingMixin, self).get_kwargs() + kwargs['trainer'] = 'chatterbot.trainers.ListTrainer' + kwargs['show_training_progress'] = False + return kwargs + + def assert_response_duration(self, maximum_duration, test_kwargs): + """ + Assert that the response time did not exceed the maximum allowed amount. + """ + + chatbot = ChatBot('Benchmark', **test_kwargs) + chatbot.train(STATEMENT_LIST) + + duration = utils.get_response_time(chatbot) + + self.logger.info('Duration was %f seconds' % duration) + + if duration > maximum_duration: + raise AssertionError( + '{duration} was greater than the maximum allowed ' + 'response time of {maximum_duration}'.format( + duration=duration, + maximum_duration=maximum_duration + ) + ) + + +class SqlBenchmarkingTests(BenchmarkingMixin, ChatBotSQLTestCase): + """ + Benchmarking tests for SQL storage. + """ + + def get_kwargs(self): + kwargs = super(SqlBenchmarkingTests, self).get_kwargs() + kwargs['storage_adapter'] = 'chatterbot.storage.SQLStorageAdapter' + return kwargs + + def test_levenshtein_distance_comparisons(self): + """ + Test the levenshtein distance comparison algorithm. + """ + kwargs = self.get_kwargs() + kwargs.update({ + 'logic_adapters': [ + { + 'import_path': 'chatterbot.logic.BestMatch', + 'statement_comparison_function': 'chatterbot.comparisons.levenshtein_distance', + 'response_selection_method': 'chatterbot.response_selection.get_first_response' + } + ] + }) + + self.assert_response_duration(1, kwargs) + + def test_synset_distance_comparisons(self): + """ + Test the synset distance comparison algorithm. + """ + kwargs = self.get_kwargs() + kwargs.update({ + 'logic_adapters': [ + { + 'import_path': 'chatterbot.logic.BestMatch', + 'statement_comparison_function': 'chatterbot.comparisons.synset_distance', + 'response_selection_method': 'chatterbot.response_selection.get_first_response' + } + ] + }) + + self.assert_response_duration(1.9, kwargs) + + def test_english_corpus_training(self): + """ + Test the amount of time it takes to train with the English corpus. + """ + import unittest + raise unittest.SkipTest('TODO: This test needs to be written.') + + +class MongoBenchmarkingTests(BenchmarkingMixin, ChatBotMongoTestCase): + """ + Benchmarking tests for Mongo DB storage. + """ + + def get_kwargs(self): + kwargs = super(MongoBenchmarkingTests, self).get_kwargs() + kwargs['storage_adapter'] = 'chatterbot.storage.MongoDatabaseAdapter' + return kwargs + + def test_levenshtein_distance_comparisons(self): + """ + Test the levenshtein distance comparison algorithm. + """ + kwargs = self.get_kwargs() + kwargs.update({ + 'logic_adapters': [ + { + 'import_path': 'chatterbot.logic.BestMatch', + 'statement_comparison_function': 'chatterbot.comparisons.levenshtein_distance', + 'response_selection_method': 'chatterbot.response_selection.get_first_response' + } + ] + }) + + self.assert_response_duration(1, kwargs) + + def test_synset_distance_comparisons(self): + """ + Test the synset distance comparison algorithm. + """ + kwargs = self.get_kwargs() + kwargs.update({ + 'logic_adapters': [ + { + 'import_path': 'chatterbot.logic.BestMatch', + 'statement_comparison_function': 'chatterbot.comparisons.synset_distance', + 'response_selection_method': 'chatterbot.response_selection.get_first_response' + } + ] + }) + + self.assert_response_duration(1.9, kwargs) + + def test_english_corpus_training(self): + """ + Test the amount of time it takes to train with the English corpus. + """ + import unittest + raise unittest.SkipTest('TODO: This test needs to be written.') diff --git a/tox.ini b/tox.ini index 9af9ca3b6..ad35a1abe 100644 --- a/tox.ini +++ b/tox.ini @@ -27,12 +27,6 @@ commands = python runtests.py python examples/django_app/manage.py test examples/django_app/ -[testenv:benchmark] -deps = -rrequirements.txt -commands = - python setup.py develop --no-deps - python tests/benchmarks.py - [testenv:lint] deps = flake8 commands = flake8