From 786b404f2e1af58af14ea4d3d44fb32ff809ff58 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Sat, 22 Aug 2020 00:04:12 +0000 Subject: [PATCH 1/8] [sonic-py-common] Add unit test framework --- .gitignore | 6 -- src/sonic-py-common/.gitignore | 13 ++++ src/sonic-py-common/pytest.ini | 2 + src/sonic-py-common/setup.cfg | 2 + src/sonic-py-common/setup.py | 1 + src/sonic-py-common/tests/__init__.py | 0 src/sonic-py-common/tests/device_info_test.py | 67 +++++++++++++++++++ 7 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 src/sonic-py-common/.gitignore create mode 100644 src/sonic-py-common/pytest.ini create mode 100644 src/sonic-py-common/setup.cfg create mode 100644 src/sonic-py-common/tests/__init__.py create mode 100644 src/sonic-py-common/tests/device_info_test.py diff --git a/.gitignore b/.gitignore index 307a6cf0ae46..5916ed62b395 100644 --- a/.gitignore +++ b/.gitignore @@ -28,12 +28,6 @@ platform/*/docker-*/Dockerfile # Installer-related files and directories installer/x86_64/platforms/ -src/sonic-py-common/**/*.pyc -src/sonic-py-common/.eggs/ -src/sonic-py-common/build -src/sonic-py-common/dist -src/sonic-py-common/sonic_py_common.egg-info - # Misc. files asic_config_checksum files/Aboot/boot0 diff --git a/src/sonic-py-common/.gitignore b/src/sonic-py-common/.gitignore new file mode 100644 index 000000000000..5f621ff9e2ab --- /dev/null +++ b/src/sonic-py-common/.gitignore @@ -0,0 +1,13 @@ +**/*.pyc + +# Distribution / packaging +*.egg-info/ +.eggs/ +build/ +dist/ + +# Unit test / coverage reports +.cache +.coverage +coverage.xml +htmlcov/ diff --git a/src/sonic-py-common/pytest.ini b/src/sonic-py-common/pytest.ini new file mode 100644 index 000000000000..777811f8f489 --- /dev/null +++ b/src/sonic-py-common/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --cov=sonic_py_common --cov-report html --cov-report term --cov-report xml diff --git a/src/sonic-py-common/setup.cfg b/src/sonic-py-common/setup.cfg new file mode 100644 index 000000000000..b7e478982ccf --- /dev/null +++ b/src/sonic-py-common/setup.cfg @@ -0,0 +1,2 @@ +[aliases] +test=pytest diff --git a/src/sonic-py-common/setup.py b/src/sonic-py-common/setup.py index b1294472624e..6b05ebbd7738 100644 --- a/src/sonic-py-common/setup.py +++ b/src/sonic-py-common/setup.py @@ -27,6 +27,7 @@ packages=[ 'sonic_py_common', ], + test_suite = 'setup.get_test_suite', classifiers=[ 'Intended Audience :: Developers', 'Operating System :: Linux', diff --git a/src/sonic-py-common/tests/__init__.py b/src/sonic-py-common/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/src/sonic-py-common/tests/device_info_test.py b/src/sonic-py-common/tests/device_info_test.py new file mode 100644 index 000000000000..b6700e2ef5d4 --- /dev/null +++ b/src/sonic-py-common/tests/device_info_test.py @@ -0,0 +1,67 @@ +import os +import sys + +import mock + +test_path = os.path.dirname(os.path.abspath(__file__)) +modules_path = os.path.dirname(test_path) +sys.path.insert(0, modules_path) + +from sonic_py_common import device_info + +MACHINE_CONF_CONTENTS = """\ +onie_version=2016.11-5.1.0008-9600 +onie_vendor_id=33049 +onie_machine_rev=0 +onie_arch=x86_64 +onie_config_version=1 +onie_build_date="2017-04-26T11:01+0300" +onie_partition_type=gpt +onie_kernel_version=4.10.11 +onie_firmware=auto +onie_switch_asic=mlnx +onie_skip_ethmgmt_macs=yes +onie_machine=mlnx_msn2700 +onie_platform=x86_64-mlnx_msn2700-r0""" + +EXPECTED_GET_MACHINE_INFO_RESULT = { + 'onie_arch': 'x86_64', + 'onie_skip_ethmgmt_macs': 'yes', + 'onie_platform': 'x86_64-mlnx_msn2700-r0', + 'onie_machine_rev': '0', + 'onie_version': '2016.11-5.1.0008-9600', + 'onie_machine': 'mlnx_msn2700', + 'onie_config_version': '1', + 'onie_partition_type': 'gpt', + 'onie_build_date': '"2017-04-26T11:01+0300"', + 'onie_switch_asic': 'mlnx', + 'onie_vendor_id': '33049', + 'onie_firmware': 'auto', + 'onie_kernel_version': '4.10.11' +} + + +class TestDeviceInfo(object): + @classmethod + def setup_class(cls): + print("SETUP") + + def test_get_machine_info(self): + with mock.patch('os.path.isfile') as mock_isfile: + mock_isfile.return_value = True + mocked_open = mock.mock_open(read_data=MACHINE_CONF_CONTENTS) + with mock.patch('__builtin__.open', mocked_open): + result = device_info.get_machine_info() + assert result == EXPECTED_GET_MACHINE_INFO_RESULT + mocked_open.assert_called_once_with('/host/machine.conf') + + def test_get_platform(self): + with mock.patch('sonic_py_common.device_info.get_machine_info') as mock_get_machine_info: + mock_get_machine_info.return_value = EXPECTED_GET_MACHINE_INFO_RESULT + result = device_info.get_platform() + assert result == 'x86_64-mlnx_msn2700-r0' + + @classmethod + def teardown_class(cls): + print("TEARDOWN") + os.environ["PATH"] = os.pathsep.join(os.environ["PATH"].split(os.pathsep)[:-1]) From 421e107c6791afa6d08c9479dbb10d433c65a462 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Sat, 22 Aug 2020 01:29:25 +0000 Subject: [PATCH 2/8] Cleanup unused stuff --- src/sonic-py-common/tests/device_info_test.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/sonic-py-common/tests/device_info_test.py b/src/sonic-py-common/tests/device_info_test.py index b6700e2ef5d4..95e7be85cf05 100644 --- a/src/sonic-py-common/tests/device_info_test.py +++ b/src/sonic-py-common/tests/device_info_test.py @@ -1,12 +1,6 @@ import os -import sys import mock - -test_path = os.path.dirname(os.path.abspath(__file__)) -modules_path = os.path.dirname(test_path) -sys.path.insert(0, modules_path) - from sonic_py_common import device_info MACHINE_CONF_CONTENTS = """\ From 193b84ae1d2ff5f4b9e1fdb00b008c7b17fcaffc Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Sat, 22 Aug 2020 03:15:43 +0000 Subject: [PATCH 3/8] Support Python 3 pytest in Buster slave --- sonic-slave-buster/Dockerfile.j2 | 20 +++++++++++-------- src/sonic-py-common/setup.py | 5 ++++- src/sonic-py-common/tests/device_info_test.py | 19 ++++++++++++++++-- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2 index e7ad71c53a55..6dd057f3f692 100644 --- a/sonic-slave-buster/Dockerfile.j2 +++ b/sonic-slave-buster/Dockerfile.j2 @@ -209,8 +209,10 @@ RUN apt-get update && apt-get install -y \ clang \ pylint \ python-pytest \ + python3-pytest \ gcovr \ python-pytest-cov \ + python3-pytest-cov \ python-parse \ # For snmpd default-libmysqlclient-dev \ @@ -339,6 +341,16 @@ RUN export VERSION=1.14.2 \ && echo 'export PATH=$PATH:$GOROOT/bin' >> /etc/bash.bashrc \ && rm go$VERSION.linux-*.tar.gz +# For building Python packages +RUN pip install setuptools==40.8.0 +RUN pip3 install setuptools==49.6.00 + +# For running Python unit tests +RUN pip install pytest-runner==4.4 +RUN pip3 install pytest-runner==5.2 +RUN pip install mockredispy==2.9.3 +RUN pip3 install mockredispy==2.9.3 + # For p4 build RUN pip install \ ctypesgen==1.0.2 \ @@ -357,9 +369,6 @@ RUN apt-get purge -y python-click # For sonic utilities testing RUN pip install click natsort tabulate netifaces==0.10.7 fastentrypoints -# For sonic snmpagent mock testing -RUN pip3 install mockredispy==2.9.3 - RUN pip3 install "PyYAML>=5.1" # For sonic-platform-common testing @@ -371,11 +380,6 @@ RUN pip install meld3 mock # For vs image build RUN pip install pexpect==4.6.0 -# For sonic-utilities build -RUN pip install mockredispy==2.9.3 -RUN pip install pytest-runner==4.4 -RUN pip install setuptools==40.8.0 - # For sonic-swss-common testing RUN pip install Pympler==0.8 diff --git a/src/sonic-py-common/setup.py b/src/sonic-py-common/setup.py index 6b05ebbd7738..f5c8b593966a 100644 --- a/src/sonic-py-common/setup.py +++ b/src/sonic-py-common/setup.py @@ -2,7 +2,6 @@ dependencies = [ 'natsort', - 'pyyaml', 'swsssdk>=2.0.1', ] @@ -28,6 +27,10 @@ 'sonic_py_common', ], test_suite = 'setup.get_test_suite', + tests_require=[ + 'pytest', + 'mock>=2.0.0' + ], classifiers=[ 'Intended Audience :: Developers', 'Operating System :: Linux', diff --git a/src/sonic-py-common/tests/device_info_test.py b/src/sonic-py-common/tests/device_info_test.py index 95e7be85cf05..d135db4d6228 100644 --- a/src/sonic-py-common/tests/device_info_test.py +++ b/src/sonic-py-common/tests/device_info_test.py @@ -1,8 +1,23 @@ import os +import sys + +# TODO: Remove this if/else block once we no longer support Python 2 +if sys.version_info.major == 3: + from unittest import mock +else: + # Expect the 'mock' package for python 2 + # https://pypi.python.org/pypi/mock + import mock -import mock from sonic_py_common import device_info + +# TODO: Remove this if/else block once we no longer support Python 2 +if sys.version_info.major == 3: + BUILTINS = "builtins" +else: + BUILTINS = "__builtin__" + MACHINE_CONF_CONTENTS = """\ onie_version=2016.11-5.1.0008-9600 onie_vendor_id=33049 @@ -44,7 +59,7 @@ def test_get_machine_info(self): with mock.patch('os.path.isfile') as mock_isfile: mock_isfile.return_value = True mocked_open = mock.mock_open(read_data=MACHINE_CONF_CONTENTS) - with mock.patch('__builtin__.open', mocked_open): + with mock.patch('{}.open'.format(BUILTINS), mocked_open): result = device_info.get_machine_info() assert result == EXPECTED_GET_MACHINE_INFO_RESULT mocked_open.assert_called_once_with('/host/machine.conf') From 59f93adf7923d365291180716c5d01cf901aebcb Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Sat, 22 Aug 2020 18:38:09 +0000 Subject: [PATCH 4/8] More cleanup --- src/sonic-py-common/tests/device_info_test.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/sonic-py-common/tests/device_info_test.py b/src/sonic-py-common/tests/device_info_test.py index d135db4d6228..561d96b89589 100644 --- a/src/sonic-py-common/tests/device_info_test.py +++ b/src/sonic-py-common/tests/device_info_test.py @@ -58,19 +58,18 @@ def setup_class(cls): def test_get_machine_info(self): with mock.patch('os.path.isfile') as mock_isfile: mock_isfile.return_value = True - mocked_open = mock.mock_open(read_data=MACHINE_CONF_CONTENTS) - with mock.patch('{}.open'.format(BUILTINS), mocked_open): + open_mocked = mock.mock_open(read_data=MACHINE_CONF_CONTENTS) + with mock.patch('{}.open'.format(BUILTINS), open_mocked): result = device_info.get_machine_info() assert result == EXPECTED_GET_MACHINE_INFO_RESULT - mocked_open.assert_called_once_with('/host/machine.conf') + open_mocked.assert_called_once_with('/host/machine.conf') def test_get_platform(self): - with mock.patch('sonic_py_common.device_info.get_machine_info') as mock_get_machine_info: - mock_get_machine_info.return_value = EXPECTED_GET_MACHINE_INFO_RESULT + with mock.patch('sonic_py_common.device_info.get_machine_info') as get_machine_info_mocked: + get_machine_info_mocked.return_value = EXPECTED_GET_MACHINE_INFO_RESULT result = device_info.get_platform() assert result == 'x86_64-mlnx_msn2700-r0' @classmethod def teardown_class(cls): print("TEARDOWN") - os.environ["PATH"] = os.pathsep.join(os.environ["PATH"].split(os.pathsep)[:-1]) From 52ab254c52fe667e55cc9c13cef8558dc3d4395a Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Sat, 22 Aug 2020 21:48:34 +0000 Subject: [PATCH 5/8] Align quotes; fix typo --- src/sonic-py-common/sonic_py_common/device_info.py | 2 +- src/sonic-py-common/tests/device_info_test.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/sonic-py-common/sonic_py_common/device_info.py b/src/sonic-py-common/sonic_py_common/device_info.py index 99f818b0f27e..b5aefc397fe8 100644 --- a/src/sonic-py-common/sonic_py_common/device_info.py +++ b/src/sonic-py-common/sonic_py_common/device_info.py @@ -6,7 +6,7 @@ import yaml from natsort import natsorted -# TODD: Replace with swsscommon +# TODO: Replace with swsscommon from swsssdk import ConfigDBConnector, SonicDBConfig USR_SHARE_SONIC_PATH = "/usr/share/sonic" diff --git a/src/sonic-py-common/tests/device_info_test.py b/src/sonic-py-common/tests/device_info_test.py index 561d96b89589..f3b14b0a5f13 100644 --- a/src/sonic-py-common/tests/device_info_test.py +++ b/src/sonic-py-common/tests/device_info_test.py @@ -56,19 +56,19 @@ def setup_class(cls): print("SETUP") def test_get_machine_info(self): - with mock.patch('os.path.isfile') as mock_isfile: + with mock.patch("os.path.isfile") as mock_isfile: mock_isfile.return_value = True open_mocked = mock.mock_open(read_data=MACHINE_CONF_CONTENTS) - with mock.patch('{}.open'.format(BUILTINS), open_mocked): + with mock.patch("{}.open".format(BUILTINS), open_mocked): result = device_info.get_machine_info() assert result == EXPECTED_GET_MACHINE_INFO_RESULT - open_mocked.assert_called_once_with('/host/machine.conf') + open_mocked.assert_called_once_with("/host/machine.conf") def test_get_platform(self): - with mock.patch('sonic_py_common.device_info.get_machine_info') as get_machine_info_mocked: + with mock.patch("sonic_py_common.device_info.get_machine_info") as get_machine_info_mocked: get_machine_info_mocked.return_value = EXPECTED_GET_MACHINE_INFO_RESULT result = device_info.get_platform() - assert result == 'x86_64-mlnx_msn2700-r0' + assert result == "x86_64-mlnx_msn2700-r0" @classmethod def teardown_class(cls): From a7149f392814dc665dacbd4d990460a9481a16c2 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Sat, 22 Aug 2020 23:10:23 +0000 Subject: [PATCH 6/8] Add pytest-runner to setup_requires --- src/sonic-py-common/setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sonic-py-common/setup.py b/src/sonic-py-common/setup.py index f5c8b593966a..e110e2ca6b01 100644 --- a/src/sonic-py-common/setup.py +++ b/src/sonic-py-common/setup.py @@ -26,7 +26,9 @@ packages=[ 'sonic_py_common', ], - test_suite = 'setup.get_test_suite', + setup_requires= [ + 'pytest-runner' + ], tests_require=[ 'pytest', 'mock>=2.0.0' @@ -39,5 +41,6 @@ 'Programming Language :: Python', ], keywords='SONiC sonic PYTHON python COMMON common', + test_suite = 'setup.get_test_suite' ) From 6900ea2f9a6031992cba55f11016b0b15ebd6d20 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Sun, 23 Aug 2020 00:45:29 +0000 Subject: [PATCH 7/8] Upgrade 'mock' to 3.0.5 to fix iterating a mock_open() file object --- sonic-slave-buster/Dockerfile.j2 | 7 ++++++- src/sonic-py-common/setup.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2 index 6dd057f3f692..88f905cb0f8a 100644 --- a/sonic-slave-buster/Dockerfile.j2 +++ b/sonic-slave-buster/Dockerfile.j2 @@ -351,6 +351,11 @@ RUN pip3 install pytest-runner==5.2 RUN pip install mockredispy==2.9.3 RUN pip3 install mockredispy==2.9.3 +# For Python 2 unit tests, we need 'mock'. The last version of 'mock' +# which supports Python 2 is 3.0.5. In Python 3, 'mock' is part of 'unittest' +# in the standard library +RUN pip install mock==3.0.5 + # For p4 build RUN pip install \ ctypesgen==1.0.2 \ @@ -375,7 +380,7 @@ RUN pip3 install "PyYAML>=5.1" RUN pip3 install redis # For supervisor build -RUN pip install meld3 mock +RUN pip install meld3 # For vs image build RUN pip install pexpect==4.6.0 diff --git a/src/sonic-py-common/setup.py b/src/sonic-py-common/setup.py index e110e2ca6b01..b1ba09c17d5d 100644 --- a/src/sonic-py-common/setup.py +++ b/src/sonic-py-common/setup.py @@ -31,7 +31,7 @@ ], tests_require=[ 'pytest', - 'mock>=2.0.0' + 'mock==3.0.5' # For python 2. Version >=4.0.0 drops support for py2 ], classifiers=[ 'Intended Audience :: Developers', From e697d3396201606668bfc0d99ec1b7fa614a87ed Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Sun, 23 Aug 2020 07:28:58 +0000 Subject: [PATCH 8/8] Add pyyaml back to dependencies; needed by device_info --- src/sonic-py-common/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sonic-py-common/setup.py b/src/sonic-py-common/setup.py index b1ba09c17d5d..f56ad96b04fd 100644 --- a/src/sonic-py-common/setup.py +++ b/src/sonic-py-common/setup.py @@ -2,6 +2,7 @@ dependencies = [ 'natsort', + 'pyyaml', 'swsssdk>=2.0.1', ]