Merge pull request #36 from nginxinc/e2e-tests

Prepare a POC of testing framework for e2e tests.

tests/.dockerignore

@@ -0,0 +1,2 @@
+# Ignore pytest cache
+.pytest_cache

tests/.flake8

@@ -0,0 +1,5 @@
+[flake8]
+format = pylint
+max-complexity = 10
+max-line-length = 170
+exclude = .git,__pycache__,data,.idea,.pytest_cache

tests/.gitignore

@@ -0,0 +1,13 @@
+# IntelliJ project files
+.idea
+out
+gen
+
+# Junit-style report for Jenkins integration
+results.xml
+
+# Python specific files
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+.pytest_cache

tests/Makefile

@@ -0,0 +1,12 @@
+NGINX_API =
+AWS_REGION = us-east-2
+PREFIX = test-runner
+TAG = edge
+PYTEST_ARGS =
+AWS_CREDENTIALS = ~/.aws/credentials
+
+build:
+	docker build -t $(PREFIX):$(TAG) -f docker/Dockerfile ..
+
+run-tests: build
+	docker run --rm -v $(AWS_CREDENTIALS):/root/.aws/credentials $(PREFIX):$(TAG) --nginx-api=$(NGINX_API) --aws-region=$(AWS_REGION) $(PYTEST_ARGS)

tests/README.md

@@ -0,0 +1,47 @@
+# Tests
+
+The project includes automated tests for testing the ASG Sync tool in an AWS environment (Azure support will be added later). The tests are written in Python3, use the pytest framework to run the tests and utilize boto3 to call the AWS API.
+
+Note: for now this is for internal use only, as AWS stack configuration is done outside of testing framework.
+
+Below you will find the instructions on how to run the tests against a cloud provider.
+
+## Running Tests
+
+### Prerequisites:
+
+* AWS stack prepared. 
+* AWS access key and AWS secret access key. 
+* Python3 or Docker.
+
+#### Step 1 - Set up the environment
+
+* Either create|update ~/.aws/credentials file or set the AWS_SHARED_CREDENTIALS_FILE environment variable pointing to your own location. This file is an INI formatted file with section names corresponding to profiles. Tests use 'default' profile. The file [credentials](data/credentials) is a minimal example of such a file.
+
+#### Step 2 - Run the Tests
+
+Run the tests:
+* Use local Python3 installation:
+    ```bash
+    $ cd tests
+    $ pip3 install -r requirements.txt
+    $ python3 -m pytest --nginx-api=nginx_plus_api_url
+    ```
+* Use Docker:
+    ```bash
+    $ cd tests
+    $ make run-tests AWS_CREDENTIALS=abs_path_to_creds_file NGINX_API=nginx_plus_api_url
+    ```
+
+## Configuring the Tests
+
+The table below shows various configuration options for the tests. If you use Python3 to run the tests, use the command-line arguments. If you use Docker, use the [Makefile](Makefile) variables.
+
+
+| Command-line Argument | Makefile Variable | Description | Default |
+| :----------------------- | :------------ | :------------ | :----------------------- |
+| `--nginx-api` | `NGINX_API` | The NGINX Plus API url. | `N/A` |
+| `--aws-region` | `AWS_REGION` | The AWS stack region. | `us-east-2` |
+| `N/A` | `PYTEST_ARGS` | Any additional pytest command-line arguments (i.e `-k TestSmoke`) | `""` |
+
+If you would like to use an IDE (such as PyCharm) to run the tests, use the [pytest.ini](pytest.ini) file to set the command-line arguments.

tests/conftest.py

@@ -0,0 +1,62 @@
+"""Describe overall framework configuration."""
+import pytest
+from boto3 import Session
+from botocore.client import BaseClient
+
+from tests.settings import DEFAULT_AWS_REGION
+
+
+def pytest_addoption(parser) -> None:
+    """Get cli-arguments.
+
+    :param parser: pytest parser
+    :return:
+    """
+    parser.addoption("--nginx-api", action="store", default="", help="The NGINX Plus API url.")
+    parser.addoption("--aws-region", action="store", default=DEFAULT_AWS_REGION, help="The AWS region name.")
+
+
+class CLIArguments:
+    """
+    Encapsulate CLI arguments.
+
+    Attributes:
+        nginx_api (str): NGINX Plus API url
+        aws_region (str): AWS region name
+    """
+    def __init__(self, nginx_api: str, aws_region: str):
+        self.nginx_api = nginx_api
+        self.aws_region = aws_region
+
+
+@pytest.fixture(scope="session", autouse=True)
+def cli_arguments(request) -> CLIArguments:
+    """
+    Verify the CLI arguments.
+
+    :param request: pytest fixture
+    :return: CLIArguments
+    """
+    nginx_api = request.config.getoption("--nginx-api")
+    assert nginx_api != "", "Empty NGINX Plus API url is not allowed"
+    print(f"\nTests will use NGINX Plus API url: {nginx_api}")
+
+    aws_region = request.config.getoption("--aws-region")
+    print(f"\nTests will use AWS region: {aws_region}")
+
+    return CLIArguments(nginx_api, aws_region)
+
+
+@pytest.fixture(scope="session")
+def autoscaling_client(cli_arguments) -> BaseClient:
+    """
+    Set up kubernets-client to operate in cluster.
+
+    boto3 looks for AWS credentials file and uses a `default` profile from it.
+
+    :param cli_arguments: a set of command-line arguments
+    :return:
+    """
+    session = Session(profile_name='default',
+                      region_name=cli_arguments.aws_region)
+    return session.client('autoscaling')

tests/data/credentials

@@ -0,0 +1,3 @@
+[default]
+aws_access_key_id=foo
+aws_secret_access_key=bar

tests/docker/Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.7.2-slim
+
+RUN mkdir /workspace
+
+WORKDIR /workspace
+
+COPY tests tests
+
+WORKDIR /workspace/tests
+
+RUN pip install -r requirements.txt
+
+ENTRYPOINT [ "python3", "-m", "pytest"]

tests/pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+addopts = --tb=native -r fsxX --disable-warnings
+log_cli=true

tests/requirements.txt

@@ -0,0 +1,3 @@
+requests==2.22.0
+boto3==1.11.6
+pytest==4.4.1

tests/settings.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+"""Describe project settings"""
+import os
+
+BASEDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))
+TEST_DATA = f"{PROJECT_ROOT}/data"
+DEFAULT_AWS_REGION = "us-east-2"
+# Time in seconds to ensure reconfiguration changes in cluster
+RECONFIGURATION_DELAY = 60
+NGINX_API_VERSION = 4

tests/suite/test_smoke.py

@@ -0,0 +1,118 @@
+import pytest
+import time
+import requests
+import json
+
+from botocore.waiter import WaiterModel, create_waiter_with_client
+
+from tests.settings import RECONFIGURATION_DELAY, NGINX_API_VERSION
+
+
+def wait_for_changes_in_api(req_url, desired_capacity) -> None:
+    resp = requests.get(req_url)
+    nginx_upstream = json.loads(resp.text)
+    counter = 0
+    while len(nginx_upstream['peers']) != desired_capacity and counter < 10:
+        time.sleep(RECONFIGURATION_DELAY)
+        counter = counter + 1
+        resp = requests.get(req_url)
+        nginx_upstream = json.loads(resp.text)
+
+
+def wait_for_changes_in_aws(autoscaling_client, group_name, desired_capacity) -> None:
+    waiter_name = 'autoscaling_completed'
+    argument = f"contains(AutoScalingGroups[?(starts_with(AutoScalingGroupName, `{group_name}`) == `true`)]." \
+               f"[length(Instances[?LifecycleState=='InService']) == `{desired_capacity}`][], `false`)"
+    waiter_config = {
+        "version": 2,
+        "waiters": {
+            "autoscaling_completed": {
+                "acceptors": [
+                    {
+                        "argument": argument,
+                        "expected": True,
+                        "matcher": "path",
+                        "state": "success"
+                    },
+                    {
+                        "argument": argument,
+                        "expected": False,
+                        "matcher": "path",
+                        "state": "retry"
+                    }
+                ],
+                "delay": 5,
+                "maxAttempts": 20,
+                "operation": "DescribeAutoScalingGroups"
+            }
+        }
+    }
+    waiter_model = WaiterModel(waiter_config)
+    custom_waiter = create_waiter_with_client(waiter_name, waiter_model, autoscaling_client)
+    custom_waiter.wait()
+
+
+def scale_aws_group(autoscaling_client, group_name, desired_capacity) -> dict:
+    counter = 0
+    while counter < 10:
+        try:
+            response = autoscaling_client.set_desired_capacity(
+                AutoScalingGroupName=group_name,
+                DesiredCapacity=desired_capacity,
+                HonorCooldown=True,
+            )
+            print(f"Scaling activity started: {response}")
+            return response
+        except autoscaling_client.exceptions.ScalingActivityInProgressFault:
+            print("Scaling activity is in progress, wait for 60 seconds then retry.")
+            counter = counter + 1
+            time.sleep(RECONFIGURATION_DELAY)
+    pytest.fail(f"Failed to scale the group {group_name}")
+
+
+def get_aws_group_name(autoscaling_client, group_name) -> str:
+    """
+    Get AWS unique group name.
+
+    :param autoscaling_client: AWS API
+    :param group_name:
+    :return: str
+    """
+    groups = autoscaling_client.describe_auto_scaling_groups()['AutoScalingGroups']
+    return list(filter(lambda group: group_name in group['AutoScalingGroupName'], groups))[0]['AutoScalingGroupName']
+
+
+class TestSmoke:
+    @pytest.mark.parametrize("test_data", [
+        pytest.param({'group_name': 'WebserverGroup1', 'api_url': '/http/upstreams/backend1'}, id="backend1"),
+        pytest.param({'group_name': 'WebserverGroup2', 'api_url': '/http/upstreams/backend2'}, id="backend2"),
+        pytest.param({'group_name': 'WebserverGroup3', 'api_url': '/stream/upstreams/tcp-backend'}, id="tcp-backend")
+    ])
+    def test_aws_scale_up(self, cli_arguments, autoscaling_client, test_data):
+        desired_capacity = 5
+        group_name = get_aws_group_name(autoscaling_client, test_data['group_name'])
+        scale_aws_group(autoscaling_client, group_name, desired_capacity)
+        wait_for_changes_in_aws(autoscaling_client, group_name, desired_capacity)
+        wait_for_changes_in_api(f"{cli_arguments.nginx_api}/{NGINX_API_VERSION}{test_data['api_url']}",
+                                desired_capacity)
+        resp = requests.get(f"{cli_arguments.nginx_api}/{NGINX_API_VERSION}{test_data['api_url']}")
+        nginx_upstream = json.loads(resp.text)
+        assert len(nginx_upstream['peers']) == desired_capacity,\
+            f"Expected {desired_capacity} servers, found: {nginx_upstream['peers']}"
+
+    @pytest.mark.parametrize("test_data", [
+        pytest.param({'group_name': 'WebserverGroup1', 'api_url': '/http/upstreams/backend1'}, id="backend1"),
+        pytest.param({'group_name': 'WebserverGroup2', 'api_url': '/http/upstreams/backend2'}, id="backend2"),
+        pytest.param({'group_name': 'WebserverGroup3', 'api_url': '/stream/upstreams/tcp-backend'}, id="tcp-backend")
+    ])
+    def test_aws_scale_down(self, cli_arguments, autoscaling_client, test_data):
+        desired_capacity = 1
+        group_name = get_aws_group_name(autoscaling_client, test_data['group_name'])
+        scale_aws_group(autoscaling_client, group_name, desired_capacity)
+        wait_for_changes_in_aws(autoscaling_client, group_name, desired_capacity)
+        wait_for_changes_in_api(f"{cli_arguments.nginx_api}/{NGINX_API_VERSION}{test_data['api_url']}",
+                                desired_capacity)
+        resp = requests.get(f"{cli_arguments.nginx_api}/{NGINX_API_VERSION}{test_data['api_url']}")
+        nginx_upstream = json.loads(resp.text)
+        assert len(nginx_upstream['peers']) == desired_capacity,\
+            f"Expected {desired_capacity} servers, found: {nginx_upstream['peers']}"