diff --git a/.azure-pipelines-templates/deploy_aci.yml b/.azure-pipelines-templates/deploy_aci.yml index d4dc36b75f83..3767ac350639 100644 --- a/.azure-pipelines-templates/deploy_aci.yml +++ b/.azure-pipelines-templates/deploy_aci.yml @@ -98,7 +98,7 @@ jobs: set -ex cd /CCF/build npm config set cache /ccfci/workspace_$(Build.BuildNumber)/.npm - WORKSPACE=/ccfci/workspace_$(Build.BuildNumber) ELECTION_TIMEOUT_MS=10000 ./tests.sh -VV -T Test -LE "benchmark|perf|tlstest|vegeta|suite|snp_flaky" -E "lts_compatibility" + WORKSPACE=/ccfci/workspace_$(Build.BuildNumber) ELECTION_TIMEOUT_MS=10000 ./tests.sh -VV -T Test -LE "benchmark|perf|tlstest|suite|snp_flaky" -E "lts_compatibility" # Remove irrelevant and bulky data from workspace before uploading find /ccfci/workspace_$(Build.BuildNumber) -type f -name cchost -delete find /ccfci/workspace_$(Build.BuildNumber) -type f -name "*.so" -delete diff --git a/.azure-pipelines-templates/stress-matrix.yml b/.azure-pipelines-templates/stress-matrix.yml deleted file mode 100644 index a53c6ca03c40..000000000000 --- a/.azure-pipelines-templates/stress-matrix.yml +++ /dev/null @@ -1,13 +0,0 @@ -jobs: - - template: common.yml - parameters: - target: SGX - env: - container: sgx - pool: ado-sgx-ccf-sub-backup - cmake_args: "-DCOMPILE_TARGET=sgx -DLONG_TESTS=ON" - suffix: "StressTest" - artifact_name: "StressTest" - ctest_filter: '-L "vegeta" -C "long_stress"' - depends_on: configure - installExtendedTestingTools: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 718fbf93a2d4..bd02fb677256 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,7 +90,7 @@ jobs: # Unit tests ./tests.sh --output-on-failure -L unit -j$(nproc --all) # All other acceptably fast tests, mostly end-to-end - ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|perf|protocolstest|vegeta|suite|unit" + ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|perf|protocolstest|suite|unit" # Partitions tests ./tests.sh --timeout 240 --output-on-failure -L partitions -C partitions shell: bash diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml new file mode 100644 index 000000000000..de5de6007181 --- /dev/null +++ b/.github/workflows/long-test.yml @@ -0,0 +1,102 @@ +name: Long Test + +on: + schedule: + - cron: "0 0 * * 1-5" + workflow_dispatch: + +jobs: + scan_build: + name: "Scan build" + runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] + container: + image: ghcr.io/microsoft/ccf/ci/default:build-26-06-2024 + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: "Run scan" + run: | + set -x + mkdir build + cd build + ../scripts/scan-build.sh + + long-asan: + name: ASAN + runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] + container: + image: ghcr.io/microsoft/ccf/ci/default:build-26-06-2024 + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: "Install deps" + run: | + sudo apt-get -y update + sudo apt install ansible -y + cd getting_started/setup_vm + ansible-playbook ccf-extended-testing.yml + + - name: "Build" + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + mkdir build + cd build + cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=ON -DLVI_MITIGATIONS=OFF -DSAN=ON .. + ninja + + - name: "Test" + run: | + set +x + cd build + ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark|perf" + + - name: "Upload logs" + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: logs-asan + path: | + build/workspace/*/*.config.json + build/workspace/*/out + build/workspace/*/err + if-no-files-found: ignore + + long-tsan: + name: TSAN + runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] + container: + image: ghcr.io/microsoft/ccf/ci/default:build-26-06-2024 + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: "Build" + run: | + git config --global --add safe.directory /__w/CCF/CCF + mkdir build + cd build + cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=ON -DLVI_MITIGATIONS=OFF -DTSAN=ON -DWORKER_THREADS=2 .. + ninja + + - name: "Test" + run: | + set +x + cd build + ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark|perf" + + - name: "Upload logs" + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: logs-tsan + path: | + build/workspace/*/*.config.json + build/workspace/*/out + build/workspace/*/err + if-no-files-found: ignore diff --git a/.stress.yml b/.stress.yml deleted file mode 100644 index 9ad58f2451e8..000000000000 --- a/.stress.yml +++ /dev/null @@ -1,33 +0,0 @@ -pr: - branches: - include: - - main - exclude: - - "release/[0-3].x" - paths: - include: - - .stress.yml - - .azure-pipelines-templates/stress-matrix.yml - -trigger: none - -schedules: - - cron: "0 3 * * Mon-Fri" - displayName: Stress test build - branches: - include: - - main - exclude: - - "release/[0-3].x" - always: true - -resources: - containers: - - container: sgx - image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-sgx - options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx - -jobs: - - template: .azure-pipelines-templates/configure.yml - - - template: .azure-pipelines-templates/stress-matrix.yml diff --git a/CMakeLists.txt b/CMakeLists.txt index 514b7f65aa11..56fc815ff94d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1509,21 +1509,6 @@ if(BUILD_TESTS) PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/acme_endorsement.py LABEL ACME ) - - add_e2e_test( - NAME vegeta_stress - PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/vegeta_stress.py - LABEL vegeta - ADDITIONAL_ARGS -p "samples/apps/logging/liblogging" - ) - - add_e2e_test( - NAME vegeta_long_stress - PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/vegeta_stress.py - LABEL vegeta - CONFIGURATIONS long_stress - ADDITIONAL_ARGS -p "samples/apps/logging/liblogging" --duration 45m - ) endif() add_piccolo_test( diff --git a/doc/build_apps/run_app.rst b/doc/build_apps/run_app.rst index 35f7f7568b40..41b24a5feaa7 100644 --- a/doc/build_apps/run_app.rst +++ b/doc/build_apps/run_app.rst @@ -176,37 +176,3 @@ Integration Tests The ``sandbox.sh`` script can be a helpful element of infrastructure to execute Integration Tests against a CCF test network running a particular application (see :ccf_repo:`test_install.sh ` script as example). ``test_install.sh`` illustrates how to wait for the sandbox to be :ccf_repo:`ready ` before issuing application transactions, how to shut it down cleanly, and how to trigger a recovery. Recovering a test network can be a useful way to inspect post-test application test. - -Performance Tests ------------------ - -``sandbox.sh`` can be equally useful for performance testing, for example with a load testing tool such as `vegeta `_: - -.. code-block:: bash - - $ /opt/ccf_virtual/bin/sandbox.sh --package ./liblogging.virtual.so - ... - [16:14:10.011] Node [0] = https://127.0.0.1:8000 - ... - [16:14:10.011] Keys and certificates have been copied to the common folder: /data/src/CCF/build/workspace/sandbox_common - ... - -.. code-block:: bash - - # Extracted from the output of sandbox.sh, above. - $ export SCDIR=/data/src/CCF/build/workspace/sandbox_common - $ export VEGETA=/opt/vegeta/vegeta - $ $VEGETA attack --targets sample_targets.json - --format json --duration 10s \ - --cert $SCDIR/user0_cert.pem \ - --key $SCDIR/user0_privk.pem \ - --root-certs $SCDIR/service_cert.pem | /opt/vegeta/vegeta report - -Where ``sample_targets.json`` is a file containing some sample requests to be sent as load testing, for example: - -.. code-block:: json - - {"method": "POST", "url": "https://127.0.0.1:8000/app/log/private", "header": {"Content-Type": ["application/json"]}, "body": "eyJpZCI6IDAsICJtc2ciOiAiUHJpdmF0ZSBtZXNzYWdlOiAwIn0="} - {"method": "GET", "url": "https://127.0.0.1:8000/app/log/private?id=0", "header": {"Content-Type": ["application/json"]}} - -.. _curl: https://curl.haxx.se/ diff --git a/getting_started/setup_vm/ccf-extended-testing.yml b/getting_started/setup_vm/ccf-extended-testing.yml index 93610045d5a1..51271ea6f382 100644 --- a/getting_started/setup_vm/ccf-extended-testing.yml +++ b/getting_started/setup_vm/ccf-extended-testing.yml @@ -1,8 +1,5 @@ - hosts: localhost tasks: - - import_role: - name: vegeta - tasks_from: install.yml - import_role: name: pebble tasks_from: install.yml diff --git a/getting_started/setup_vm/roles/vegeta/tasks/install.yml b/getting_started/setup_vm/roles/vegeta/tasks/install.yml deleted file mode 100644 index 080a3de42449..000000000000 --- a/getting_started/setup_vm/roles/vegeta/tasks/install.yml +++ /dev/null @@ -1,20 +0,0 @@ -- name: Include vars - include_vars: common.yml - -- name: Download Vegeta - get_url: - url: https://github.com/tsenart/vegeta/releases/download/v{{ vegeta_version }}/vegeta_{{ vegeta_version }}_linux_386.tar.gz - dest: "{{ workspace }}/vegeta.tar.gz" - become: true - -- name: Create directory for Vegeta - file: - path: "{{ vegeta_install_prefix }}" - state: directory - become: true - -- name: Unpack Vegeta - unarchive: - src: "{{ workspace }}/vegeta.tar.gz" - dest: "{{ vegeta_install_prefix }}" - become: true diff --git a/getting_started/setup_vm/roles/vegeta/vars/common.yml b/getting_started/setup_vm/roles/vegeta/vars/common.yml deleted file mode 100644 index 92270953a319..000000000000 --- a/getting_started/setup_vm/roles/vegeta/vars/common.yml +++ /dev/null @@ -1,3 +0,0 @@ -workspace: "/tmp/" -vegeta_version: 12.8.4 -vegeta_install_prefix: "/opt/vegeta" diff --git a/tests/generate_vegeta_targets.py b/tests/generate_vegeta_targets.py deleted file mode 100644 index 4d646efd7a7a..000000000000 --- a/tests/generate_vegeta_targets.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the Apache 2.0 License. -import argparse -import json -import base64 -import sys -import urllib.parse -from collections import abc - - -def build_vegeta_target(hostname, path, body=None, method="POST"): - target = {} - target["method"] = method - target["url"] = urllib.parse.urljoin(f"https://{hostname}", path) - target["header"] = {"Content-Type": ["application/json"]} - if body is not None: - # Bodies must be base64 encoded strings - target["body"] = base64.b64encode(json.dumps(body).encode()).decode() - return target - - -def write_vegeta_target_line(f, *args, **kwargs): - target = build_vegeta_target(*args, **kwargs) - f.write(json.dumps(target)) - f.write("\n") - - -# Quick and dirty solution for building request bodies that vary -def recursive_format(obj, i): - if isinstance(obj, str): - return obj.format(i=i) - elif isinstance(obj, abc.Sequence): - return [recursive_format(e, i) for e in obj] - elif isinstance(obj, abc.Mapping): - return {recursive_format(k, i): recursive_format(v, i) for k, v in obj.items()} - else: - return obj - - -def nan_replacer(i): - def fun(s): - if s == "NaN": - return i - return float(s) - - return fun - - -def append_targets(file, args): - for i in range(args.range_start, args.range_end): - format_args = {"i": i} - path = args.uri_path.format(**format_args) - if args.body is not None: - body = recursive_format( - json.loads(args.body, parse_constant=nan_replacer(i)), **format_args - ) - else: - body = None - write_vegeta_target_line( - file, - hostname=args.hostname, - path=path, - method=args.method, - body=body, - ) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - - subparsers = parser.add_subparsers( - title="targets", - description="Instructions to generate a list of targets. Can be repeated multiple times", - ) - targets_s = "targets" - targets_parser = subparsers.add_parser( - targets_s, formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - - targets_parser.add_argument("--uri-path", type=str, help="Path to desired endpoint") - targets_parser.add_argument( - "--range-start", type=int, default=0, help="First index for range of targets" - ) - targets_parser.add_argument( - "--range-end", type=int, default=1, help="End index for range of targets" - ) - targets_parser.add_argument( - "--method", type=str, default="POST", help="HTTP method to be called" - ) - targets_parser.add_argument( - "--hostname", - type=str, - default="127.0.0.1:8000", - help="Base hostname to submit target to", - ) - targets_parser.add_argument("--body", default=None, help="JSON body of request") - - remaining_args = sys.argv[1:] - steps = [] - while remaining_args: - try: - start_index = remaining_args.index(targets_s) - except ValueError: - start_index = None - if start_index != 0: - raise RuntimeError( - f"Unable to parse args - next section doesn't begin with '{targets_s}'" - ) - - try: - end_index = remaining_args.index(targets_s, start_index + 1) - except ValueError: - end_index = len(remaining_args) - - parse_slice = remaining_args[start_index:end_index] - args, rest = targets_parser.parse_known_args(parse_slice) - steps += [args] - remaining_args = remaining_args[end_index:] - - for step in steps: - append_targets(sys.stdout, step) diff --git a/tests/vegeta_stress.py b/tests/vegeta_stress.py deleted file mode 100644 index c040b23bbb6f..000000000000 --- a/tests/vegeta_stress.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the Apache 2.0 License. -import infra.network -import infra.e2e_args -import subprocess -import threading -import time -import generate_vegeta_targets as TargetGenerator -from loguru import logger as LOG - -VEGETA_BIN = "/opt/vegeta/vegeta" - - -def print_memory_stats(node, shutdown_event): - with node.client() as c: - while not shutdown_event.is_set(): - r = c.get("/node/memory") - LOG.warning(r.body.json()) - time.sleep(10) - - -def run(args, additional_attack_args): - # Test that vegeta is available - subprocess.run([VEGETA_BIN, "-version"], capture_output=True, check=True) - - with infra.network.network( - args.nodes, - args.binary_dir, - args.debug_nodes, - args.perf_nodes, - pdb=args.pdb, - ) as network: - network.start_and_open(args) - - primary, _ = network.find_primary() - primary_hostname = ( - f"{primary.get_public_rpc_host()}:{primary.get_public_rpc_port()}" - ) - - vegeta_targets = "vegeta_targets" - with open(vegeta_targets, "w", encoding="utf-8") as f: - for i in range(10): - TargetGenerator.write_vegeta_target_line( - f, - primary_hostname, - "/app/log/private", - body={"id": i, "msg": f"Private message: {i}"}, - ) - - for i in range(10): - TargetGenerator.write_vegeta_target_line( - f, primary_hostname, f"/app/log/private?id={i}", method="GET" - ) - - for i in range(10): - TargetGenerator.write_vegeta_target_line( - f, - primary_hostname, - "/app/log/public", - body={"id": i, "msg": f"Public message: {i}"}, - ) - - for i in range(10): - TargetGenerator.write_vegeta_target_line( - f, primary_hostname, f"/app/log/public?id={i}", method="GET" - ) - - attack_cmd = [VEGETA_BIN, "attack"] - attack_cmd += ["--targets", vegeta_targets] - attack_cmd += ["--format", "json"] - attack_cmd += ["--duration", "10s"] - sa = primary.session_auth("user0") - attack_cmd += ["--cert", sa["session_auth"].cert] - attack_cmd += ["--key", sa["session_auth"].key] - attack_cmd += ["--root-certs", primary.session_ca(False)["ca"]] - attack_cmd += additional_attack_args - - attack_cmd_s = " ".join(attack_cmd) - LOG.warning(f"Starting: {attack_cmd_s}") - vegeta_run = subprocess.Popen(attack_cmd, stdout=subprocess.PIPE) - - tee_split = subprocess.Popen( - ["tee", "vegeta_results.bin"], - stdin=vegeta_run.stdout, - stdout=subprocess.PIPE, - ) - - report_cmd = [VEGETA_BIN, "report", "--every", "5s"] - vegeta_report = subprocess.Popen(report_cmd, stdin=tee_split.stdout) - - # Start a second thread which will print the primary's memory stats at regular intervals - shutdown_event = threading.Event() - memory_thread = threading.Thread( - target=print_memory_stats, args=(primary, shutdown_event) - ) - memory_thread.start() - - LOG.info("Waiting for completion...") - vegeta_report.communicate() - - LOG.info("Shutting down...") - shutdown_event.set() - memory_thread.join() - - LOG.success("Done!") - - -if __name__ == "__main__": - - def add(parser): - pass - - args, unknown_args = infra.e2e_args.cli_args(add=add, accept_unknown=True) - args.package = "samples/apps/logging/liblogging" - args.nodes = infra.e2e_args.min_nodes(args, f=1) - run(args, unknown_args)