diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index 23ae120f712..c9acd055685 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -37,3 +37,52 @@ jobs: if git add --update && git commit --no-edit --allow-empty --message "Set Version: $(cat package/version)"; then git push origin master fi + + post-performance-tests: + name: 'Performace Tests' + runs-on: [self-hosted, linux, normal] + continue-on-error: true + steps: + - uses: actions/checkout@v3 + - name: 'Set up Docker Test Image' + env: + BASE_OS: ubuntu + BASE_DISTRO: jammy + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} + BENCHER_PROJECT: k-framework + BENCHER_ADAPTER: json + run: | + set -euxo pipefail + workspace=$(pwd) + docker run --name ci-${GITHUB_SHA} \ + --rm -it --detach \ + -e BENCHER_API_TOKEN=$BENCHER_API_TOKEN \ + -e BENCHER_PROJECT=$BENCHER_PROJECT \ + -e BENCHER_ADAPTER=$BENCHER_ADAPTER \ + -e GITHUB_HEAD_REF=$GITHUB_HEAD_REF \ + -e GITHUB_BASE_REF=$GITHUB_BASE_REF \ + -e GITHUB_TOKEN=$GITHUB_TOKEN \ + -e GITHUB_ACTIONS=true \ + -e GITHUB_EVENT_NAME=$GITHUB_EVENT_NAME \ + -e GITHUB_REPOSITORY=$GITHUB_REPOSITORY \ + -e GITHUB_REF=$GITHUB_REF \ + --workdir /opt/workspace \ + -v "${workspace}:/opt/workspace" \ + ${BASE_OS}:${BASE_DISTRO} + - name: 'Getting Performance Tests Results' + run: | + set -euxo pipefail + docker exec -t ci-${GITHUB_SHA} /bin/bash -c './k-distribution/tests/profiling/post_results_to_develop.py' + - name: 'Posting Performance Tests Results' + run: | + set -euxo pipefail + docker exec -t ci-${GITHUB_SHA} /bin/bash -c 'bencher run \ + --if-branch "develop" --else-if-branch "master" \ + --file .profiling-results.json --err --ci-only-on-alert \ + --github-actions "${GITHUB_TOKEN}" 'echo "Exporting report"' + - name: 'Tear down Docker' + if: always() + run: | + docker stop --time=0 ci-${GITHUB_SHA} + docker container rm --force ci-${GITHUB_SHA} || true \ No newline at end of file diff --git a/bencher_0.3.10_amd64.deb b/bencher_0.3.10_amd64.deb deleted file mode 100644 index dd0f8675b14..00000000000 Binary files a/bencher_0.3.10_amd64.deb and /dev/null differ diff --git a/k-distribution/tests/profiling/Makefile b/k-distribution/tests/profiling/Makefile index 30bbec6c2cd..fe4d8116925 100644 --- a/k-distribution/tests/profiling/Makefile +++ b/k-distribution/tests/profiling/Makefile @@ -1,4 +1,4 @@ -EXCLUDE=Makefile kprofiling.mak setup_profiling.sh +EXCLUDE=Makefile kprofiling.mak setup_profiling.sh post_results_to_develop.py SUBDIRS=$(filter-out $(EXCLUDE), $(wildcard *)) SUBCLEAN=$(addsuffix .clean,$(SUBDIRS)) diff --git a/k-distribution/tests/profiling/post_results_to_develop.py b/k-distribution/tests/profiling/post_results_to_develop.py new file mode 100755 index 00000000000..37104b56def --- /dev/null +++ b/k-distribution/tests/profiling/post_results_to_develop.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +import json +import subprocess + +COMMIT_SHA='' +FROM_BRANCH='' +TO_BRANCH='develop' + +def execute_command(command): + # Try to run the command + try: + result = subprocess.run(command,stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True) + + # Check for errors + if result.returncode != 0: + print("Error:", result.stderr) + exit(1) + + except subprocess.CalledProcessError as e: + # Handle any errors or exceptions here + print("Error:", e) + exit(1) + + return result.stdout + +# git command to get the last commit in develop +commit_command = [ 'git', 'log', '--format=\"%H\"', '-n', '1' ] +COMMIT_SHA = execute_command(commit_command).strip('\"').strip('\"\n') + +# curl command to get the branch name of last commit in develop +API_URL = 'https://api.github.com/repos/runtimeverification/k/commits/' \ + + COMMIT_SHA + '/pulls' +branch_command = ['curl', '-L', '-H', 'Accept:', 'application/vnd.github+json', + '-H', '\"Authorization', 'Bearer', '${GITHUB_TOKEN}\"', '-H', + '\"X-GitHub-Api-Version:', '2022-11-28\"', API_URL] +FROM_BRANCH = json.loads(execute_command(branch_command))[0]['head']['label'] + +print("Exporting last bencher report from", FROM_BRANCH, "to", TO_BRANCH) + +# This command will generate a JSON file with a list containing the last reports +# sorted in descendenting order for the project. +bencher_command = ["bencher", "report", "list", "--project", "k-framework", + "--sort", "date_time", "--direction", "desc", "--per-page", + "255"] +data = json.loads(execute_command(bencher_command)) + +# Collect all elemnts where the key 'project' is 'k_framework' +k_framework = [item for item in data if item['project']['slug'] == 'k-framework' + and item['branch']['slug'] == FROM_BRANCH] + +# Append the last 6 reports to the list, they correspond to the last performance +# execution on the last commit in FROM_BRANCH +def get_name_and_value(item): + return item['metric_kind']['slug'], item['benchmarks'][0]['metric']['value'] + +data = {} +for i in range(0,6): + item = k_framework[i] + results = item['results'] + benchmark_name = results[0][0]['benchmarks'][0]['name'] + metric_name_memory, value_memory = get_name_and_value(results[0][0]) + metric_name_size, value_size = get_name_and_value(results[0][1]) + + branch = item['branch'] + print("Appended:", benchmark_name, "created in", item['created'], + "on branch", branch['name'], "with version", branch['version']['number']) + + data.update({benchmark_name: {metric_name_size: {"value": value_size}, + metric_name_memory: {"value": value_memory}} + }) + +json.dump(data, open('.profiling-results.json', 'w'), indent=4)