Adding CI to release workflow to create a develop baseline performa…

…nce on Bencher (#3627)

This modification is necessary to create a baseline for newer branches
to compare the performance of their changes against what we already have
in develop.

This modification is necessary to create and populate the develop's
branch as the baseline for newer branches.
This script `post_results_to_develop.py` works by getting the last
benchmarks reports of the last branch merged into develop, aggregating
them in one file, and posting them on the develop branch of Bencher.

Hopefully, this is a temporary fix, as this was already accepted as a
bug/possible feature by Bencher's maintainer
bencherdev/bencher#187.

This PR, unlike #3625,
doesn't need to rerun all `Performance Tests` on the develop or master
branch.

.github/workflows/develop.yml

@@ -37,3 +37,52 @@ jobs:
           if git add --update && git commit --no-edit --allow-empty --message "Set Version: $(cat package/version)"; then
             git push origin master
           fi
+
+  post-performance-tests:
+    name: 'Performace Tests'
+    runs-on: [self-hosted, linux, normal]
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v3
+      - name: 'Set up Docker Test Image'
+        env:
+          BASE_OS: ubuntu
+          BASE_DISTRO: jammy
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
+          BENCHER_PROJECT: k-framework
+          BENCHER_ADAPTER: json
+        run: |
+          set -euxo pipefail
+          workspace=$(pwd)
+          docker run --name ci-${GITHUB_SHA}                                   \
+            --rm -it --detach                                                  \
+            -e BENCHER_API_TOKEN=$BENCHER_API_TOKEN                            \
+            -e BENCHER_PROJECT=$BENCHER_PROJECT                                \
+            -e BENCHER_ADAPTER=$BENCHER_ADAPTER                                \
+            -e GITHUB_HEAD_REF=$GITHUB_HEAD_REF                                \
+            -e GITHUB_BASE_REF=$GITHUB_BASE_REF                                \
+            -e GITHUB_TOKEN=$GITHUB_TOKEN                                      \
+            -e GITHUB_ACTIONS=true                                             \
+            -e GITHUB_EVENT_NAME=$GITHUB_EVENT_NAME                            \
+            -e GITHUB_REPOSITORY=$GITHUB_REPOSITORY                            \
+            -e GITHUB_REF=$GITHUB_REF                                          \
+            --workdir /opt/workspace                                           \
+            -v "${workspace}:/opt/workspace"                                   \
+            ${BASE_OS}:${BASE_DISTRO}
+      - name: 'Getting Performance Tests Results'
+        run: |
+          set -euxo pipefail
+          docker exec -t ci-${GITHUB_SHA} /bin/bash -c './k-distribution/tests/profiling/post_results_to_develop.py'
+      - name: 'Posting Performance Tests Results'
+        run: |
+          set -euxo pipefail
+          docker exec -t ci-${GITHUB_SHA} /bin/bash -c 'bencher run            \
+            --if-branch "develop" --else-if-branch "master"                    \
+            --file .profiling-results.json --err --ci-only-on-alert            \
+            --github-actions "${GITHUB_TOKEN}" 'echo "Exporting report"'
+      - name: 'Tear down Docker'
+        if: always()
+        run: |
+          docker stop --time=0 ci-${GITHUB_SHA}
+          docker container rm --force ci-${GITHUB_SHA} || true

k-distribution/tests/profiling/Makefile

@@ -1,4 +1,4 @@
-EXCLUDE=Makefile kprofiling.mak setup_profiling.sh
+EXCLUDE=Makefile kprofiling.mak setup_profiling.sh post_results_to_develop.py
 SUBDIRS=$(filter-out $(EXCLUDE), $(wildcard *))
 
 SUBCLEAN=$(addsuffix .clean,$(SUBDIRS))

k-distribution/tests/profiling/post_results_to_develop.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+import json
+import subprocess
+
+COMMIT_SHA=''
+FROM_BRANCH=''
+TO_BRANCH='develop'
+
+def execute_command(command):
+    # Try to run the command
+    try:
+        result = subprocess.run(command,stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE, text=True)
+
+        # Check for errors
+        if result.returncode != 0:
+            print("Error:", result.stderr)
+            exit(1)
+
+    except subprocess.CalledProcessError as e:
+        # Handle any errors or exceptions here
+        print("Error:", e)
+        exit(1)
+
+    return result.stdout
+
+# git command to get the last commit in develop
+commit_command = [ 'git', 'log', '--format=\"%H\"', '-n', '1' ]
+COMMIT_SHA = execute_command(commit_command).strip('\"').strip('\"\n')
+
+# curl command to get the branch name of last commit in develop
+API_URL = 'https://api.github.com/repos/runtimeverification/k/commits/' \
+        + COMMIT_SHA + '/pulls'
+branch_command = ['curl', '-L', '-H', 'Accept:', 'application/vnd.github+json',
+                  '-H', '\"Authorization', 'Bearer', '${GITHUB_TOKEN}\"', '-H',
+                  '\"X-GitHub-Api-Version:', '2022-11-28\"', API_URL]
+FROM_BRANCH = json.loads(execute_command(branch_command))[0]['head']['label']
+
+print("Exporting last bencher report from", FROM_BRANCH, "to", TO_BRANCH)
+
+# This command will generate a JSON file with a list containing the last reports
+# sorted in descendenting order for the project.
+bencher_command = ["bencher", "report", "list", "--project", "k-framework",
+                   "--sort", "date_time", "--direction", "desc", "--per-page",
+                   "255"]
+data = json.loads(execute_command(bencher_command))
+
+# Collect all elemnts where the key 'project' is 'k_framework'
+k_framework = [item for item in data if item['project']['slug'] == 'k-framework'
+               and item['branch']['slug'] == FROM_BRANCH]
+
+# Append the last 6 reports to the list, they correspond to the last performance
+# execution on the last commit in FROM_BRANCH
+def get_name_and_value(item):
+    return item['metric_kind']['slug'], item['benchmarks'][0]['metric']['value']
+
+data = {}
+for i in range(0,6):
+    item = k_framework[i]
+    results = item['results']
+    benchmark_name = results[0][0]['benchmarks'][0]['name']
+    metric_name_memory, value_memory = get_name_and_value(results[0][0])
+    metric_name_size, value_size = get_name_and_value(results[0][1])
+
+    branch = item['branch']
+    print("Appended:", benchmark_name, "created in", item['created'],
+          "on branch", branch['name'], "with version", branch['version']['number'])
+
+    data.update({benchmark_name: {metric_name_size: {"value": value_size},
+                                  metric_name_memory: {"value": value_memory}}
+                 })
+
+json.dump(data, open('.profiling-results.json', 'w'), indent=4)