-
Notifications
You must be signed in to change notification settings - Fork 67
92 lines (76 loc) · 2.3 KB
/
scenario-runner.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
---
name: 🚂 Scenario Runner
on:
# Trigger update
# push:
workflow_dispatch:
inputs:
experiments_dir:
type: choice
description: 'Experiments Directory'
required: false
default: eval6
options:
- eval6
- eval5
- eval1-4
debug_enabled:
type: boolean
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
required: false
default: false
jobs:
matrix_generator:
name: Matrix Generator
runs-on: [self-hosted]
outputs:
matrix: ${{ steps.generate-matrix.outputs.matrix }}
steps:
- name: 🐄 Got git?
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: 👩💻 Generating Matrix
id: generate-matrix
run: |
echo "::set-output name=matrix::$(find ./scenario_configs/${{ github.event.inputs.experiments_dir }} -type f -name "*.json" | jq -cnR '[inputs | select(length>0)]')"
matrix_job:
runs-on: [self-hosted]
needs: matrix_generator
strategy:
fail-fast: false
matrix:
scenario_path: ${{ fromJson(needs.matrix_generator.outputs.matrix) }}
steps:
- name: 🐄 Got git?
uses: actions/checkout@v3
- name: 🐍 Setup Environment
uses: ./.github/actions/evaluations_environment_setup
- name: ⚒️ Install Prerequisites
run: |
source .venv/bin/activate
- name: 🚀 Run Scenario Evaluation
timeout-minutes: 30
env:
ARMORY_INSTALL: "/tmp"
HOME: "/tmp"
run: |
mkdir -p "/tmp/logs"
mkdir -p "${HOME}/armory/outputs/"
mkdir -p "${HOME}/.armory/"
armory configure --use-defaults
pytest \
-c pyproject.toml \
--verbose \
-s ./tests/end_to_end/test_e2e_scenarios.py \
--scenario-path ${{ matrix.scenario_path }} \
--github-ci | tee "scenario_evaluation.log"
- name: 📁 Archiving Artifacts
uses: actions/upload-artifact@v3
# if: ${{ !failure() }}
continue-on-error: true
with:
name: evaluation-artifacts
retention-days: 1
path: |
/tmp/.armory/outputs/