-
Notifications
You must be signed in to change notification settings - Fork 65
/
gen-interop.sh
executable file
·370 lines (315 loc) · 16.4 KB
/
gen-interop.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
#! /bin/bash
# Configuration Data -- order matters in these arrays. A new entry requires an entry in ALL "ta_" arrays
ta_tlas=("acapy" "credo" "aries-vcx")
ta_names=("Aries Cloud Agent Python" "Credo-TS" "Aries VCX")
ta_shortnames=("ACA-Py" "Credo" "VCX")
ta_scopes=("AIP 1, 2" "AIP 1" "AIP 1")
ta_exceptions=("None" "Revocation" "Revocation")
ta_urls=(https://github.com/hyperledger/aries-cloudagent-python \
https://github.com/openwallet-foundation/credo-ts \
https://github.com/hyperledger/aries-vcx)
workflows=".github/workflows/test-harness-*"
aath_guide="docs/guide"
# Clean up files made when testing the information web site.
# We don't want these files checked into the website
if [[ "$1" == "clean" ]]; then
rm -rf ${aath_guide} docs/acapy.md docs/aries-vcx.md docs/credo.md docs/README.md
exit 0
fi
unameOut="$(uname -s)"
case "${unameOut}" in
Linux*) machine=Linux;;
Darwin*) machine=Mac;;
CYGWIN*) machine=Cygwin;;
MINGW*) machine=MinGw;;
*) machine="UNKNOWN:${unameOut}"
esac
usage () {
cat << EOF
Usage: gen-interop.sh [--help]
Generates the Aries Interop tests documentation pages based on the test agents setup and the workflows
defined in the repo. The workflows are in the .github/workflows folder and must be named "test-harness-<name>.yml".
The generator iterates over each workflow file, extracting data from it -- either live
data or from comments. Further, it curl's the related allure "latest" files to get some
data about the last run - tests run, tests passed and date. All of the data goes into
a set of arrays indexed by the workflow file number.
From there, a README file is created summarizing by Test Agent the run results, and
a file per Test Agent is generated that gives more details about the tests run for that agent.
Basic process:
Collect all the runset data in arrays
Generate the header for the summary page (README.md)
Iterate through the test agents configured in this script.
Print a summary per test agent showing the results against all other agents.
Iterate again through the test agents
Print a details file about about the current test agent (<ta_shortname>.md)
Print a table listing each runset the test agent is a part of
Iterate over the runsets
Print details about each runset the test agent is a part of
To add information about a workflow, insert comments into the YAML with identifiers RUNSET_NAME,
Scope, Summary, and Current.
To exclude a workflow, add a comment "# SKIP", and the process will not include the workflow
EOF
exit
}
if [ "$1" == "--help" ]; then
usage
fi
default_value () {
# Helper function set a value to a default if no value is provided
if [ "$2" == "" ]; then
echo $1
else
echo $2
fi
}
list_agents () {
# Helper function -- list the agents with links -- used in the header function
count=0
for agent in "${ta_shortnames[@]}"; do
echo "- [${ta_names[$count]}](${ta_urls[$count]}) ($agent)"
count=$(expr ${count} + 1)
done
}
aries_interop_header () {
# The summary page header -- markdown format
cat << EOF
This web site shows the current status of Aries Interoperability between Aries frameworks and agents.
The latest interoperability test results are below.
The following test agents are currently being tested:
$(list_agents)
## Latest Interoperability Results
EOF
}
aries_interop_footer () {
# The summary page footer -- markdown format
cat << EOF
In the table above, each row is a test agent, its columns the results of tests executed in combination with other test agents.
The last column ("All Tests") shows the results of all tests run for the given test agent in any role. The link on each test
agent name provides more details about results for all test combinations for that test agent. On
that page are links to a full history of the test runs and full details on every executed test.
**Notes:**
- Where the row and column are the same Test Agent, the results include only the tests where the Test Agent plays ALL of the roles (ACME, Bob, Faber and Mallory)
- The results in the "All Tests" column include tests involving the "Test Agent" in ANY of the roles.
- Wondering what the results mean? Please read the brief [introduction to Aries interoperability](aries-interop-intro.md) for some background.
- Select the "Test Agent" links to drill down into the tests being run for each Test Agent.
EOF
}
aries_interop_summary_table_header () {
# The header for the table on the summary page -- static columns, plus one per test agent
printf "| Test Agent | Scope | Exceptions "
for agent in "${ta_shortnames[@]}"; do
printf "| $agent "
done
printf "| **All Tests** |\\n"
printf "| ----- | ----- | ----- "
for agent in "${ta_shortnames[@]}"; do
printf "| :----: "
done
printf "| :----: |\\n"
}
sum_print_tests () {
# Inserts the value of a cell by summing the number of tests involving the two agents
# passed in as args 1 and 2. Prints the result at the end.
# Special handling if just arg 1 -- only include runsets where ALL the agents are of type arg 1
# In all cases SKIP-ped runsets are NOT included in the results.
passed=0
total=0
file_num=0
for file in ${workflows}; do
if [[ -z "$2" ]]; then
if [[ ${ACME[$file_num]} =~ $1 && ${BOB[$file_num]} =~ $1 && ${FABER[$file_num]} =~ $1 && ${MALLORY[$file_num]} =~ $1 && "${SKIP[$file_num]}" == "" ]]; then
passed=$(expr $passed + ${PASSED[$file_num]} )
total=$(expr $total + ${TOTAL_CASES[$file_num]} )
fi
else
agents=${ACME[$file_num]}${BOB[$file_num]}${FABER[$file_num]}${MALLORY[$file_num]}
if [[ $agents =~ $1 && $agents =~ $2 && "${SKIP[$file_num]}" == "" ]]; then
passed=$(expr $passed + ${PASSED[$file_num]} )
total=$(expr $total + ${TOTAL_CASES[$file_num]} )
fi
fi
file_num=$(expr ${file_num} + 1 )
done
if [ ${total} -eq 0 ]; then
percent=0
else
percent=$((${passed}*100/${total}))
fi
bold=""
if [ "$1" == "$2" ]; then
bold="**"
fi
printf "| ${bold}%d / %d<br>%d%%${bold} " ${passed} ${total} ${percent}
}
aries_interop_summary_table () {
# Prints the data rows of the summary page table -- iterating over the list of agents
# Calls the "sum_print_tests" function to sum across runsets for a test agent.
# Special handling when the agent and other_agent are the same -- call "sum_print_tests" with just one arg
# After, call "sum_print_tests" a last time with agent for both args to get total across all tests.
count=0
for agent in "${ta_tlas[@]}"; do
printf "| [${ta_shortnames[$count]}](${agent}.md)"
printf "| ${ta_scopes[$count]} "
printf "| ${ta_exceptions[$count]} "
count=$(expr $count + 1)
for other_agent in "${ta_tlas[@]}"; do
if [[ "$agent" == "$other_agent" ]]; then
sum_print_tests $agent
else
sum_print_tests $agent $other_agent
fi
done
sum_print_tests $agent $agent
printf "|\\n"
done
}
# Inline code starts here
# Copy the MD guide files from the root a folder in the docs folder
mkdir -p ${aath_guide}
cp aries-backchannels/README.md aries-backchannels.md
cp LICENSE ${aath_guide}/LICENSE.md
# Fix links in the copied Guide documents
for file in *.md; do
sed ${file} -e "s/LICENSE/LICENSE.md/" \
-e "s#(\.*/docs/assets#(../assets#g" \
-e "s#(docs/assets#(../assets#g" \
-e "s#../aries-agent-test-harness/aries-backchannels/README.md#aries-backchannels.md#g" \
-e "s#aries-backchannels/README.md#./aries-backchannels.md#g" \
> ${aath_guide}/${file}
done
# Collect all the data about each runset into a series of arrays - one per data element
count=0
for file in ${workflows}; do
RUNSET[$count]=$(echo $file | sed "s/.*ness-//" | sed "s/.yml//")
# set -x
RUNSET_NAME[$count]=$( default_value ${RUNSET[$count]} "$(grep RUNSET_NAME $file | sed 's/^.*: //' | sed 's/["]//g')" )
SCOPE[$count]=$(grep "# Scope" $file | sed 's/^.*: //' | sed 's/["]//g')
EXCEPTIONS[$count]=$(grep "# Exceptions" $file | sed 's/^.*: //' | sed 's/["]//g')
# Optional -- to skip a runset from inclusion in the results.
SKIP[$count]=$(grep "# SKIP" $file )
RUNSET_LINK[$count]=${RUNSET}
SUMMARY[$count]=$(grep "^#" $file | sed 's/^#[ ]*//' | sed '/Current/,$d' | sed '1,/Summary/d' | sed 's/$/\\n/' | sed 's/^ //')
CURRENT_STATUS[$count]=$(grep "^#" $file | sed 's/^#[ ]*//' | sed '/^End/,$d' | sed -n '/Current/,$p' | sed '1d' | sed 's/^ //')
ALLURE_PROJECT[$count]=$(grep "REPORT_PROJECT" $file | sed -n '1p' | sed 's/.*: //' | sed 's/ *$//' )
ALLURE_LINK[$count]="https://allure.vonx.io/allure-docker-service-ui/projects/${ALLURE_PROJECT[$count]}/reports/latest"
ALLURE_BEHAVIORS_LINK[$count]="https://allure.vonx.io/api/allure-docker-service/projects/${ALLURE_PROJECT[$count]}/reports/latest/index.html?redirect=false#behaviors"
ALLURE_SUMMARY[$count]=$(curl --silent https://allure.vonx.io/api/allure-docker-service/projects/${ALLURE_PROJECT[$count]}/reports/latest/widgets/summary.json)
ALLURE_ENVIRONMENT[$count]=$(curl --silent https://allure.vonx.io/api/allure-docker-service/projects/${ALLURE_PROJECT[$count]}/reports/latest/widgets/environment.json)
ACME[$count]=$(echo ${ALLURE_ENVIRONMENT[$count]} | sed 's/acme.agent.*/acme.agent/' | sed 's/-agent-backchannel.*//' | sed 's/.*\[ "//' )
BOB[$count]=$(echo ${ALLURE_ENVIRONMENT[$count]} | sed 's/.*acme.agent/acme.agent/' | sed 's/bob.agent.*/bob.agent/' |sed 's/-agent-backchannel.*//' | sed 's/.*\[ "//' )
FABER[$count]=$(echo ${ALLURE_ENVIRONMENT[$count]} | sed 's/.*bob.agent/bob.agent/' | sed 's/faber.agent.*/faber.agent/' |sed 's/-agent-backchannel.*//' | sed 's/.*\[ "//' )
MALLORY[$count]=$(echo ${ALLURE_ENVIRONMENT[$count]} | sed 's/.*faber.agent/faber.agent/' | sed 's/mallory.agent.*/mallory.agent/' |sed 's/-agent-backchannel.*//' | sed 's/.*\[ "//' )
ACME_VERSION[$count]=$(echo ${ALLURE_ENVIRONMENT[$count]} | sed 's/.*role.acme//' | sed 's/acme.agent.*//' | sed 's/\\"//g' | sed 's/.* \[ "//' | sed 's/".*//' )
BOB_VERSION[$count]=$(echo ${ALLURE_ENVIRONMENT[$count]} | sed 's/.*role.bob//' | sed 's/bob.agent.*//'| sed 's/\\"//g' | sed 's/.* \[ "//' | sed 's/".*//' )
FABER_VERSION[$count]=$(echo ${ALLURE_ENVIRONMENT[$count]} | sed 's/.*role.faber//' | sed 's/faber.agent.*//' | sed 's/\\"//g' | sed 's/.* \[ "//' | sed 's/".*//' )
MALLORY_VERSION[$count]=$(echo ${ALLURE_ENVIRONMENT[$count]} | sed 's/.*role.mallory//' | sed 's/mallory.agent.*//' | sed 's/\\"//g' | sed 's/.* \[ "//' | sed 's/".*//' )
TOTAL_CASES[$count]=$( echo ${ALLURE_SUMMARY[$count]} | sed 's/.*"total" : \([0-9]*\).*/\1/' )
PASSED[$count]=$( echo ${ALLURE_SUMMARY[$count]} | sed 's/.*"passed" : \([0-9]*\).*/\1/' )
FAILED[$count]=$( echo ${ALLURE_SUMMARY[$count]} | sed 's/.*"failed" : \([0-9]*\).*/\1/' )
if [[ ${#TOTAL_CASES[$count]} -gt 4 ]] || [[ ${#PASSED[$count]} -gt 4 ]] || [[ ${#FAILED[$count]} -gt 4 ]]; then
# Special case: No results found for the runset
TOTAL_CASES[$count]=0
PASSED[$count]=0
FAILED[$count]=0
fi
if [ "${TOTAL_CASES[$count]}" -eq "0" ]; then
PERCENT[$count]=0
else
PERCENT[$count]=$((${PASSED[$count]}*100/${TOTAL_CASES[$count]}))
fi
# Figure out the date/time of the last test for the runset
epoch_seconds=$(echo ${ALLURE_SUMMARY[$count]} | sed 's/.*"stop" : \([0-9]\{10\}\).*/\1/' )
if [ ${#epoch_seconds} -gt 10 ]; then
# Special case: No results found for the runset
epoch_seconds=$(date +%s)
fi
if [ ${machine} == 'Mac' ]; then
ALLURE_DATE[$count]=$( date -j -f %s ${epoch_seconds} )
else
ALLURE_DATE[$count]=$( date -d@${epoch_seconds} )
fi
echo File: $file
# echo ALLURE_PROJECT: ${ALLURE_PROJECT[$count]}
# echo ALLURE_LINK: ${ALLURE_LINK[$count]}
# echo ALLURE_BEHAVIORS_LINK: ${ALLURE_BEHAVIORS_LINK[$count]}
# echo ALLURE_SUMMARY: ${ALLURE_SUMMARY[$count]}
# echo ALLURE_DATE: ${ALLURE_DATE[$count]}
# echo ALLURE_ENVIRONMENT: ${ALLURE_ENVIRONMENT[$count]}
echo -n PASSED: ${PASSED[$count]} /
echo -n TOTAL_CASES: ${TOTAL_CASES[$count]} =
echo PERCENT: ${PERCENT[$count]}
# echo ACME: ${ACME_VERSION[$count]}
# echo BOB: ${BOB_VERSION[$count]}
# echo FABER: ${FABER_VERSION[$count]}
# echo MALLORY: ${MALLORY_VERSION[$count]}
# echo ${ALLURE_DATE[$count]}
count=$(expr ${count} + 1)
done
# First write the summary file
outfile=docs/README.md
# Remove the file if it exists
rm -f $outfile
# Title of the page
echo -e '# Aries Interoperability Information\n' >>$outfile
# Header part of the page
aries_interop_header >>$outfile
# Summary table header
aries_interop_summary_table_header >>$outfile
# Summary table data
aries_interop_summary_table >>$outfile
# Finished with the table -- add in the footer and done with the summary file
aries_interop_footer >>$outfile
echo -e "\\n*Results last updated: $(date | sed 's/ / /g')*" >>$outfile
echo "" >>$outfile
# Iterate through the test agents and create a file per test agent.
ta_num=0
for agent in "${ta_tlas[@]}"; do
outfile=docs/${agent}.md
# Remove the file if it exists
rm -f $outfile
# echo -e "---\\nsort: ${ta_num} # follow a certain sequence of letters or numbers\\n---" >>$outfile
echo -e "# ${ta_names[$ta_num]} Interoperability\\n" >>$outfile
echo -e "## Runsets with ${ta_shortnames[$ta_num]}\\n" >>$outfile
# Print a table of details
echo "| Runset | ACME<br>(Issuer) | Bob<br>(Holder) | Faber<br>(Verifier) | Mallory<br>(Holder) | Scope | Results | " >>$outfile
echo "| ------ | :--------------: | :-------------: | :----------------: | :-----------------: | ----- | :-----: | " >>$outfile
runset_num=0
# Iterate through the workflows and find the ones involving this test agent -- ignoring skipped files
# Insert the data into the table
for file in ${workflows}; do
agents=${ACME[$runset_num]}${BOB[$runset_num]}${FABER[$runset_num]}${MALLORY[$runset_num]}
if [[ $agents =~ $agent && "${SKIP[$runset_num]}" == "" ]]; then
echo -e "| [${RUNSET[$runset_num]}](#runset-${RUNSET[$runset_num]}) | ${ACME[$runset_num]}<br>${ACME_VERSION[$runset_num]} | ${BOB[$runset_num]}<br>${BOB_VERSION[$runset_num]} | ${FABER[$runset_num]}<br>${FABER_VERSION[$runset_num]} | ${MALLORY[$runset_num]}<br>${MALLORY_VERSION[$runset_num]} | ${SCOPE[$runset_num]} | [**${PASSED[$runset_num]} / ${TOTAL_CASES[$runset_num]}<br>${PERCENT[$runset_num]}%**](${ALLURE_BEHAVIORS_LINK[$runset_num]}) |" >>$outfile
fi
runset_num=$( expr ${runset_num} + 1)
done
# Iterate again through the workflows to write out a section per workflow
echo -e "\\n## Runset Notes" >>$outfile
runset_num=0
for file in ${workflows}; do
agents=${ACME[$runset_num]}${BOB[$runset_num]}${FABER[$runset_num]}${MALLORY[$runset_num]}
if [[ $agents =~ $agent && "${SKIP[$runset_num]}" == "" ]]; then
echo -e "\\n### Runset **${RUNSET[$runset_num]}**\\n" >>$outfile
echo -e "Runset Name: ${RUNSET_NAME[$runset_num]}" >>$outfile
echo -e "\\n\`\`\`tip\\n**Latest results: ${PASSED[$runset_num]} out of ${TOTAL_CASES[$runset_num]} (${PERCENT[$runset_num]}%)**\\n" >>$outfile
echo -e "\\n*Last run: ${ALLURE_DATE[$runset_num]}*\\n\`\`\`\\n" >>$outfile
echo -e "#### Current Runset Status" >>$outfile
# Print the status text from the workflow file -- or a default message
if [ "${CURRENT_STATUS[$runset_num]}" == "" ]; then
echo -e "\`\`\`warning\\nNo test status note is available for this runset. Please update: $file.\\n\`\`\`\\n" >>$outfile
else
echo -e "${CURRENT_STATUS[$runset_num]}\\n" >>$outfile
fi
# Links to the allure results
echo -e "#### Runset Details\\n" >>$outfile
echo -e "- [Results by executed Aries RFCs](${ALLURE_BEHAVIORS_LINK[$runset_num]})" >>$outfile
echo -e "- [Test execution history](${ALLURE_LINK[$runset_num]})\\n" >>$outfile
fi
runset_num=$( expr ${runset_num} + 1)
done
# Link to the summary page
echo "Jump back to the [interoperability summary](./README.md)." >>$outfile
# And we're done for this file
echo "" >>$outfile
ta_num=$( expr ${ta_num} + 1)
done