From 0ba47c7e675ae26d29c671d76cdb16f02b44435b Mon Sep 17 00:00:00 2001 From: Long Vu Date: Thu, 15 Jul 2021 14:06:45 -0400 Subject: [PATCH 1/3] stress-test.ipynb: fix incorrect calculation of number of failed tests Below is the result from a nightly run on production. It found 4 failed tests but in fact none were failing since failed_results was empty. ``` =================================== FAILURES =================================== _____________________ notebooks/stress-tests.ipynb::Cell 2 _____________________ Notebook cell execution failed Cell 2: Cell execution caused an exception Input: # NBVAL_IGNORE_OUTPUT test_statuses = [] failed_results = '' for bird in TEST_WPS_BIRDS: bird_url = f"{TWITCHER_URL}/{bird}/wps?service=wps&request=getcapabilities" expect_status_code = 200 results = stress_test_requests(bird_url, runs=TEST_RUNS, code=expect_status_code, max_err_code=TEST_MAX_ERR_CODE, max_avg_time=TEST_MAX_AVG_TIME, abort_retries=TEST_TIMEOUT_RETRY, abort_timeout=TEST_TIMEOUT_ABORT) test_statuses.append(results.status) print(results) if results.status: failed_results = f"{failed_results}\n{results}" failed_tests = len([status != 0 for status in test_statuses]) assert not failed_tests, f"Failed {failed_tests} tests. Failed results: {failed_results}" print("\nAll tests passed!") Traceback: --------------------------------------------------------------------------- AssertionError Traceback (most recent call last) in 15 failed_results = f"{failed_results}\n{results}" 16 failed_tests = len([status != 0 for status in test_statuses]) ---> 17 assert not failed_tests, f"Failed {failed_tests} tests. Failed results: {failed_results}" 18 19 print("\nAll tests passed!") AssertionError: Failed 4 tests. Failed results: =========================== short test summary info ============================ FAILED notebooks/stress-tests.ipynb::Cell 2 ============= 1 failed, 174 passed, 1 skipped in 428.88s (0:07:08) ============= ``` --- notebooks/stress-tests.ipynb | 841 +++++++++++++++++------------------ 1 file changed, 420 insertions(+), 421 deletions(-) diff --git a/notebooks/stress-tests.ipynb b/notebooks/stress-tests.ipynb index 830024d0..5abaee71 100644 --- a/notebooks/stress-tests.ipynb +++ b/notebooks/stress-tests.ipynb @@ -171,7 +171,7 @@ " f\"Maximum number of timeout ({self.timeout_abort}s) requests exceeded ({self.timeout_count}).\",\n", " \"Test was aborted to avoid further delays.\"\n", " ]\n", - " summary.append(f\"Test {'succeeded' if results.status == 0 else 'failed'}.\")\n", + " summary.append(f\"Test {'succeeded' if results.status == 0 else 'failed'} (status={self.status}).\")\n", " summary = (\"\\n\" + offset * \" \").join(summary)\n", " return cleandoc(f\"\"\"\n", " Stress Test:\n", @@ -328,114 +328,114 @@ " url: https://lvupavicsdev.ouranos.ca/twitcher/ows/proxy/finch/wps?service=wps&request=getcapabilities\n", " args: {}\n", " Times:\n", - " min: 0.162s\n", - " avg: 0.200s\n", - " max: 0.285s\n", + " min: 0.141s\n", + " avg: 0.199s\n", + " max: 0.322s\n", " Results:\n", " Run Codes Delta Times\n", - " 1 200 0.000s 0.172s\n", - " 2 200 0.067s 0.165s\n", - " 3 200 0.023s 0.263s\n", - " 4 200 0.048s 0.187s\n", - " 5 200 0.028s 0.164s\n", - " 6 200 0.075s 0.168s\n", - " 7 200 0.090s 0.270s\n", - " 8 200 0.085s 0.178s\n", - " 9 200 0.062s 0.183s\n", - " 10 200 0.029s 0.282s\n", - " 11 200 0.022s 0.180s\n", - " 12 200 0.015s 0.178s\n", - " 13 200 0.080s 0.172s\n", - " 14 200 0.096s 0.285s\n", - " 15 200 0.030s 0.173s\n", - " 16 200 0.035s 0.174s\n", - " 17 200 0.029s 0.169s\n", - " 18 200 0.063s 0.274s\n", - " 19 200 0.056s 0.176s\n", - " 20 200 0.026s 0.167s\n", - " 21 200 0.019s 0.170s\n", - " 22 200 0.099s 0.276s\n", - " 23 200 0.088s 0.174s\n", - " 24 200 0.083s 0.187s\n", - " 25 200 0.095s 0.268s\n", - " 26 200 0.059s 0.179s\n", - " 27 200 0.004s 0.183s\n", - " 28 200 0.078s 0.174s\n", - " 29 200 0.076s 0.267s\n", - " 30 200 0.001s 0.180s\n", - " 31 200 0.036s 0.167s\n", - " 32 200 0.065s 0.181s\n", - " 33 200 0.059s 0.270s\n", - " 34 200 0.074s 0.180s\n", - " 35 200 0.038s 0.162s\n", - " 36 200 0.077s 0.171s\n", - " 37 200 0.073s 0.265s\n", - " 38 200 0.036s 0.179s\n", - " 39 200 0.074s 0.166s\n", - " 40 200 0.066s 0.280s\n", - " 41 200 0.094s 0.171s\n", - " 42 200 0.069s 0.173s\n", - " 43 200 0.047s 0.180s\n", - " 44 200 0.077s 0.263s\n", - " 45 200 0.041s 0.169s\n", - " 46 200 0.027s 0.172s\n", - " 47 200 0.040s 0.165s\n", - " 48 200 0.056s 0.268s\n", - " 49 200 0.072s 0.175s\n", - " 50 200 0.079s 0.172s\n", - " 51 200 0.032s 0.279s\n", - " 52 200 0.049s 0.184s\n", - " 53 200 0.083s 0.180s\n", - " 54 200 0.003s 0.173s\n", - " 55 200 0.086s 0.259s\n", - " 56 200 0.019s 0.175s\n", - " 57 200 0.072s 0.172s\n", - " 58 200 0.018s 0.176s\n", - " 59 200 0.091s 0.275s\n", - " 60 200 0.016s 0.173s\n", - " 61 200 0.061s 0.169s\n", - " 62 200 0.013s 0.179s\n", - " 63 200 0.046s 0.260s\n", - " 64 200 0.027s 0.186s\n", - " 65 200 0.045s 0.182s\n", - " 66 200 0.061s 0.168s\n", - " 67 200 0.050s 0.264s\n", - " 68 200 0.048s 0.178s\n", - " 69 200 0.086s 0.176s\n", - " 70 200 0.063s 0.178s\n", - " 71 200 0.056s 0.272s\n", - " 72 200 0.069s 0.179s\n", - " 73 200 0.086s 0.175s\n", - " 74 200 0.049s 0.176s\n", - " 75 200 0.079s 0.284s\n", - " 76 200 0.011s 0.173s\n", - " 77 200 0.027s 0.166s\n", - " 78 200 0.091s 0.180s\n", - " 79 200 0.046s 0.281s\n", - " 80 200 0.020s 0.173s\n", - " 81 200 0.085s 0.176s\n", - " 82 200 0.059s 0.272s\n", - " 83 200 0.028s 0.179s\n", - " 84 200 0.030s 0.173s\n", - " 85 200 0.055s 0.221s\n", - " 86 200 0.009s 0.263s\n", - " 87 200 0.066s 0.179s\n", - " 88 200 0.087s 0.171s\n", - " 89 200 0.039s 0.174s\n", - " 90 200 0.067s 0.260s\n", - " 91 200 0.056s 0.177s\n", - " 92 200 0.046s 0.180s\n", - " 93 200 0.100s 0.169s\n", - " 94 200 0.073s 0.266s\n", - " 95 200 0.008s 0.175s\n", - " 96 200 0.044s 0.175s\n", - " 97 200 0.003s 0.272s\n", - " 98 200 0.070s 0.163s\n", - " 99 200 0.021s 0.177s\n", - " 100 200 0.016s 0.195s\n", + " 1 200 0.000s 0.225s\n", + " 2 200 0.070s 0.160s\n", + " 3 200 0.089s 0.186s\n", + " 4 200 0.032s 0.273s\n", + " 5 200 0.025s 0.181s\n", + " 6 200 0.060s 0.173s\n", + " 7 200 0.097s 0.181s\n", + " 8 200 0.034s 0.268s\n", + " 9 200 0.093s 0.176s\n", + " 10 200 0.035s 0.174s\n", + " 11 200 0.012s 0.174s\n", + " 12 200 0.073s 0.273s\n", + " 13 200 0.016s 0.172s\n", + " 14 200 0.041s 0.172s\n", + " 15 200 0.027s 0.179s\n", + " 16 200 0.077s 0.271s\n", + " 17 200 0.091s 0.172s\n", + " 18 200 0.004s 0.178s\n", + " 19 200 0.060s 0.172s\n", + " 20 200 0.075s 0.275s\n", + " 21 200 0.040s 0.179s\n", + " 22 200 0.099s 0.170s\n", + " 23 200 0.043s 0.176s\n", + " 24 200 0.079s 0.288s\n", + " 25 200 0.028s 0.172s\n", + " 26 200 0.076s 0.169s\n", + " 27 200 0.029s 0.187s\n", + " 28 200 0.082s 0.274s\n", + " 29 200 0.073s 0.178s\n", + " 30 200 0.060s 0.177s\n", + " 31 200 0.070s 0.172s\n", + " 32 200 0.032s 0.273s\n", + " 33 200 0.081s 0.169s\n", + " 34 200 0.073s 0.171s\n", + " 35 200 0.097s 0.180s\n", + " 36 200 0.094s 0.278s\n", + " 37 200 0.067s 0.182s\n", + " 38 200 0.071s 0.179s\n", + " 39 200 0.093s 0.175s\n", + " 40 200 0.002s 0.297s\n", + " 41 200 0.048s 0.160s\n", + " 42 200 0.081s 0.174s\n", + " 43 200 0.027s 0.180s\n", + " 44 200 0.082s 0.275s\n", + " 45 200 0.052s 0.171s\n", + " 46 200 0.061s 0.173s\n", + " 47 200 0.056s 0.172s\n", + " 48 200 0.061s 0.286s\n", + " 49 200 0.001s 0.161s\n", + " 50 200 0.041s 0.167s\n", + " 51 200 0.095s 0.175s\n", + " 52 200 0.100s 0.272s\n", + " 53 200 0.011s 0.178s\n", + " 54 200 0.015s 0.166s\n", + " 55 200 0.098s 0.170s\n", + " 56 200 0.001s 0.276s\n", + " 57 200 0.073s 0.166s\n", + " 58 200 0.086s 0.182s\n", + " 59 200 0.008s 0.171s\n", + " 60 200 0.010s 0.267s\n", + " 61 200 0.045s 0.166s\n", + " 62 200 0.073s 0.184s\n", + " 63 200 0.084s 0.165s\n", + " 64 200 0.008s 0.270s\n", + " 65 200 0.094s 0.167s\n", + " 66 200 0.004s 0.174s\n", + " 67 200 0.014s 0.167s\n", + " 68 200 0.067s 0.266s\n", + " 69 200 0.007s 0.176s\n", + " 70 200 0.017s 0.175s\n", + " 71 200 0.100s 0.176s\n", + " 72 200 0.079s 0.287s\n", + " 73 200 0.080s 0.176s\n", + " 74 200 0.057s 0.173s\n", + " 75 200 0.067s 0.162s\n", + " 76 200 0.056s 0.272s\n", + " 77 200 0.021s 0.166s\n", + " 78 200 0.030s 0.177s\n", + " 79 200 0.055s 0.172s\n", + " 80 200 0.080s 0.288s\n", + " 81 200 0.051s 0.172s\n", + " 82 200 0.077s 0.159s\n", + " 83 200 0.056s 0.177s\n", + " 84 200 0.049s 0.322s\n", + " 85 200 0.004s 0.180s\n", + " 86 200 0.060s 0.166s\n", + " 87 200 0.075s 0.177s\n", + " 88 200 0.073s 0.320s\n", + " 89 200 0.028s 0.201s\n", + " 90 200 0.075s 0.168s\n", + " 91 200 0.077s 0.209s\n", + " 92 200 0.075s 0.141s\n", + " 93 200 0.039s 0.147s\n", + " 94 200 0.094s 0.267s\n", + " 95 200 0.031s 0.162s\n", + " 96 200 0.070s 0.158s\n", + " 97 200 0.060s 0.147s\n", + " 98 200 0.051s 0.219s\n", + " 99 200 0.016s 0.144s\n", + " 100 200 0.063s 0.245s\n", " Summary:\n", " All passing conditions have been achieved.\n", - " Test succeeded.\n", + " Test succeeded (status=0).\n", "\n", "Stress Test with [100] calls to [https://lvupavicsdev.ouranos.ca/twitcher/ows/proxy/flyingpigeon/wps?service=wps&request=getcapabilities]\n", "Progress: 0/100\n", @@ -464,114 +464,114 @@ " url: https://lvupavicsdev.ouranos.ca/twitcher/ows/proxy/flyingpigeon/wps?service=wps&request=getcapabilities\n", " args: {}\n", " Times:\n", - " min: 0.109s\n", + " min: 0.104s\n", " avg: 0.122s\n", - " max: 0.149s\n", + " max: 0.176s\n", " Results:\n", " Run Codes Delta Times\n", - " 1 200 0.000s 0.121s\n", - " 2 200 0.030s 0.117s\n", - " 3 200 0.075s 0.130s\n", - " 4 200 0.060s 0.113s\n", - " 5 200 0.074s 0.117s\n", - " 6 200 0.076s 0.126s\n", - " 7 200 0.009s 0.124s\n", - " 8 200 0.019s 0.123s\n", - " 9 200 0.057s 0.118s\n", - " 10 200 0.091s 0.117s\n", - " 11 200 0.041s 0.135s\n", - " 12 200 0.069s 0.124s\n", - " 13 200 0.097s 0.122s\n", - " 14 200 0.054s 0.124s\n", - " 15 200 0.091s 0.116s\n", - " 16 200 0.028s 0.118s\n", - " 17 200 0.033s 0.120s\n", - " 18 200 0.075s 0.115s\n", - " 19 200 0.083s 0.113s\n", - " 20 200 0.076s 0.125s\n", - " 21 200 0.062s 0.114s\n", - " 22 200 0.044s 0.115s\n", - " 23 200 0.058s 0.126s\n", - " 24 200 0.090s 0.124s\n", - " 25 200 0.001s 0.120s\n", - " 26 200 0.070s 0.114s\n", - " 27 200 0.022s 0.119s\n", - " 28 200 0.098s 0.120s\n", - " 29 200 0.096s 0.117s\n", - " 30 200 0.029s 0.116s\n", - " 31 200 0.072s 0.123s\n", - " 32 200 0.013s 0.127s\n", - " 33 200 0.097s 0.113s\n", - " 34 200 0.049s 0.127s\n", - " 35 200 0.099s 0.123s\n", - " 36 200 0.093s 0.140s\n", - " 37 200 0.004s 0.118s\n", - " 38 200 0.055s 0.109s\n", - " 39 200 0.089s 0.122s\n", - " 40 200 0.042s 0.115s\n", - " 41 200 0.039s 0.135s\n", - " 42 200 0.029s 0.113s\n", - " 43 200 0.042s 0.116s\n", - " 44 200 0.046s 0.122s\n", - " 45 200 0.049s 0.132s\n", - " 46 200 0.097s 0.114s\n", - " 47 200 0.089s 0.138s\n", - " 48 200 0.009s 0.137s\n", - " 49 200 0.035s 0.114s\n", - " 50 200 0.021s 0.111s\n", - " 51 200 0.029s 0.112s\n", - " 52 200 0.002s 0.147s\n", - " 53 200 0.077s 0.122s\n", - " 54 200 0.067s 0.119s\n", - " 55 200 0.085s 0.119s\n", - " 56 200 0.066s 0.117s\n", - " 57 200 0.070s 0.124s\n", - " 58 200 0.070s 0.114s\n", - " 59 200 0.065s 0.121s\n", - " 60 200 0.060s 0.129s\n", - " 61 200 0.051s 0.115s\n", - " 62 200 0.052s 0.126s\n", - " 63 200 0.031s 0.122s\n", - " 64 200 0.031s 0.117s\n", - " 65 200 0.087s 0.120s\n", - " 66 200 0.021s 0.124s\n", - " 67 200 0.097s 0.121s\n", - " 68 200 0.075s 0.117s\n", - " 69 200 0.001s 0.115s\n", - " 70 200 0.089s 0.113s\n", - " 71 200 0.089s 0.125s\n", - " 72 200 0.024s 0.122s\n", - " 73 200 0.062s 0.129s\n", - " 74 200 0.043s 0.125s\n", - " 75 200 0.086s 0.121s\n", - " 76 200 0.069s 0.121s\n", - " 77 200 0.030s 0.128s\n", - " 78 200 0.088s 0.120s\n", - " 79 200 0.054s 0.123s\n", - " 80 200 0.057s 0.128s\n", - " 81 200 0.044s 0.124s\n", - " 82 200 0.018s 0.127s\n", - " 83 200 0.067s 0.121s\n", - " 84 200 0.096s 0.115s\n", - " 85 200 0.012s 0.117s\n", - " 86 200 0.078s 0.119s\n", - " 87 200 0.097s 0.138s\n", - " 88 200 0.034s 0.118s\n", - " 89 200 0.048s 0.127s\n", - " 90 200 0.035s 0.115s\n", - " 91 200 0.044s 0.119s\n", - " 92 200 0.009s 0.121s\n", - " 93 200 0.051s 0.149s\n", - " 94 200 0.084s 0.118s\n", - " 95 200 0.072s 0.122s\n", - " 96 200 0.011s 0.125s\n", - " 97 200 0.015s 0.123s\n", - " 98 200 0.054s 0.112s\n", - " 99 200 0.035s 0.117s\n", - " 100 200 0.027s 0.136s\n", + " 1 200 0.000s 0.104s\n", + " 2 200 0.098s 0.130s\n", + " 3 200 0.025s 0.127s\n", + " 4 200 0.020s 0.126s\n", + " 5 200 0.075s 0.126s\n", + " 6 200 0.001s 0.176s\n", + " 7 200 0.048s 0.120s\n", + " 8 200 0.086s 0.124s\n", + " 9 200 0.067s 0.124s\n", + " 10 200 0.065s 0.128s\n", + " 11 200 0.073s 0.125s\n", + " 12 200 0.085s 0.116s\n", + " 13 200 0.075s 0.123s\n", + " 14 200 0.038s 0.126s\n", + " 15 200 0.093s 0.117s\n", + " 16 200 0.096s 0.115s\n", + " 17 200 0.049s 0.121s\n", + " 18 200 0.069s 0.118s\n", + " 19 200 0.063s 0.123s\n", + " 20 200 0.099s 0.125s\n", + " 21 200 0.062s 0.120s\n", + " 22 200 0.045s 0.122s\n", + " 23 200 0.062s 0.116s\n", + " 24 200 0.045s 0.114s\n", + " 25 200 0.062s 0.118s\n", + " 26 200 0.022s 0.122s\n", + " 27 200 0.067s 0.116s\n", + " 28 200 0.090s 0.123s\n", + " 29 200 0.068s 0.125s\n", + " 30 200 0.015s 0.114s\n", + " 31 200 0.006s 0.108s\n", + " 32 200 0.100s 0.118s\n", + " 33 200 0.059s 0.124s\n", + " 34 200 0.087s 0.127s\n", + " 35 200 0.041s 0.125s\n", + " 36 200 0.050s 0.124s\n", + " 37 200 0.008s 0.139s\n", + " 38 200 0.075s 0.115s\n", + " 39 200 0.077s 0.115s\n", + " 40 200 0.097s 0.118s\n", + " 41 200 0.095s 0.119s\n", + " 42 200 0.049s 0.113s\n", + " 43 200 0.035s 0.121s\n", + " 44 200 0.014s 0.126s\n", + " 45 200 0.050s 0.116s\n", + " 46 200 0.021s 0.118s\n", + " 47 200 0.015s 0.142s\n", + " 48 200 0.048s 0.117s\n", + " 49 200 0.020s 0.121s\n", + " 50 200 0.075s 0.119s\n", + " 51 200 0.031s 0.126s\n", + " 52 200 0.003s 0.125s\n", + " 53 200 0.061s 0.116s\n", + " 54 200 0.007s 0.128s\n", + " 55 200 0.094s 0.111s\n", + " 56 200 0.010s 0.118s\n", + " 57 200 0.099s 0.133s\n", + " 58 200 0.092s 0.112s\n", + " 59 200 0.025s 0.125s\n", + " 60 200 0.056s 0.116s\n", + " 61 200 0.051s 0.123s\n", + " 62 200 0.068s 0.122s\n", + " 63 200 0.059s 0.126s\n", + " 64 200 0.014s 0.129s\n", + " 65 200 0.038s 0.130s\n", + " 66 200 0.041s 0.110s\n", + " 67 200 0.047s 0.125s\n", + " 68 200 0.068s 0.116s\n", + " 69 200 0.062s 0.115s\n", + " 70 200 0.072s 0.122s\n", + " 71 200 0.093s 0.116s\n", + " 72 200 0.008s 0.118s\n", + " 73 200 0.079s 0.122s\n", + " 74 200 0.060s 0.124s\n", + " 75 200 0.028s 0.124s\n", + " 76 200 0.028s 0.125s\n", + " 77 200 0.094s 0.127s\n", + " 78 200 0.073s 0.122s\n", + " 79 200 0.046s 0.115s\n", + " 80 200 0.053s 0.119s\n", + " 81 200 0.023s 0.117s\n", + " 82 200 0.036s 0.124s\n", + " 83 200 0.093s 0.120s\n", + " 84 200 0.061s 0.114s\n", + " 85 200 0.068s 0.127s\n", + " 86 200 0.091s 0.118s\n", + " 87 200 0.074s 0.131s\n", + " 88 200 0.084s 0.128s\n", + " 89 200 0.077s 0.108s\n", + " 90 200 0.044s 0.119s\n", + " 91 200 0.001s 0.119s\n", + " 92 200 0.007s 0.113s\n", + " 93 200 0.031s 0.117s\n", + " 94 200 0.017s 0.137s\n", + " 95 200 0.077s 0.120s\n", + " 96 200 0.006s 0.119s\n", + " 97 200 0.026s 0.118s\n", + " 98 200 0.019s 0.121s\n", + " 99 200 0.095s 0.116s\n", + " 100 200 0.005s 0.123s\n", " Summary:\n", " All passing conditions have been achieved.\n", - " Test succeeded.\n", + " Test succeeded (status=0).\n", "\n", "Stress Test with [100] calls to [https://lvupavicsdev.ouranos.ca/twitcher/ows/proxy/raven/wps?service=wps&request=getcapabilities]\n", "Progress: 0/100\n" @@ -606,114 +606,114 @@ " url: https://lvupavicsdev.ouranos.ca/twitcher/ows/proxy/raven/wps?service=wps&request=getcapabilities\n", " args: {}\n", " Times:\n", - " min: 0.195s\n", + " min: 0.203s\n", " avg: 0.226s\n", - " max: 0.379s\n", + " max: 0.370s\n", " Results:\n", " Run Codes Delta Times\n", - " 1 200 0.000s 0.205s\n", - " 2 200 0.077s 0.199s\n", - " 3 200 0.005s 0.201s\n", - " 4 200 0.062s 0.197s\n", - " 5 200 0.015s 0.204s\n", - " 6 200 0.011s 0.195s\n", - " 7 200 0.084s 0.379s\n", - " 8 200 0.071s 0.230s\n", - " 9 200 0.048s 0.243s\n", - " 10 200 0.054s 0.252s\n", - " 11 200 0.075s 0.277s\n", - " 12 200 0.001s 0.242s\n", - " 13 200 0.012s 0.248s\n", - " 14 200 0.033s 0.234s\n", - " 15 200 0.044s 0.241s\n", - " 16 200 0.084s 0.248s\n", - " 17 200 0.090s 0.251s\n", - " 18 200 0.064s 0.237s\n", - " 19 200 0.048s 0.244s\n", - " 20 200 0.085s 0.241s\n", - " 21 200 0.071s 0.242s\n", - " 22 200 0.068s 0.245s\n", - " 23 200 0.093s 0.227s\n", - " 24 200 0.070s 0.251s\n", - " 25 200 0.009s 0.230s\n", - " 26 200 0.060s 0.234s\n", - " 27 200 0.100s 0.234s\n", - " 28 200 0.053s 0.236s\n", - " 29 200 0.006s 0.234s\n", - " 30 200 0.009s 0.228s\n", - " 31 200 0.004s 0.237s\n", - " 32 200 0.092s 0.245s\n", - " 33 200 0.064s 0.239s\n", - " 34 200 0.007s 0.240s\n", - " 35 200 0.028s 0.236s\n", - " 36 200 0.032s 0.230s\n", - " 37 200 0.100s 0.226s\n", - " 38 200 0.049s 0.229s\n", - " 39 200 0.069s 0.223s\n", - " 40 200 0.036s 0.230s\n", - " 41 200 0.039s 0.223s\n", - " 42 200 0.020s 0.230s\n", - " 43 200 0.097s 0.245s\n", - " 44 200 0.053s 0.223s\n", - " 45 200 0.063s 0.247s\n", - " 46 200 0.067s 0.240s\n", - " 47 200 0.024s 0.232s\n", - " 48 200 0.053s 0.232s\n", - " 49 200 0.078s 0.226s\n", - " 50 200 0.070s 0.231s\n", - " 51 200 0.075s 0.223s\n", - " 52 200 0.080s 0.230s\n", - " 53 200 0.068s 0.235s\n", - " 54 200 0.005s 0.213s\n", - " 55 200 0.014s 0.223s\n", - " 56 200 0.081s 0.214s\n", - " 57 200 0.012s 0.235s\n", - " 58 200 0.023s 0.227s\n", - " 59 200 0.076s 0.213s\n", - " 60 200 0.024s 0.220s\n", - " 61 200 0.068s 0.230s\n", - " 62 200 0.049s 0.209s\n", - " 63 200 0.066s 0.219s\n", - " 64 200 0.038s 0.218s\n", - " 65 200 0.098s 0.222s\n", - " 66 200 0.072s 0.210s\n", - " 67 200 0.079s 0.205s\n", - " 68 200 0.003s 0.209s\n", - " 69 200 0.036s 0.207s\n", - " 70 200 0.067s 0.234s\n", - " 71 200 0.042s 0.234s\n", - " 72 200 0.068s 0.217s\n", - " 73 200 0.081s 0.226s\n", - " 74 200 0.041s 0.215s\n", - " 75 200 0.060s 0.229s\n", - " 76 200 0.005s 0.218s\n", - " 77 200 0.090s 0.208s\n", - " 78 200 0.036s 0.214s\n", - " 79 200 0.011s 0.211s\n", - " 80 200 0.036s 0.216s\n", - " 81 200 0.010s 0.198s\n", - " 82 200 0.063s 0.201s\n", - " 83 200 0.029s 0.215s\n", - " 84 200 0.006s 0.216s\n", - " 85 200 0.067s 0.215s\n", - " 86 200 0.089s 0.203s\n", - " 87 200 0.080s 0.212s\n", - " 88 200 0.029s 0.208s\n", - " 89 200 0.004s 0.207s\n", - " 90 200 0.037s 0.203s\n", - " 91 200 0.063s 0.207s\n", - " 92 200 0.004s 0.209s\n", - " 93 200 0.018s 0.206s\n", - " 94 200 0.055s 0.215s\n", - " 95 200 0.081s 0.216s\n", - " 96 200 0.074s 0.218s\n", - " 97 200 0.083s 0.242s\n", - " 98 200 0.061s 0.208s\n", - " 99 200 0.088s 0.206s\n", - " 100 200 0.085s 0.216s\n", + " 1 200 0.000s 0.215s\n", + " 2 200 0.093s 0.221s\n", + " 3 200 0.076s 0.210s\n", + " 4 200 0.030s 0.215s\n", + " 5 200 0.025s 0.222s\n", + " 6 200 0.058s 0.204s\n", + " 7 200 0.003s 0.210s\n", + " 8 200 0.060s 0.207s\n", + " 9 200 0.094s 0.217s\n", + " 10 200 0.092s 0.211s\n", + " 11 200 0.061s 0.208s\n", + " 12 200 0.066s 0.208s\n", + " 13 200 0.094s 0.217s\n", + " 14 200 0.003s 0.208s\n", + " 15 200 0.022s 0.205s\n", + " 16 200 0.049s 0.212s\n", + " 17 200 0.069s 0.204s\n", + " 18 200 0.013s 0.204s\n", + " 19 200 0.034s 0.204s\n", + " 20 200 0.037s 0.204s\n", + " 21 200 0.058s 0.212s\n", + " 22 200 0.049s 0.203s\n", + " 23 200 0.055s 0.205s\n", + " 24 200 0.083s 0.214s\n", + " 25 200 0.063s 0.209s\n", + " 26 200 0.047s 0.209s\n", + " 27 200 0.094s 0.207s\n", + " 28 200 0.023s 0.370s\n", + " 29 200 0.096s 0.238s\n", + " 30 200 0.012s 0.257s\n", + " 31 200 0.069s 0.264s\n", + " 32 200 0.001s 0.242s\n", + " 33 200 0.073s 0.240s\n", + " 34 200 0.084s 0.241s\n", + " 35 200 0.056s 0.242s\n", + " 36 200 0.047s 0.233s\n", + " 37 200 0.011s 0.252s\n", + " 38 200 0.097s 0.237s\n", + " 39 200 0.084s 0.247s\n", + " 40 200 0.084s 0.248s\n", + " 41 200 0.003s 0.247s\n", + " 42 200 0.075s 0.238s\n", + " 43 200 0.018s 0.246s\n", + " 44 200 0.065s 0.235s\n", + " 45 200 0.059s 0.237s\n", + " 46 200 0.052s 0.234s\n", + " 47 200 0.055s 0.234s\n", + " 48 200 0.026s 0.237s\n", + " 49 200 0.095s 0.241s\n", + " 50 200 0.099s 0.242s\n", + " 51 200 0.095s 0.230s\n", + " 52 200 0.048s 0.253s\n", + " 53 200 0.083s 0.239s\n", + " 54 200 0.069s 0.231s\n", + " 55 200 0.091s 0.233s\n", + " 56 200 0.066s 0.237s\n", + " 57 200 0.011s 0.243s\n", + " 58 200 0.047s 0.237s\n", + " 59 200 0.020s 0.233s\n", + " 60 200 0.023s 0.233s\n", + " 61 200 0.030s 0.237s\n", + " 62 200 0.036s 0.234s\n", + " 63 200 0.010s 0.230s\n", + " 64 200 0.056s 0.225s\n", + " 65 200 0.085s 0.237s\n", + " 66 200 0.013s 0.220s\n", + " 67 200 0.050s 0.225s\n", + " 68 200 0.027s 0.229s\n", + " 69 200 0.026s 0.232s\n", + " 70 200 0.028s 0.230s\n", + " 71 200 0.037s 0.215s\n", + " 72 200 0.002s 0.225s\n", + " 73 200 0.021s 0.239s\n", + " 74 200 0.095s 0.231s\n", + " 75 200 0.007s 0.216s\n", + " 76 200 0.008s 0.218s\n", + " 77 200 0.052s 0.218s\n", + " 78 200 0.020s 0.225s\n", + " 79 200 0.003s 0.220s\n", + " 80 200 0.068s 0.219s\n", + " 81 200 0.078s 0.221s\n", + " 82 200 0.031s 0.219s\n", + " 83 200 0.077s 0.220s\n", + " 84 200 0.083s 0.220s\n", + " 85 200 0.008s 0.218s\n", + " 86 200 0.037s 0.217s\n", + " 87 200 0.031s 0.216s\n", + " 88 200 0.045s 0.218s\n", + " 89 200 0.008s 0.219s\n", + " 90 200 0.084s 0.223s\n", + " 91 200 0.005s 0.222s\n", + " 92 200 0.004s 0.214s\n", + " 93 200 0.046s 0.209s\n", + " 94 200 0.100s 0.218s\n", + " 95 200 0.038s 0.219s\n", + " 96 200 0.049s 0.212s\n", + " 97 200 0.078s 0.228s\n", + " 98 200 0.045s 0.214s\n", + " 99 200 0.063s 0.213s\n", + " 100 200 0.056s 0.212s\n", " Summary:\n", " All passing conditions have been achieved.\n", - " Test succeeded.\n", + " Test succeeded (status=0).\n", "\n", "Stress Test with [100] calls to [https://lvupavicsdev.ouranos.ca/twitcher/ows/proxy/hummingbird/wps?service=wps&request=getcapabilities]\n", "Progress: 0/100\n", @@ -742,114 +742,114 @@ " url: https://lvupavicsdev.ouranos.ca/twitcher/ows/proxy/hummingbird/wps?service=wps&request=getcapabilities\n", " args: {}\n", " Times:\n", - " min: 0.087s\n", - " avg: 0.099s\n", - " max: 0.150s\n", + " min: 0.082s\n", + " avg: 0.106s\n", + " max: 0.208s\n", " Results:\n", " Run Codes Delta Times\n", - " 1 200 0.000s 0.091s\n", - " 2 200 0.090s 0.090s\n", - " 3 200 0.085s 0.096s\n", - " 4 200 0.064s 0.093s\n", - " 5 200 0.065s 0.091s\n", - " 6 200 0.059s 0.089s\n", - " 7 200 0.037s 0.105s\n", - " 8 200 0.097s 0.098s\n", - " 9 200 0.016s 0.097s\n", - " 10 200 0.100s 0.090s\n", - " 11 200 0.094s 0.106s\n", - " 12 200 0.011s 0.098s\n", - " 13 200 0.019s 0.106s\n", - " 14 200 0.093s 0.097s\n", - " 15 200 0.042s 0.096s\n", - " 16 200 0.022s 0.092s\n", - " 17 200 0.052s 0.102s\n", - " 18 200 0.095s 0.096s\n", - " 19 200 0.089s 0.105s\n", - " 20 200 0.007s 0.097s\n", - " 21 200 0.024s 0.101s\n", - " 22 200 0.068s 0.105s\n", - " 23 200 0.035s 0.098s\n", - " 24 200 0.012s 0.104s\n", - " 25 200 0.009s 0.093s\n", - " 26 200 0.015s 0.088s\n", - " 27 200 0.041s 0.090s\n", - " 28 200 0.027s 0.104s\n", - " 29 200 0.092s 0.101s\n", - " 30 200 0.058s 0.093s\n", - " 31 200 0.071s 0.092s\n", - " 32 200 0.071s 0.098s\n", - " 33 200 0.045s 0.097s\n", - " 34 200 0.082s 0.098s\n", - " 35 200 0.003s 0.093s\n", - " 36 200 0.055s 0.090s\n", - " 37 200 0.031s 0.099s\n", - " 38 200 0.032s 0.093s\n", - " 39 200 0.073s 0.102s\n", - " 40 200 0.049s 0.102s\n", - " 41 200 0.080s 0.095s\n", - " 42 200 0.026s 0.087s\n", - " 43 200 0.067s 0.099s\n", - " 44 200 0.051s 0.100s\n", - " 45 200 0.082s 0.097s\n", - " 46 200 0.035s 0.103s\n", - " 47 200 0.060s 0.097s\n", - " 48 200 0.039s 0.094s\n", - " 49 200 0.037s 0.098s\n", - " 50 200 0.072s 0.107s\n", - " 51 200 0.016s 0.099s\n", - " 52 200 0.039s 0.097s\n", - " 53 200 0.007s 0.096s\n", - " 54 200 0.076s 0.097s\n", - " 55 200 0.079s 0.106s\n", - " 56 200 0.078s 0.102s\n", - " 57 200 0.058s 0.099s\n", - " 58 200 0.034s 0.102s\n", - " 59 200 0.052s 0.098s\n", - " 60 200 0.007s 0.107s\n", - " 61 200 0.033s 0.097s\n", - " 62 200 0.092s 0.097s\n", - " 63 200 0.055s 0.099s\n", - " 64 200 0.089s 0.103s\n", - " 65 200 0.033s 0.101s\n", - " 66 200 0.054s 0.096s\n", - " 67 200 0.044s 0.150s\n", - " 68 200 0.073s 0.093s\n", - " 69 200 0.078s 0.095s\n", - " 70 200 0.092s 0.097s\n", - " 71 200 0.022s 0.097s\n", - " 72 200 0.032s 0.097s\n", - " 73 200 0.034s 0.102s\n", - " 74 200 0.031s 0.094s\n", - " 75 200 0.040s 0.102s\n", - " 76 200 0.017s 0.093s\n", - " 77 200 0.044s 0.104s\n", - " 78 200 0.070s 0.097s\n", - " 79 200 0.065s 0.099s\n", - " 80 200 0.067s 0.101s\n", - " 81 200 0.049s 0.100s\n", - " 82 200 0.020s 0.103s\n", - " 83 200 0.064s 0.096s\n", - " 84 200 0.021s 0.090s\n", - " 85 200 0.030s 0.101s\n", - " 86 200 0.077s 0.093s\n", - " 87 200 0.018s 0.108s\n", - " 88 200 0.095s 0.101s\n", - " 89 200 0.078s 0.099s\n", - " 90 200 0.070s 0.102s\n", - " 91 200 0.016s 0.101s\n", - " 92 200 0.013s 0.094s\n", - " 93 200 0.082s 0.102s\n", - " 94 200 0.092s 0.145s\n", - " 95 200 0.007s 0.095s\n", - " 96 200 0.004s 0.090s\n", - " 97 200 0.076s 0.093s\n", - " 98 200 0.088s 0.093s\n", - " 99 200 0.086s 0.112s\n", - " 100 200 0.048s 0.100s\n", + " 1 200 0.000s 0.101s\n", + " 2 200 0.048s 0.098s\n", + " 3 200 0.009s 0.096s\n", + " 4 200 0.057s 0.104s\n", + " 5 200 0.080s 0.100s\n", + " 6 200 0.023s 0.099s\n", + " 7 200 0.054s 0.106s\n", + " 8 200 0.070s 0.092s\n", + " 9 200 0.074s 0.092s\n", + " 10 200 0.037s 0.100s\n", + " 11 200 0.092s 0.091s\n", + " 12 200 0.023s 0.104s\n", + " 13 200 0.071s 0.096s\n", + " 14 200 0.086s 0.096s\n", + " 15 200 0.016s 0.093s\n", + " 16 200 0.049s 0.111s\n", + " 17 200 0.036s 0.182s\n", + " 18 200 0.074s 0.092s\n", + " 19 200 0.064s 0.093s\n", + " 20 200 0.062s 0.104s\n", + " 21 200 0.076s 0.102s\n", + " 22 200 0.066s 0.107s\n", + " 23 200 0.056s 0.093s\n", + " 24 200 0.055s 0.100s\n", + " 25 200 0.034s 0.096s\n", + " 26 200 0.003s 0.104s\n", + " 27 200 0.094s 0.095s\n", + " 28 200 0.073s 0.093s\n", + " 29 200 0.053s 0.099s\n", + " 30 200 0.014s 0.095s\n", + " 31 200 0.090s 0.095s\n", + " 32 200 0.005s 0.088s\n", + " 33 200 0.026s 0.090s\n", + " 34 200 0.003s 0.098s\n", + " 35 200 0.095s 0.089s\n", + " 36 200 0.055s 0.097s\n", + " 37 200 0.022s 0.103s\n", + " 38 200 0.063s 0.101s\n", + " 39 200 0.004s 0.100s\n", + " 40 200 0.088s 0.087s\n", + " 41 200 0.091s 0.098s\n", + " 42 200 0.043s 0.095s\n", + " 43 200 0.100s 0.099s\n", + " 44 200 0.094s 0.101s\n", + " 45 200 0.088s 0.100s\n", + " 46 200 0.084s 0.102s\n", + " 47 200 0.004s 0.107s\n", + " 48 200 0.087s 0.088s\n", + " 49 200 0.052s 0.095s\n", + " 50 200 0.053s 0.106s\n", + " 51 200 0.013s 0.090s\n", + " 52 200 0.034s 0.093s\n", + " 53 200 0.037s 0.098s\n", + " 54 200 0.026s 0.091s\n", + " 55 200 0.037s 0.094s\n", + " 56 200 0.031s 0.097s\n", + " 57 200 0.086s 0.100s\n", + " 58 200 0.035s 0.094s\n", + " 59 200 0.067s 0.095s\n", + " 60 200 0.049s 0.101s\n", + " 61 200 0.046s 0.098s\n", + " 62 200 0.080s 0.097s\n", + " 63 200 0.075s 0.099s\n", + " 64 200 0.036s 0.100s\n", + " 65 200 0.042s 0.096s\n", + " 66 200 0.015s 0.101s\n", + " 67 200 0.050s 0.092s\n", + " 68 200 0.086s 0.093s\n", + " 69 200 0.043s 0.107s\n", + " 70 200 0.026s 0.097s\n", + " 71 200 0.007s 0.098s\n", + " 72 200 0.039s 0.096s\n", + " 73 200 0.042s 0.099s\n", + " 74 200 0.066s 0.101s\n", + " 75 200 0.099s 0.097s\n", + " 76 200 0.060s 0.099s\n", + " 77 200 0.065s 0.097s\n", + " 78 200 0.040s 0.094s\n", + " 79 200 0.092s 0.099s\n", + " 80 200 0.054s 0.177s\n", + " 81 200 0.020s 0.208s\n", + " 82 200 0.069s 0.167s\n", + " 83 200 0.045s 0.175s\n", + " 84 200 0.064s 0.119s\n", + " 85 200 0.039s 0.120s\n", + " 86 200 0.090s 0.194s\n", + " 87 200 0.078s 0.101s\n", + " 88 200 0.065s 0.106s\n", + " 89 200 0.080s 0.143s\n", + " 90 200 0.022s 0.126s\n", + " 91 200 0.019s 0.137s\n", + " 92 200 0.045s 0.101s\n", + " 93 200 0.008s 0.103s\n", + " 94 200 0.019s 0.103s\n", + " 95 200 0.088s 0.123s\n", + " 96 200 0.100s 0.163s\n", + " 97 200 0.014s 0.190s\n", + " 98 200 0.021s 0.082s\n", + " 99 200 0.054s 0.095s\n", + " 100 200 0.076s 0.098s\n", " Summary:\n", " All passing conditions have been achieved.\n", - " Test succeeded.\n", + " Test succeeded (status=0).\n", "\n", "All tests passed!\n" ] @@ -859,20 +859,19 @@ "# NBVAL_IGNORE_OUTPUT\n", "\n", "\n", - "test_statuses = []\n", - "failed_results = ''\n", + "failed_count = 0\n", + "failed_results = \"\"\n", "for bird in TEST_WPS_BIRDS:\n", " bird_url = f\"{TWITCHER_URL}/{bird}/wps?service=wps&request=getcapabilities\"\n", " expect_status_code = 200\n", " results = stress_test_requests(bird_url, runs=TEST_RUNS, code=expect_status_code,\n", " max_err_code=TEST_MAX_ERR_CODE, max_avg_time=TEST_MAX_AVG_TIME,\n", " abort_retries=TEST_TIMEOUT_RETRY, abort_timeout=TEST_TIMEOUT_ABORT)\n", - " test_statuses.append(results.status)\n", " print(results)\n", - " if results.status:\n", + " if results.status != 0:\n", + " failed_count += 1\n", " failed_results = f\"{failed_results}\\n{results}\"\n", - "failed_tests = len([status != 0 for status in test_statuses])\n", - "assert not failed_tests, f\"Failed {failed_tests} tests. Failed results: {failed_results}\"\n", + "assert failed_count == 0, f\"Failed {failed_count} tests. Failed results: {failed_results}\"\n", "\n", "print(\"\\nAll tests passed!\")" ] From 9c034a6eb0716d98b30ac17f41e189aa412d6357 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Thu, 15 Jul 2021 14:49:41 -0400 Subject: [PATCH 2/3] stress-test.ipynb: fix incorrect variable access Hopefully fix ``` ============================= test session starts ============================== platform linux -- Python 3.7.10, pytest-6.2.4, py-1.10.0, pluggy-0.13.1 rootdir: /home/jenkins/agent/workspace/ailed-tests-in-stress-test.ipynb plugins: anyio-3.1.0, dash-1.20.0, nbval-0.9.6, tornasync-0.6.0.post2 collected 0 items INTERNALERROR> Traceback (most recent call last): INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/_pytest/main.py", line 269, in wrap_session INTERNALERROR> session.exitstatus = doit(config, session) or 0 INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/_pytest/main.py", line 322, in _main INTERNALERROR> config.hook.pytest_collection(session=session) INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/hooks.py", line 286, in __call__ INTERNALERROR> return self._hookexec(self, self.get_hookimpls(), kwargs) INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/manager.py", line 93, in _hookexec INTERNALERROR> return self._inner_hookexec(hook, methods, kwargs) INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/manager.py", line 87, in INTERNALERROR> firstresult=hook.spec.opts.get("firstresult") if hook.spec else False, INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/callers.py", line 208, in _multicall INTERNALERROR> return outcome.get_result() INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/callers.py", line 80, in get_result INTERNALERROR> raise ex[1].with_traceback(ex[2]) INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/callers.py", line 187, in _multicall INTERNALERROR> res = hook_impl.function(*args) INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/_pytest/main.py", line 333, in pytest_collection INTERNALERROR> session.perform_collect() INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/_pytest/main.py", line 620, in perform_collect INTERNALERROR> rep = collect_one_node(self) INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/_pytest/runner.py", line 457, in collect_one_node INTERNALERROR> ihook.pytest_collectstart(collector=collector) INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/hooks.py", line 286, in __call__ INTERNALERROR> return self._hookexec(self, self.get_hookimpls(), kwargs) INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/manager.py", line 93, in _hookexec INTERNALERROR> return self._inner_hookexec(hook, methods, kwargs) INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/manager.py", line 87, in INTERNALERROR> firstresult=hook.spec.opts.get("firstresult") if hook.spec else False, INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/callers.py", line 208, in _multicall INTERNALERROR> return outcome.get_result() INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/callers.py", line 80, in get_result INTERNALERROR> raise ex[1].with_traceback(ex[2]) INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/callers.py", line 187, in _multicall INTERNALERROR> res = hook_impl.function(*args) INTERNALERROR> File "/home/jenkins/agent/workspace/ailed-tests-in-stress-test.ipynb/conftest.py", line 3, in pytest_collectstart INTERNALERROR> collector.skip_compare += 'text/html', 'application/javascript', INTERNALERROR> AttributeError: 'Session' object has no attribute 'skip_compare' ============================ no tests ran in 0.01s ============================= ``` --- notebooks/stress-tests.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/notebooks/stress-tests.ipynb b/notebooks/stress-tests.ipynb index 5abaee71..579dcb56 100644 --- a/notebooks/stress-tests.ipynb +++ b/notebooks/stress-tests.ipynb @@ -153,25 +153,25 @@ " in enumerate(zip(self.codes, self.delta, self.times))]\n", " lines = \"\\n\".join(data)\n", " summary = \"Undefined failure result status condition encountered.\"\n", - " if results.status == 0:\n", + " if self.status == 0:\n", " summary = [\n", " \"All passing conditions have been achieved.\",\n", " ]\n", - " elif results.status == -1:\n", + " elif self.status == -1:\n", " summary = [\n", " f\"Detected {self.sum_err_code} erroneous HTTP codes not equal to expected {self.code}.\"\n", " ]\n", - " elif results.status == -2:\n", + " elif self.status == -2:\n", " summary = [\n", " f\"Detected regression with long request time.\",\n", " f\"Expected max-avg-time: ({self.max_avg_time:.3f}s <= {self.max_time:.3f}s).\"\n", " ]\n", - " elif results.status == -3:\n", + " elif self.status == -3:\n", " summary = [\n", " f\"Maximum number of timeout ({self.timeout_abort}s) requests exceeded ({self.timeout_count}).\",\n", " \"Test was aborted to avoid further delays.\"\n", " ]\n", - " summary.append(f\"Test {'succeeded' if results.status == 0 else 'failed'} (status={self.status}).\")\n", + " summary.append(f\"Test {'succeeded' if self.status == 0 else 'failed'} (status={self.status}).\")\n", " summary = (\"\\n\" + offset * \" \").join(summary)\n", " return cleandoc(f\"\"\"\n", " Stress Test:\n", From da918ecb843fdb0fc09191610a231557271fde77 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Thu, 15 Jul 2021 15:08:04 -0400 Subject: [PATCH 3/3] conftest.py: note another case of AttributeError: 'Session' object has no attribute 'skip_compare' ``` ============================= test session starts ============================== platform linux -- Python 3.7.10, pytest-6.2.4, py-1.10.0, pluggy-0.13.1 rootdir: /home/jenkins/agent/workspace/ailed-tests-in-stress-test.ipynb plugins: anyio-3.1.0, dash-1.20.0, nbval-0.9.6, tornasync-0.6.0.post2 collected 0 items INTERNALERROR> Traceback (most recent call last): INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/_pytest/main.py", line 269, in wrap_session INTERNALERROR> session.exitstatus = doit(config, session) or 0 INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/_pytest/main.py", line 322, in _main INTERNALERROR> config.hook.pytest_collection(session=session) INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/hooks.py", line 286, in __call__ INTERNALERROR> return self._hookexec(self, self.get_hookimpls(), kwargs) INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/manager.py", line 93, in _hookexec INTERNALERROR> return self._inner_hookexec(hook, methods, kwargs) INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/manager.py", line 87, in INTERNALERROR> firstresult=hook.spec.opts.get("firstresult") if hook.spec else False, INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/callers.py", line 208, in _multicall INTERNALERROR> return outcome.get_result() INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/callers.py", line 80, in get_result INTERNALERROR> raise ex[1].with_traceback(ex[2]) INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/callers.py", line 187, in _multicall INTERNALERROR> res = hook_impl.function(*args) INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/_pytest/main.py", line 333, in pytest_collection INTERNALERROR> session.perform_collect() INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/_pytest/main.py", line 620, in perform_collect INTERNALERROR> rep = collect_one_node(self) INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/_pytest/runner.py", line 457, in collect_one_node INTERNALERROR> ihook.pytest_collectstart(collector=collector) INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/hooks.py", line 286, in __call__ INTERNALERROR> return self._hookexec(self, self.get_hookimpls(), kwargs) INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/manager.py", line 93, in _hookexec INTERNALERROR> return self._inner_hookexec(hook, methods, kwargs) INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/manager.py", line 87, in INTERNALERROR> firstresult=hook.spec.opts.get("firstresult") if hook.spec else False, INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/callers.py", line 208, in _multicall INTERNALERROR> return outcome.get_result() INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/callers.py", line 80, in get_result INTERNALERROR> raise ex[1].with_traceback(ex[2]) INTERNALERROR> File "/opt/conda/envs/birdy/lib/python3.7/site-packages/pluggy/callers.py", line 187, in _multicall INTERNALERROR> res = hook_impl.function(*args) INTERNALERROR> File "/home/jenkins/agent/workspace/ailed-tests-in-stress-test.ipynb/conftest.py", line 3, in pytest_collectstart INTERNALERROR> collector.skip_compare += 'text/html', 'application/javascript', INTERNALERROR> AttributeError: 'Session' object has no attribute 'skip_compare' ============================ no tests ran in 0.01s ============================= ``` --- conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conftest.py b/conftest.py index 2499bd39..51632802 100644 --- a/conftest.py +++ b/conftest.py @@ -1,3 +1,5 @@ def pytest_collectstart(collector): + # Make sure ancestor folder name do not end with `.ipynb`, else we have + # AttributeError: 'Session' object has no attribute 'skip_compare'. if collector.fspath and collector.fspath.ext == '.ipynb': collector.skip_compare += 'text/html', 'application/javascript',