Allow for 10 tests to fail using the new EasyBlock easybuilders/easyb…

…uild-easyblocks#2742

easybuild/easyconfigs/p/PyTorch/PyTorch-1.11.0-foss-2021a-CUDA-11.3.1.eb

@@ -111,6 +111,10 @@ excluded_tests = {
 
 runtest = 'cd test && PYTHONUNBUFFERED=1 %(python)s run_test.py --continue-through-error  --verbose %(excluded_tests)s'
 
+# several tests are known to be flaky, and fail in some contexts (like having multiple GPUs available),
+# so we allow up to 10 (out of ~90k) tests to fail before treating the installation to be faulty
+max_failed_tests = 10
+
 # The readelf sanity check command can be taken out once the TestRPATH test from 
 # https://github.com/pytorch/pytorch/pull/68912 is accepted, since it is then checked as part of the PyTorch test suite
 local_libcaffe2 = "$EBROOTPYTORCH/lib/python%%(pyshortver)s/site-packages/torch/lib/libcaffe2_nvrtc.%s" % SHLIB_EXT