From bdb6e7207751f25b1d64f9962bc35f9867c88bbc Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Wed, 16 Feb 2022 11:55:33 -0600 Subject: [PATCH] Fix failure to generate worker_id when xdist is not present (#4799) Signed-off-by: Jason Lowe --- integration_tests/src/main/python/conftest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/integration_tests/src/main/python/conftest.py b/integration_tests/src/main/python/conftest.py index bd5d4870fc2..e398b579889 100644 --- a/integration_tests/src/main/python/conftest.py +++ b/integration_tests/src/main/python/conftest.py @@ -234,11 +234,17 @@ def std_input_path(request): yield path @pytest.fixture -def spark_tmp_path(request, worker_id): +def spark_tmp_path(request): debug = request.config.getoption('debug_tmp_path') ret = request.config.getoption('tmp_path') if ret is None: ret = '/tmp/pyspark_tests/' + worker_id = 'main' + try: + import xdist + worker_id = xdist.plugin.get_xdist_worker_id(request) + except ImportError: + pass pid = os.getpid() hostname = os.uname()[1] ret = f'{ret}/{hostname}-{worker_id}-{pid}-{random.randrange(0, 1<<31)}/'