test: update remaining python-* to use --gunzip

This also fixes an issue with installRequirements: make sure to clean out venv if we aren't done yet and the context was cancelled

Signed-off-by: Nick Mitchell <nickm@us.ibm.com>

pkg/runtime/needs/install_requirements.go

@@ -95,13 +95,29 @@ pip%s install %s %s -r %s %s 1>&2`, version, venvPath, venvPath, version, versio
 	cmd.Stdout = os.Stderr // Stderr so as not to collide with `lunchpail needs` stdout
 	cmd.Stderr = os.Stderr
 
+	alreadyCleanedUp := false
+	installSuccessful := false
+	go func() {
+		select {
+		case <-ctx.Done():
+			if !installSuccessful && !alreadyCleanedUp {
+				if err := os.RemoveAll(venvPath); err != nil {
+					fmt.Fprintln(os.Stderr, "Unable to clean up venv cache directory after pip install failure", err)
+				}
+				alreadyCleanedUp = true
+			}
+		}
+	}()
+
 	if err := cmd.Run(); err != nil {
 		// Clean up the venv cache directory, since we failed at populating it
 		if err := os.RemoveAll(venvPath); err != nil {
 			fmt.Fprintln(os.Stderr, "Unable to clean up venv cache directory after pip install failure", err)
 		}
+		alreadyCleanedUp = true
 		return path, err
 	}
+	installSuccessful = true
 
 	return path, nil
 }

tests/tests/python-code-header-cleanser/pail/command

@@ -0,0 +1 @@
+python3.10 main.py

tests/tests/python-code-header-cleanser/pail/image

@@ -0,0 +1 @@
+docker.io/python:3.10

tests/tests/python-code-header-cleanser/pail/requirements.txt

@@ -2,6 +2,6 @@ data-prep-toolkit==0.2.2.dev1
 scancode-toolkit ; platform_system != 'Darwin'
 
 # we can probably update to 18+, but we will have to re-generate expected output as pyarrow 18 seems to have resulted in a binary format change
-pyarrow<18
+pyarrow<17
 
 setuptools

tests/tests/python-code-header-cleanser/post.sh

@@ -25,4 +25,4 @@ function validate {
     rm -f "$actual"
 }
 
-validate task.1.txt "$DATA"/expected/test1.parquet.gz
+validate test1.parquet "$DATA"/expected/test1.parquet.gz

tests/tests/python-code-header-cleanser/settings.sh

@@ -6,4 +6,4 @@ NUM_DESIRED_OUTPUTS=0
 # the default is --yaml. we don't want that
 source_from=" "
 
-up_args='<(gunzip -c "$TEST_PATH"/pail/test-data/input/test1.parquet.gz)'
+up_args='--gunzip "$TEST_PATH"/pail/test-data/input/test1.parquet.gz'

tests/tests/python-language-doc-chunk/pail/requirements.txt

@@ -4,6 +4,5 @@ llama-index-core>=0.11.0,<0.12.0
 # sigh. see https://github.com/run-llama/llama_index/issues/17016
 pydantic==2.9.2
 
-# we can probably update to 18+, but we will have to re-generate expected output as pyarrow 18 seems to have resulted in a binary format change
-pyarrow<18
+pyarrow
 

tests/tests/python-language-doc-chunk/post.sh

@@ -4,8 +4,8 @@ DATA="$TEST_PATH"/pail/test-data
 
 for i in $(seq 1 1)
 do
-    actual=task.${i}_0.parquet # pkg/boot/up.go currently downloads named pipes (see up_args in settings.sh) to cwd
-    expected="$DATA"/expected/test$i.parquet.gz
+    actual=test${i}_0.parquet
+    expected="$DATA"/expected/test${i}_0.parquet.gz
 
     while true
     do

tests/tests/python-language-doc-chunk/settings.sh

@@ -6,4 +6,4 @@ NUM_DESIRED_OUTPUTS=0
 # the default is --yaml. we don't want that
 source_from=" "
 
-up_args='<(gunzip -c "$TEST_PATH"/pail/test-data/input/test1.parquet.gz)'
+up_args='--gunzip "$TEST_PATH"/pail/test-data/input/test1.parquet.gz'

tests/tests/python-language-doc-quality/post.sh

@@ -4,7 +4,7 @@ DATA="$TEST_PATH"/pail/test-data
 
 for i in $(seq 1 1)
 do
-    actual=task.$i.txt # pkg/boot/up.go currently downloads named pipes (see up_args in settings.sh) to cwd
+    actual=test$i.parquet
     expected="$DATA"/expected/test$i.parquet.gz
 
     while true

tests/tests/python-language-doc-quality/settings.sh

@@ -6,4 +6,4 @@ NUM_DESIRED_OUTPUTS=0
 # the default is --yaml. we don't want that
 source_from=" "
 
-up_args='<(gunzip -c "$TEST_PATH"/pail/test-data/input/test1.parquet.gz)'
+up_args='--gunzip "$TEST_PATH"/pail/test-data/input/test1.parquet.gz'

tests/tests/python-language-lang-id/pail/requirements.txt

@@ -7,6 +7,5 @@ langcodes==3.3.0
 huggingface-hub >= 0.21.4, <1.0.0
 numpy==1.26.4
 
-# we can probably update to 18+, but we will have to re-generate expected output as pyarrow 18 seems to have resulted in a binary format change
-pyarrow<18
+pyarrow
 

tests/tests/python-language-lang-id/post.sh

@@ -4,7 +4,7 @@ DATA="$TEST_PATH"/pail/test-data/sm
 
 for i in $(seq 1 3)
 do
-    actual=task.$i.txt # pkg/boot/up.go currently downloads named pipes (see up_args in settings.sh) to cwd
+    actual=test_0$i.parquet
     expected="$DATA"/expected/test_0$i.parquet.gz
 
     while true

tests/tests/python-language-lang-id/settings.sh

@@ -6,4 +6,4 @@ NUM_DESIRED_OUTPUTS=0
 # the default is --yaml. we don't want that
 source_from=" "
 
-up_args='<(gunzip -c "$TEST_PATH"/pail/test-data/sm/input/test_01.parquet.gz) <(gunzip -c "$TEST_PATH"/pail/test-data/sm/input/test_02.parquet.gz) <(gunzip -c "$TEST_PATH"/pail/test-data/sm/input/test_03.parquet.gz)'
+up_args='--gunzip "$TEST_PATH"/pail/test-data/sm/input/test_01.parquet.gz "$TEST_PATH"/pail/test-data/sm/input/test_02.parquet.gz "$TEST_PATH"/pail/test-data/sm/input/test_03.parquet.gz'

tests/tests/python-language-pii-redactor/pail/requirements.txt

@@ -7,7 +7,5 @@ presidio-anonymizer>=2.2.355
 flair>=0.14.0
 pandas>=2.2.2
 
-# we can probably update to 18+, but we will have to re-generate expected output as pyarrow 18 seems to have resulted in a binary format change
-pyarrow<18
-
+pyarrow
 spacy

tests/tests/python-language-pii-redactor/post.sh

@@ -2,7 +2,7 @@
 
 DATA="$TEST_PATH"/pail/test-data
 
-actual=task.1.txt # pkg/boot/up.go currently downloads named pipes (see up_args in settings.sh) to cwd
+actual=1.parquet
 expected="$DATA"/xs/1.expected.parquet.gz
 
 while true

tests/tests/python-language-pii-redactor/settings.sh

@@ -6,4 +6,4 @@ NUM_DESIRED_OUTPUTS=0
 # the default is --yaml. we don't want that
 source_from=" "
 
-up_args='<(gunzip -c "$TEST_PATH"/pail/test-data/xs/1.parquet.gz)'
+up_args='--gunzip "$TEST_PATH"/pail/test-data/xs/1.parquet.gz'

tests/tests/python-universal-doc-id/pail/command

@@ -0,0 +1 @@
+python3.12 main.py

tests/tests/python-universal-doc-id/pail/image

@@ -0,0 +1 @@
+docker.io/python:3.12

tests/tests/python-universal-doc-id/pail/requirements.txt

@@ -1,6 +1,4 @@
 data-prep-toolkit==0.2.2.dev0
 
-# we can probably update to 18+, but we will have to re-generate expected output as pyarrow 18 seems to have resulted in a binary format change
-pyarrow<18
-
+pyarrow<17
 setuptools

tests/tests/python-universal-doc-id/post.sh

@@ -25,4 +25,4 @@ function validate {
     rm -f "$actual"
 }
 
-validate task.1.txt "$DATA"/expected/sample1.parquet.gz
+validate sample1.parquet "$DATA"/expected/sample1.parquet.gz

tests/tests/python-universal-doc-id/settings.sh

@@ -6,4 +6,4 @@ NUM_DESIRED_OUTPUTS=0
 # the default is --yaml. we don't want that
 source_from=" "
 
-up_args='<(gunzip -c "$TEST_PATH"/pail/test-data/input/sample1.parquet.gz)'
+up_args='--gunzip "$TEST_PATH"/pail/test-data/input/sample1.parquet.gz'

tests/tests/python-universal-ededup/pail/command

@@ -0,0 +1 @@
+python3.12 main.py

tests/tests/python-universal-ededup/pail/image

@@ -0,0 +1 @@
+docker.io/python:3.12

tests/tests/python-universal-ededup/pail/requirements.txt

@@ -3,6 +3,6 @@ mmh3==4.1.0
 xxhash==3.4.1
 
 # we can probably update to 18+, but we will have to re-generate expected output as pyarrow 18 seems to have resulted in a binary format change
-pyarrow<18
+pyarrow<17
 
 setuptools

tests/tests/python-universal-ededup/post.sh

@@ -25,4 +25,4 @@ function validate {
     rm -f "$actual"
 }
 
-validate task.1.txt "$DATA"/expected/sample1.parquet.gz
+validate sample1.parquet "$DATA"/expected/sample1.parquet.gz

tests/tests/python-universal-ededup/settings.sh

@@ -6,4 +6,4 @@ NUM_DESIRED_OUTPUTS=0
 # the default is --yaml. we don't want that
 source_from=" "
 
-up_args='<(gunzip -c "$TEST_PATH"/pail/test-data/input/sample1.parquet.gz)'
+up_args='--gunzip "$TEST_PATH"/pail/test-data/input/sample1.parquet.gz'

tests/tests/python-universal-filter/pail/requirements.txt

@@ -1,5 +1,4 @@
-duckdb==0.10.1
+duckdb==1.1.3
 
-# we can probably update to 18+, but we will have to re-generate expected output as pyarrow 18 seems to have resulted in a binary format change
-pyarrow<18
+pyarrow
 

tests/tests/python-universal-filter/post.sh

@@ -25,4 +25,4 @@ function validate {
     rm -f "$actual"
 }
 
-validate task.1.txt "$DATA"/expected/test1.parquet.gz
+validate test1.parquet "$DATA"/expected/test1.parquet.gz

tests/tests/python-universal-filter/settings.sh

@@ -6,4 +6,4 @@ NUM_DESIRED_OUTPUTS=0
 # the default is --yaml. we don't want that
 source_from=" "
 
-up_args='<(gunzip -c "$TEST_PATH"/pail/test-data/input/test1.parquet.gz)'
+up_args='--gunzip "$TEST_PATH"/pail/test-data/input/test1.parquet.gz'

tests/tests/python-universal-resize/pail/requirements.txt

@@ -1,3 +1,2 @@
-# we can probably update to 18+, but we will have to re-generate expected output as pyarrow 18 seems to have resulted in a binary format change
-pyarrow<18
+pyarrow
 

tests/tests/python-universal-resize/post.sh

@@ -29,9 +29,9 @@ function validate {
     rm -f "$actual"
 }
 
-validate task.1_0.parquet "$DATA"/expected/task.1_0.parquet.gz
-validate task.1_1.parquet "$DATA"/expected/task.1_1.parquet.gz
-validate task.2_0.parquet "$DATA"/expected/task.2_0.parquet.gz
-validate task.2_1.parquet "$DATA"/expected/task.2_1.parquet.gz
-validate task.3_0.parquet "$DATA"/expected/task.3_0.parquet.gz
-validate task.3_1.parquet "$DATA"/expected/task.3_1.parquet.gz
+validate test1_0.parquet "$DATA"/expected/test1_0.parquet.gz
+validate test1_1.parquet "$DATA"/expected/test1_1.parquet.gz
+validate test2_0.parquet "$DATA"/expected/test2_0.parquet.gz
+validate test2_1.parquet "$DATA"/expected/test2_1.parquet.gz
+validate test3_0.parquet "$DATA"/expected/test3_0.parquet.gz
+validate test3_1.parquet "$DATA"/expected/test3_1.parquet.gz

tests/tests/python-universal-resize/settings.sh

@@ -6,4 +6,4 @@ NUM_DESIRED_OUTPUTS=0
 # the default is --yaml. we don't want that
 source_from=" "
 
-up_args='-e max_rows_per_table=125 <(gunzip -c "$TEST_PATH"/pail/test-data/input/test1.parquet.gz) <(gunzip -c "$TEST_PATH"/pail/test-data/input/test2.parquet.gz) <(gunzip -c "$TEST_PATH"/pail/test-data/input/test3.parquet.gz)'
+up_args='-e max_rows_per_table=125 --gunzip "$TEST_PATH"/pail/test-data/input/test1.parquet.gz "$TEST_PATH"/pail/test-data/input/test2.parquet.gz "$TEST_PATH"/pail/test-data/input/test3.parquet.gz'