test: update python-language-text-encoder to newer pyarrow

and specify python3.12 Signed-off-by: Nick Mitchell <nickm@us.ibm.com>
IBM · Dec 1, 2024 · 321fb2f · 321fb2f
1 parent 5f9df15
commit 321fb2f
Show file tree

Hide file tree

Showing 10 changed files with 26 additions and 4 deletions.
diff --git a/pkg/fe/builder/overlay/filesystem.go b/pkg/fe/builder/overlay/filesystem.go
@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"slices"
 	"strings"
 
@@ -193,6 +194,19 @@ func (b filesystemBuilder) addMetadata(spec *hlir.Spec, sourcePath string) (appV
 				} else {
 					spec.Needs = append(spec.Needs, hlir.Needs{Name: "python", Version: "latest", Requirements: req})
 				}
+			case "requirements_linux_ci.txt":
+				if os.Getenv("CI") != "" && runtime.GOOS == "linux" {
+					if req, rerr := b.readString(path); rerr != nil {
+						err = rerr
+						return
+					} else {
+						if needsIdx := slices.IndexFunc(spec.Needs, func(needs hlir.Needs) bool { return needs.Name == "python" }); needsIdx >= 0 {
+							// splice out prior requirements.txt Needs
+							spec.Needs = append(spec.Needs[:needsIdx], spec.Needs[needsIdx+1:]...)
+						}
+						spec.Needs = append(spec.Needs, hlir.Needs{Name: "python", Version: "latest", Requirements: req})
+					}
+				}
 			case "memory", "memory.txt":
 				if mem, rerr := b.readString(path); rerr != nil {
 					err = rerr

diff --git a/tests/tests/python-language-text-encoder/pail/command b/tests/tests/python-language-text-encoder/pail/command
@@ -0,0 +1 @@
+python3.12 main.py
diff --git a/tests/tests/python-language-text-encoder/pail/image b/tests/tests/python-language-text-encoder/pail/image
@@ -0,0 +1 @@
+docker.io/python:3.12
diff --git a/tests/tests/python-language-text-encoder/pail/requirements.txt b/tests/tests/python-language-text-encoder/pail/requirements.txt
@@ -1,5 +1,4 @@
 sentence-transformers==3.0.1
 
-# we can probably update to 18+, but we will have to re-generate expected output as pyarrow 18 seems to have resulted in a binary format change
-pyarrow<18
+pyarrow
 
diff --git a/tests/tests/python-language-text-encoder/pail/requirements_linux_ci.txt b/tests/tests/python-language-text-encoder/pail/requirements_linux_ci.txt
@@ -0,0 +1,7 @@
+sentence-transformers==3.0.1
+
+pyarrow
+
+# TODO this is only for CI, github actions runs out of disk space installing nvidia pips
+--extra-index-url https://download.pytorch.org/whl/cpu
+torch==2.5.1+cpu
diff --git a/tests/tests/python-language-text-encoder/pail/test-data/expected/test1.parquet.gz b/tests/tests/python-language-text-encoder/pail/test-data/expected/test1.parquet.gz
diff --git a/tests/tests/python-language-text-encoder/pail/test-data/input/test1.parquet b/tests/tests/python-language-text-encoder/pail/test-data/input/test1.parquet
diff --git a/tests/tests/python-language-text-encoder/pail/test-data/input/test1.parquet.gz b/tests/tests/python-language-text-encoder/pail/test-data/input/test1.parquet.gz
diff --git a/tests/tests/python-language-text-encoder/post.sh b/tests/tests/python-language-text-encoder/post.sh
@@ -11,7 +11,7 @@ do
 
     ext=${b##*.}
     bb=${b%%.*}
-    actual="$(dirname $i)"/"$bb".output.$ext
+    actual=$bb.parquet
     expected="$DATA"/expected/$bb.parquet.gz
 
     while true

diff --git a/tests/tests/python-language-text-encoder/settings.sh b/tests/tests/python-language-text-encoder/settings.sh
@@ -6,4 +6,4 @@ NUM_DESIRED_OUTPUTS=0
 # the default is --yaml. we don't want that
 source_from=" "
 
-up_args='"$TEST_PATH"/pail/test-data/input/test1.parquet'
+up_args='--gunzip "$TEST_PATH"/pail/test-data/input/test1.parquet.gz'