test: port python-universal-tokenization to avoid yaml

Signed-off-by: Nick Mitchell <nickm@us.ibm.com>
IBM · Nov 28, 2024 · e305acf · e305acf
1 parent 27ef22d
commit e305acf
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 19 deletions.
diff --git a/tests/tests/python-universal-tokenization/pail/app.yaml b/tests/tests/python-universal-tokenization/pail/app.yaml
diff --git a/...l-tokenization/pail/data/requirements.txt → ...versal-tokenization/pail/requirements.txt b/...l-tokenization/pail/data/requirements.txt → ...versal-tokenization/pail/requirements.txt
diff --git a/tests/tests/python-universal-tokenization/settings.sh b/tests/tests/python-universal-tokenization/settings.sh
@@ -3,4 +3,7 @@ api=workqueue
 expected=("Done with num_files=1 num_rows=3 num_tokenized_rows=3 num_empty_rows=0 num_tokens=45 num_chars=193" "Done with num_files=1 num_rows=2 num_tokenized_rows=2 num_empty_rows=0 num_tokens=28 num_chars=132")
 NUM_DESIRED_OUTPUTS=0
 
+# the default is --yaml. we don't want that
+source_from=" "
+
 up_args='<(gunzip -c "$TEST_PATH"/pail/test-data/ds01/input/lang=en/pq01.parquet.gz) <(gunzip -c "$TEST_PATH"/pail/test-data/ds01/input/lang=en/pq02.parquet.gz)'