Reorder default data splits to have validation before test

src/datasets/data_files.py

@@ -31,48 +31,31 @@ class EmptyDatasetError(FileNotFoundError):
 
 SPLIT_PATTERN_SHARDED = "data/{split}-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*"
 
-TRAIN_KEYWORDS = ["train", "training"]
-TEST_KEYWORDS = ["test", "testing", "eval", "evaluation"]
-VALIDATION_KEYWORDS = ["validation", "valid", "dev", "val"]
+SPLIT_KEYWORDS = {
+    Split.TRAIN: ["train", "training"],
+    Split.VALIDATION: ["validation", "valid", "dev", "val"],
+    Split.TEST: ["test", "testing", "eval", "evaluation"],
+}
 NON_WORDS_CHARS = "-._ 0-9"
 KEYWORDS_IN_FILENAME_BASE_PATTERNS = ["**[{sep}/]{keyword}[{sep}]*", "{keyword}[{sep}]*"]
 KEYWORDS_IN_DIR_NAME_BASE_PATTERNS = ["{keyword}[{sep}/]**", "**[{sep}/]{keyword}[{sep}/]**"]
 
-DEFAULT_SPLITS = [Split.TRAIN, Split.TEST, Split.VALIDATION]
+DEFAULT_SPLITS = [Split.TRAIN, Split.VALIDATION, Split.TEST]
 DEFAULT_PATTERNS_SPLIT_IN_FILENAME = {
-    Split.TRAIN: [
-        pattern.format(keyword=keyword, sep=NON_WORDS_CHARS)
-        for keyword in TRAIN_KEYWORDS
-        for pattern in KEYWORDS_IN_FILENAME_BASE_PATTERNS
-    ],
-    Split.TEST: [
+    split: [
         pattern.format(keyword=keyword, sep=NON_WORDS_CHARS)
-        for keyword in TEST_KEYWORDS
+        for keyword in SPLIT_KEYWORDS[split]
         for pattern in KEYWORDS_IN_FILENAME_BASE_PATTERNS
-    ],
-    Split.VALIDATION: [
-        pattern.format(keyword=keyword, sep=NON_WORDS_CHARS)
-        for keyword in VALIDATION_KEYWORDS
-        for pattern in KEYWORDS_IN_FILENAME_BASE_PATTERNS
-    ],
+    ]
+    for split in DEFAULT_SPLITS
 }
-
 DEFAULT_PATTERNS_SPLIT_IN_DIR_NAME = {
-    Split.TRAIN: [
-        pattern.format(keyword=keyword, sep=NON_WORDS_CHARS)
-        for keyword in TRAIN_KEYWORDS
-        for pattern in KEYWORDS_IN_DIR_NAME_BASE_PATTERNS
-    ],
-    Split.TEST: [
+    split: [
         pattern.format(keyword=keyword, sep=NON_WORDS_CHARS)
-        for keyword in TEST_KEYWORDS
+        for keyword in SPLIT_KEYWORDS[split]
         for pattern in KEYWORDS_IN_DIR_NAME_BASE_PATTERNS
-    ],
-    Split.VALIDATION: [
-        pattern.format(keyword=keyword, sep=NON_WORDS_CHARS)
-        for keyword in VALIDATION_KEYWORDS
-        for pattern in KEYWORDS_IN_DIR_NAME_BASE_PATTERNS
-    ],
+    ]
+    for split in DEFAULT_SPLITS
 }
 
 DEFAULT_PATTERNS_ALL = {

tests/test_data_files.py

@@ -546,27 +546,30 @@ def ls(self, path, detail=True, refresh=True, **kwargs):
     [
         # === Main cases ===
         # file named after split at the root
-        {"train": "train.txt", "test": "test.txt", "validation": "valid.txt"},
+        {"train": "train.txt", "validation": "valid.txt", "test": "test.txt"},
         # file named after split in a directory
         {
             "train": "data/train.txt",
-            "test": "data/test.txt",
             "validation": "data/valid.txt",
+            "test": "data/test.txt",
         },
         # directory named after split
         {
             "train": "train/split.txt",
-            "test": "test/split.txt",
             "validation": "valid/split.txt",
+            "test": "test/split.txt",
         },
         # sharded splits
         {
             "train": [f"data/train_{i}.txt" for i in range(3)],
+            "validation": [f"data/validation_{i}.txt" for i in range(3)],
             "test": [f"data/test_{i}.txt" for i in range(3)],
         },
         # sharded splits with standard format (+ custom split name)
         {
             "train": [f"data/train-0000{i}-of-00003.txt" for i in range(3)],
+            "validation": [f"data/validation-0000{i}-of-00003.txt" for i in range(3)],
+            "test": [f"data/test-0000{i}-of-00003.txt" for i in range(3)],
             "random": [f"data/random-0000{i}-of-00003.txt" for i in range(3)],
         },
         # === Secondary cases ===
@@ -594,7 +597,7 @@ def ls(self, path, detail=True, refresh=True, **kwargs):
         {"validation": "val.txt"},
         {"validation": "data/val.txt"},
         # With other extensions
-        {"train": "train.parquet", "test": "test.parquet", "validation": "valid.parquet"},
+        {"train": "train.parquet", "validation": "valid.parquet", "test": "test.parquet"},
         # With "dev" or "eval" without separators
         {"train": "developers_list.txt"},
         {"train": "data/seqeval_results.txt"},