[CI] Fix

legacy/examples/benchmark/clue/mrc/run_cmrc2018.py

@@ -233,7 +233,7 @@ def run(args):
     set_seed(args)
 
     train_examples, dev_examples, test_examples = load_dataset(
-        "clue", "cmrc2018", split=["train", "validation", "test"]
+        "clue", "cmrc2018", split=["train", "validation", "test"], trust_remote_code=True
     )
 
     column_names = train_examples.column_names

legacy/examples/information_extraction/msra_ner/train.py

@@ -88,9 +88,9 @@ def do_train(args):
 
     # Create dataset, tokenizer and dataloader.
     if args.dataset == "peoples_daily_ner":
-        raw_datasets = load_dataset(args.dataset)
+        raw_datasets = load_dataset(args.dataset, trust_remote_code=True)
     else:
-        raw_datasets = load_dataset(args.dataset)
+        raw_datasets = load_dataset(args.dataset, trust_remote_code=True)
 
     AutoForTokenClassification, AutoTokenizer = MODEL_CLASSES[args.model_type]
     tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)

legacy/examples/machine_reading_comprehension/SQuAD/run_squad.py

@@ -230,11 +230,11 @@ def run(args):
     tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path)
 
     if args.version_2_with_negative:
-        train_examples = load_dataset("squad_v2", split="train")
-        dev_examples = load_dataset("squad_v2", split="validation")
+        train_examples = load_dataset("squad_v2", split="train", trust_remote_code=True)
+        dev_examples = load_dataset("squad_v2", split="validation", trust_remote_code=True)
     else:
-        train_examples = load_dataset("squad", split="train")
-        dev_examples = load_dataset("squad", split="validation")
+        train_examples = load_dataset("squad", split="train", trust_remote_code=True)
+        dev_examples = load_dataset("squad", split="validation", trust_remote_code=True)
     set_seed(args)
     if rank == 0:
         if os.path.exists(args.model_name_or_path):

legacy/model_zoo/bert/run_glue_trainer.py

@@ -92,7 +92,7 @@ def do_train():
 
     sentence1_key, sentence2_key = task_to_keys[model_args.task_name]
 
-    train_ds = load_dataset("glue", model_args.task_name, split="train")
+    train_ds = load_dataset("glue", model_args.task_name, split="train", trust_remote_code=True)
     columns = train_ds.column_names
     is_regression = model_args.task_name == "stsb"
     label_list = None

legacy/model_zoo/bert/static_ipu/run_squad.py

@@ -208,9 +208,9 @@ def load_squad_dataset(args):
     tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
     features_fn = prepare_train_features if args.is_training else prepare_validation_features
     if args.is_training:
-        raw_dataset = load_dataset("squad", split="train")
+        raw_dataset = load_dataset("squad", split="train", trust_remote_code=True)
     else:
-        raw_dataset = load_dataset("squad", split="validation")
+        raw_dataset = load_dataset("squad", split="validation", trust_remote_code=True)
     column_names = raw_dataset.column_names
     dataset = raw_dataset.map(
         partial(features_fn, tokenizer=tokenizer, args=args), batched=True, remove_columns=column_names, num_proc=4

legacy/model_zoo/ernie-1.0/finetune/run_qa.py

@@ -87,7 +87,10 @@ def main():
 
     dataset_config = data_args.dataset.split(" ")
     raw_datasets = load_dataset(
-        dataset_config[0], None if len(dataset_config) <= 1 else dataset_config[1], cache_dir=model_args.cache_dir
+        dataset_config[0],
+        None if len(dataset_config) <= 1 else dataset_config[1],
+        cache_dir=model_args.cache_dir,
+        trust_remote_code=True,
     )
 
     label_list = getattr(raw_datasets["train"], "label_list", None)

paddlenlp/datasets/dataset.py

@@ -51,9 +51,9 @@
     new_path = os.path.split(path)[-1]
     new_path = os.path.join(ppnlp_path, "hf_datasets", new_path + ".py")
     if os.path.exists(new_path):
-        return origin_load_dataset(new_path, *args, **kwargs)
+        return origin_load_dataset(new_path, trust_remote_code=True, *args, **kwargs)
     else:
-        return origin_load_dataset(path, *args, **kwargs)
+        return origin_load_dataset(path, trust_remote_code=True, *args, **kwargs)
 
 
 datasets.load_dataset = load_from_ppnlp