From fd5cdaeea6eafac32e9d967327bfa3dc0e0d962d Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Mon, 23 Jan 2023 14:33:18 -0500 Subject: [PATCH] Models docstring (#21225) * Clean all models * Style * Last to remove * address review comments * Address review comments --- src/transformers/modeling_flax_utils.py | 3 +- .../models/albert/modeling_albert.py | 23 ++---- .../models/albert/modeling_flax_albert.py | 19 ++--- .../models/albert/modeling_tf_albert.py | 23 ++---- .../models/altclip/modeling_altclip.py | 29 ++++---- .../modeling_audio_spectrogram_transformer.py | 5 +- src/transformers/models/bart/modeling_bart.py | 26 +++---- .../models/bart/modeling_flax_bart.py | 36 ++++------ .../models/bart/modeling_tf_bart.py | 16 ++--- src/transformers/models/beit/modeling_beit.py | 6 +- .../models/beit/modeling_flax_beit.py | 16 ++--- src/transformers/models/bert/modeling_bert.py | 10 +-- .../models/bert/modeling_flax_bert.py | 26 +++---- .../models/bert/modeling_tf_bert.py | 20 ++---- .../modeling_bert_generation.py | 8 +-- .../models/big_bird/modeling_big_bird.py | 30 +++----- .../models/big_bird/modeling_flax_big_bird.py | 20 ++---- .../modeling_bigbird_pegasus.py | 16 ++--- .../models/biogpt/modeling_biogpt.py | 5 +- src/transformers/models/bit/modeling_bit.py | 4 +- .../models/blenderbot/modeling_blenderbot.py | 23 +++--- .../blenderbot/modeling_flax_blenderbot.py | 43 ++++++----- .../blenderbot/modeling_tf_blenderbot.py | 46 +++++++----- .../modeling_blenderbot_small.py | 24 +++---- .../modeling_flax_blenderbot_small.py | 63 ++++++++-------- .../modeling_tf_blenderbot_small.py | 55 ++++++++------ src/transformers/models/blip/modeling_blip.py | 36 +++++----- .../models/bloom/modeling_bloom.py | 7 +- .../models/camembert/modeling_camembert.py | 9 +-- .../models/camembert/modeling_tf_camembert.py | 10 +-- .../models/canine/modeling_canine.py | 15 ++-- .../chinese_clip/modeling_chinese_clip.py | 22 +++--- src/transformers/models/clip/modeling_clip.py | 36 +++++----- .../models/clip/modeling_flax_clip.py | 32 ++++----- .../models/clip/modeling_tf_clip.py | 24 +++---- .../models/clipseg/modeling_clipseg.py | 32 ++++----- .../models/codegen/modeling_codegen.py | 5 +- .../modeling_conditional_detr.py | 6 +- .../models/convbert/modeling_convbert.py | 9 +-- .../models/convbert/modeling_tf_convbert.py | 9 +-- .../models/convnext/modeling_convnext.py | 2 +- .../models/convnext/modeling_tf_convnext.py | 10 +-- src/transformers/models/ctrl/modeling_ctrl.py | 18 ++--- .../models/ctrl/modeling_tf_ctrl.py | 6 +- src/transformers/models/cvt/modeling_cvt.py | 2 +- .../models/cvt/modeling_tf_cvt.py | 4 +- .../data2vec/modeling_data2vec_audio.py | 5 +- .../models/data2vec/modeling_data2vec_text.py | 9 +-- .../data2vec/modeling_data2vec_vision.py | 2 +- .../data2vec/modeling_tf_data2vec_vision.py | 4 +- .../models/deberta/modeling_deberta.py | 2 +- .../models/deberta/modeling_tf_deberta.py | 8 +-- .../models/deberta_v2/modeling_deberta_v2.py | 2 +- .../deberta_v2/modeling_tf_deberta_v2.py | 8 +-- .../modeling_deformable_detr.py | 5 +- src/transformers/models/deit/modeling_deit.py | 10 +-- .../models/deit/modeling_tf_deit.py | 10 +-- src/transformers/models/detr/modeling_detr.py | 16 ++--- .../models/dinat/modeling_dinat.py | 4 +- .../models/distilbert/modeling_distilbert.py | 12 +--- .../distilbert/modeling_flax_distilbert.py | 13 +--- .../distilbert/modeling_tf_distilbert.py | 9 +-- .../models/donut/modeling_donut_swin.py | 2 +- src/transformers/models/dpr/modeling_dpr.py | 2 +- .../models/dpr/modeling_tf_dpr.py | 2 +- src/transformers/models/dpt/modeling_dpt.py | 10 +-- .../modeling_efficientformer.py | 4 -- .../models/electra/modeling_electra.py | 17 ++--- .../models/electra/modeling_flax_electra.py | 20 ++---- .../models/electra/modeling_tf_electra.py | 13 +--- .../models/ernie/modeling_ernie.py | 15 ++-- src/transformers/models/esm/modeling_esm.py | 7 +- .../models/esm/modeling_esmfold.py | 3 +- .../models/esm/modeling_tf_esm.py | 7 +- .../models/flaubert/modeling_flaubert.py | 9 +-- .../models/flaubert/modeling_tf_flaubert.py | 9 +-- .../models/flava/modeling_flava.py | 39 +++++----- src/transformers/models/fnet/modeling_fnet.py | 17 ++--- src/transformers/models/fsmt/modeling_fsmt.py | 8 +-- .../models/funnel/modeling_funnel.py | 12 +--- .../models/funnel/modeling_tf_funnel.py | 14 +--- src/transformers/models/git/modeling_git.py | 7 +- src/transformers/models/glpn/modeling_glpn.py | 6 +- .../models/gpt2/modeling_flax_gpt2.py | 5 +- src/transformers/models/gpt2/modeling_gpt2.py | 13 +--- .../models/gpt2/modeling_tf_gpt2.py | 10 +-- .../models/gpt_neo/modeling_flax_gpt_neo.py | 11 +-- .../models/gpt_neo/modeling_gpt_neo.py | 6 +- .../models/gpt_neox/modeling_gpt_neox.py | 6 +- .../modeling_gpt_neox_japanese.py | 11 ++- .../models/gptj/modeling_flax_gptj.py | 5 +- src/transformers/models/gptj/modeling_gptj.py | 2 +- .../models/gptj/modeling_tf_gptj.py | 7 +- .../models/groupvit/modeling_groupvit.py | 4 +- .../models/groupvit/modeling_tf_groupvit.py | 4 +- .../models/hubert/modeling_hubert.py | 12 ++-- .../models/hubert/modeling_tf_hubert.py | 12 ++-- .../models/ibert/modeling_ibert.py | 9 +-- .../models/imagegpt/modeling_imagegpt.py | 16 ++--- .../models/jukebox/modeling_jukebox.py | 8 +-- .../models/layoutlm/modeling_layoutlm.py | 2 +- .../models/layoutlm/modeling_tf_layoutlm.py | 2 +- .../models/layoutlmv2/modeling_layoutlmv2.py | 19 +++-- .../models/layoutlmv3/modeling_layoutlmv3.py | 4 +- .../layoutlmv3/modeling_tf_layoutlmv3.py | 2 +- src/transformers/models/led/modeling_led.py | 18 ++--- .../models/led/modeling_tf_led.py | 10 ++- .../models/levit/modeling_levit.py | 2 +- src/transformers/models/lilt/modeling_lilt.py | 2 +- .../models/longformer/modeling_longformer.py | 14 ++-- .../longformer/modeling_tf_longformer.py | 8 +-- .../models/longt5/modeling_flax_longt5.py | 33 ++++----- .../models/longt5/modeling_longt5.py | 11 ++- src/transformers/models/luke/modeling_luke.py | 25 +++---- .../models/lxmert/modeling_lxmert.py | 5 +- .../models/lxmert/modeling_tf_lxmert.py | 4 +- .../models/m2m_100/modeling_m2m_100.py | 14 ++-- .../models/marian/modeling_flax_marian.py | 29 ++++---- .../models/marian/modeling_marian.py | 23 +++--- .../models/marian/modeling_tf_marian.py | 14 ++-- .../models/markuplm/modeling_markuplm.py | 11 ++- .../mask2former/modeling_mask2former.py | 1 - .../models/maskformer/modeling_maskformer.py | 14 ++-- .../models/mbart/modeling_flax_mbart.py | 35 ++++----- .../models/mbart/modeling_mbart.py | 22 +++--- .../models/mbart/modeling_tf_mbart.py | 18 +++-- .../models/mctct/modeling_mctct.py | 3 - .../megatron_bert/modeling_megatron_bert.py | 21 ++---- .../models/mobilebert/modeling_mobilebert.py | 8 +-- .../mobilebert/modeling_tf_mobilebert.py | 17 ++--- .../mobilenet_v1/modeling_mobilenet_v1.py | 2 +- .../mobilenet_v2/modeling_mobilenet_v2.py | 6 +- .../models/mobilevit/modeling_mobilevit.py | 6 +- .../models/mobilevit/modeling_tf_mobilevit.py | 6 +- .../models/mpnet/modeling_mpnet.py | 9 +-- .../models/mpnet/modeling_tf_mpnet.py | 9 +-- .../models/mt5/modeling_flax_mt5.py | 13 ++-- src/transformers/models/mt5/modeling_mt5.py | 31 ++++---- .../models/mt5/modeling_tf_mt5.py | 13 ++-- src/transformers/models/mvp/modeling_mvp.py | 28 ++++---- src/transformers/models/nat/modeling_nat.py | 4 +- .../models/nezha/modeling_nezha.py | 17 ++--- .../nystromformer/modeling_nystromformer.py | 7 -- .../models/oneformer/modeling_oneformer.py | 5 +- .../models/openai/modeling_openai.py | 10 +-- .../models/openai/modeling_tf_openai.py | 10 +-- .../models/opt/modeling_flax_opt.py | 8 +-- src/transformers/models/opt/modeling_opt.py | 19 +++-- .../models/opt/modeling_tf_opt.py | 9 +-- .../models/owlvit/modeling_owlvit.py | 34 ++++----- .../models/pegasus/modeling_flax_pegasus.py | 63 ++++++++-------- .../models/pegasus/modeling_pegasus.py | 23 +++--- .../models/pegasus/modeling_tf_pegasus.py | 14 ++-- .../models/pegasus_x/modeling_pegasus_x.py | 17 +++-- .../models/perceiver/modeling_perceiver.py | 25 ++++--- .../models/plbart/modeling_plbart.py | 18 ++--- .../models/poolformer/modeling_poolformer.py | 2 +- .../models/prophetnet/modeling_prophetnet.py | 31 ++++---- .../models/qdqbert/modeling_qdqbert.py | 17 ++--- src/transformers/models/rag/modeling_rag.py | 12 ++-- .../models/rag/modeling_tf_rag.py | 12 ++-- .../models/realm/modeling_realm.py | 23 +++--- .../models/reformer/modeling_reformer.py | 2 +- .../models/regnet/modeling_regnet.py | 2 +- .../models/regnet/modeling_tf_regnet.py | 2 +- .../models/rembert/modeling_rembert.py | 13 +--- .../models/rembert/modeling_tf_rembert.py | 10 +-- .../models/resnet/modeling_resnet.py | 2 +- .../models/resnet/modeling_tf_resnet.py | 2 +- .../models/retribert/modeling_retribert.py | 2 +- .../models/roberta/modeling_flax_roberta.py | 13 +--- .../models/roberta/modeling_roberta.py | 9 +-- .../models/roberta/modeling_tf_roberta.py | 10 +-- .../modeling_flax_roberta_prelayernorm.py | 10 +-- .../modeling_roberta_prelayernorm.py | 9 +-- .../modeling_tf_roberta_prelayernorm.py | 10 +-- .../models/roc_bert/modeling_roc_bert.py | 24 +++---- .../models/roformer/modeling_flax_roformer.py | 12 +--- .../models/roformer/modeling_roformer.py | 13 +--- .../models/roformer/modeling_tf_roformer.py | 10 +-- .../models/segformer/modeling_segformer.py | 6 +- .../models/segformer/modeling_tf_segformer.py | 6 +- src/transformers/models/sew/modeling_sew.py | 8 +-- .../models/sew_d/modeling_sew_d.py | 8 +-- .../modeling_flax_speech_encoder_decoder.py | 12 ++-- .../modeling_speech_encoder_decoder.py | 12 ++-- .../speech_to_text/modeling_speech_to_text.py | 10 +-- .../modeling_tf_speech_to_text.py | 6 +- .../models/splinter/modeling_splinter.py | 5 +- .../squeezebert/modeling_squeezebert.py | 9 +-- src/transformers/models/swin/modeling_swin.py | 4 +- .../models/swin/modeling_tf_swin.py | 4 +- .../models/swin2sr/modeling_swin2sr.py | 2 +- .../models/swinv2/modeling_swinv2.py | 4 +- .../modeling_switch_transformers.py | 19 +++-- .../models/t5/modeling_flax_t5.py | 33 ++++----- src/transformers/models/t5/modeling_t5.py | 19 +++-- src/transformers/models/t5/modeling_tf_t5.py | 15 ++-- .../models/tapas/modeling_tapas.py | 28 ++++---- .../models/tapas/modeling_tf_tapas.py | 27 ++++--- .../timesformer/modeling_timesformer.py | 10 +-- .../transfo_xl/modeling_tf_transfo_xl.py | 4 -- .../models/transfo_xl/modeling_transfo_xl.py | 6 +- .../models/trocr/modeling_trocr.py | 5 +- .../models/unispeech/modeling_unispeech.py | 9 ++- .../unispeech_sat/modeling_unispeech_sat.py | 9 ++- .../models/upernet/modeling_upernet.py | 2 +- src/transformers/models/van/modeling_van.py | 2 +- .../models/videomae/modeling_videomae.py | 14 ++-- src/transformers/models/vilt/modeling_vilt.py | 4 +- .../modeling_flax_vision_encoder_decoder.py | 18 ++--- .../modeling_tf_vision_encoder_decoder.py | 8 +-- .../modeling_vision_encoder_decoder.py | 12 ++-- .../modeling_flax_vision_text_dual_encoder.py | 12 ++-- .../modeling_vision_text_dual_encoder.py | 14 ++-- .../visual_bert/modeling_visual_bert.py | 26 +++---- .../models/vit/modeling_flax_vit.py | 10 +-- .../models/vit/modeling_tf_vit.py | 2 +- src/transformers/models/vit/modeling_vit.py | 6 +- .../models/vit_hybrid/modeling_vit_hybrid.py | 2 +- .../models/vit_mae/modeling_tf_vit_mae.py | 4 +- .../models/vit_mae/modeling_vit_mae.py | 4 +- .../models/vit_msn/modeling_vit_msn.py | 4 +- .../models/wav2vec2/modeling_flax_wav2vec2.py | 20 +++--- .../models/wav2vec2/modeling_tf_wav2vec2.py | 13 ++-- .../models/wav2vec2/modeling_wav2vec2.py | 8 +-- .../modeling_wav2vec2_conformer.py | 8 +-- .../models/wavlm/modeling_wavlm.py | 8 +-- .../models/whisper/modeling_tf_whisper.py | 16 ++--- .../models/whisper/modeling_whisper.py | 15 ++-- .../models/x_clip/modeling_x_clip.py | 12 ++-- .../models/xglm/modeling_flax_xglm.py | 5 +- .../models/xglm/modeling_tf_xglm.py | 5 +- src/transformers/models/xglm/modeling_xglm.py | 7 +- .../models/xlm/modeling_tf_xlm.py | 9 +-- src/transformers/models/xlm/modeling_xlm.py | 13 +--- .../xlm_prophetnet/modeling_xlm_prophetnet.py | 33 ++++----- .../xlm_roberta/modeling_flax_xlm_roberta.py | 13 +--- .../xlm_roberta/modeling_tf_xlm_roberta.py | 12 +--- .../xlm_roberta/modeling_xlm_roberta.py | 9 +-- .../xlm_roberta_xl/modeling_xlm_roberta_xl.py | 13 +--- .../models/xlnet/modeling_tf_xlnet.py | 12 +--- .../models/xlnet/modeling_xlnet.py | 16 ++--- .../models/yolos/modeling_yolos.py | 4 +- src/transformers/models/yoso/modeling_yoso.py | 7 -- src/transformers/utils/doc.py | 72 +++++++++---------- 246 files changed, 1285 insertions(+), 1890 deletions(-) diff --git a/src/transformers/modeling_flax_utils.py b/src/transformers/modeling_flax_utils.py index ade1a5063bf1b6..7dc8f4ae48915e 100644 --- a/src/transformers/modeling_flax_utils.py +++ b/src/transformers/modeling_flax_utils.py @@ -1124,10 +1124,9 @@ def overwrite_call_docstring(model_class, docstring): model_class.__call__ = add_start_docstrings_to_model_forward(docstring)(model_class.__call__) -def append_call_sample_docstring(model_class, tokenizer_class, checkpoint, output_type, config_class, mask=None): +def append_call_sample_docstring(model_class, checkpoint, output_type, config_class, mask=None): model_class.__call__ = copy_func(model_class.__call__) model_class.__call__ = add_code_sample_docstrings( - processor_class=tokenizer_class, checkpoint=checkpoint, output_type=output_type, config_class=config_class, diff --git a/src/transformers/models/albert/modeling_albert.py b/src/transformers/models/albert/modeling_albert.py index 6ba582fa725207..dc9559ac3624ef 100755 --- a/src/transformers/models/albert/modeling_albert.py +++ b/src/transformers/models/albert/modeling_albert.py @@ -50,7 +50,6 @@ _CHECKPOINT_FOR_DOC = "albert-base-v2" _CONFIG_FOR_DOC = "AlbertConfig" -_TOKENIZER_FOR_DOC = "AlbertTokenizer" ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ @@ -582,7 +581,7 @@ class AlbertForPreTrainingOutput(ModelOutput): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`AlbertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -677,7 +676,6 @@ def _prune_heads(self, heads_to_prune: Dict[int, List[int]]) -> None: @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPooling, config_class=_CONFIG_FOR_DOC, @@ -818,10 +816,10 @@ def forward( Example: ```python - >>> from transformers import AlbertTokenizer, AlbertForPreTraining + >>> from transformers import AutoTokenizer, AlbertForPreTraining >>> import torch - >>> tokenizer = AlbertTokenizer.from_pretrained("albert-base-v2") + >>> tokenizer = AutoTokenizer.from_pretrained("albert-base-v2") >>> model = AlbertForPreTraining.from_pretrained("albert-base-v2") >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) @@ -967,9 +965,9 @@ def forward( ```python >>> import torch - >>> from transformers import AlbertTokenizer, AlbertForMaskedLM + >>> from transformers import AutoTokenizer, AlbertForMaskedLM - >>> tokenizer = AlbertTokenizer.from_pretrained("albert-base-v2") + >>> tokenizer = AutoTokenizer.from_pretrained("albert-base-v2") >>> model = AlbertForMaskedLM.from_pretrained("albert-base-v2") >>> # add mask_token @@ -1048,7 +1046,6 @@ def __init__(self, config: AlbertConfig): @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="textattack/albert-base-v2-imdb", output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1157,15 +1154,9 @@ def __init__(self, config: AlbertConfig): @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, - checkpoint="vumichien/tiny-albert", + checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, - expected_output=( - "['LABEL_1', 'LABEL_1', 'LABEL_1', 'LABEL_0', 'LABEL_1', 'LABEL_0', 'LABEL_1', 'LABEL_1', " - "'LABEL_0', 'LABEL_1', 'LABEL_0', 'LABEL_0', 'LABEL_1', 'LABEL_1']" - ), - expected_loss=0.66, ) def forward( self, @@ -1243,7 +1234,6 @@ def __init__(self, config: AlbertConfig): @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="twmkn9/albert-base-v2-squad2", output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1347,7 +1337,6 @@ def __init__(self, config: AlbertConfig): @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/albert/modeling_flax_albert.py b/src/transformers/models/albert/modeling_flax_albert.py index 03ff7d01705625..e55038c8acff72 100644 --- a/src/transformers/models/albert/modeling_flax_albert.py +++ b/src/transformers/models/albert/modeling_flax_albert.py @@ -50,7 +50,6 @@ _CHECKPOINT_FOR_DOC = "albert-base-v2" _CONFIG_FOR_DOC = "AlbertConfig" -_TOKENIZER_FOR_DOC = "AlbertTokenizer" @flax.struct.dataclass @@ -122,7 +121,7 @@ class FlaxAlbertForPreTrainingOutput(ModelOutput): input_ids (`numpy.ndarray` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`AlbertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -680,9 +679,7 @@ class FlaxAlbertModel(FlaxAlbertPreTrainedModel): module_class = FlaxAlbertModule -append_call_sample_docstring( - FlaxAlbertModel, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutputWithPooling, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxAlbertModel, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutputWithPooling, _CONFIG_FOR_DOC) class FlaxAlbertForPreTrainingModule(nn.Module): @@ -757,9 +754,9 @@ class FlaxAlbertForPreTraining(FlaxAlbertPreTrainedModel): Example: ```python - >>> from transformers import AlbertTokenizer, FlaxAlbertForPreTraining + >>> from transformers import AutoTokenizer, FlaxAlbertForPreTraining - >>> tokenizer = AlbertTokenizer.from_pretrained("albert-base-v2") + >>> tokenizer = AutoTokenizer.from_pretrained("albert-base-v2") >>> model = FlaxAlbertForPreTraining.from_pretrained("albert-base-v2") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="np") @@ -834,9 +831,7 @@ class FlaxAlbertForMaskedLM(FlaxAlbertPreTrainedModel): module_class = FlaxAlbertForMaskedLMModule -append_call_sample_docstring( - FlaxAlbertForMaskedLM, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxMaskedLMOutput, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxAlbertForMaskedLM, _CHECKPOINT_FOR_DOC, FlaxMaskedLMOutput, _CONFIG_FOR_DOC) class FlaxAlbertForSequenceClassificationModule(nn.Module): @@ -906,7 +901,6 @@ class FlaxAlbertForSequenceClassification(FlaxAlbertPreTrainedModel): append_call_sample_docstring( FlaxAlbertForSequenceClassification, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSequenceClassifierOutput, _CONFIG_FOR_DOC, @@ -983,7 +977,6 @@ class FlaxAlbertForMultipleChoice(FlaxAlbertPreTrainedModel): ) append_call_sample_docstring( FlaxAlbertForMultipleChoice, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxMultipleChoiceModelOutput, _CONFIG_FOR_DOC, @@ -1054,7 +1047,6 @@ class FlaxAlbertForTokenClassification(FlaxAlbertPreTrainedModel): append_call_sample_docstring( FlaxAlbertForTokenClassification, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxTokenClassifierOutput, _CONFIG_FOR_DOC, @@ -1123,7 +1115,6 @@ class FlaxAlbertForQuestionAnswering(FlaxAlbertPreTrainedModel): append_call_sample_docstring( FlaxAlbertForQuestionAnswering, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxQuestionAnsweringModelOutput, _CONFIG_FOR_DOC, diff --git a/src/transformers/models/albert/modeling_tf_albert.py b/src/transformers/models/albert/modeling_tf_albert.py index e23ea64059fdaa..823c7c48bbb572 100644 --- a/src/transformers/models/albert/modeling_tf_albert.py +++ b/src/transformers/models/albert/modeling_tf_albert.py @@ -61,7 +61,6 @@ _CHECKPOINT_FOR_DOC = "albert-base-v2" _CONFIG_FOR_DOC = "AlbertConfig" -_TOKENIZER_FOR_DOC = "AlbertTokenizer" TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ "albert-base-v1", @@ -738,7 +737,7 @@ class TFAlbertForPreTrainingOutput(ModelOutput): input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`AlbertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -802,7 +801,6 @@ def __init__(self, config: AlbertConfig, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutputWithPooling, config_class=_CONFIG_FOR_DOC, @@ -895,9 +893,9 @@ def call( ```python >>> import tensorflow as tf - >>> from transformers import AlbertTokenizer, TFAlbertForPreTraining + >>> from transformers import AutoTokenizer, TFAlbertForPreTraining - >>> tokenizer = AlbertTokenizer.from_pretrained("albert-base-v2") + >>> tokenizer = AutoTokenizer.from_pretrained("albert-base-v2") >>> model = TFAlbertForPreTraining.from_pretrained("albert-base-v2") >>> input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] @@ -1015,9 +1013,9 @@ def call( ```python >>> import tensorflow as tf - >>> from transformers import AlbertTokenizer, TFAlbertForMaskedLM + >>> from transformers import AutoTokenizer, TFAlbertForMaskedLM - >>> tokenizer = AlbertTokenizer.from_pretrained("albert-base-v2") + >>> tokenizer = AutoTokenizer.from_pretrained("albert-base-v2") >>> model = TFAlbertForMaskedLM.from_pretrained("albert-base-v2") >>> # add mask_token @@ -1101,7 +1099,6 @@ def __init__(self, config: AlbertConfig, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="vumichien/albert-base-v2-imdb", output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1196,15 +1193,9 @@ def __init__(self, config: AlbertConfig, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, - checkpoint="vumichien/tiny-albert", + checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFTokenClassifierOutput, config_class=_CONFIG_FOR_DOC, - expected_output=( - "['LABEL_1', 'LABEL_1', 'LABEL_1', 'LABEL_0', 'LABEL_1', 'LABEL_0', 'LABEL_1', 'LABEL_1', " - "'LABEL_0', 'LABEL_1', 'LABEL_0', 'LABEL_0', 'LABEL_1', 'LABEL_1']" - ), - expected_loss=0.66, ) def call( self, @@ -1285,7 +1276,6 @@ def __init__(self, config: AlbertConfig, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="vumichien/albert-base-v2-squad2", output_type=TFQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1400,7 +1390,6 @@ def dummy_inputs(self): @unpack_inputs @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/altclip/modeling_altclip.py b/src/transformers/models/altclip/modeling_altclip.py index a64077f46d7e1d..3e52e2b2d93ad7 100755 --- a/src/transformers/models/altclip/modeling_altclip.py +++ b/src/transformers/models/altclip/modeling_altclip.py @@ -37,7 +37,6 @@ logger = logging.get_logger(__name__) -_TOKENIZER_FOR_DOC = "XLMRobertaTokenizer" _CHECKPOINT_FOR_DOC = "BAAI/AltCLIP" _CONFIG_FOR_DOC = "AltCLIPConfig" @@ -68,7 +67,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`XLMRobertaTokenizerFast`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -98,7 +97,7 @@ Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`CLIPImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. @@ -115,7 +114,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`XLMRobertaTokenizerFast`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -133,7 +132,7 @@ [What are position IDs?](../glossary#position-ids) pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`CLIPImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. return_loss (`bool`, *optional*): Whether or not to return the contrastive loss. output_attentions (`bool`, *optional*): @@ -1181,10 +1180,10 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import AltCLIPProcessor, AltCLIPVisionModel + >>> from transformers import AutoProcessor, AltCLIPVisionModel >>> model = AltCLIPVisionModel.from_pretrained("BAAI/AltCLIP") - >>> processor = AltCLIPProcessor.from_pretrained("BAAI/AltCLIP") + >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1422,10 +1421,10 @@ def forward( Examples: ```python - >>> from transformers import AltCLIPProcessor, AltCLIPTextModel + >>> from transformers import AutoProcessor, AltCLIPTextModel >>> model = AltCLIPTextModel.from_pretrained("BAAI/AltCLIP") - >>> processor = AltCLIPProcessor.from_pretrained("BAAI/AltCLIP") + >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP") >>> texts = ["it's a cat", "it's a dog"] @@ -1526,10 +1525,10 @@ def get_text_features( Examples: ```python - >>> from transformers import AltCLIPProcessor, AltCLIPModel + >>> from transformers import AutoProcessor, AltCLIPModel >>> model = AltCLIPModel.from_pretrained("BAAI/AltCLIP") - >>> processor = AltCLIPProcessor.from_pretrained("BAAI/AltCLIP") + >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP") >>> inputs = processor(text=["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") >>> text_features = model.get_text_features(**inputs) ```""" @@ -1572,10 +1571,10 @@ def get_image_features( ```python >>> from PIL import Image >>> import requests - >>> from transformers import AltCLIPProcessor, AltCLIPModel + >>> from transformers import AutoProcessor, AltCLIPModel >>> model = AltCLIPModel.from_pretrained("BAAI/AltCLIP") - >>> processor = AltCLIPProcessor.from_pretrained("BAAI/AltCLIP") + >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(images=image, return_tensors="pt") @@ -1622,10 +1621,10 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import AltCLIPProcessor, AltCLIPModel + >>> from transformers import AutoProcessor, AltCLIPModel >>> model = AltCLIPModel.from_pretrained("BAAI/AltCLIP") - >>> processor = AltCLIPProcessor.from_pretrained("BAAI/AltCLIP") + >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor( diff --git a/src/transformers/models/audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.py b/src/transformers/models/audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.py index 9cf24f31e5f9a8..de67d477445069 100644 --- a/src/transformers/models/audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.py +++ b/src/transformers/models/audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.py @@ -34,7 +34,6 @@ # General docstring _CONFIG_FOR_DOC = "ASTConfig" -_FEAT_EXTRACTOR_FOR_DOC = "ASTFeatureExtractor" # Base docstring _CHECKPOINT_FOR_DOC = "MIT/ast-finetuned-audioset-10-10-0.4593" @@ -418,7 +417,7 @@ def _set_gradient_checkpointing(self, module: ASTEncoder, value: bool = False) - AUDIO_SPECTROGRAM_TRANSFORMER_INPUTS_DOCSTRING = r""" Args: input_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`ASTFeatureExtractor`]. See + Pixel values. Pixel values can be obtained using [`AutoFeatureExtractor`]. See [`ASTFeatureExtractor.__call__`] for details. head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): @@ -468,7 +467,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(AUDIO_SPECTROGRAM_TRANSFORMER_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_FEAT_EXTRACTOR_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPooling, config_class=_CONFIG_FOR_DOC, @@ -558,7 +556,6 @@ def __init__(self, config: ASTConfig) -> None: @add_start_docstrings_to_model_forward(AUDIO_SPECTROGRAM_TRANSFORMER_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_FEAT_EXTRACTOR_FOR_DOC, checkpoint=_SEQ_CLASS_CHECKPOINT, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/bart/modeling_bart.py b/src/transformers/models/bart/modeling_bart.py index 313eb249367f29..136a5eebc40090 100755 --- a/src/transformers/models/bart/modeling_bart.py +++ b/src/transformers/models/bart/modeling_bart.py @@ -50,7 +50,6 @@ _CHECKPOINT_FOR_DOC = "facebook/bart-base" _CONFIG_FOR_DOC = "BartConfig" -_TOKENIZER_FOR_DOC = "BartTokenizer" # Base model docstring _EXPECTED_OUTPUT_SHAPE = [1, 8, 768] @@ -563,10 +562,10 @@ def __init_subclass__(self): Summarization example: ```python - >>> from transformers import BartTokenizer, BartForConditionalGeneration + >>> from transformers import AutoTokenizer, BartForConditionalGeneration >>> model = BartForConditionalGeneration.from_pretrained("facebook/bart-large-cnn") - >>> tokenizer = BartTokenizer.from_pretrained("facebook/bart-large-cnn") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/bart-large-cnn") >>> ARTICLE_TO_SUMMARIZE = ( ... "PG&E stated it scheduled the blackouts in response to forecasts for high winds " @@ -584,9 +583,9 @@ def __init_subclass__(self): Mask filling example: ```python - >>> from transformers import BartTokenizer, BartForConditionalGeneration + >>> from transformers import AutoTokenizer, BartForConditionalGeneration - >>> tokenizer = BartTokenizer.from_pretrained("facebook/bart-base") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/bart-base") >>> model = BartForConditionalGeneration.from_pretrained("facebook/bart-base") >>> TXT = "My friends are but they eat too many carbs." @@ -608,7 +607,7 @@ def __init_subclass__(self): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -622,7 +621,7 @@ def __init_subclass__(self): decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -758,7 +757,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -956,7 +955,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1190,7 +1189,6 @@ def get_decoder(self): @add_start_docstrings_to_model_forward(BART_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1483,7 +1481,6 @@ def __init__(self, config: BartConfig, **kwargs): @add_start_docstrings_to_model_forward(BART_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_SEQUENCE_CLASSIFICATION, output_type=Seq2SeqSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1611,7 +1608,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(BART_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_QA, output_type=Seq2SeqQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1789,7 +1785,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1854,9 +1850,9 @@ def forward( Example: ```python - >>> from transformers import BartTokenizer, BartForCausalLM + >>> from transformers import AutoTokenizer, BartForCausalLM - >>> tokenizer = BartTokenizer.from_pretrained("facebook/bart-base") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/bart-base") >>> model = BartForCausalLM.from_pretrained("facebook/bart-base", add_cross_attention=False) >>> assert model.config.is_decoder, f"{model.__class__} has to be configured as a decoder." >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") diff --git a/src/transformers/models/bart/modeling_flax_bart.py b/src/transformers/models/bart/modeling_flax_bart.py index 4963f9548071ba..01b2bf8ecec144 100644 --- a/src/transformers/models/bart/modeling_flax_bart.py +++ b/src/transformers/models/bart/modeling_flax_bart.py @@ -55,7 +55,6 @@ _CHECKPOINT_FOR_DOC = "facebook/bart-base" _CONFIG_FOR_DOC = "BartConfig" -_TOKENIZER_FOR_DOC = "BartTokenizer" BART_START_DOCSTRING = r""" @@ -98,7 +97,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -112,7 +111,7 @@ decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -149,7 +148,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -178,7 +177,7 @@ decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -1019,10 +1018,10 @@ def encode( Example: ```python - >>> from transformers import BartTokenizer, FlaxBartForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxBartForConditionalGeneration >>> model = FlaxBartForConditionalGeneration.from_pretrained("facebook/bart-large-cnn") - >>> tokenizer = BartTokenizer.from_pretrained("facebook/bart-large-cnn") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/bart-large-cnn") >>> text = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, max_length=1024, return_tensors="jax") @@ -1086,10 +1085,10 @@ def decode( ```python >>> import jax.numpy as jnp - >>> from transformers import BartTokenizer, FlaxBartForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxBartForConditionalGeneration >>> model = FlaxBartForConditionalGeneration.from_pretrained("facebook/bart-large-cnn") - >>> tokenizer = BartTokenizer.from_pretrained("facebook/bart-large-cnn") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/bart-large-cnn") >>> text = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, max_length=1024, return_tensors="jax") @@ -1247,9 +1246,7 @@ class FlaxBartModel(FlaxBartPreTrainedModel): module_class = FlaxBartModule -append_call_sample_docstring( - FlaxBartModel, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqModelOutput, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxBartModel, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqModelOutput, _CONFIG_FOR_DOC) class FlaxBartForConditionalGenerationModule(nn.Module): @@ -1355,10 +1352,10 @@ def decode( ```python >>> import jax.numpy as jnp - >>> from transformers import BartTokenizer, FlaxBartForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxBartForConditionalGeneration >>> model = FlaxBartForConditionalGeneration.from_pretrained("facebook/bart-large-cnn") - >>> tokenizer = BartTokenizer.from_pretrained("facebook/bart-large-cnn") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/bart-large-cnn") >>> text = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, max_length=1024, return_tensors="jax") @@ -1511,10 +1508,10 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): Summarization example: ```python - >>> from transformers import BartTokenizer, FlaxBartForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxBartForConditionalGeneration >>> model = FlaxBartForConditionalGeneration.from_pretrained("facebook/bart-large-cnn") - >>> tokenizer = BartTokenizer.from_pretrained("facebook/bart-large-cnn") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/bart-large-cnn") >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors="np") @@ -1528,10 +1525,10 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): ```python >>> import jax - >>> from transformers import BartTokenizer, FlaxBartForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxBartForConditionalGeneration >>> model = FlaxBartForConditionalGeneration.from_pretrained("facebook/bart-large") - >>> tokenizer = BartTokenizer.from_pretrained("facebook/bart-large") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/bart-large") >>> TXT = "My friends are but they eat too many carbs." >>> input_ids = tokenizer([TXT], return_tensors="jax")["input_ids"] @@ -1647,7 +1644,6 @@ class FlaxBartForSequenceClassification(FlaxBartPreTrainedModel): append_call_sample_docstring( FlaxBartForSequenceClassification, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqSequenceClassifierOutput, _CONFIG_FOR_DOC, @@ -1734,7 +1730,6 @@ class FlaxBartForQuestionAnswering(FlaxBartPreTrainedModel): append_call_sample_docstring( FlaxBartForQuestionAnswering, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqQuestionAnsweringModelOutput, _CONFIG_FOR_DOC, @@ -1997,7 +1992,6 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): append_call_sample_docstring( FlaxBartForCausalLM, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxCausalLMOutputWithCrossAttentions, _CONFIG_FOR_DOC, diff --git a/src/transformers/models/bart/modeling_tf_bart.py b/src/transformers/models/bart/modeling_tf_bart.py index 355db78d84e89a..44b230be0eb9b0 100644 --- a/src/transformers/models/bart/modeling_tf_bart.py +++ b/src/transformers/models/bart/modeling_tf_bart.py @@ -57,7 +57,6 @@ _CHECKPOINT_FOR_DOC = "facebook/bart-large" _CONFIG_FOR_DOC = "BartConfig" -_TOKENIZER_FOR_DOC = "BartTokenizer" LARGE_NEGATIVE = -1e8 @@ -558,10 +557,10 @@ def serving(self, inputs): Summarization example: ```python - >>> from transformers import BartTokenizer, TFBartForConditionalGeneration + >>> from transformers import AutoTokenizer, TFBartForConditionalGeneration >>> model = TFBartForConditionalGeneration.from_pretrained("facebook/bart-large") - >>> tokenizer = BartTokenizer.from_pretrained("facebook/bart-large") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/bart-large") >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors="tf") @@ -574,9 +573,9 @@ def serving(self, inputs): Mask filling example: ```python - >>> from transformers import BartTokenizer, TFBartForConditionalGeneration + >>> from transformers import AutoTokenizer, TFBartForConditionalGeneration - >>> tokenizer = BartTokenizer.from_pretrained("facebook/bart-large") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/bart-large") >>> TXT = "My friends are but they eat too many carbs." >>> model = TFBartForConditionalGeneration.from_pretrained("facebook/bart-large") @@ -607,7 +606,7 @@ def serving(self, inputs): decoder_input_ids (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -716,7 +715,7 @@ def call( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -883,7 +882,7 @@ def call( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1202,7 +1201,6 @@ def get_decoder(self): @add_start_docstrings_to_model_forward(BART_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFSeq2SeqModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/beit/modeling_beit.py b/src/transformers/models/beit/modeling_beit.py index fd06d2db8f7895..803f602f92a2f7 100755 --- a/src/transformers/models/beit/modeling_beit.py +++ b/src/transformers/models/beit/modeling_beit.py @@ -592,7 +592,7 @@ def _set_gradient_checkpointing(self, module, value=False): BEIT_INPUTS_DOCSTRING = r""" Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`BeitImageProcessor`]. See + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`BeitImageProcessor.__call__`] for details. head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): @@ -767,7 +767,7 @@ def forward( Examples: ```python - >>> from transformers import BeitImageProcessor, BeitForMaskedImageModeling + >>> from transformers import AutoImageProcessor, BeitForMaskedImageModeling >>> import torch >>> from PIL import Image >>> import requests @@ -775,7 +775,7 @@ def forward( >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = BeitImageProcessor.from_pretrained("microsoft/beit-base-patch16-224-pt22k") + >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-patch16-224-pt22k") >>> model = BeitForMaskedImageModeling.from_pretrained("microsoft/beit-base-patch16-224-pt22k") >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2 diff --git a/src/transformers/models/beit/modeling_flax_beit.py b/src/transformers/models/beit/modeling_flax_beit.py index 4a866584fb3b3f..8e9f87b29448ed 100644 --- a/src/transformers/models/beit/modeling_flax_beit.py +++ b/src/transformers/models/beit/modeling_flax_beit.py @@ -102,8 +102,8 @@ class FlaxBeitModelOutputWithPooling(FlaxBaseModelOutputWithPooling): BEIT_INPUTS_DOCSTRING = r""" Args: pixel_values (`numpy.ndarray` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`BeitImageProcessor`]. See - [`BeitImageProcessor.__call__`] for details. + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See + [`AutoImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned @@ -756,14 +756,14 @@ class FlaxBeitModel(FlaxBeitPreTrainedModel): Examples: ```python - >>> from transformers import BeitImageProcessor, FlaxBeitModel + >>> from transformers import AutoImageProcessor, FlaxBeitModel >>> from PIL import Image >>> import requests >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = BeitImageProcessor.from_pretrained("microsoft/beit-base-patch16-224-pt22k-ft22k") + >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-patch16-224-pt22k-ft22k") >>> model = FlaxBeitModel.from_pretrained("microsoft/beit-base-patch16-224-pt22k-ft22k") >>> inputs = image_processor(images=image, return_tensors="np") @@ -843,14 +843,14 @@ class FlaxBeitForMaskedImageModeling(FlaxBeitPreTrainedModel): Examples: ```python - >>> from transformers import BeitImageProcessor, BeitForMaskedImageModeling + >>> from transformers import AutoImageProcessor, BeitForMaskedImageModeling >>> from PIL import Image >>> import requests >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = BeitImageProcessor.from_pretrained("microsoft/beit-base-patch16-224-pt22k") + >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-patch16-224-pt22k") >>> model = BeitForMaskedImageModeling.from_pretrained("microsoft/beit-base-patch16-224-pt22k") >>> inputs = image_processor(images=image, return_tensors="np") @@ -927,14 +927,14 @@ class FlaxBeitForImageClassification(FlaxBeitPreTrainedModel): Example: ```python - >>> from transformers import BeitImageProcessor, FlaxBeitForImageClassification + >>> from transformers import AutoImageProcessor, FlaxBeitForImageClassification >>> from PIL import Image >>> import requests >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = BeitImageProcessor.from_pretrained("microsoft/beit-base-patch16-224") + >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-patch16-224") >>> model = FlaxBeitForImageClassification.from_pretrained("microsoft/beit-base-patch16-224") >>> inputs = image_processor(images=image, return_tensors="np") diff --git a/src/transformers/models/bert/modeling_bert.py b/src/transformers/models/bert/modeling_bert.py index 4537d98497ef02..61355216653cb0 100755 --- a/src/transformers/models/bert/modeling_bert.py +++ b/src/transformers/models/bert/modeling_bert.py @@ -820,7 +820,7 @@ class BertForPreTrainingOutput(ModelOutput): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1104,10 +1104,10 @@ def forward( Example: ```python - >>> from transformers import BertTokenizer, BertForPreTraining + >>> from transformers import AutoTokenizer, BertForPreTraining >>> import torch - >>> tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") >>> model = BertForPreTraining.from_pretrained("bert-base-uncased") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") @@ -1449,10 +1449,10 @@ def forward( Example: ```python - >>> from transformers import BertTokenizer, BertForNextSentencePrediction + >>> from transformers import AutoTokenizer, BertForNextSentencePrediction >>> import torch - >>> tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") >>> model = BertForNextSentencePrediction.from_pretrained("bert-base-uncased") >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." diff --git a/src/transformers/models/bert/modeling_flax_bert.py b/src/transformers/models/bert/modeling_flax_bert.py index 0206e56c171bac..818a3ee0896452 100644 --- a/src/transformers/models/bert/modeling_flax_bert.py +++ b/src/transformers/models/bert/modeling_flax_bert.py @@ -55,7 +55,6 @@ _CHECKPOINT_FOR_DOC = "bert-base-uncased" _CONFIG_FOR_DOC = "BertConfig" -_TOKENIZER_FOR_DOC = "BertTokenizer" remat = nn_partitioning.remat @@ -142,7 +141,7 @@ class FlaxBertForPreTrainingOutput(ModelOutput): input_ids (`numpy.ndarray` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1032,9 +1031,7 @@ class FlaxBertModel(FlaxBertPreTrainedModel): module_class = FlaxBertModule -append_call_sample_docstring( - FlaxBertModel, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutputWithPooling, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxBertModel, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutputWithPooling, _CONFIG_FOR_DOC) class FlaxBertForPreTrainingModule(nn.Module): @@ -1116,9 +1113,9 @@ class FlaxBertForPreTraining(FlaxBertPreTrainedModel): Example: ```python - >>> from transformers import BertTokenizer, FlaxBertForPreTraining + >>> from transformers import AutoTokenizer, FlaxBertForPreTraining - >>> tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") >>> model = FlaxBertForPreTraining.from_pretrained("bert-base-uncased") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="np") @@ -1201,9 +1198,7 @@ class FlaxBertForMaskedLM(FlaxBertPreTrainedModel): module_class = FlaxBertForMaskedLMModule -append_call_sample_docstring( - FlaxBertForMaskedLM, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxMaskedLMOutput, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxBertForMaskedLM, _CHECKPOINT_FOR_DOC, FlaxMaskedLMOutput, _CONFIG_FOR_DOC) class FlaxBertForNextSentencePredictionModule(nn.Module): @@ -1273,9 +1268,9 @@ class FlaxBertForNextSentencePrediction(FlaxBertPreTrainedModel): Example: ```python - >>> from transformers import BertTokenizer, FlaxBertForNextSentencePrediction + >>> from transformers import AutoTokenizer, FlaxBertForNextSentencePrediction - >>> tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") >>> model = FlaxBertForNextSentencePrediction.from_pretrained("bert-base-uncased") >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." @@ -1372,7 +1367,6 @@ class FlaxBertForSequenceClassification(FlaxBertPreTrainedModel): append_call_sample_docstring( FlaxBertForSequenceClassification, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSequenceClassifierOutput, _CONFIG_FOR_DOC, @@ -1455,7 +1449,7 @@ class FlaxBertForMultipleChoice(FlaxBertPreTrainedModel): FlaxBertForMultipleChoice, BERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) append_call_sample_docstring( - FlaxBertForMultipleChoice, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxMultipleChoiceModelOutput, _CONFIG_FOR_DOC + FlaxBertForMultipleChoice, _CHECKPOINT_FOR_DOC, FlaxMultipleChoiceModelOutput, _CONFIG_FOR_DOC ) @@ -1530,7 +1524,7 @@ class FlaxBertForTokenClassification(FlaxBertPreTrainedModel): append_call_sample_docstring( - FlaxBertForTokenClassification, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxTokenClassifierOutput, _CONFIG_FOR_DOC + FlaxBertForTokenClassification, _CHECKPOINT_FOR_DOC, FlaxTokenClassifierOutput, _CONFIG_FOR_DOC ) @@ -1604,7 +1598,6 @@ class FlaxBertForQuestionAnswering(FlaxBertPreTrainedModel): append_call_sample_docstring( FlaxBertForQuestionAnswering, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxQuestionAnsweringModelOutput, _CONFIG_FOR_DOC, @@ -1715,7 +1708,6 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): append_call_sample_docstring( FlaxBertForCausalLM, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxCausalLMOutputWithCrossAttentions, _CONFIG_FOR_DOC, diff --git a/src/transformers/models/bert/modeling_tf_bert.py b/src/transformers/models/bert/modeling_tf_bert.py index bb67224ea8d606..834b6237d36e4a 100644 --- a/src/transformers/models/bert/modeling_tf_bert.py +++ b/src/transformers/models/bert/modeling_tf_bert.py @@ -67,7 +67,6 @@ _CHECKPOINT_FOR_DOC = "bert-base-uncased" _CONFIG_FOR_DOC = "BertConfig" -_TOKENIZER_FOR_DOC = "BertTokenizer" # TokenClassification docstring _CHECKPOINT_FOR_TOKEN_CLASSIFICATION = "dbmdz/bert-large-cased-finetuned-conll03-english" @@ -1008,7 +1007,7 @@ class TFBertForPreTrainingOutput(ModelOutput): input_ids (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -1072,7 +1071,6 @@ def __init__(self, config: BertConfig, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1220,9 +1218,9 @@ def call( ```python >>> import tensorflow as tf - >>> from transformers import BertTokenizer, TFBertForPreTraining + >>> from transformers import AutoTokenizer, TFBertForPreTraining - >>> tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") >>> model = TFBertForPreTraining.from_pretrained("bert-base-uncased") >>> input_ids = tokenizer("Hello, my dog is cute", add_special_tokens=True, return_tensors="tf") >>> # Batch size 1 @@ -1308,7 +1306,6 @@ def get_prefix_bias_name(self) -> str: @unpack_inputs @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1408,7 +1405,6 @@ def prepare_inputs_for_generation(self, input_ids, past_key_values=None, attenti @unpack_inputs @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFCausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1546,9 +1542,9 @@ def call( ```python >>> import tensorflow as tf - >>> from transformers import BertTokenizer, TFBertForNextSentencePrediction + >>> from transformers import AutoTokenizer, TFBertForNextSentencePrediction - >>> tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") >>> model = TFBertForNextSentencePrediction.from_pretrained("bert-base-uncased") >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." @@ -1627,7 +1623,6 @@ def __init__(self, config: BertConfig, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_SEQUENCE_CLASSIFICATION, output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1723,7 +1718,6 @@ def dummy_inputs(self) -> Dict[str, tf.Tensor]: @unpack_inputs @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1856,7 +1850,6 @@ def __init__(self, config: BertConfig, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_TOKEN_CLASSIFICATION, output_type=TFTokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1948,10 +1941,11 @@ def __init__(self, config: BertConfig, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_QA, output_type=TFQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, + qa_target_start_index=_QA_TARGET_START_INDEX, + qa_target_end_index=_QA_TARGET_END_INDEX, expected_output=_QA_EXPECTED_OUTPUT, expected_loss=_QA_EXPECTED_LOSS, ) diff --git a/src/transformers/models/bert_generation/modeling_bert_generation.py b/src/transformers/models/bert_generation/modeling_bert_generation.py index eec18e03d65bd1..54044195e961ad 100755 --- a/src/transformers/models/bert_generation/modeling_bert_generation.py +++ b/src/transformers/models/bert_generation/modeling_bert_generation.py @@ -40,7 +40,6 @@ _CHECKPOINT_FOR_DOC = "google/bert_for_seq_generation_L-24_bbc_encoder" _CONFIG_FOR_DOC = "BertGenerationConfig" -_TOKENIZER_FOR_DOC = "BertGenerationTokenizer" # Copied from transformers.models.bert.modeling_bert.BertSelfOutput with Bert->BertGeneration @@ -633,7 +632,7 @@ def _set_gradient_checkpointing(self, module, value=False): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BertGenerationTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -717,7 +716,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(BERT_GENERATION_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPastAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -929,10 +927,10 @@ def forward( Example: ```python - >>> from transformers import BertGenerationTokenizer, BertGenerationDecoder, BertGenerationConfig + >>> from transformers import AutoTokenizer, BertGenerationDecoder, BertGenerationConfig >>> import torch - >>> tokenizer = BertGenerationTokenizer.from_pretrained("google/bert_for_seq_generation_L-24_bbc_encoder") + >>> tokenizer = AutoTokenizer.from_pretrained("google/bert_for_seq_generation_L-24_bbc_encoder") >>> config = BertGenerationConfig.from_pretrained("google/bert_for_seq_generation_L-24_bbc_encoder") >>> config.is_decoder = True >>> model = BertGenerationDecoder.from_pretrained( diff --git a/src/transformers/models/big_bird/modeling_big_bird.py b/src/transformers/models/big_bird/modeling_big_bird.py index bc3d037b6c617e..ff5eeac628d4c2 100755 --- a/src/transformers/models/big_bird/modeling_big_bird.py +++ b/src/transformers/models/big_bird/modeling_big_bird.py @@ -53,7 +53,6 @@ _CHECKPOINT_FOR_DOC = "google/bigbird-roberta-base" _CONFIG_FOR_DOC = "BigBirdConfig" -_TOKENIZER_FOR_DOC = "BigBirdTokenizer" BIG_BIRD_PRETRAINED_MODEL_ARCHIVE_LIST = [ "google/bigbird-roberta-base", @@ -1803,7 +1802,7 @@ def _set_gradient_checkpointing(self, module, value=False): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BigBirdTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1980,7 +1979,6 @@ def set_attention_type(self, value: str): @add_start_docstrings_to_model_forward(BIG_BIRD_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -2315,10 +2313,10 @@ def forward( Example: ```python - >>> from transformers import BigBirdTokenizer, BigBirdForPreTraining + >>> from transformers import AutoTokenizer, BigBirdForPreTraining >>> import torch - >>> tokenizer = BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base") + >>> tokenizer = AutoTokenizer.from_pretrained("google/bigbird-roberta-base") >>> model = BigBirdForPreTraining.from_pretrained("google/bigbird-roberta-base") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") @@ -2420,10 +2418,10 @@ def forward( ```python >>> import torch - >>> from transformers import BigBirdTokenizer, BigBirdForMaskedLM + >>> from transformers import AutoTokenizer, BigBirdForMaskedLM >>> from datasets import load_dataset - >>> tokenizer = BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base") + >>> tokenizer = AutoTokenizer.from_pretrained("google/bigbird-roberta-base") >>> model = BigBirdForMaskedLM.from_pretrained("google/bigbird-roberta-base") >>> squad_ds = load_dataset("squad_v2", split="train") # doctest: +IGNORE_RESULT @@ -2539,7 +2537,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(BIG_BIRD_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -2717,10 +2714,10 @@ def forward( ```python >>> import torch - >>> from transformers import BigBirdTokenizer, BigBirdForSequenceClassification + >>> from transformers import AutoTokenizer, BigBirdForSequenceClassification >>> from datasets import load_dataset - >>> tokenizer = BigBirdTokenizer.from_pretrained("l-yohai/bigbird-roberta-base-mnli") + >>> tokenizer = AutoTokenizer.from_pretrained("l-yohai/bigbird-roberta-base-mnli") >>> model = BigBirdForSequenceClassification.from_pretrained("l-yohai/bigbird-roberta-base-mnli") >>> squad_ds = load_dataset("squad_v2", split="train") # doctest: +IGNORE_RESULT @@ -2822,7 +2819,6 @@ def __init__(self, config): BIG_BIRD_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -2918,15 +2914,9 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(BIG_BIRD_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, - checkpoint="vumichien/token-classification-bigbird-roberta-base-random", + checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, - expected_output=( - "['LABEL_1', 'LABEL_1', 'LABEL_1', 'LABEL_1', 'LABEL_1', 'LABEL_1', 'LABEL_1', 'LABEL_1', " - "'LABEL_1', 'LABEL_1', 'LABEL_1', 'LABEL_1']" - ), - expected_loss=0.54, ) def forward( self, @@ -3053,10 +3043,10 @@ def forward( ```python >>> import torch - >>> from transformers import BigBirdTokenizer, BigBirdForQuestionAnswering + >>> from transformers import AutoTokenizer, BigBirdForQuestionAnswering >>> from datasets import load_dataset - >>> tokenizer = BigBirdTokenizer.from_pretrained("abhinavkulkarni/bigbird-roberta-base-finetuned-squad") + >>> tokenizer = AutoTokenizer.from_pretrained("abhinavkulkarni/bigbird-roberta-base-finetuned-squad") >>> model = BigBirdForQuestionAnswering.from_pretrained("abhinavkulkarni/bigbird-roberta-base-finetuned-squad") >>> squad_ds = load_dataset("squad_v2", split="train") # doctest: +IGNORE_RESULT diff --git a/src/transformers/models/big_bird/modeling_flax_big_bird.py b/src/transformers/models/big_bird/modeling_flax_big_bird.py index 49fb61118d30df..2dfa871b118191 100644 --- a/src/transformers/models/big_bird/modeling_flax_big_bird.py +++ b/src/transformers/models/big_bird/modeling_flax_big_bird.py @@ -53,7 +53,6 @@ _CHECKPOINT_FOR_DOC = "google/bigbird-roberta-base" _CONFIG_FOR_DOC = "BigBirdConfig" -_TOKENIZER_FOR_DOC = "BigBirdTokenizer" remat = nn_partitioning.remat @@ -159,7 +158,7 @@ class FlaxBigBirdForQuestionAnsweringModelOutput(ModelOutput): input_ids (`numpy.ndarray` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BigBirdTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1834,9 +1833,7 @@ class FlaxBigBirdModel(FlaxBigBirdPreTrainedModel): module_class = FlaxBigBirdModule -append_call_sample_docstring( - FlaxBigBirdModel, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutputWithPooling, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxBigBirdModel, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutputWithPooling, _CONFIG_FOR_DOC) # Copied from transformers.models.bert.modeling_flax_bert.FlaxBertForPreTrainingModule with Bert->BigBird @@ -1920,9 +1917,9 @@ class FlaxBigBirdForPreTraining(FlaxBigBirdPreTrainedModel): Example: ```python - >>> from transformers import BigBirdTokenizer, FlaxBigBirdForPreTraining + >>> from transformers import AutoTokenizer, FlaxBigBirdForPreTraining - >>> tokenizer = BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base") + >>> tokenizer = AutoTokenizer.from_pretrained("google/bigbird-roberta-base") >>> model = FlaxBigBirdForPreTraining.from_pretrained("google/bigbird-roberta-base") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="np") @@ -2007,9 +2004,7 @@ class FlaxBigBirdForMaskedLM(FlaxBigBirdPreTrainedModel): module_class = FlaxBigBirdForMaskedLMModule -append_call_sample_docstring( - FlaxBigBirdForMaskedLM, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxMaskedLMOutput, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxBigBirdForMaskedLM, _CHECKPOINT_FOR_DOC, FlaxMaskedLMOutput, _CONFIG_FOR_DOC) class FlaxBigBirdClassificationHead(nn.Module): @@ -2101,7 +2096,6 @@ class FlaxBigBirdForSequenceClassification(FlaxBigBirdPreTrainedModel): append_call_sample_docstring( FlaxBigBirdForSequenceClassification, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSequenceClassifierOutput, _CONFIG_FOR_DOC, @@ -2201,7 +2195,6 @@ def __init__( ) append_call_sample_docstring( FlaxBigBirdForMultipleChoice, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxMultipleChoiceModelOutput, _CONFIG_FOR_DOC, @@ -2282,7 +2275,6 @@ class FlaxBigBirdForTokenClassification(FlaxBigBirdPreTrainedModel): append_call_sample_docstring( FlaxBigBirdForTokenClassification, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxTokenClassifierOutput, _CONFIG_FOR_DOC, @@ -2465,7 +2457,6 @@ def prepare_question_mask(q_lengths, maxlen: int): append_call_sample_docstring( FlaxBigBirdForQuestionAnswering, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxBigBirdForQuestionAnsweringModelOutput, _CONFIG_FOR_DOC, @@ -2578,7 +2569,6 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): append_call_sample_docstring( FlaxBigBirdForCausalLM, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxCausalLMOutputWithCrossAttentions, _CONFIG_FOR_DOC, diff --git a/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py b/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py index d94e0614d9f305..ee784c9e8a6f4e 100755 --- a/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py +++ b/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py @@ -1636,10 +1636,10 @@ def dummy_inputs(self): Summarization example: ```python - >>> from transformers import PegasusTokenizer, BigBirdPegasusForConditionalGeneration + >>> from transformers import AutoTokenizer, BigBirdPegasusForConditionalGeneration >>> model = BigBirdPegasusForConditionalGeneration.from_pretrained("google/bigbird-pegasus-large-arxiv") - >>> tokenizer = PegasusTokenizer.from_pretrained("google/bigbird-pegasus-large-arxiv") + >>> tokenizer = AutoTokenizer.from_pretrained("google/bigbird-pegasus-large-arxiv") >>> ARTICLE_TO_SUMMARIZE = ( ... "The dominant sequence transduction models are based on complex recurrent or convolutional neural " @@ -1664,7 +1664,7 @@ def dummy_inputs(self): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1815,7 +1815,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -2141,7 +2141,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -2965,7 +2965,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -3030,9 +3030,9 @@ def forward( Example: ```python - >>> from transformers import PegasusTokenizer, BigBirdPegasusForCausalLM + >>> from transformers import AutoTokenizer, BigBirdPegasusForCausalLM - >>> tokenizer = PegasusTokenizer.from_pretrained("google/bigbird-pegasus-large-arxiv") + >>> tokenizer = AutoTokenizer.from_pretrained("google/bigbird-pegasus-large-arxiv") >>> model = BigBirdPegasusForCausalLM.from_pretrained( ... "google/bigbird-pegasus-large-arxiv", add_cross_attention=False ... ) diff --git a/src/transformers/models/biogpt/modeling_biogpt.py b/src/transformers/models/biogpt/modeling_biogpt.py index 95f297d011ad83..716db23c68ac42 100755 --- a/src/transformers/models/biogpt/modeling_biogpt.py +++ b/src/transformers/models/biogpt/modeling_biogpt.py @@ -35,7 +35,6 @@ _CHECKPOINT_FOR_DOC = "microsoft/biogpt" _CONFIG_FOR_DOC = "BioGptConfig" -_TOKENIZER_FOR_DOC = "BioGptTokenizer" BIOGPT_PRETRAINED_MODEL_ARCHIVE_LIST = [ "microsoft/biogpt", @@ -385,7 +384,7 @@ def _set_gradient_checkpointing(self, module, value=False): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BioGptTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -487,7 +486,6 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em @add_start_docstrings_to_model_forward(BIOGPT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPastAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -644,7 +642,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(BIOGPT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/bit/modeling_bit.py b/src/transformers/models/bit/modeling_bit.py index 6cadafc9a5df73..7ebe461e5be01a 100644 --- a/src/transformers/models/bit/modeling_bit.py +++ b/src/transformers/models/bit/modeling_bit.py @@ -687,8 +687,8 @@ def _set_gradient_checkpointing(self, module, value=False): BIT_INPUTS_DOCSTRING = r""" Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`BitImageProcessor.__call__`] + for details. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for diff --git a/src/transformers/models/blenderbot/modeling_blenderbot.py b/src/transformers/models/blenderbot/modeling_blenderbot.py index f19a26f68266cb..830fb04bda58b9 100755 --- a/src/transformers/models/blenderbot/modeling_blenderbot.py +++ b/src/transformers/models/blenderbot/modeling_blenderbot.py @@ -50,7 +50,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "BlenderbotConfig" -_TOKENIZER_FOR_DOC = "BlenderbotTokenizer" _CHECKPOINT_FOR_DOC = "facebook/blenderbot-400M-distill" @@ -519,11 +518,11 @@ def dummy_inputs(self): Conversation example: ```python - >>> from transformers import BlenderbotTokenizer, BlenderbotForConditionalGeneration + >>> from transformers import AutoTokenizer, BlenderbotForConditionalGeneration >>> mname = "facebook/blenderbot-400M-distill" >>> model = BlenderbotForConditionalGeneration.from_pretrained(mname) - >>> tokenizer = BlenderbotTokenizer.from_pretrained(mname) + >>> tokenizer = AutoTokenizer.from_pretrained(mname) >>> UTTERANCE = "My friends are cool but they eat too many carbs." >>> print("Human: ", UTTERANCE) Human: My friends are cool but they eat too many carbs. @@ -555,7 +554,7 @@ def dummy_inputs(self): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -569,7 +568,7 @@ def dummy_inputs(self): decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -692,7 +691,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -891,7 +890,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1161,10 +1160,10 @@ def forward( Example: ```python - >>> from transformers import BlenderbotTokenizer, BlenderbotModel + >>> from transformers import AutoTokenizer, BlenderbotModel >>> model = BlenderbotModel.from_pretrained("facebook/blenderbot-400M-distill") - >>> tokenizer = BlenderbotTokenizer.from_pretrained("facebook/blenderbot-400M-distill") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/blenderbot-400M-distill") >>> inputs = tokenizer("Studies have been shown that owning a dog is good for you", return_tensors="pt") >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids # Batch size 1 @@ -1486,7 +1485,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1551,9 +1550,9 @@ def forward( Example: ```python - >>> from transformers import BlenderbotTokenizer, BlenderbotForCausalLM + >>> from transformers import AutoTokenizer, BlenderbotForCausalLM - >>> tokenizer = BlenderbotTokenizer.from_pretrained("facebook/blenderbot-400M-distill") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/blenderbot-400M-distill") >>> model = BlenderbotForCausalLM.from_pretrained( ... "facebook/blenderbot-400M-distill", add_cross_attention=False ... ) diff --git a/src/transformers/models/blenderbot/modeling_flax_blenderbot.py b/src/transformers/models/blenderbot/modeling_flax_blenderbot.py index 8f26e80c9f8257..baadfd973e69fd 100644 --- a/src/transformers/models/blenderbot/modeling_flax_blenderbot.py +++ b/src/transformers/models/blenderbot/modeling_flax_blenderbot.py @@ -52,7 +52,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "BlenderbotConfig" -_TOKENIZER_FOR_DOC = "BlenderbotTokenizer" _CHECKPOINT_FOR_DOC = "facebook/blenderbot-400M-distill" @@ -84,7 +83,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -98,7 +97,7 @@ decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -135,7 +134,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -164,7 +163,7 @@ decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -992,10 +991,10 @@ def encode( Example: ```python - >>> from transformers import BlenderbotTokenizer, FlaxBlenderbotForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxBlenderbotForConditionalGeneration >>> model = FlaxBlenderbotForConditionalGeneration.from_pretrained("facebook/blenderbot-400M-distill") - >>> tokenizer = BlenderbotTokenizer.from_pretrained("facebook/blenderbot-400M-distill") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/blenderbot-400M-distill") >>> text = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, max_length=1024, return_tensors="jax") @@ -1061,10 +1060,10 @@ def decode( ```python >>> import jax.numpy as jnp - >>> from transformers import BlenderbotTokenizer, FlaxBlenderbotForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxBlenderbotForConditionalGeneration >>> model = FlaxBlenderbotForConditionalGeneration.from_pretrained("facebook/blenderbot-400M-distill") - >>> tokenizer = BlenderbotTokenizer.from_pretrained("facebook/blenderbot-400M-distill") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/blenderbot-400M-distill") >>> text = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, max_length=1024, return_tensors="jax") @@ -1222,9 +1221,7 @@ class FlaxBlenderbotModel(FlaxBlenderbotPreTrainedModel): module_class = FlaxBlenderbotModule -append_call_sample_docstring( - FlaxBlenderbotModel, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqModelOutput, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxBlenderbotModel, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqModelOutput, _CONFIG_FOR_DOC) # Copied from transformers.models.bart.modeling_flax_bart.FlaxBartForConditionalGenerationModule with Bart->Blenderbot @@ -1331,10 +1328,10 @@ def decode( ```python >>> import jax.numpy as jnp - >>> from transformers import BlenderbotTokenizer, FlaxBlenderbotForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxBlenderbotForConditionalGeneration >>> model = FlaxBlenderbotForConditionalGeneration.from_pretrained("facebook/blenderbot-400M-distill") - >>> tokenizer = BlenderbotTokenizer.from_pretrained("facebook/blenderbot-400M-distill") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/blenderbot-400M-distill") >>> text = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, max_length=1024, return_tensors="jax") @@ -1486,17 +1483,19 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): Conversation example:: - >>> from transformers import BlenderbotTokenizer, FlaxBlenderbotForConditionalGeneration, BlenderbotConfig + ```py + >>> from transformers import AutoTokenizer, FlaxBlenderbotForConditionalGeneration - >>> model = FlaxBlenderbotForConditionalGeneration.from_pretrained('facebook/blenderbot-400M-distill') >>> - tokenizer = BlenderbotTokenizer.from_pretrained('facebook/blenderbot-400M-distill') + >>> model = FlaxBlenderbotForConditionalGeneration.from_pretrained("facebook/blenderbot-400M-distill") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/blenderbot-400M-distill") - >>> UTTERANCE = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer([UTTERANCE], - max_length=1024, return_tensors='np') + >>> UTTERANCE = "My friends are cool but they eat too many carbs." + >>> inputs = tokenizer([UTTERANCE], max_length=1024, return_tensors="np") - >>> # Generate Reply >>> reply_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=5, - early_stopping=True).sequences >>> print([tokenizer.decode(g, skip_special_tokens=True, - clean_up_tokenization_spaces=False) for g in reply_ids]) + >>> # Generate Reply + >>> reply_ids = model.generate(inputs["input_ids"], num_beams=4, max_length=5, early_stopping=True).sequences + >>> print([tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in reply_ids]) + ``` """ overwrite_call_docstring( diff --git a/src/transformers/models/blenderbot/modeling_tf_blenderbot.py b/src/transformers/models/blenderbot/modeling_tf_blenderbot.py index c28cc66cadec05..23693b24bd6941 100644 --- a/src/transformers/models/blenderbot/modeling_tf_blenderbot.py +++ b/src/transformers/models/blenderbot/modeling_tf_blenderbot.py @@ -55,7 +55,6 @@ _CHECKPOINT_FOR_DOC = "facebook/blenderbot-400M-distill" _CONFIG_FOR_DOC = "BlenderbotConfig" -_TOKENIZER_FOR_DOC = "BlenderbotTokenizer" LARGE_NEGATIVE = -1e8 @@ -536,18 +535,30 @@ def serving(self, inputs): BLENDERBOT_GENERATION_EXAMPLE = r""" Conversation example:: - >>> from transformers import BlenderbotTokenizer, TFBlenderbotForConditionalGeneration >>> mname = - 'facebook/blenderbot-400M-distill' >>> model = TFBlenderbotForConditionalGeneration.from_pretrained(mname) >>> - tokenizer = BlenderbotTokenizer.from_pretrained(mname) >>> UTTERANCE = "My friends are cool but they eat too - many carbs." >>> print("Human: ", UTTERANCE) >>> inputs = tokenizer([UTTERANCE], return_tensors='tf') >>> - reply_ids = model.generate(**inputs) >>> print("Bot: ", tokenizer.batch_decode(reply_ids, - skip_special_tokens=True)[0]) - - >>> REPLY = "I'm not sure" >>> print("Human: ", REPLY) >>> NEXT_UTTERANCE = ( ... "My friends are cool but they - eat too many carbs. That's unfortunate. " ... "Are they trying to lose weight or are they just trying to - be healthier? " ... " I'm not sure." ... ) >>> inputs = tokenizer([NEXT_UTTERANCE], return_tensors='tf') - >>> next_reply_ids = model.generate(**inputs) >>> print("Bot: ", tokenizer.batch_decode(next_reply_ids, - skip_special_tokens=True)[0]) + ```py + >>> from transformers import AutoTokenizer, TFBlenderbotForConditionalGeneration + + >>> mname = "facebook/blenderbot-400M-distill" + >>> model = TFBlenderbotForConditionalGeneration.from_pretrained(mname) + >>> tokenizer = AutoTokenizer.from_pretrained(mname) + >>> UTTERANCE = "My friends are cool but they eat too many carbs." + >>> print("Human: ", UTTERANCE) + + >>> inputs = tokenizer([UTTERANCE], return_tensors="tf") + >>> reply_ids = model.generate(**inputs) + >>> print("Bot: ", tokenizer.batch_decode(reply_ids, skip_special_tokens=True)[0]) + + >>> REPLY = "I'm not sure" + >>> print("Human: ", REPLY) + >>> NEXT_UTTERANCE = ( + ... "My friends are cool but they eat too many carbs. That's unfortunate. " + ... "Are they trying to lose weight or are they just trying to be healthier? " + ... " I'm not sure." + ... ) + >>> inputs = tokenizer([NEXT_UTTERANCE], return_tensors="tf") + >>> next_reply_ids = model.generate(**inputs) + >>> print("Bot: ", tokenizer.batch_decode(next_reply_ids, skip_special_tokens=True)[0]) + ``` """ BLENDERBOT_INPUTS_DOCSTRING = r""" @@ -555,7 +566,7 @@ def serving(self, inputs): input_ids (`tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -569,7 +580,7 @@ def serving(self, inputs): decoder_input_ids (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -681,7 +692,7 @@ def call( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -861,7 +872,7 @@ def call( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1185,7 +1196,6 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P @unpack_inputs @add_start_docstrings_to_model_forward(BLENDERBOT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFSeq2SeqModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/blenderbot_small/modeling_blenderbot_small.py b/src/transformers/models/blenderbot_small/modeling_blenderbot_small.py index 5b8614de12dc95..633675dc56720d 100755 --- a/src/transformers/models/blenderbot_small/modeling_blenderbot_small.py +++ b/src/transformers/models/blenderbot_small/modeling_blenderbot_small.py @@ -47,8 +47,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "BlenderbotSmallConfig" -_TOKENIZER_FOR_DOC = "BlenderbotSmallTokenizer" -_CHECKPOINT_FOR_DOC = "facebook/blenderbot_small-90M" BLENDERBOT_SMALL_PRETRAINED_MODEL_ARCHIVE_LIST = [ @@ -517,11 +515,11 @@ def dummy_inputs(self): Conversation example: ```python - >>> from transformers import BlenderbotSmallTokenizer, BlenderbotSmallForConditionalGeneration + >>> from transformers import AutoTokenizer, BlenderbotSmallForConditionalGeneration >>> mname = "facebook/blenderbot_small-90M" >>> model = BlenderbotSmallForConditionalGeneration.from_pretrained(mname) - >>> tokenizer = BlenderbotSmallTokenizer.from_pretrained(mname) + >>> tokenizer = AutoTokenizer.from_pretrained(mname) >>> UTTERANCE = "My friends are cool but they eat too many carbs." >>> print("Human: ", UTTERANCE) Human: My friends are cool but they eat too many carbs. @@ -553,7 +551,7 @@ def dummy_inputs(self): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -567,7 +565,7 @@ def dummy_inputs(self): decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -690,7 +688,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -887,7 +885,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1142,10 +1140,10 @@ def forward( Example: ```python - >>> from transformers import BlenderbotSmallTokenizer, BlenderbotSmallModel + >>> from transformers import AutoTokenizer, BlenderbotSmallModel >>> model = BlenderbotSmallModel.from_pretrained("facebook/blenderbot_small-90M") - >>> tokenizer = BlenderbotSmallTokenizer.from_pretrained("facebook/blenderbot_small-90M") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/blenderbot_small-90M") >>> inputs = tokenizer("Studies have been shown that owning a dog is good for you", return_tensors="pt") >>> decoder_inputs = tokenizer("Studies show that", return_tensors="pt") # Batch size 1 @@ -1453,7 +1451,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1518,9 +1516,9 @@ def forward( Example: ```python - >>> from transformers import BlenderbotSmallTokenizer, BlenderbotSmallForCausalLM + >>> from transformers import AutoTokenizer, BlenderbotSmallForCausalLM - >>> tokenizer = BlenderbotSmallTokenizer.from_pretrained("facebook/blenderbot_small-90M") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/blenderbot_small-90M") >>> model = BlenderbotSmallForCausalLM.from_pretrained( ... "facebook/blenderbot_small-90M", add_cross_attention=False ... ) diff --git a/src/transformers/models/blenderbot_small/modeling_flax_blenderbot_small.py b/src/transformers/models/blenderbot_small/modeling_flax_blenderbot_small.py index 4115460b7a3ed4..78947481faa5f3 100644 --- a/src/transformers/models/blenderbot_small/modeling_flax_blenderbot_small.py +++ b/src/transformers/models/blenderbot_small/modeling_flax_blenderbot_small.py @@ -54,7 +54,6 @@ _CHECKPOINT_FOR_DOC = "facebook/blenderbot_small-90M" _CONFIG_FOR_DOC = "BlenderbotSmallConfig" -_TOKENIZER_FOR_DOC = "BlenderbotSmallTokenizer" BLENDERBOT_SMALL_START_DOCSTRING = r""" This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the @@ -96,7 +95,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -110,7 +109,7 @@ decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -147,7 +146,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -176,7 +175,7 @@ decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -990,10 +989,10 @@ def encode( Example: ```python - >>> from transformers import BlenderbotSmallTokenizer, FlaxBlenderbotSmallForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxBlenderbotSmallForConditionalGeneration >>> model = FlaxBlenderbotSmallForConditionalGeneration.from_pretrained("facebook/blenderbot_small-90M") - >>> tokenizer = BlenderbotSmallTokenizer.from_pretrained("facebook/blenderbot_small-90M") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/blenderbot_small-90M") >>> text = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, max_length=1024, return_tensors="np") @@ -1059,10 +1058,10 @@ def decode( ```python >>> import jax.numpy as jnp - >>> from transformers import BlenderbotSmallTokenizer, FlaxBlenderbotSmallForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxBlenderbotSmallForConditionalGeneration >>> model = FlaxBlenderbotSmallForConditionalGeneration.from_pretrained("facebook/blenderbot_small-90M") - >>> tokenizer = BlenderbotSmallTokenizer.from_pretrained("facebook/blenderbot_small-90M") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/blenderbot_small-90M") >>> text = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, max_length=1024, return_tensors="np") @@ -1219,9 +1218,7 @@ class FlaxBlenderbotSmallModel(FlaxBlenderbotSmallPreTrainedModel): module_class = FlaxBlenderbotSmallModule -append_call_sample_docstring( - FlaxBlenderbotSmallModel, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqModelOutput, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxBlenderbotSmallModel, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqModelOutput, _CONFIG_FOR_DOC) # Copied from transformers.models.bart.modeling_flax_bart.FlaxBartForConditionalGenerationModule with Bart->BlenderbotSmall @@ -1329,10 +1326,10 @@ def decode( ```python >>> import jax.numpy as jnp - >>> from transformers import BlenderbotSmallTokenizer, FlaxBlenderbotSmallForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxBlenderbotSmallForConditionalGeneration >>> model = FlaxBlenderbotSmallForConditionalGeneration.from_pretrained("facebook/blenderbot_small-90M") - >>> tokenizer = BlenderbotSmallTokenizer.from_pretrained("facebook/blenderbot_small-90M") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/blenderbot_small-90M") >>> text = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, max_length=1024, return_tensors="np") @@ -1484,30 +1481,38 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): Summarization example: - >>> from transformers import BlenderbotSmallTokenizer, FlaxBlenderbotSmallForConditionalGeneration + ```py + >>> from transformers import AutoTokenizer, FlaxBlenderbotSmallForConditionalGeneration - >>> model = FlaxBlenderbotSmallForConditionalGeneration.from_pretrained('facebook/blenderbot_small-90M') >>> - tokenizer = BlenderbotSmallTokenizer.from_pretrained('facebook/blenderbot_small-90M') + >>> model = FlaxBlenderbotSmallForConditionalGeneration.from_pretrained("facebook/blenderbot_small-90M") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/blenderbot_small-90M") - >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs." >>> inputs = - tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='np') + >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs." + >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors="np") - >>> # Generate Summary >>> summary_ids = model.generate(inputs['input_ids']).sequences >>> - print(tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)) + >>> # Generate Summary + >>> summary_ids = model.generate(inputs["input_ids"]).sequences + >>> print(tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)) + ``` Mask filling example: - >>> from transformers import BlenderbotSmallTokenizer, FlaxBlenderbotSmallForConditionalGeneration >>> - tokenizer = BlenderbotSmallTokenizer.from_pretrained('facebook/blenderbot_small-90M') >>> TXT = "My friends are - but they eat too many carbs." + ```py + >>> from transformers import AutoTokenizer, FlaxBlenderbotSmallForConditionalGeneration + + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/blenderbot_small-90M") + >>> TXT = "My friends are but they eat too many carbs." - >>> model = FlaxBlenderbotSmallForConditionalGeneration.from_pretrained('facebook/blenderbot_small-90M') >>> - input_ids = tokenizer([TXT], return_tensors='np')['input_ids'] >>> logits = model(input_ids).logits + >>> model = FlaxBlenderbotSmallForConditionalGeneration.from_pretrained("facebook/blenderbot_small-90M") + >>> input_ids = tokenizer([TXT], return_tensors="np")["input_ids"] + >>> logits = model(input_ids).logits - >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item() >>> probs = - jax.nn.softmax(logits[0, masked_index], axis=0) >>> values, predictions = jax.lax.top_k(probs) + >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item() + >>> probs = jax.nn.softmax(logits[0, masked_index], axis=0) + >>> values, predictions = jax.lax.top_k(probs) - >>> tokenizer.decode(predictions).split() + >>> tokenizer.decode(predictions).split() + ``` """ overwrite_call_docstring( diff --git a/src/transformers/models/blenderbot_small/modeling_tf_blenderbot_small.py b/src/transformers/models/blenderbot_small/modeling_tf_blenderbot_small.py index 384ad68c336386..0ef4e1e8beef5c 100644 --- a/src/transformers/models/blenderbot_small/modeling_tf_blenderbot_small.py +++ b/src/transformers/models/blenderbot_small/modeling_tf_blenderbot_small.py @@ -54,7 +54,6 @@ _CHECKPOINT_FOR_DOC = "facebook/blenderbot_small-90M" _CONFIG_FOR_DOC = "BlenderbotSmallConfig" -_TOKENIZER_FOR_DOC = "BlenderbotSmallTokenizer" LARGE_NEGATIVE = -1e8 @@ -536,23 +535,34 @@ def serving(self, inputs): BLENDERBOT_SMALL_GENERATION_EXAMPLE = r""" Conversation example:: - >>> from transformers import BlenderbotSmallTokenizer, TFBlenderbotSmallForConditionalGeneration >>> mname = - 'facebook/blenderbot_small-90M' >>> model = BlenderbotSmallForConditionalGeneration.from_pretrained(mname) >>> - tokenizer = BlenderbotSmallTokenizer.from_pretrained(mname) - - >>> UTTERANCE = "My friends are cool but they eat too many carbs." >>> print("Human: ", UTTERANCE) >>> inputs = - tokenizer([UTTERANCE], return_tensors='tf') - - >>> reply_ids = model.generate(**inputs) >>> print("Bot: ", tokenizer.batch_decode(reply_ids, - skip_special_tokens=True)[0]) what kind of carbs do they eat? i don't know much about carbs. - - >>> REPLY = "I'm not sure" >>> print("Human: ", REPLY) >>> NEXT_UTTERANCE = ( ... "My friends are cool but they - eat too many carbs. " ... "what kind of carbs do they eat? i don't know much about carbs. " ... - "I'm not sure." ... ) - - >>> inputs = tokenizer([NEXT_UTTERANCE], return_tensors='tf') >>> inputs.pop("token_type_ids") >>> - next_reply_ids = model.generate(**inputs) >>> print("Bot: ", tokenizer.batch_decode(next_reply_ids, - skip_special_tokens=True)[0]) + ```py + >>> from transformers import AutoTokenizer, TFBlenderbotSmallForConditionalGeneration + + >>> mname = "facebook/blenderbot_small-90M" + >>> model = BlenderbotSmallForConditionalGeneration.from_pretrained(mname) + >>> tokenizer = AutoTokenizer.from_pretrained(mname) + + >>> UTTERANCE = "My friends are cool but they eat too many carbs." + >>> print("Human: ", UTTERANCE) + >>> inputs = tokenizer([UTTERANCE], return_tensors="tf") + + >>> reply_ids = model.generate(**inputs) + >>> print("Bot: ", tokenizer.batch_decode(reply_ids, skip_special_tokens=True)[0]) + what kind of carbs do they eat? i don't know much about carbs. + + >>> REPLY = "I'm not sure" + >>> print("Human: ", REPLY) + >>> NEXT_UTTERANCE = ( + ... "My friends are cool but they eat too many carbs. " + ... "what kind of carbs do they eat? i don't know much about carbs. " + ... "I'm not sure." + ... ) + + >>> inputs = tokenizer([NEXT_UTTERANCE], return_tensors="tf") + >>> inputs.pop("token_type_ids") + >>> next_reply_ids = model.generate(**inputs) + >>> print("Bot: ", tokenizer.batch_decode(next_reply_ids, skip_special_tokens=True)[0]) + ``` """ BLENDERBOT_SMALL_INPUTS_DOCSTRING = r""" @@ -560,7 +570,7 @@ def serving(self, inputs): input_ids (`tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -574,7 +584,7 @@ def serving(self, inputs): decoder_input_ids (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -688,7 +698,7 @@ def call( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -869,7 +879,7 @@ def call( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1179,7 +1189,6 @@ def get_decoder(self): @unpack_inputs @add_start_docstrings_to_model_forward(BLENDERBOT_SMALL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFSeq2SeqModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index bdef8c442e7b7c..ca4c5bcb619b35 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -483,7 +483,7 @@ def _set_gradient_checkpointing(self, module, value=False): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BlipProcessor`]. See [`BlipProcessor.__call__`] for details. + Indices can be obtained using [`AutoProcessor`]. See [`BlipProcessor.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): @@ -529,7 +529,7 @@ def _set_gradient_checkpointing(self, module, value=False): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BlipProcessor`]. See [`BlipProcessor.__call__`] for details. + Indices can be obtained using [`AutoProcessor`]. See [`BlipProcessor.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): @@ -772,10 +772,10 @@ def get_text_features( Examples: ```python - >>> from transformers import BlipProcessor, BlipModel + >>> from transformers import AutoProcessor, BlipModel >>> model = BlipModel.from_pretrained("Salesforce/blip-image-captioning-base") - >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") + >>> processor = AutoProcessor.from_pretrained("Salesforce/blip-image-captioning-base") >>> inputs = processor(text=["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") >>> text_features = model.get_text_features(**inputs) @@ -810,10 +810,10 @@ def get_image_features( ```python >>> from PIL import Image >>> import requests - >>> from transformers import BlipProcessor, BlipModel + >>> from transformers import AutoProcessor, BlipModel >>> model = BlipModel.from_pretrained("Salesforce/blip-image-captioning-base") - >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") + >>> processor = AutoProcessor.from_pretrained("Salesforce/blip-image-captioning-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -855,10 +855,10 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import BlipProcessor, BlipModel + >>> from transformers import AutoProcessor, BlipModel >>> model = BlipModel.from_pretrained("Salesforce/blip-image-captioning-base") - >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") + >>> processor = AutoProcessor.from_pretrained("Salesforce/blip-image-captioning-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -978,9 +978,9 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import BlipProcessor, BlipForConditionalGeneration + >>> from transformers import AutoProcessor, BlipForConditionalGeneration - >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") + >>> processor = AutoProcessor.from_pretrained("Salesforce/blip-image-captioning-base") >>> model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" @@ -1054,10 +1054,10 @@ def generate( ```python >>> from PIL import Image >>> import requests - >>> from transformers import BlipProcessor, BlipForConditionalGeneration + >>> from transformers import AutoProcessor, BlipForConditionalGeneration >>> model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base") - >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") + >>> processor = AutoProcessor.from_pretrained("Salesforce/blip-image-captioning-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1169,10 +1169,10 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import BlipProcessor, BlipForQuestionAnswering + >>> from transformers import AutoProcessor, BlipForQuestionAnswering >>> model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base") - >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base") + >>> processor = AutoProcessor.from_pretrained("Salesforce/blip-vqa-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1284,10 +1284,10 @@ def generate( ```python >>> from PIL import Image >>> import requests - >>> from transformers import BlipProcessor, BlipForQuestionAnswering + >>> from transformers import AutoProcessor, BlipForQuestionAnswering >>> model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base") - >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base") + >>> processor = AutoProcessor.from_pretrained("Salesforce/blip-vqa-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1403,10 +1403,10 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import BlipProcessor, BlipForImageTextRetrieval + >>> from transformers import AutoProcessor, BlipForImageTextRetrieval >>> model = BlipForImageTextRetrieval.from_pretrained("Salesforce/blip-itm-base-coco") - >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-itm-base-coco") + >>> processor = AutoProcessor.from_pretrained("Salesforce/blip-itm-base-coco") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) diff --git a/src/transformers/models/bloom/modeling_bloom.py b/src/transformers/models/bloom/modeling_bloom.py index d12dcdc69e2f64..ad7d0567106b67 100644 --- a/src/transformers/models/bloom/modeling_bloom.py +++ b/src/transformers/models/bloom/modeling_bloom.py @@ -41,7 +41,6 @@ _CHECKPOINT_FOR_DOC = "bigscience/bloom-560m" _CONFIG_FOR_DOC = "BloomConfig" -_TOKENIZER_FOR_DOC = "BloomTokenizerFast" BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST = [ "bigscience/bigscience-small-testing", @@ -570,7 +569,7 @@ def _convert_to_bloom_cache( If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. - Indices can be obtained using [`BloomTokenizerFast`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -672,7 +671,6 @@ def set_input_embeddings(self, new_embeddings: torch.Tensor): @add_start_docstrings_to_model_forward(BLOOM_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPastAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -863,7 +861,6 @@ def prepare_inputs_for_generation( @add_start_docstrings_to_model_forward(BLOOM_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -994,7 +991,6 @@ def __init__(self, config: BloomConfig): @add_start_docstrings_to_model_forward(BLOOM_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutputWithPast, config_class=_CONFIG_FOR_DOC, @@ -1131,7 +1127,6 @@ def __init__(self, config: BloomConfig): @add_start_docstrings_to_model_forward(BLOOM_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/camembert/modeling_camembert.py b/src/transformers/models/camembert/modeling_camembert.py index e7c22b917af685..e7755dd0305324 100644 --- a/src/transformers/models/camembert/modeling_camembert.py +++ b/src/transformers/models/camembert/modeling_camembert.py @@ -50,7 +50,6 @@ _CHECKPOINT_FOR_DOC = "camembert-base" _CONFIG_FOR_DOC = "CamembertConfig" -_TOKENIZER_FOR_DOC = "CamembertTokenizer" CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ "camembert-base", @@ -642,7 +641,7 @@ def update_keys_to_ignore(self, config, del_keys_to_ignore): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`CamembertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -794,7 +793,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(CAMEMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -966,7 +964,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(CAMEMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1056,7 +1053,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(CAMEMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="cardiffnlp/twitter-roberta-base-emotion", output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1158,7 +1154,6 @@ def __init__(self, config): CAMEMBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1257,7 +1252,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(CAMEMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="Jean-Baptiste/roberta-large-ner-english", output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1341,7 +1335,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(CAMEMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="deepset/roberta-base-squad2", output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/camembert/modeling_tf_camembert.py b/src/transformers/models/camembert/modeling_tf_camembert.py index f0cf089b3c2c15..29f435bdb194b2 100644 --- a/src/transformers/models/camembert/modeling_tf_camembert.py +++ b/src/transformers/models/camembert/modeling_tf_camembert.py @@ -62,7 +62,6 @@ _CHECKPOINT_FOR_DOC = "camembert-base" _CONFIG_FOR_DOC = "CamembertConfig" -_TOKENIZER_FOR_DOC = "CamembertTokenizer" TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ # See all CamemBERT models at https://huggingface.co/models?filter=camembert @@ -116,7 +115,7 @@ input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`CamembertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -927,7 +926,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(CAMEMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1088,7 +1086,6 @@ def get_prefix_bias_name(self): @unpack_inputs @add_start_docstrings_to_model_forward(CAMEMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1204,7 +1201,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(CAMEMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="cardiffnlp/twitter-roberta-base-emotion", output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1296,7 +1292,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(CAMEMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="ydshieh/roberta-large-ner-english", output_type=TFTokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1396,7 +1391,6 @@ def dummy_inputs(self): CAMEMBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1507,7 +1501,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(CAMEMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="ydshieh/roberta-base-squad2", output_type=TFQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1626,7 +1619,6 @@ def prepare_inputs_for_generation(self, input_ids, past_key_values=None, attenti @unpack_inputs @add_start_docstrings_to_model_forward(CAMEMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFCausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/canine/modeling_canine.py b/src/transformers/models/canine/modeling_canine.py index 39ab5843423409..9cabe705b4486a 100644 --- a/src/transformers/models/canine/modeling_canine.py +++ b/src/transformers/models/canine/modeling_canine.py @@ -51,7 +51,6 @@ _CHECKPOINT_FOR_DOC = "google/canine-s" _CONFIG_FOR_DOC = "CanineConfig" -_TOKENIZER_FOR_DOC = "CanineTokenizer" CANINE_PRETRAINED_MODEL_ARCHIVE_LIST = [ "google/canine-s", @@ -941,7 +940,7 @@ def _set_gradient_checkpointing(self, module, value=False): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`CanineTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1100,7 +1099,6 @@ def _repeat_molecules(self, molecules: torch.Tensor, char_seq_length: torch.Tens @add_start_docstrings_to_model_forward(CANINE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=CanineModelOutputWithPooling, config_class=_CONFIG_FOR_DOC, @@ -1282,12 +1280,9 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(CANINE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, - checkpoint="vicl/canine-c-finetuned-cola", + checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, - expected_output="'LABEL_0'", - expected_loss=0.82, ) def forward( self, @@ -1381,7 +1376,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(CANINE_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1496,10 +1490,10 @@ def forward( Example: ```python - >>> from transformers import CanineTokenizer, CanineForTokenClassification + >>> from transformers import AutoTokenizer, CanineForTokenClassification >>> import torch - >>> tokenizer = CanineTokenizer.from_pretrained("google/canine-s") + >>> tokenizer = AutoTokenizer.from_pretrained("google/canine-s") >>> model = CanineForTokenClassification.from_pretrained("google/canine-s") >>> inputs = tokenizer( @@ -1579,7 +1573,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(CANINE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="Splend1dchan/canine-c-squad", output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/chinese_clip/modeling_chinese_clip.py b/src/transformers/models/chinese_clip/modeling_chinese_clip.py index cf15f8c26f8a3c..ca574f4ececb56 100644 --- a/src/transformers/models/chinese_clip/modeling_chinese_clip.py +++ b/src/transformers/models/chinese_clip/modeling_chinese_clip.py @@ -47,7 +47,6 @@ _CHECKPOINT_FOR_DOC = "OFA-Sys/chinese-clip-vit-base-patch16" _CONFIG_FOR_DOC = "ChineseCLIPConfig" -_TOKENIZER_FOR_DOC = "BertTokenizer" CHINESE_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST = [ "OFA-Sys/chinese-clip-vit-base-patch16", @@ -762,7 +761,7 @@ def _set_gradient_checkpointing(self, module, value=False): input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -810,7 +809,7 @@ def _set_gradient_checkpointing(self, module, value=False): Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`ChineseCLIPImageProcessor`]. See [`ChineseCLIPImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`ChineseCLIPImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. @@ -827,7 +826,7 @@ def _set_gradient_checkpointing(self, module, value=False): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -853,7 +852,7 @@ def _set_gradient_checkpointing(self, module, value=False): [What are position IDs?](../glossary#position-ids) pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`ChineseCLIPImageProcessor`]. See [`ChineseCLIPImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`ChineseCLIPImageProcessor.__call__`] for details. return_loss (`bool`, *optional*): Whether or not to return the contrastive loss. output_attentions (`bool`, *optional*): @@ -1152,7 +1151,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(CHINESE_CLIP_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1400,10 +1398,10 @@ def get_text_features( Examples: ```python - >>> from transformers import BertTokenizer, ChineseCLIPModel + >>> from transformers import AutoTokenizer, ChineseCLIPModel >>> model = ChineseCLIPModel.from_pretrained("OFA-Sys/chinese-clip-vit-base-patch16") - >>> tokenizer = BertTokenizer.from_pretrained("OFA-Sys/chinese-clip-vit-base-patch16") + >>> tokenizer = AutoTokenizer.from_pretrained("OFA-Sys/chinese-clip-vit-base-patch16") >>> inputs = tokenizer(["杰尼龟", "妙蛙种子", "小火龙", "皮卡丘"], padding=True, return_tensors="pt") >>> text_features = model.get_text_features(**inputs) @@ -1449,10 +1447,10 @@ def get_image_features( ```python >>> from PIL import Image >>> import requests - >>> from transformers import ChineseCLIPProcessor, ChineseCLIPModel + >>> from transformers import AutoProcessor, ChineseCLIPModel >>> model = ChineseCLIPModel.from_pretrained("OFA-Sys/chinese-clip-vit-base-patch16") - >>> processor = ChineseCLIPProcessor.from_pretrained("OFA-Sys/chinese-clip-vit-base-patch16") + >>> processor = AutoProcessor.from_pretrained("OFA-Sys/chinese-clip-vit-base-patch16") >>> url = "https://clip-cn-beijing.oss-cn-beijing.aliyuncs.com/pokemon.jpeg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1503,10 +1501,10 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import ChineseCLIPProcessor, ChineseCLIPModel + >>> from transformers import AutoProcessor, ChineseCLIPModel >>> model = ChineseCLIPModel.from_pretrained("OFA-Sys/chinese-clip-vit-base-patch16") - >>> processor = ChineseCLIPProcessor.from_pretrained("OFA-Sys/chinese-clip-vit-base-patch16") + >>> processor = AutoProcessor.from_pretrained("OFA-Sys/chinese-clip-vit-base-patch16") >>> url = "https://clip-cn-beijing.oss-cn-beijing.aliyuncs.com/pokemon.jpeg" >>> image = Image.open(requests.get(url, stream=True).raw) diff --git a/src/transformers/models/clip/modeling_clip.py b/src/transformers/models/clip/modeling_clip.py index 9731eeb6d081b9..75474ed6cd079d 100644 --- a/src/transformers/models/clip/modeling_clip.py +++ b/src/transformers/models/clip/modeling_clip.py @@ -491,7 +491,7 @@ def _set_gradient_checkpointing(self, module, value=False): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -521,7 +521,7 @@ def _set_gradient_checkpointing(self, module, value=False): Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`CLIPImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. @@ -538,7 +538,7 @@ def _set_gradient_checkpointing(self, module, value=False): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -556,7 +556,7 @@ def _set_gradient_checkpointing(self, module, value=False): [What are position IDs?](../glossary#position-ids) pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`CLIPImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. return_loss (`bool`, *optional*): Whether or not to return the contrastive loss. output_attentions (`bool`, *optional*): @@ -800,10 +800,10 @@ def forward( Examples: ```python - >>> from transformers import CLIPTokenizer, CLIPTextModel + >>> from transformers import AutoTokenizer, CLIPTextModel >>> model = CLIPTextModel.from_pretrained("openai/clip-vit-base-patch32") - >>> tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-base-patch32") + >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32") >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") @@ -915,10 +915,10 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import CLIPProcessor, CLIPVisionModel + >>> from transformers import AutoProcessor, CLIPVisionModel >>> model = CLIPVisionModel.from_pretrained("openai/clip-vit-base-patch32") - >>> processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32") + >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -993,10 +993,10 @@ def get_text_features( Examples: ```python - >>> from transformers import CLIPTokenizer, CLIPModel + >>> from transformers import AutoTokenizer, CLIPModel >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32") - >>> tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-base-patch32") + >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32") >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") >>> text_features = model.get_text_features(**inputs) @@ -1040,10 +1040,10 @@ def get_image_features( ```python >>> from PIL import Image >>> import requests - >>> from transformers import CLIPProcessor, CLIPModel + >>> from transformers import AutoProcessor, CLIPModel >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32") - >>> processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32") + >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1092,10 +1092,10 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import CLIPProcessor, CLIPModel + >>> from transformers import AutoProcessor, CLIPModel >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32") - >>> processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32") + >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1209,10 +1209,10 @@ def forward( Examples: ```python - >>> from transformers import CLIPTokenizer, CLIPTextModelWithProjection + >>> from transformers import AutoTokenizer, CLIPTextModelWithProjection >>> model = CLIPTextModelWithProjection.from_pretrained("openai/clip-vit-base-patch32") - >>> tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-base-patch32") + >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32") >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") @@ -1286,10 +1286,10 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import CLIPProcessor, CLIPVisionModelWithProjection + >>> from transformers import AutoProcessor, CLIPVisionModelWithProjection >>> model = CLIPVisionModelWithProjection.from_pretrained("openai/clip-vit-base-patch32") - >>> processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32") + >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) diff --git a/src/transformers/models/clip/modeling_flax_clip.py b/src/transformers/models/clip/modeling_flax_clip.py index 5c823e046eba1a..ea4ff88a2c6b23 100644 --- a/src/transformers/models/clip/modeling_flax_clip.py +++ b/src/transformers/models/clip/modeling_flax_clip.py @@ -78,7 +78,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -108,7 +108,7 @@ Args: pixel_values (`numpy.ndarray` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`CLIPImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. @@ -125,7 +125,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -143,7 +143,7 @@ [What are position IDs?](../glossary#position-ids) pixel_values (`numpy.ndarray` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`CLIPImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. @@ -834,7 +834,7 @@ def get_text_features( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -846,10 +846,10 @@ def get_text_features( Examples: ```python - >>> from transformers import CLIPTokenizer, FlaxCLIPModel + >>> from transformers import AutoTokenizer, FlaxCLIPModel >>> model = FlaxCLIPModel.from_pretrained("openai/clip-vit-base-patch32") - >>> tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-base-patch32") + >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32") >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="np") >>> text_features = model.get_text_features(**inputs) @@ -893,7 +893,7 @@ def get_image_features( Args: pixel_values (`numpy.ndarray` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained - using [`CLIPImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. + using [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. Returns: image_features (`jnp.ndarray` of shape `(batch_size, output_dim`): The image embeddings obtained by @@ -904,10 +904,10 @@ def get_image_features( ```python >>> from PIL import Image >>> import requests - >>> from transformers import CLIPProcessor, FlaxCLIPModel + >>> from transformers import AutoProcessor, FlaxCLIPModel >>> model = FlaxCLIPModel.from_pretrained("openai/clip-vit-base-patch32") - >>> processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32") + >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -976,10 +976,10 @@ class FlaxCLIPTextModel(FlaxCLIPTextPreTrainedModel): Example: ```python - >>> from transformers import CLIPTokenizer, FlaxCLIPTextModel + >>> from transformers import AutoTokenizer, FlaxCLIPTextModel >>> model = FlaxCLIPTextModel.from_pretrained("openai/clip-vit-base-patch32") - >>> tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-base-patch32") + >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32") >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="np") @@ -1031,10 +1031,10 @@ class FlaxCLIPVisionModel(FlaxCLIPVisionPreTrainedModel): ```python >>> from PIL import Image >>> import requests - >>> from transformers import CLIPProcessor, FlaxCLIPVisionModel + >>> from transformers import AutoProcessor, FlaxCLIPVisionModel >>> model = FlaxCLIPVisionModel.from_pretrained("openai/clip-vit-base-patch32") - >>> processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32") + >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1158,10 +1158,10 @@ class FlaxCLIPModel(FlaxCLIPPreTrainedModel): >>> import jax >>> from PIL import Image >>> import requests - >>> from transformers import CLIPProcessor, FlaxCLIPModel + >>> from transformers import AutoProcessor, FlaxCLIPModel >>> model = FlaxCLIPModel.from_pretrained("openai/clip-vit-base-patch32") - >>> processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32") + >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) diff --git a/src/transformers/models/clip/modeling_tf_clip.py b/src/transformers/models/clip/modeling_tf_clip.py index e5f3c15a8cf49c..9ea7e7690428a8 100644 --- a/src/transformers/models/clip/modeling_tf_clip.py +++ b/src/transformers/models/clip/modeling_tf_clip.py @@ -992,7 +992,7 @@ class TFCLIPPreTrainedModel(TFPreTrainedModel): CLIP_VISION_INPUTS_DOCSTRING = r""" Args: pixel_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`CLIPImageProcessor`]. See + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used @@ -1019,7 +1019,7 @@ class TFCLIPPreTrainedModel(TFPreTrainedModel): [What are input IDs?](../glossary#input-ids) pixel_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` `Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`CLIPImageProcessor`]. See + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. attention_mask (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: @@ -1079,10 +1079,10 @@ def call( Examples: ```python - >>> from transformers import CLIPTokenizer, TFCLIPTextModel + >>> from transformers import AutoTokenizer, TFCLIPTextModel >>> model = TFCLIPTextModel.from_pretrained("openai/clip-vit-base-patch32") - >>> tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-base-patch32") + >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32") >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="tf") @@ -1187,10 +1187,10 @@ def call( ```python >>> from PIL import Image >>> import requests - >>> from transformers import CLIPProcessor, TFCLIPVisionModel + >>> from transformers import AutoProcessor, TFCLIPVisionModel >>> model = TFCLIPVisionModel.from_pretrained("openai/clip-vit-base-patch32") - >>> processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32") + >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1291,10 +1291,10 @@ def get_text_features( Examples: ```python - >>> from transformers import CLIPTokenizer, TFCLIPModel + >>> from transformers import AutoTokenizer, TFCLIPModel >>> model = TFCLIPModel.from_pretrained("openai/clip-vit-base-patch32") - >>> tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-base-patch32") + >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32") >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="tf") >>> text_features = model.get_text_features(**inputs) @@ -1331,10 +1331,10 @@ def get_image_features( ```python >>> from PIL import Image >>> import requests - >>> from transformers import CLIPProcessor, TFCLIPModel + >>> from transformers import AutoProcessor, TFCLIPModel >>> model = TFCLIPModel.from_pretrained("openai/clip-vit-base-patch32") - >>> processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32") + >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1377,10 +1377,10 @@ def call( >>> import tensorflow as tf >>> from PIL import Image >>> import requests - >>> from transformers import CLIPProcessor, TFCLIPModel + >>> from transformers import AutoProcessor, TFCLIPModel >>> model = TFCLIPModel.from_pretrained("openai/clip-vit-base-patch32") - >>> processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32") + >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) diff --git a/src/transformers/models/clipseg/modeling_clipseg.py b/src/transformers/models/clipseg/modeling_clipseg.py index 96b475aca88cfc..0c97d95bedfa38 100644 --- a/src/transformers/models/clipseg/modeling_clipseg.py +++ b/src/transformers/models/clipseg/modeling_clipseg.py @@ -500,7 +500,7 @@ def _set_gradient_checkpointing(self, module, value=False): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -530,7 +530,7 @@ def _set_gradient_checkpointing(self, module, value=False): Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`CLIPImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. @@ -547,7 +547,7 @@ def _set_gradient_checkpointing(self, module, value=False): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -565,7 +565,7 @@ def _set_gradient_checkpointing(self, module, value=False): [What are position IDs?](../glossary#position-ids) pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`CLIPImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. return_loss (`bool`, *optional*): Whether or not to return the contrastive loss. output_attentions (`bool`, *optional*): @@ -808,9 +808,9 @@ def forward( Examples: ```python - >>> from transformers import CLIPTokenizer, CLIPSegTextModel + >>> from transformers import AutoTokenizer, CLIPSegTextModel - >>> tokenizer = CLIPTokenizer.from_pretrained("CIDAS/clipseg-rd64-refined") + >>> tokenizer = AutoTokenizer.from_pretrained("CIDAS/clipseg-rd64-refined") >>> model = CLIPSegTextModel.from_pretrained("CIDAS/clipseg-rd64-refined") >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") @@ -919,9 +919,9 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import CLIPSegProcessor, CLIPSegVisionModel + >>> from transformers import AutoProcessor, CLIPSegVisionModel - >>> processor = CLIPSegProcessor.from_pretrained("CIDAS/clipseg-rd64-refined") + >>> processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined") >>> model = CLIPSegVisionModel.from_pretrained("CIDAS/clipseg-rd64-refined") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" @@ -995,9 +995,9 @@ def get_text_features( Examples: ```python - >>> from transformers import CLIPTokenizer, CLIPSegModel + >>> from transformers import AutoTokenizer, CLIPSegModel - >>> tokenizer = CLIPTokenizer.from_pretrained("CIDAS/clipseg-rd64-refined") + >>> tokenizer = AutoTokenizer.from_pretrained("CIDAS/clipseg-rd64-refined") >>> model = CLIPSegModel.from_pretrained("CIDAS/clipseg-rd64-refined") >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") @@ -1042,9 +1042,9 @@ def get_image_features( ```python >>> from PIL import Image >>> import requests - >>> from transformers import CLIPSegProcessor, CLIPSegModel + >>> from transformers import AutoProcessor, CLIPSegModel - >>> processor = CLIPSegProcessor.from_pretrained("CIDAS/clipseg-rd64-refined") + >>> processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined") >>> model = CLIPSegModel.from_pretrained("CIDAS/clipseg-rd64-refined") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" @@ -1094,9 +1094,9 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import CLIPSegProcessor, CLIPSegModel + >>> from transformers import AutoProcessor, CLIPSegModel - >>> processor = CLIPSegProcessor.from_pretrained("CIDAS/clipseg-rd64-refined") + >>> processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined") >>> model = CLIPSegModel.from_pretrained("CIDAS/clipseg-rd64-refined") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" @@ -1401,11 +1401,11 @@ def forward( Examples: ```python - >>> from transformers import CLIPSegProcessor, CLIPSegForImageSegmentation + >>> from transformers import AutoProcessor, CLIPSegForImageSegmentation >>> from PIL import Image >>> import requests - >>> processor = CLIPSegProcessor.from_pretrained("CIDAS/clipseg-rd64-refined") + >>> processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined") >>> model = CLIPSegForImageSegmentation.from_pretrained("CIDAS/clipseg-rd64-refined") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" diff --git a/src/transformers/models/codegen/modeling_codegen.py b/src/transformers/models/codegen/modeling_codegen.py index 52e62cb73737f8..92a7d9506cd9a2 100644 --- a/src/transformers/models/codegen/modeling_codegen.py +++ b/src/transformers/models/codegen/modeling_codegen.py @@ -32,7 +32,6 @@ _CHECKPOINT_FOR_DOC = "Salesforce/codegen-2B-mono" _CONFIG_FOR_DOC = "CodeGenConfig" -_TOKENIZER_FOR_DOC = "GPT2Tokenizer" CODEGEN_PRETRAINED_MODEL_ARCHIVE_LIST = [ @@ -374,7 +373,7 @@ def _set_gradient_checkpointing(self, module, value=False): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`GPT2Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoProcenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -448,7 +447,6 @@ def set_input_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(CODEGEN_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPast, config_class=_CONFIG_FOR_DOC, @@ -662,7 +660,6 @@ def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwarg @add_start_docstrings_to_model_forward(CODEGEN_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/conditional_detr/modeling_conditional_detr.py b/src/transformers/models/conditional_detr/modeling_conditional_detr.py index 78052317877d24..5068c003f95b49 100644 --- a/src/transformers/models/conditional_detr/modeling_conditional_detr.py +++ b/src/transformers/models/conditional_detr/modeling_conditional_detr.py @@ -1109,8 +1109,8 @@ def _set_gradient_checkpointing(self, module, value=False): pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. - Pixel values can be obtained using [`ConditionalDetrImageProcessor`]. See - [`ConditionalDetrImageProcessor.__call__`] for details. + Pixel values can be obtained using [`AutoImageProcessor`]. See [`ConditionalDetrImageProcessor.__call__`] + for details. pixel_mask (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: @@ -1911,7 +1911,7 @@ def forward( >>> # forward pass >>> outputs = model(**inputs) - >>> # Use the `post_process_panoptic_segmentation` method of `ConditionalDetrImageProcessor` to retrieve post-processed panoptic segmentation maps + >>> # Use the `post_process_panoptic_segmentation` method of the `image_processor` to retrieve post-processed panoptic segmentation maps >>> # Segmentation results are returned as a list of dictionaries >>> result = image_processor.post_process_panoptic_segmentation(outputs, target_sizes=[(300, 500)]) >>> # A tensor of shape (height, width) where each value denotes a segment id, filled with -1 if no segment is found diff --git a/src/transformers/models/convbert/modeling_convbert.py b/src/transformers/models/convbert/modeling_convbert.py index 5922e652788d33..6a3e81e25eed12 100755 --- a/src/transformers/models/convbert/modeling_convbert.py +++ b/src/transformers/models/convbert/modeling_convbert.py @@ -44,7 +44,6 @@ _CHECKPOINT_FOR_DOC = "YituTech/conv-bert-base" _CONFIG_FOR_DOC = "ConvBertConfig" -_TOKENIZER_FOR_DOC = "ConvBertTokenizer" CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ "YituTech/conv-bert-base", @@ -710,7 +709,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`ConvBertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -793,7 +792,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(CONVBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -899,7 +897,6 @@ def set_output_embeddings(self, word_embeddings): @add_start_docstrings_to_model_forward(CONVBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1005,7 +1002,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(CONVBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1105,7 +1101,6 @@ def __init__(self, config): CONVBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1203,7 +1198,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(CONVBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1283,7 +1277,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(CONVBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/convbert/modeling_tf_convbert.py b/src/transformers/models/convbert/modeling_tf_convbert.py index a714e265c896ea..3976be69eb5b86 100644 --- a/src/transformers/models/convbert/modeling_tf_convbert.py +++ b/src/transformers/models/convbert/modeling_tf_convbert.py @@ -57,7 +57,6 @@ _CHECKPOINT_FOR_DOC = "YituTech/conv-bert-base" _CONFIG_FOR_DOC = "ConvBertConfig" -_TOKENIZER_FOR_DOC = "ConvBertTokenizer" TF_CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ "YituTech/conv-bert-base", @@ -682,7 +681,7 @@ class TFConvBertPreTrainedModel(TFPreTrainedModel): input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`ConvBertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -746,7 +745,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(CONVBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutput, config_class=_CONFIG_FOR_DOC, @@ -863,7 +861,6 @@ def get_prefix_bias_name(self): @unpack_inputs @add_start_docstrings_to_model_forward(CONVBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -971,7 +968,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(CONVBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1064,7 +1060,6 @@ def dummy_inputs(self): CONVBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1178,7 +1173,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(CONVBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFTokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1256,7 +1250,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(CONVBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/convnext/modeling_convnext.py b/src/transformers/models/convnext/modeling_convnext.py index dd6764a8d6c31d..5e60ddfe6d99c1 100755 --- a/src/transformers/models/convnext/modeling_convnext.py +++ b/src/transformers/models/convnext/modeling_convnext.py @@ -315,7 +315,7 @@ def _set_gradient_checkpointing(self, module, value=False): Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + [`ConvNextImageProcessor.__call__`] for details. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for diff --git a/src/transformers/models/convnext/modeling_tf_convnext.py b/src/transformers/models/convnext/modeling_tf_convnext.py index 8906fa6b476e33..671de6ca9b46ed 100644 --- a/src/transformers/models/convnext/modeling_tf_convnext.py +++ b/src/transformers/models/convnext/modeling_tf_convnext.py @@ -432,7 +432,7 @@ def serving(self, inputs): CONVNEXT_INPUTS_DOCSTRING = r""" Args: pixel_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`ConvNextImageProcessor`]. See + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ConvNextImageProcessor.__call__`] for details. output_hidden_states (`bool`, *optional*): @@ -470,14 +470,14 @@ def call( Examples: ```python - >>> from transformers import ConvNextImageProcessor, TFConvNextModel + >>> from transformers import AutoImageProcessor, TFConvNextModel >>> from PIL import Image >>> import requests >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = ConvNextImageProcessor.from_pretrained("facebook/convnext-tiny-224") + >>> image_processor = AutoImageProcessor.from_pretrained("facebook/convnext-tiny-224") >>> model = TFConvNextModel.from_pretrained("facebook/convnext-tiny-224") >>> inputs = image_processor(images=image, return_tensors="tf") @@ -561,7 +561,7 @@ def call( Examples: ```python - >>> from transformers import ConvNextImageProcessor, TFConvNextForImageClassification + >>> from transformers import AutoImageProcessor, TFConvNextForImageClassification >>> import tensorflow as tf >>> from PIL import Image >>> import requests @@ -569,7 +569,7 @@ def call( >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = ConvNextImageProcessor.from_pretrained("facebook/convnext-tiny-224") + >>> image_processor = AutoImageProcessor.from_pretrained("facebook/convnext-tiny-224") >>> model = TFConvNextForImageClassification.from_pretrained("facebook/convnext-tiny-224") >>> inputs = image_processor(images=image, return_tensors="tf") diff --git a/src/transformers/models/ctrl/modeling_ctrl.py b/src/transformers/models/ctrl/modeling_ctrl.py index fffa4e141413fa..58c1859f2a06df 100644 --- a/src/transformers/models/ctrl/modeling_ctrl.py +++ b/src/transformers/models/ctrl/modeling_ctrl.py @@ -260,7 +260,7 @@ def _init_weights(self, module): If `past_key_values` is used, only input IDs that do not have their past calculated should be passed as `input_ids`. - Indices can be obtained using [`CTRLTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -371,10 +371,10 @@ def forward( Example: ```python - >>> from transformers import CTRLTokenizer, CTRLModel + >>> from transformers import AutoTokenizer, CTRLModel >>> import torch - >>> tokenizer = CTRLTokenizer.from_pretrained("ctrl") + >>> tokenizer = AutoTokenizer.from_pretrained("ctrl") >>> model = CTRLModel.from_pretrained("ctrl") >>> # CTRL was trained with control codes as the first token @@ -561,9 +561,9 @@ def forward( ```python >>> import torch - >>> from transformers import CTRLTokenizer, CTRLLMHeadModel + >>> from transformers import AutoTokenizer, CTRLLMHeadModel - >>> tokenizer = CTRLTokenizer.from_pretrained("ctrl") + >>> tokenizer = AutoTokenizer.from_pretrained("ctrl") >>> model = CTRLLMHeadModel.from_pretrained("ctrl") >>> # CTRL was trained with control codes as the first token @@ -687,9 +687,9 @@ def forward( ```python >>> import torch - >>> from transformers import CTRLTokenizer, CTRLForSequenceClassification + >>> from transformers import AutoTokenizer, CTRLForSequenceClassification - >>> tokenizer = CTRLTokenizer.from_pretrained("ctrl") + >>> tokenizer = AutoTokenizer.from_pretrained("ctrl") >>> model = CTRLForSequenceClassification.from_pretrained("ctrl") >>> # CTRL was trained with control codes as the first token @@ -722,9 +722,9 @@ def forward( ```python >>> import torch - >>> from transformers import CTRLTokenizer, CTRLForSequenceClassification + >>> from transformers import AutoTokenizer, CTRLForSequenceClassification - >>> tokenizer = CTRLTokenizer.from_pretrained("ctrl") + >>> tokenizer = AutoTokenizer.from_pretrained("ctrl") >>> model = CTRLForSequenceClassification.from_pretrained("ctrl", problem_type="multi_label_classification") >>> # CTRL was trained with control codes as the first token diff --git a/src/transformers/models/ctrl/modeling_tf_ctrl.py b/src/transformers/models/ctrl/modeling_tf_ctrl.py index 64765edab02373..116bb4ca665bdd 100644 --- a/src/transformers/models/ctrl/modeling_tf_ctrl.py +++ b/src/transformers/models/ctrl/modeling_tf_ctrl.py @@ -41,7 +41,6 @@ _CHECKPOINT_FOR_DOC = "ctrl" _CONFIG_FOR_DOC = "CTRLConfig" -_TOKENIZER_FOR_DOC = "CTRLTokenizer" TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST = [ "ctrl" @@ -468,7 +467,7 @@ class TFCTRLPreTrainedModel(TFPreTrainedModel): If `past` is used, only input IDs that do not have their past calculated should be passed as `input_ids`. - Indices can be obtained using [`CTRLTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -537,7 +536,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(CTRL_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutputWithPast, config_class=_CONFIG_FOR_DOC, @@ -651,7 +649,6 @@ def prepare_inputs_for_generation(self, input_ids, past_key_values=None, use_cac @unpack_inputs @add_start_docstrings_to_model_forward(CTRL_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC, @@ -756,7 +753,6 @@ def get_output_embeddings(self): @unpack_inputs @add_start_docstrings_to_model_forward(CTRL_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/cvt/modeling_cvt.py b/src/transformers/models/cvt/modeling_cvt.py index d0f22d22e73e18..220e40396c9b8b 100644 --- a/src/transformers/models/cvt/modeling_cvt.py +++ b/src/transformers/models/cvt/modeling_cvt.py @@ -573,7 +573,7 @@ def _init_weights(self, module): CVT_INPUTS_DOCSTRING = r""" Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`CvtImageProcessor`]. See [`CvtImageProcessor.__call__`] + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`CvtImageProcessor.__call__`] for details. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for diff --git a/src/transformers/models/cvt/modeling_tf_cvt.py b/src/transformers/models/cvt/modeling_tf_cvt.py index 17880eaa9d8245..c82cade5247982 100644 --- a/src/transformers/models/cvt/modeling_tf_cvt.py +++ b/src/transformers/models/cvt/modeling_tf_cvt.py @@ -766,8 +766,8 @@ def serving(self, inputs): TFCVT_INPUTS_DOCSTRING = r""" Args: pixel_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`CvtImageProcessor.__call__`] + for details. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for diff --git a/src/transformers/models/data2vec/modeling_data2vec_audio.py b/src/transformers/models/data2vec/modeling_data2vec_audio.py index 965d250faca299..8ad1c9f0e9fb77 100755 --- a/src/transformers/models/data2vec/modeling_data2vec_audio.py +++ b/src/transformers/models/data2vec/modeling_data2vec_audio.py @@ -792,8 +792,8 @@ def _set_gradient_checkpointing(self, module, value=False): input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install - soundfile*). To prepare the array into *input_values*, the [`Wav2Vec2Processor`] should be used for padding - and conversion into a tensor of type *torch.FloatTensor*. See [`Wav2Vec2Processor.__call__`] for details. + soundfile*). To prepare the array into *input_values*, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type *torch.FloatTensor*. See [`Wav2Vec2Processor.__call__`] for details. attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`: @@ -907,7 +907,6 @@ def _mask_hidden_states( config_class=_CONFIG_FOR_DOC, modality="audio", expected_output=_EXPECTED_OUTPUT_SHAPE, - processor_class="AutoProcessor", ) def forward( self, diff --git a/src/transformers/models/data2vec/modeling_data2vec_text.py b/src/transformers/models/data2vec/modeling_data2vec_text.py index 29930094f380f3..b3c5cba4f226c1 100644 --- a/src/transformers/models/data2vec/modeling_data2vec_text.py +++ b/src/transformers/models/data2vec/modeling_data2vec_text.py @@ -53,7 +53,6 @@ # General docstring _CHECKPOINT_FOR_DOC = "facebook/data2vec-text-base" _CONFIG_FOR_DOC = "Data2VecTextConfig" -_TOKENIZER_FOR_DOC = "RobertaTokenizer" DATA2VEC_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST = [ @@ -649,7 +648,7 @@ def update_keys_to_ignore(self, config, del_keys_to_ignore): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`RobertaTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -744,7 +743,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(DATA2VECTEXT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1066,7 +1064,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(DATA2VECTEXT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1185,7 +1182,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(DATA2VECTEXT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1284,7 +1280,6 @@ def __init__(self, config): DATA2VECTEXT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1382,7 +1377,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(DATA2VECTEXT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1486,7 +1480,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(DATA2VECTEXT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/data2vec/modeling_data2vec_vision.py b/src/transformers/models/data2vec/modeling_data2vec_vision.py index ad1ee93d99e243..c1b85c7f093265 100644 --- a/src/transformers/models/data2vec/modeling_data2vec_vision.py +++ b/src/transformers/models/data2vec/modeling_data2vec_vision.py @@ -605,7 +605,7 @@ def _set_gradient_checkpointing(self, module, value=False): DATA2VEC_VISION_INPUTS_DOCSTRING = r""" Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`BeitImageProcessor`]. See + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`BeitImageProcessor.__call__`] for details. head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): diff --git a/src/transformers/models/data2vec/modeling_tf_data2vec_vision.py b/src/transformers/models/data2vec/modeling_tf_data2vec_vision.py index 7749e6ecb34aec..cfb81ceb396e05 100644 --- a/src/transformers/models/data2vec/modeling_tf_data2vec_vision.py +++ b/src/transformers/models/data2vec/modeling_tf_data2vec_vision.py @@ -847,8 +847,8 @@ def serving(self, inputs): DATA2VEC_VISION_INPUTS_DOCSTRING = r""" Args: - pixel_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`BeitImageProcessor`]. See + pixel_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` `Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `(batch_size, num_channels, height, width)`): + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`BeitImageProcessor.__call__`] for details. head_mask (`np.ndarray` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): diff --git a/src/transformers/models/deberta/modeling_deberta.py b/src/transformers/models/deberta/modeling_deberta.py index 7dd2b0767c8409..ecfd0c53e6a6d5 100644 --- a/src/transformers/models/deberta/modeling_deberta.py +++ b/src/transformers/models/deberta/modeling_deberta.py @@ -867,7 +867,7 @@ def _set_gradient_checkpointing(self, module, value=False): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`DebertaTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) diff --git a/src/transformers/models/deberta/modeling_tf_deberta.py b/src/transformers/models/deberta/modeling_tf_deberta.py index 9aeee82574015f..c86b900042c059 100644 --- a/src/transformers/models/deberta/modeling_tf_deberta.py +++ b/src/transformers/models/deberta/modeling_tf_deberta.py @@ -48,7 +48,6 @@ _CONFIG_FOR_DOC = "DebertaConfig" -_TOKENIZER_FOR_DOC = "DebertaTokenizer" _CHECKPOINT_FOR_DOC = "kamalkraj/deberta-base" TF_DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST = [ @@ -1048,7 +1047,7 @@ class TFDebertaPreTrainedModel(TFPreTrainedModel): input_ids (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`DebertaTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1100,7 +1099,6 @@ def __init__(self, config: DebertaConfig, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(DEBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1158,7 +1156,6 @@ def get_lm_head(self) -> tf.keras.layers.Layer: @unpack_inputs @add_start_docstrings_to_model_forward(DEBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1243,7 +1240,6 @@ def __init__(self, config: DebertaConfig, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(DEBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1325,7 +1321,6 @@ def __init__(self, config: DebertaConfig, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(DEBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFTokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1402,7 +1397,6 @@ def __init__(self, config: DebertaConfig, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(DEBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/deberta_v2/modeling_deberta_v2.py b/src/transformers/models/deberta_v2/modeling_deberta_v2.py index bc38defb86451a..7d73a49ef9281f 100644 --- a/src/transformers/models/deberta_v2/modeling_deberta_v2.py +++ b/src/transformers/models/deberta_v2/modeling_deberta_v2.py @@ -966,7 +966,7 @@ def _set_gradient_checkpointing(self, module, value=False): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`DebertaV2Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) diff --git a/src/transformers/models/deberta_v2/modeling_tf_deberta_v2.py b/src/transformers/models/deberta_v2/modeling_tf_deberta_v2.py index 08420ea07cee11..885212be7389be 100644 --- a/src/transformers/models/deberta_v2/modeling_tf_deberta_v2.py +++ b/src/transformers/models/deberta_v2/modeling_tf_deberta_v2.py @@ -47,7 +47,6 @@ _CONFIG_FOR_DOC = "DebertaV2Config" -_TOKENIZER_FOR_DOC = "DebertaV2Tokenizer" _CHECKPOINT_FOR_DOC = "kamalkraj/deberta-v2-xlarge" TF_DEBERTA_V2_PRETRAINED_MODEL_ARCHIVE_LIST = [ @@ -1141,7 +1140,7 @@ class TFDebertaV2PreTrainedModel(TFPreTrainedModel): input_ids (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`DebertaV2Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1194,7 +1193,6 @@ def __init__(self, config: DebertaV2Config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(DEBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1253,7 +1251,6 @@ def get_lm_head(self) -> tf.keras.layers.Layer: @unpack_inputs @add_start_docstrings_to_model_forward(DEBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1339,7 +1336,6 @@ def __init__(self, config: DebertaV2Config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(DEBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1422,7 +1418,6 @@ def __init__(self, config: DebertaV2Config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(DEBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFTokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1500,7 +1495,6 @@ def __init__(self, config: DebertaV2Config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(DEBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/deformable_detr/modeling_deformable_detr.py b/src/transformers/models/deformable_detr/modeling_deformable_detr.py index 0d0e247bfd191d..dd66ef7acaf74f 100755 --- a/src/transformers/models/deformable_detr/modeling_deformable_detr.py +++ b/src/transformers/models/deformable_detr/modeling_deformable_detr.py @@ -242,7 +242,7 @@ class DeformableDetrObjectDetectionOutput(ModelOutput): pred_boxes (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)`): Normalized boxes coordinates for all queries, represented as (center_x, center_y, width, height). These values are normalized in [0, 1], relative to the size of each individual image in the batch (disregarding - possible padding). You can use [`~AutoImageProcessor.post_process_object_detection`] to retrieve the + possible padding). You can use [`~DeformableDetrProcessor.post_process_object_detection`] to retrieve the unnormalized bounding boxes. auxiliary_outputs (`list[Dict]`, *optional*): Optional, only returned when auxilary losses are activated (i.e. `config.auxiliary_loss` is set to `True`) @@ -1086,7 +1086,8 @@ def _set_gradient_checkpointing(self, module, value=False): pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. - Pixel values can be obtained using [`AutoImageProcessor`]. See [`AutoImageProcessor.__call__`] for details. + Pixel values can be obtained using [`AutoImageProcessor`]. See [`DeformableDetrImageProcessor.__call__`] + for details. pixel_mask (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: diff --git a/src/transformers/models/deit/modeling_deit.py b/src/transformers/models/deit/modeling_deit.py index 4d4a9d6bc98581..ca849f230e6c5e 100644 --- a/src/transformers/models/deit/modeling_deit.py +++ b/src/transformers/models/deit/modeling_deit.py @@ -431,7 +431,7 @@ def _set_gradient_checkpointing(self, module: DeiTEncoder, value: bool = False) DEIT_INPUTS_DOCSTRING = r""" Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`DeiTImageProcessor`]. See + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`DeiTImageProcessor.__call__`] for details. head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): @@ -608,7 +608,7 @@ def forward( Examples: ```python - >>> from transformers import DeiTImageProcessor, DeiTForMaskedImageModeling + >>> from transformers import AutoImageProcessor, DeiTForMaskedImageModeling >>> import torch >>> from PIL import Image >>> import requests @@ -616,7 +616,7 @@ def forward( >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = DeiTImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224") + >>> image_processor = AutoImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224") >>> model = DeiTForMaskedImageModeling.from_pretrained("facebook/deit-base-distilled-patch16-224") >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2 @@ -718,7 +718,7 @@ def forward( Examples: ```python - >>> from transformers import DeiTImageProcessor, DeiTForImageClassification + >>> from transformers import AutoImageProcessor, DeiTForImageClassification >>> import torch >>> from PIL import Image >>> import requests @@ -729,7 +729,7 @@ def forward( >>> # note: we are loading a DeiTForImageClassificationWithTeacher from the hub here, >>> # so the head will be randomly initialized, hence the predictions will be random - >>> image_processor = DeiTImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224") + >>> image_processor = AutoImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224") >>> model = DeiTForImageClassification.from_pretrained("facebook/deit-base-distilled-patch16-224") >>> inputs = image_processor(images=image, return_tensors="pt") diff --git a/src/transformers/models/deit/modeling_tf_deit.py b/src/transformers/models/deit/modeling_tf_deit.py index 1adb432eba3304..161f2518d068a2 100644 --- a/src/transformers/models/deit/modeling_tf_deit.py +++ b/src/transformers/models/deit/modeling_tf_deit.py @@ -613,7 +613,7 @@ def serving(self, inputs): DEIT_INPUTS_DOCSTRING = r""" Args: pixel_values (`tf.Tensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`DeiTImageProcessor`]. See + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`DeiTImageProcessor.__call__`] for details. head_mask (`tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): @@ -784,7 +784,7 @@ def call( Examples: ```python - >>> from transformers import DeiTImageProcessor, TFDeiTForMaskedImageModeling + >>> from transformers import AutoImageProcessor, TFDeiTForMaskedImageModeling >>> import tensorflow as tf >>> from PIL import Image >>> import requests @@ -792,7 +792,7 @@ def call( >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = DeiTImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224") + >>> image_processor = AutoImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224") >>> model = TFDeiTForMaskedImageModeling.from_pretrained("facebook/deit-base-distilled-patch16-224") >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2 @@ -915,7 +915,7 @@ def call( Examples: ```python - >>> from transformers import DeiTImageProcessor, TFDeiTForImageClassification + >>> from transformers import AutoImageProcessor, TFDeiTForImageClassification >>> import tensorflow as tf >>> from PIL import Image >>> import requests @@ -926,7 +926,7 @@ def call( >>> # note: we are loading a TFDeiTForImageClassificationWithTeacher from the hub here, >>> # so the head will be randomly initialized, hence the predictions will be random - >>> image_processor = DeiTImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224") + >>> image_processor = AutoImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224") >>> model = TFDeiTForImageClassification.from_pretrained("facebook/deit-base-distilled-patch16-224") >>> inputs = image_processor(images=image, return_tensors="tf") diff --git a/src/transformers/models/detr/modeling_detr.py b/src/transformers/models/detr/modeling_detr.py index fc6011d315c645..814a09c37b9f91 100644 --- a/src/transformers/models/detr/modeling_detr.py +++ b/src/transformers/models/detr/modeling_detr.py @@ -868,7 +868,7 @@ def _set_gradient_checkpointing(self, module, value=False): pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. - Pixel values can be obtained using [`DetrImageProcessor`]. See [`DetrImageProcessor.__call__`] for details. + Pixel values can be obtained using [`AutoImageProcessor`]. See [`DetrImageProcessor.__call__`] for details. pixel_mask (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: @@ -1252,14 +1252,14 @@ def forward( Examples: ```python - >>> from transformers import DetrImageProcessor, DetrModel + >>> from transformers import AutoImageProcessor, DetrModel >>> from PIL import Image >>> import requests >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = DetrImageProcessor.from_pretrained("facebook/detr-resnet-50") + >>> image_processor = AutoImageProcessor.from_pretrained("facebook/detr-resnet-50") >>> model = DetrModel.from_pretrained("facebook/detr-resnet-50") >>> # prepare image for the model @@ -1419,7 +1419,7 @@ def forward( Examples: ```python - >>> from transformers import DetrImageProcessor, DetrForObjectDetection + >>> from transformers import AutoImageProcessor, DetrForObjectDetection >>> import torch >>> from PIL import Image >>> import requests @@ -1427,7 +1427,7 @@ def forward( >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = DetrImageProcessor.from_pretrained("facebook/detr-resnet-50") + >>> image_processor = AutoImageProcessor.from_pretrained("facebook/detr-resnet-50") >>> model = DetrForObjectDetection.from_pretrained("facebook/detr-resnet-50") >>> inputs = image_processor(images=image, return_tensors="pt") @@ -1597,13 +1597,13 @@ def forward( >>> import torch >>> import numpy - >>> from transformers import DetrImageProcessor, DetrForSegmentation + >>> from transformers import AutoImageProcessor, DetrForSegmentation >>> from transformers.image_transforms import rgb_to_id >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = DetrImageProcessor.from_pretrained("facebook/detr-resnet-50-panoptic") + >>> image_processor = AutoImageProcessor.from_pretrained("facebook/detr-resnet-50-panoptic") >>> model = DetrForSegmentation.from_pretrained("facebook/detr-resnet-50-panoptic") >>> # prepare image for the model @@ -1612,7 +1612,7 @@ def forward( >>> # forward pass >>> outputs = model(**inputs) - >>> # Use the `post_process_panoptic_segmentation` method of `DetrImageProcessor` to retrieve post-processed panoptic segmentation maps + >>> # Use the `post_process_panoptic_segmentation` method of the `image_processor` to retrieve post-processed panoptic segmentation maps >>> # Segmentation results are returned as a list of dictionaries >>> result = image_processor.post_process_panoptic_segmentation(outputs, target_sizes=[(300, 500)]) diff --git a/src/transformers/models/dinat/modeling_dinat.py b/src/transformers/models/dinat/modeling_dinat.py index e9b65cd3836d60..7176fae899d0f2 100644 --- a/src/transformers/models/dinat/modeling_dinat.py +++ b/src/transformers/models/dinat/modeling_dinat.py @@ -678,8 +678,8 @@ def _set_gradient_checkpointing(self, module: DinatEncoder, value: bool = False) DINAT_INPUTS_DOCSTRING = r""" Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] + for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned diff --git a/src/transformers/models/distilbert/modeling_distilbert.py b/src/transformers/models/distilbert/modeling_distilbert.py index d6bb3eb6691132..214c5d916621ea 100755 --- a/src/transformers/models/distilbert/modeling_distilbert.py +++ b/src/transformers/models/distilbert/modeling_distilbert.py @@ -54,7 +54,6 @@ logger = logging.get_logger(__name__) _CHECKPOINT_FOR_DOC = "distilbert-base-uncased" _CONFIG_FOR_DOC = "DistilBertConfig" -_TOKENIZER_FOR_DOC = "DistilBertTokenizer" DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ "distilbert-base-uncased", @@ -428,7 +427,7 @@ def _init_weights(self, module: nn.Module): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`DistilBertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -538,7 +537,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(DISTILBERT_INPUTS_DOCSTRING.format("batch_size, num_choices")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC, @@ -638,7 +636,6 @@ def set_output_embeddings(self, new_embeddings: nn.Module): @add_start_docstrings_to_model_forward(DISTILBERT_INPUTS_DOCSTRING.format("batch_size, num_choices")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -736,7 +733,6 @@ def resize_position_embeddings(self, new_num_position_embeddings: int): @add_start_docstrings_to_model_forward(DISTILBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -854,7 +850,6 @@ def resize_position_embeddings(self, new_num_position_embeddings: int): @add_start_docstrings_to_model_forward(DISTILBERT_INPUTS_DOCSTRING.format("batch_size, num_choices")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, @@ -971,7 +966,6 @@ def resize_position_embeddings(self, new_num_position_embeddings: int): @add_start_docstrings_to_model_forward(DISTILBERT_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1090,10 +1084,10 @@ def forward( Examples: ```python - >>> from transformers import DistilBertTokenizer, DistilBertForMultipleChoice + >>> from transformers import AutoTokenizer, DistilBertForMultipleChoice >>> import torch - >>> tokenizer = DistilBertTokenizer.from_pretrained("distilbert-base-cased") + >>> tokenizer = AutoTokenizer.from_pretrained("distilbert-base-cased") >>> model = DistilBertForMultipleChoice.from_pretrained("distilbert-base-cased") >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." diff --git a/src/transformers/models/distilbert/modeling_flax_distilbert.py b/src/transformers/models/distilbert/modeling_flax_distilbert.py index 28f76194d7bb52..984931bb4003ea 100644 --- a/src/transformers/models/distilbert/modeling_flax_distilbert.py +++ b/src/transformers/models/distilbert/modeling_flax_distilbert.py @@ -42,7 +42,6 @@ _CHECKPOINT_FOR_DOC = "distilbert-base-uncased" _CONFIG_FOR_DOC = "DistilBertConfig" -_TOKENIZER_FOR_DOC = "DistilBertTokenizer" FLAX_DISTILBERT_START_DOCSTRING = r""" @@ -72,7 +71,7 @@ input_ids (`numpy.ndarray` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -536,7 +535,7 @@ class FlaxDistilBertModel(FlaxDistilBertPreTrainedModel): module_class = FlaxDistilBertModule -append_call_sample_docstring(FlaxDistilBertModel, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, None, _CONFIG_FOR_DOC) +append_call_sample_docstring(FlaxDistilBertModel, _CHECKPOINT_FOR_DOC, None, _CONFIG_FOR_DOC) class FlaxDistilBertForMaskedLMModule(nn.Module): @@ -609,9 +608,7 @@ class FlaxDistilBertForMaskedLM(FlaxDistilBertPreTrainedModel): module_class = FlaxDistilBertForMaskedLMModule -append_call_sample_docstring( - FlaxDistilBertForMaskedLM, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxMaskedLMOutput, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxDistilBertForMaskedLM, _CHECKPOINT_FOR_DOC, FlaxMaskedLMOutput, _CONFIG_FOR_DOC) class FlaxDistilBertForSequenceClassificationModule(nn.Module): @@ -680,7 +677,6 @@ class FlaxDistilBertForSequenceClassification(FlaxDistilBertPreTrainedModel): append_call_sample_docstring( FlaxDistilBertForSequenceClassification, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSequenceClassifierOutput, _CONFIG_FOR_DOC, @@ -763,7 +759,6 @@ class FlaxDistilBertForMultipleChoice(FlaxDistilBertPreTrainedModel): ) append_call_sample_docstring( FlaxDistilBertForMultipleChoice, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxMultipleChoiceModelOutput, _CONFIG_FOR_DOC, @@ -826,7 +821,6 @@ class FlaxDistilBertForTokenClassification(FlaxDistilBertPreTrainedModel): append_call_sample_docstring( FlaxDistilBertForTokenClassification, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxTokenClassifierOutput, _CONFIG_FOR_DOC, @@ -896,7 +890,6 @@ class FlaxDistilBertForQuestionAnswering(FlaxDistilBertPreTrainedModel): append_call_sample_docstring( FlaxDistilBertForQuestionAnswering, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxQuestionAnsweringModelOutput, _CONFIG_FOR_DOC, diff --git a/src/transformers/models/distilbert/modeling_tf_distilbert.py b/src/transformers/models/distilbert/modeling_tf_distilbert.py index 9f9f5cfec49177..95c3aef4261572 100644 --- a/src/transformers/models/distilbert/modeling_tf_distilbert.py +++ b/src/transformers/models/distilbert/modeling_tf_distilbert.py @@ -58,7 +58,6 @@ _CHECKPOINT_FOR_DOC = "distilbert-base-uncased" _CONFIG_FOR_DOC = "DistilBertConfig" -_TOKENIZER_FOR_DOC = "DistilBertTokenizer" TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ "distilbert-base-uncased", @@ -492,7 +491,7 @@ def serving(self, inputs): input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`DistilBertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -542,7 +541,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(DISTILBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutput, config_class=_CONFIG_FOR_DOC, @@ -644,7 +642,6 @@ def get_prefix_bias_name(self): @unpack_inputs @add_start_docstrings_to_model_forward(DISTILBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -731,7 +728,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(DISTILBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -812,7 +808,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(DISTILBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFTokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -905,7 +900,6 @@ def dummy_inputs(self): DISTILBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1013,7 +1007,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(DISTILBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/donut/modeling_donut_swin.py b/src/transformers/models/donut/modeling_donut_swin.py index 223ab999a57c20..9c5ab57dc4e86b 100644 --- a/src/transformers/models/donut/modeling_donut_swin.py +++ b/src/transformers/models/donut/modeling_donut_swin.py @@ -847,7 +847,7 @@ def _set_gradient_checkpointing(self, module, value=False): Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + [`DonutImageProcessor.__call__`] for details. head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: diff --git a/src/transformers/models/dpr/modeling_dpr.py b/src/transformers/models/dpr/modeling_dpr.py index 6ab3c68a391a9e..471376322eda25 100644 --- a/src/transformers/models/dpr/modeling_dpr.py +++ b/src/transformers/models/dpr/modeling_dpr.py @@ -371,7 +371,7 @@ class DPRPretrainedReader(DPRPreTrainedModel): DPR is a model with absolute position embeddings so it's usually advised to pad the inputs on the right rather than the left. - Indices can be obtained using [`DPRTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) diff --git a/src/transformers/models/dpr/modeling_tf_dpr.py b/src/transformers/models/dpr/modeling_tf_dpr.py index 96ee761b819d08..28aba892e9b8f6 100644 --- a/src/transformers/models/dpr/modeling_tf_dpr.py +++ b/src/transformers/models/dpr/modeling_tf_dpr.py @@ -457,7 +457,7 @@ def serving(self, inputs): DPR is a model with absolute position embeddings so it's usually advised to pad the inputs on the right rather than the left. - Indices can be obtained using [`DPRTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) diff --git a/src/transformers/models/dpt/modeling_dpt.py b/src/transformers/models/dpt/modeling_dpt.py index fc1fdd0dbcb4ea..87f8d026891df3 100755 --- a/src/transformers/models/dpt/modeling_dpt.py +++ b/src/transformers/models/dpt/modeling_dpt.py @@ -839,7 +839,7 @@ def _set_gradient_checkpointing(self, module, value=False): DPT_INPUTS_DOCSTRING = r""" Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`ViTImageProcessor`]. See [`ViTImageProcessor.__call__`] + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`DPTImageProcessor.__call__`] for details. head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): @@ -1085,7 +1085,7 @@ def forward( Examples: ```python - >>> from transformers import DPTImageProcessor, DPTForDepthEstimation + >>> from transformers import AutoImageProcessor, DPTForDepthEstimation >>> import torch >>> import numpy as np >>> from PIL import Image @@ -1094,7 +1094,7 @@ def forward( >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = DPTImageProcessor.from_pretrained("Intel/dpt-large") + >>> image_processor = AutoImageProcessor.from_pretrained("Intel/dpt-large") >>> model = DPTForDepthEstimation.from_pretrained("Intel/dpt-large") >>> # prepare image for the model @@ -1255,14 +1255,14 @@ def forward( Examples: ```python - >>> from transformers import DPTImageProcessor, DPTForSemanticSegmentation + >>> from transformers import AutoImageProcessor, DPTForSemanticSegmentation >>> from PIL import Image >>> import requests >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = DPTImageProcessor.from_pretrained("Intel/dpt-large-ade") + >>> image_processor = AutoImageProcessor.from_pretrained("Intel/dpt-large-ade") >>> model = DPTForSemanticSegmentation.from_pretrained("Intel/dpt-large-ade") >>> inputs = image_processor(images=image, return_tensors="pt") diff --git a/src/transformers/models/efficientformer/modeling_efficientformer.py b/src/transformers/models/efficientformer/modeling_efficientformer.py index 376f429c3f4f1a..7c81cf4dba11f9 100644 --- a/src/transformers/models/efficientformer/modeling_efficientformer.py +++ b/src/transformers/models/efficientformer/modeling_efficientformer.py @@ -40,7 +40,6 @@ # General docstring _CONFIG_FOR_DOC = "EfficientFormerConfig" -_FEAT_EXTRACTOR_FOR_DOC = "EfficientFormerImageProcessor" # Base docstring _CHECKPOINT_FOR_DOC = "efficientformer-l1-300" @@ -554,7 +553,6 @@ def __init__(self, config: EfficientFormerConfig): @add_start_docstrings_to_model_forward(EFFICIENTFORMER_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_FEAT_EXTRACTOR_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPooling, config_class=_CONFIG_FOR_DOC, @@ -620,7 +618,6 @@ def __init__(self, config: EfficientFormerConfig): @add_start_docstrings_to_model_forward(EFFICIENTFORMER_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_FEAT_EXTRACTOR_FOR_DOC, checkpoint=_IMAGE_CLASS_CHECKPOINT, output_type=ImageClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -753,7 +750,6 @@ def __init__(self, config: EfficientFormerConfig): @add_start_docstrings_to_model_forward(EFFICIENTFORMER_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_FEAT_EXTRACTOR_FOR_DOC, checkpoint=_IMAGE_CLASS_CHECKPOINT, output_type=EfficientFormerForImageClassificationWithTeacherOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/electra/modeling_electra.py b/src/transformers/models/electra/modeling_electra.py index 3d5f17f71691e7..22114e28ef7360 100644 --- a/src/transformers/models/electra/modeling_electra.py +++ b/src/transformers/models/electra/modeling_electra.py @@ -52,7 +52,6 @@ _CHECKPOINT_FOR_DOC = "google/electra-small-discriminator" _CONFIG_FOR_DOC = "ElectraConfig" -_TOKENIZER_FOR_DOC = "ElectraTokenizer" ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST = [ "google/electra-small-generator", @@ -747,7 +746,7 @@ class ElectraForPreTrainingOutput(ModelOutput): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`ElectraTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -839,7 +838,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(ELECTRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -975,7 +973,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(ELECTRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="bhadresh-savani/electra-base-emotion", output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1098,11 +1095,11 @@ def forward( Examples: ```python - >>> from transformers import ElectraForPreTraining, ElectraTokenizerFast + >>> from transformers import ElectraForPreTraining, AutoTokenizer >>> import torch >>> discriminator = ElectraForPreTraining.from_pretrained("google/electra-base-discriminator") - >>> tokenizer = ElectraTokenizerFast.from_pretrained("google/electra-base-discriminator") + >>> tokenizer = AutoTokenizer.from_pretrained("google/electra-base-discriminator") >>> sentence = "The quick brown fox jumps over the lazy dog" >>> fake_sentence = "The quick brown fox fake over the lazy dog" @@ -1188,7 +1185,6 @@ def set_output_embeddings(self, word_embeddings): @add_start_docstrings_to_model_forward(ELECTRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="google/electra-small-generator", output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1275,7 +1271,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(ELECTRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="bhadresh-savani/electra-base-discriminator-finetuned-conll03-english", output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1357,7 +1352,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(ELECTRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="bhadresh-savani/electra-base-squad2", output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1463,7 +1457,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(ELECTRA_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1608,10 +1601,10 @@ def forward( Example: ```python - >>> from transformers import ElectraTokenizer, ElectraForCausalLM, ElectraConfig + >>> from transformers import AutoTokenizer, ElectraForCausalLM, ElectraConfig >>> import torch - >>> tokenizer = ElectraTokenizer.from_pretrained("google/electra-base-generator") + >>> tokenizer = AutoTokenizer.from_pretrained("google/electra-base-generator") >>> config = ElectraConfig.from_pretrained("google/electra-base-generator") >>> config.is_decoder = True >>> model = ElectraForCausalLM.from_pretrained("google/electra-base-generator", config=config) diff --git a/src/transformers/models/electra/modeling_flax_electra.py b/src/transformers/models/electra/modeling_flax_electra.py index a7ecb97414aadb..093a6e69acf059 100644 --- a/src/transformers/models/electra/modeling_flax_electra.py +++ b/src/transformers/models/electra/modeling_flax_electra.py @@ -53,7 +53,6 @@ _CHECKPOINT_FOR_DOC = "google/electra-small-discriminator" _CONFIG_FOR_DOC = "ElectraConfig" -_TOKENIZER_FOR_DOC = "ElectraTokenizer" remat = nn_partitioning.remat @@ -111,7 +110,7 @@ class FlaxElectraForPreTrainingOutput(ModelOutput): input_ids (`numpy.ndarray` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`ElectraTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -924,9 +923,7 @@ class FlaxElectraModel(FlaxElectraPreTrainedModel): module_class = FlaxElectraModule -append_call_sample_docstring( - FlaxElectraModel, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutput, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxElectraModel, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutput, _CONFIG_FOR_DOC) class FlaxElectraTiedDense(nn.Module): @@ -1013,9 +1010,7 @@ class FlaxElectraForMaskedLM(FlaxElectraPreTrainedModel): module_class = FlaxElectraForMaskedLMModule -append_call_sample_docstring( - FlaxElectraForMaskedLM, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxMaskedLMOutput, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxElectraForMaskedLM, _CHECKPOINT_FOR_DOC, FlaxMaskedLMOutput, _CONFIG_FOR_DOC) class FlaxElectraForPreTrainingModule(nn.Module): @@ -1085,9 +1080,9 @@ class FlaxElectraForPreTraining(FlaxElectraPreTrainedModel): Example: ```python - >>> from transformers import ElectraTokenizer, FlaxElectraForPreTraining + >>> from transformers import AutoTokenizer, FlaxElectraForPreTraining - >>> tokenizer = ElectraTokenizer.from_pretrained("google/electra-small-discriminator") + >>> tokenizer = AutoTokenizer.from_pretrained("google/electra-small-discriminator") >>> model = FlaxElectraForPreTraining.from_pretrained("google/electra-small-discriminator") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="np") @@ -1176,7 +1171,6 @@ class FlaxElectraForTokenClassification(FlaxElectraPreTrainedModel): append_call_sample_docstring( FlaxElectraForTokenClassification, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxTokenClassifierOutput, _CONFIG_FOR_DOC, @@ -1328,7 +1322,6 @@ class FlaxElectraForMultipleChoice(FlaxElectraPreTrainedModel): ) append_call_sample_docstring( FlaxElectraForMultipleChoice, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxMultipleChoiceModelOutput, _CONFIG_FOR_DOC, @@ -1400,7 +1393,6 @@ class FlaxElectraForQuestionAnswering(FlaxElectraPreTrainedModel): append_call_sample_docstring( FlaxElectraForQuestionAnswering, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxQuestionAnsweringModelOutput, _CONFIG_FOR_DOC, @@ -1494,7 +1486,6 @@ class FlaxElectraForSequenceClassification(FlaxElectraPreTrainedModel): append_call_sample_docstring( FlaxElectraForSequenceClassification, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSequenceClassifierOutput, _CONFIG_FOR_DOC, @@ -1605,7 +1596,6 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): append_call_sample_docstring( FlaxElectraForCausalLM, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxCausalLMOutputWithCrossAttentions, _CONFIG_FOR_DOC, diff --git a/src/transformers/models/electra/modeling_tf_electra.py b/src/transformers/models/electra/modeling_tf_electra.py index 4aff3466dae561..b782cc987bef26 100644 --- a/src/transformers/models/electra/modeling_tf_electra.py +++ b/src/transformers/models/electra/modeling_tf_electra.py @@ -62,7 +62,6 @@ _CHECKPOINT_FOR_DOC = "google/electra-small-discriminator" _CONFIG_FOR_DOC = "ElectraConfig" -_TOKENIZER_FOR_DOC = "ElectraTokenizer" TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST = [ "google/electra-small-generator", @@ -885,7 +884,7 @@ class TFElectraForPreTrainingOutput(ModelOutput): input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`ElectraTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -945,7 +944,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(ELECTRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutputWithPastAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1063,9 +1061,9 @@ def call( ```python >>> import tensorflow as tf - >>> from transformers import ElectraTokenizer, TFElectraForPreTraining + >>> from transformers import AutoTokenizer, TFElectraForPreTraining - >>> tokenizer = ElectraTokenizer.from_pretrained("google/electra-small-discriminator") + >>> tokenizer = AutoTokenizer.from_pretrained("google/electra-small-discriminator") >>> model = TFElectraForPreTraining.from_pretrained("google/electra-small-discriminator") >>> input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 >>> outputs = model(input_ids) @@ -1173,7 +1171,6 @@ def get_prefix_bias_name(self): @unpack_inputs @add_start_docstrings_to_model_forward(ELECTRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="google/electra-small-generator", output_type=TFMaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1285,7 +1282,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(ELECTRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="bhadresh-savani/electra-base-emotion", output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1379,7 +1375,6 @@ def dummy_inputs(self): @unpack_inputs @add_start_docstrings_to_model_forward(ELECTRA_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1496,7 +1491,6 @@ def __init__(self, config, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(ELECTRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="bhadresh-savani/electra-base-discriminator-finetuned-conll03-english", output_type=TFTokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1578,7 +1572,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(ELECTRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="bhadresh-savani/electra-base-squad2", output_type=TFQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/ernie/modeling_ernie.py b/src/transformers/models/ernie/modeling_ernie.py index 5ec40af77f0c62..631dc9f3fadc2c 100644 --- a/src/transformers/models/ernie/modeling_ernie.py +++ b/src/transformers/models/ernie/modeling_ernie.py @@ -54,7 +54,6 @@ _CHECKPOINT_FOR_DOC = "nghuyong/ernie-1.0-base-zh" _CONFIG_FOR_DOC = "ErnieConfig" -_TOKENIZER_FOR_DOC = "BertTokenizer" ERNIE_PRETRAINED_MODEL_ARCHIVE_LIST = [ @@ -740,7 +739,7 @@ class ErnieForPreTrainingOutput(ModelOutput): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -839,7 +838,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(ERNIE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1040,10 +1038,10 @@ def forward( Example: ```python - >>> from transformers import BertTokenizer, ErnieForPreTraining + >>> from transformers import AutoTokenizer, ErnieForPreTraining >>> import torch - >>> tokenizer = BertTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh") + >>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh") >>> model = ErnieForPreTraining.from_pretrained("nghuyong/ernie-1.0-base-zh") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") @@ -1121,7 +1119,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(ERNIE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1272,7 +1269,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(ERNIE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1401,10 +1397,10 @@ def forward( Example: ```python - >>> from transformers import BertTokenizer, ErnieForNextSentencePrediction + >>> from transformers import AutoTokenizer, ErnieForNextSentencePrediction >>> import torch - >>> tokenizer = BertTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh") + >>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh") >>> model = ErnieForNextSentencePrediction.from_pretrained("nghuyong/ernie-1.0-base-zh") >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." @@ -1584,7 +1580,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(ERNIE_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/esm/modeling_esm.py b/src/transformers/models/esm/modeling_esm.py index 0e9f792c29ddbe..dc25903743bb4d 100755 --- a/src/transformers/models/esm/modeling_esm.py +++ b/src/transformers/models/esm/modeling_esm.py @@ -39,7 +39,6 @@ _CHECKPOINT_FOR_DOC = "facebook/esm2_t6_8M_UR50D" _CONFIG_FOR_DOC = "EsmConfig" -_TOKENIZER_FOR_DOC = "EsmTokenizer" ESM_PRETRAINED_MODEL_ARCHIVE_LIST = [ "facebook/esm2_t6_8M_UR50D", @@ -731,7 +730,7 @@ def _init_weights(self, module): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`EsmTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -824,7 +823,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(ESM_INPUTS_DOCSTRING.format("(batch_size, sequence_length)")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -988,7 +986,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(ESM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1096,7 +1093,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(ESM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1192,7 +1188,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(ESM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/esm/modeling_esmfold.py b/src/transformers/models/esm/modeling_esmfold.py index 943730a2ffbf66..d37891df35ade3 100644 --- a/src/transformers/models/esm/modeling_esmfold.py +++ b/src/transformers/models/esm/modeling_esmfold.py @@ -54,7 +54,6 @@ logger = logging.get_logger(__name__) _CHECKPOINT_FOR_DOC = "facebook/esmfold_v1" _CONFIG_FOR_DOC = "EsmConfig" -_TOKENIZER_FOR_DOC = "EsmTokenizer" @dataclass @@ -143,7 +142,7 @@ class EsmForProteinFoldingOutput(ModelOutput): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`EsmTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) diff --git a/src/transformers/models/esm/modeling_tf_esm.py b/src/transformers/models/esm/modeling_tf_esm.py index 770b6640bc99c3..27476c61b012cb 100644 --- a/src/transformers/models/esm/modeling_tf_esm.py +++ b/src/transformers/models/esm/modeling_tf_esm.py @@ -49,7 +49,6 @@ _CHECKPOINT_FOR_DOC = "facebook/esm2_t6_8M_UR50D" _CONFIG_FOR_DOC = "EsmConfig" -_TOKENIZER_FOR_DOC = "EsmTokenizer" TF_ESM_PRETRAINED_MODEL_ARCHIVE_LIST = [ "facebook/esm2_t6_8M_UR50D", @@ -730,7 +729,7 @@ class TFEsmPreTrainedModel(TFPreTrainedModel): input_ids (`tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`EsmTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -995,7 +994,6 @@ def __init__(self, config: EsmConfig, add_pooling_layer=True, *inputs, **kwargs) @unpack_inputs @add_start_docstrings_to_model_forward(ESM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1119,7 +1117,6 @@ def get_lm_head(self): @unpack_inputs @add_start_docstrings_to_model_forward(ESM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1265,7 +1262,6 @@ def __init__(self, config): @unpack_inputs @add_start_docstrings_to_model_forward(ESM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1361,7 +1357,6 @@ def __init__(self, config): @unpack_inputs @add_start_docstrings_to_model_forward(ESM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFTokenClassifierOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/flaubert/modeling_flaubert.py b/src/transformers/models/flaubert/modeling_flaubert.py index 9b747e7170b276..8e48a21285eac1 100644 --- a/src/transformers/models/flaubert/modeling_flaubert.py +++ b/src/transformers/models/flaubert/modeling_flaubert.py @@ -51,7 +51,6 @@ _CHECKPOINT_FOR_DOC = "flaubert/flaubert_base_cased" _CONFIG_FOR_DOC = "FlaubertConfig" -_TOKENIZER_FOR_DOC = "FlaubertTokenizer" FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ "flaubert/flaubert_small_cased", @@ -238,7 +237,7 @@ def ff_chunk(self, input): input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`FlaubertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -478,7 +477,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(FLAUBERT_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC, @@ -688,7 +686,6 @@ def prepare_inputs_for_generation(self, input_ids, **kwargs): @add_start_docstrings_to_model_forward(FLAUBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -769,7 +766,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(FLAUBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -873,7 +869,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(FLAUBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -957,7 +952,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(FLAUBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1232,7 +1226,6 @@ def __init__(self, config, *inputs, **kwargs): FLAUBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/flaubert/modeling_tf_flaubert.py b/src/transformers/models/flaubert/modeling_tf_flaubert.py index 05b6922795ba58..d52f41a57d65c5 100644 --- a/src/transformers/models/flaubert/modeling_tf_flaubert.py +++ b/src/transformers/models/flaubert/modeling_tf_flaubert.py @@ -62,7 +62,6 @@ _CHECKPOINT_FOR_DOC = "flaubert/flaubert_base_cased" _CONFIG_FOR_DOC = "FlaubertConfig" -_TOKENIZER_FOR_DOC = "FlaubertTokenizer" TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ # See all Flaubert models at https://huggingface.co/models?filter=flaubert @@ -115,7 +114,7 @@ input_ids (`Numpy array` or `tf.Tensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`FlaubertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -250,7 +249,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(FLAUBERT_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutput, config_class=_CONFIG_FOR_DOC, @@ -808,7 +806,6 @@ def prepare_inputs_for_generation(self, inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(FLAUBERT_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFFlaubertWithLMHeadModelOutput, config_class=_CONFIG_FOR_DOC, @@ -881,7 +878,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(FLAUBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -968,7 +964,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(FLAUBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1073,7 +1068,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(FLAUBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFTokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1182,7 +1176,6 @@ def dummy_inputs(self): FLAUBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/flava/modeling_flava.py b/src/transformers/models/flava/modeling_flava.py index 41c9e726c8ae08..52f34eca18447a 100644 --- a/src/transformers/models/flava/modeling_flava.py +++ b/src/transformers/models/flava/modeling_flava.py @@ -54,7 +54,6 @@ _CONFIG_CLASS_FOR_IMAGE_MODEL_DOC = "FlavaImageConfig" _CONFIG_CLASS_FOR_TEXT_MODEL_DOC = "FlavaTextConfig" _CONFIG_CLASS_FOR_MULTIMODAL_MODEL_DOC = "FlavaMultimodalConfig" -_TOKENIZER_FOR_DOC = "BertTokenizer" _EXPECTED_IMAGE_OUTPUT_SHAPE = [1, 197, 768] FLAVA_PRETRAINED_MODEL_ARCHIVE_LIST = [ @@ -749,7 +748,7 @@ def forward(self, hidden_states: torch.Tensor): FLAVA_IMAGE_INPUTS_DOCSTRING_BASE = r""" Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`FlavaImageProcessor`]. See + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`FlavaImageProcessor.__call__`] for details. bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, image_num_patches)`): @@ -764,7 +763,7 @@ def forward(self, hidden_states: torch.Tensor): FLAVA_TEXT_INPUTS_DOCSTRING_BASE = r""" Args: input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`BertTokenizer`]. See + Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -806,7 +805,7 @@ def forward(self, hidden_states: torch.Tensor): Args: input_ids_masked (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. These ones are the masked version of the original task - to be used with MLM. Indices can be obtained using [`BertTokenizer`] along with + to be used with MLM. Indices can be obtained using [`AutoTokenizer`] along with [`DataCollatorForMaskedLanguageModeling`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1022,7 +1021,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(FLAVA_TEXT_INPUTS_DOCSTRING.format("batch_size, text_seq_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPooling, config_class=_CONFIG_CLASS_FOR_TEXT_MODEL_DOC, @@ -1127,7 +1125,6 @@ class PreTrainedModel FLAVA_MULTIMODAL_INPUTS_DOCSTRING.format("batch_size, image_num_patches + text_seq_len") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPooling, config_class=_CONFIG_CLASS_FOR_MULTIMODAL_MODEL_DOC, @@ -1259,10 +1256,10 @@ def get_text_features( Examples: ```python - >>> from transformers import FlavaProcessor, FlavaModel + >>> from transformers import AutoProcessor, FlavaModel >>> model = FlavaModel.from_pretrained("{0}") - >>> processor = FlavaProcessor.from_pretrained("{0}") + >>> processor = AutoProcessor.from_pretrained("{0}") >>> inputs = processor( ... text=["a photo of a cat", "a photo of a dog"], max_length=77, padding="max_length", return_tensors="pt" @@ -1308,10 +1305,10 @@ def get_image_features( ```python >>> from PIL import Image >>> import requests - >>> from transformers import FlavaProcessor, FlavaModel + >>> from transformers import AutoProcessor, FlavaModel >>> model = FlavaModel.from_pretrained("{0}") - >>> processor = FlavaProcessor.from_pretrained("{0}") + >>> processor = AutoProcessor.from_pretrained("{0}") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1364,10 +1361,10 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import FlavaProcessor, FlavaModel + >>> from transformers import AutoProcessor, FlavaModel >>> model = FlavaModel.from_pretrained("facebook/flava-full") - >>> processor = FlavaProcessor.from_pretrained("facebook/flava-full") + >>> processor = AutoProcessor.from_pretrained("facebook/flava-full") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1566,17 +1563,17 @@ def get_codebook_indices(self, pixel_values: torch.Tensor) -> torch.Tensor: """ Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Codebook pixel values can be obtained using [`FlavaImageProcessor`] by passing + Pixel values. Codebook pixel values can be obtained using [`AutoImageProcessor`] by passing `return_codebook_pixels=True`. See [`FlavaImageProcessor.__call__`] for details. Examples: ```python >>> from PIL import Image >>> import requests - >>> from transformers import FlavaImageProcessor, FlavaImageCodebook + >>> from transformers import AutoImageProcessor, FlavaImageCodebook >>> model = FlavaImageCodebook.from_pretrained("{0}") - >>> image_processor = FlavaImageProcessor.from_pretrained("{0}") + >>> image_processor = AutoImageProcessor.from_pretrained("{0}") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1600,7 +1597,7 @@ def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: """ Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Codebook pixel values can be obtained using [`FlavaImageProcessor`] by passing + Pixel values. Codebook pixel values can be obtained using [`AutoImageProcessor`] by passing `return_codebook_pixels=True`. See [`FlavaImageProcessor.__call__`] for details. Examples: @@ -1608,10 +1605,10 @@ def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: ```python >>> from PIL import Image >>> import requests - >>> from transformers import FlavaImageProcessor, FlavaImageCodebook + >>> from transformers import AutoImageProcessor, FlavaImageCodebook >>> model = FlavaImageCodebook.from_pretrained("{0}") - >>> image_processor = FlavaImageProcessor.from_pretrained("{0}") + >>> image_processor = AutoImageProcessor.from_pretrained("{0}") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1799,13 +1796,13 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import FlavaForPreTraining, FlavaProcessor + >>> from transformers import FlavaForPreTraining, AutoProcessor >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> model = FlavaForPreTraining.from_pretrained("facebook/flava-full") - >>> processor = FlavaProcessor.from_pretrained("facebook/flava-full") + >>> processor = AutoProcessor.from_pretrained("facebook/flava-full") >>> text = ["a photo of a cat"] @@ -1895,7 +1892,7 @@ def forward( if codebook_pixel_values is None: raise ValueError( "`codebook_pixel_value` are required to generate `mim_labels` if loss is expected. " - "Call `FlavaProcessor` with `return_codebook_pixels` set to True" + "Call `AutoProcessor` with `return_codebook_pixels` set to True" ) mim_labels = self.image_codebook.get_codebook_indices(codebook_pixel_values) # Unimodal MIM Loss diff --git a/src/transformers/models/fnet/modeling_fnet.py b/src/transformers/models/fnet/modeling_fnet.py index 672fe356450026..75684025443b04 100755 --- a/src/transformers/models/fnet/modeling_fnet.py +++ b/src/transformers/models/fnet/modeling_fnet.py @@ -58,7 +58,6 @@ _CHECKPOINT_FOR_DOC = "google/fnet-base" _CONFIG_FOR_DOC = "FNetConfig" -_TOKENIZER_FOR_DOC = "FNetTokenizer" FNET_PRETRAINED_MODEL_ARCHIVE_LIST = [ "google/fnet-base", @@ -479,7 +478,7 @@ class FNetForPreTrainingOutput(ModelOutput): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`FNetTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -541,7 +540,6 @@ def set_input_embeddings(self, value): @add_start_docstrings_to_model_forward(FNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC, @@ -673,10 +671,10 @@ def forward( Example: ```python - >>> from transformers import FNetTokenizer, FNetForPreTraining + >>> from transformers import AutoTokenizer, FNetForPreTraining >>> import torch - >>> tokenizer = FNetTokenizer.from_pretrained("google/fnet-base") + >>> tokenizer = AutoTokenizer.from_pretrained("google/fnet-base") >>> model = FNetForPreTraining.from_pretrained("google/fnet-base") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) @@ -737,7 +735,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(FNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -824,10 +821,10 @@ def forward( Example: ```python - >>> from transformers import FNetTokenizer, FNetForNextSentencePrediction + >>> from transformers import AutoTokenizer, FNetForNextSentencePrediction >>> import torch - >>> tokenizer = FNetTokenizer.from_pretrained("google/fnet-base") + >>> tokenizer = AutoTokenizer.from_pretrained("google/fnet-base") >>> model = FNetForNextSentencePrediction.from_pretrained("google/fnet-base") >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light." @@ -897,7 +894,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(FNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -982,7 +978,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(FNET_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1064,7 +1059,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(FNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1133,7 +1127,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(FNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/fsmt/modeling_fsmt.py b/src/transformers/models/fsmt/modeling_fsmt.py index 4ad4c4f6cae9ee..ac22dea6fcc079 100644 --- a/src/transformers/models/fsmt/modeling_fsmt.py +++ b/src/transformers/models/fsmt/modeling_fsmt.py @@ -59,7 +59,6 @@ _CHECKPOINT_FOR_DOC = "facebook/wmt19-ru-en" _CONFIG_FOR_DOC = "FSMTConfig" -_TOKENIZER_FOR_DOC = "FSMTTokenizer" # See all FSMT models at https://huggingface.co/models?filter=fsmt @@ -200,11 +199,11 @@ Translation example:: ```python - >>> from transformers import FSMTTokenizer, FSMTForConditionalGeneration + >>> from transformers import AutoTokenizer, FSMTForConditionalGeneration >>> mname = "facebook/wmt19-ru-en" >>> model = FSMTForConditionalGeneration.from_pretrained(mname) - >>> tokenizer = FSMTTokenizer.from_pretrained(mname) + >>> tokenizer = AutoTokenizer.from_pretrained(mname) >>> src_text = "Машинное обучение - это здорово, не так ли?" >>> input_ids = tokenizer(src_text, return_tensors="pt").input_ids @@ -234,7 +233,7 @@ decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`FSMTTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -1058,7 +1057,6 @@ def get_decoder(self): @add_start_docstrings_to_model_forward(FSMT_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/funnel/modeling_funnel.py b/src/transformers/models/funnel/modeling_funnel.py index f560baa729a09f..2c760b3cf2243c 100644 --- a/src/transformers/models/funnel/modeling_funnel.py +++ b/src/transformers/models/funnel/modeling_funnel.py @@ -47,7 +47,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "FunnelConfig" -_TOKENIZER_FOR_DOC = "FunnelTokenizer" _CHECKPOINT_FOR_DOC = "funnel-transformer/small" FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST = [ @@ -942,7 +941,6 @@ def set_input_embeddings(self, new_embeddings: nn.Embedding) -> None: @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="funnel-transformer/small-base", output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1020,7 +1018,6 @@ def set_input_embeddings(self, new_embeddings: nn.Embedding) -> None: @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1144,10 +1141,10 @@ def forward( Examples: ```python - >>> from transformers import FunnelTokenizer, FunnelForPreTraining + >>> from transformers import AutoTokenizer, FunnelForPreTraining >>> import torch - >>> tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/small") + >>> tokenizer = AutoTokenizer.from_pretrained("funnel-transformer/small") >>> model = FunnelForPreTraining.from_pretrained("funnel-transformer/small") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") @@ -1212,7 +1209,6 @@ def set_output_embeddings(self, new_embeddings: nn.Embedding) -> None: @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1287,7 +1283,6 @@ def __init__(self, config: FunnelConfig) -> None: @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="funnel-transformer/small-base", output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1378,7 +1373,6 @@ def __init__(self, config: FunnelConfig) -> None: @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="funnel-transformer/small-base", output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1465,7 +1459,6 @@ def __init__(self, config: FunnelConfig) -> None: @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1538,7 +1531,6 @@ def __init__(self, config: FunnelConfig) -> None: @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/funnel/modeling_tf_funnel.py b/src/transformers/models/funnel/modeling_tf_funnel.py index 1d9f933b7862c2..e2def568d1cdba 100644 --- a/src/transformers/models/funnel/modeling_tf_funnel.py +++ b/src/transformers/models/funnel/modeling_tf_funnel.py @@ -58,7 +58,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "FunnelConfig" -_TOKENIZER_FOR_DOC = "FunnelTokenizer" TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST = [ "funnel-transformer/small", # B4-4-4H768 @@ -1060,7 +1059,7 @@ class TFFunnelForPreTrainingOutput(ModelOutput): input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`FunnelTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -1114,7 +1113,6 @@ def __init__(self, config: FunnelConfig, *inputs, **kwargs) -> None: @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="funnel-transformer/small-base", output_type=TFBaseModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1164,7 +1162,6 @@ def __init__(self, config: FunnelConfig, *inputs, **kwargs) -> None: @unpack_inputs @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="funnel-transformer/small", output_type=TFBaseModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1236,10 +1233,10 @@ def call( Examples: ```python - >>> from transformers import FunnelTokenizer, TFFunnelForPreTraining + >>> from transformers import AutoTokenizer, TFFunnelForPreTraining >>> import torch - >>> tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/small") + >>> tokenizer = AutoTokenizer.from_pretrained("funnel-transformer/small") >>> model = TFFunnelForPreTraining.from_pretrained("funnel-transformer/small") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="tf") @@ -1293,7 +1290,6 @@ def get_prefix_bias_name(self) -> str: @unpack_inputs @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="funnel-transformer/small", output_type=TFMaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1366,7 +1362,6 @@ def __init__(self, config: FunnelConfig, *inputs, **kwargs) -> None: @unpack_inputs @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="funnel-transformer/small-base", output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1451,7 +1446,6 @@ def dummy_inputs(self): @unpack_inputs @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="funnel-transformer/small-base", output_type=TFMultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1561,7 +1555,6 @@ def __init__(self, config: FunnelConfig, *inputs, **kwargs) -> None: @unpack_inputs @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="funnel-transformer/small", output_type=TFTokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1638,7 +1631,6 @@ def __init__(self, config: FunnelConfig, *inputs, **kwargs) -> None: @unpack_inputs @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="funnel-transformer/small", output_type=TFQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/git/modeling_git.py b/src/transformers/models/git/modeling_git.py index 9bb421e12f6fb1..403c67a73d2c38 100644 --- a/src/transformers/models/git/modeling_git.py +++ b/src/transformers/models/git/modeling_git.py @@ -43,7 +43,6 @@ _CHECKPOINT_FOR_DOC = "microsoft/git-base" _CONFIG_FOR_DOC = "GitConfig" -_TOKENIZER_FOR_DOC = "BertTokenizer" GIT_PRETRAINED_MODEL_ARCHIVE_LIST = [ "microsoft/git-base", @@ -558,7 +557,7 @@ def _set_gradient_checkpointing(self, module, value=False): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -578,7 +577,7 @@ def _set_gradient_checkpointing(self, module, value=False): pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + [`CLIPImageProcessor.__call__`] for details. head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: @@ -916,7 +915,7 @@ def custom_forward(*inputs): Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`CLIPImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. diff --git a/src/transformers/models/glpn/modeling_glpn.py b/src/transformers/models/glpn/modeling_glpn.py index 3da4d1de73fef8..e05168adebe95f 100755 --- a/src/transformers/models/glpn/modeling_glpn.py +++ b/src/transformers/models/glpn/modeling_glpn.py @@ -463,7 +463,7 @@ def _init_weights(self, module): Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`GLPNImageProcessor`]. See [`GLPNImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`GLPNImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned @@ -711,7 +711,7 @@ def forward( Examples: ```python - >>> from transformers import GLPNImageProcessor, GLPNForDepthEstimation + >>> from transformers import AutoImageProcessor, GLPNForDepthEstimation >>> import torch >>> import numpy as np >>> from PIL import Image @@ -720,7 +720,7 @@ def forward( >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = GLPNImageProcessor.from_pretrained("vinvino02/glpn-kitti") + >>> image_processor = AutoImageProcessor.from_pretrained("vinvino02/glpn-kitti") >>> model = GLPNForDepthEstimation.from_pretrained("vinvino02/glpn-kitti") >>> # prepare image for the model diff --git a/src/transformers/models/gpt2/modeling_flax_gpt2.py b/src/transformers/models/gpt2/modeling_flax_gpt2.py index 796ca8ac1a1ca2..8e375b6dbe6035 100644 --- a/src/transformers/models/gpt2/modeling_flax_gpt2.py +++ b/src/transformers/models/gpt2/modeling_flax_gpt2.py @@ -37,7 +37,6 @@ _CHECKPOINT_FOR_DOC = "gpt2" _CONFIG_FOR_DOC = "GPT2Config" -_TOKENIZER_FOR_DOC = "GPT2Tokenizer" GPT2_START_DOCSTRING = r""" @@ -80,7 +79,7 @@ input_ids (`numpy.ndarray` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length`. Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`GPT2Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -670,7 +669,6 @@ class FlaxGPT2Model(FlaxGPT2PreTrainedModel): append_call_sample_docstring( FlaxGPT2Model, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutputWithPastAndCrossAttentions, _CONFIG_FOR_DOC, @@ -774,7 +772,6 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): append_call_sample_docstring( FlaxGPT2LMHeadModel, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxCausalLMOutputWithCrossAttentions, _CONFIG_FOR_DOC, diff --git a/src/transformers/models/gpt2/modeling_gpt2.py b/src/transformers/models/gpt2/modeling_gpt2.py index 38a58a2a43fc6c..5fe33bbca50935 100644 --- a/src/transformers/models/gpt2/modeling_gpt2.py +++ b/src/transformers/models/gpt2/modeling_gpt2.py @@ -51,7 +51,6 @@ _CHECKPOINT_FOR_DOC = "gpt2" _CONFIG_FOR_DOC = "GPT2Config" -_TOKENIZER_FOR_DOC = "GPT2Tokenizer" GPT2_PRETRAINED_MODEL_ARCHIVE_LIST = [ "gpt2", @@ -551,7 +550,7 @@ class GPT2DoubleHeadsModelOutput(ModelOutput): If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. - Indices can be obtained using [`GPT2Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -735,7 +734,6 @@ def _prune_heads(self, heads_to_prune): @add_start_docstrings_to_model_forward(GPT2_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPastAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1013,7 +1011,6 @@ def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwarg @add_start_docstrings_to_model_forward(GPT2_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1223,9 +1220,9 @@ def forward( ```python >>> import torch - >>> from transformers import GPT2Tokenizer, GPT2DoubleHeadsModel + >>> from transformers import AutoTokenizer, GPT2DoubleHeadsModel - >>> tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + >>> tokenizer = AutoTokenizer.from_pretrained("gpt2") >>> model = GPT2DoubleHeadsModel.from_pretrained("gpt2") >>> # Add a [CLS] to the vocabulary (we should train it also!) @@ -1343,12 +1340,9 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(GPT2_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="microsoft/DialogRPT-updown", output_type=SequenceClassifierOutputWithPast, config_class=_CONFIG_FOR_DOC, - expected_output="'LABEL_0'", - expected_loss=5.28, ) def forward( self, @@ -1478,7 +1472,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(GPT2_INPUTS_DOCSTRING) # fmt: off @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="brad1141/gpt2-finetuned-comp2", output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/gpt2/modeling_tf_gpt2.py b/src/transformers/models/gpt2/modeling_tf_gpt2.py index 563d1878f13597..c08e864b8d37f9 100644 --- a/src/transformers/models/gpt2/modeling_tf_gpt2.py +++ b/src/transformers/models/gpt2/modeling_tf_gpt2.py @@ -56,7 +56,6 @@ _CHECKPOINT_FOR_DOC = "gpt2" _CONFIG_FOR_DOC = "GPT2Config" -_TOKENIZER_FOR_DOC = "GPT2Tokenizer" TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST = [ "gpt2", @@ -655,7 +654,7 @@ class TFGPT2DoubleHeadsModelOutput(ModelOutput): If `past_key_values` is used, only input IDs that do not have their past calculated should be passed as `input_ids`. - Indices can be obtained using [`GPT2Tokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -726,7 +725,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(GPT2_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutputWithPastAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -855,7 +853,6 @@ def prepare_inputs_for_generation(self, inputs, past_key_values=None, use_cache= @unpack_inputs @add_start_docstrings_to_model_forward(GPT2_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFCausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1006,9 +1003,9 @@ def call( ```python >>> import tensorflow as tf - >>> from transformers import GPT2Tokenizer, TFGPT2DoubleHeadsModel + >>> from transformers import AutoTokenizer, TFGPT2DoubleHeadsModel - >>> tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + >>> tokenizer = AutoTokenizer.from_pretrained("gpt2") >>> model = TFGPT2DoubleHeadsModel.from_pretrained("gpt2") >>> # Add a [CLS] to the vocabulary (we should train it also!) @@ -1130,7 +1127,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(GPT2_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="microsoft/DialogRPT-updown", output_type=TFSequenceClassifierOutputWithPast, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/gpt_neo/modeling_flax_gpt_neo.py b/src/transformers/models/gpt_neo/modeling_flax_gpt_neo.py index bb29b3c6499f05..0749911f7a15fa 100644 --- a/src/transformers/models/gpt_neo/modeling_flax_gpt_neo.py +++ b/src/transformers/models/gpt_neo/modeling_flax_gpt_neo.py @@ -34,7 +34,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "GPTNeoConfig" -_TOKENIZER_FOR_DOC = "GPT2Tokenizer" _CHECKPOINT_FOR_DOC = "EleutherAI/gpt-neo-1.3B" @@ -78,7 +77,7 @@ input_ids (`numpy.ndarray` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length`. Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`GPTNeoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -593,9 +592,7 @@ class FlaxGPTNeoModel(FlaxGPTNeoPreTrainedModel): module_class = FlaxGPTNeoModule -append_call_sample_docstring( - FlaxGPTNeoModel, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutput, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxGPTNeoModel, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutput, _CONFIG_FOR_DOC) class FlaxGPTNeoForCausalLMModule(nn.Module): @@ -684,6 +681,4 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): return model_kwargs -append_call_sample_docstring( - FlaxGPTNeoForCausalLM, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxCausalLMOutput, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxGPTNeoForCausalLM, _CHECKPOINT_FOR_DOC, FlaxCausalLMOutput, _CONFIG_FOR_DOC) diff --git a/src/transformers/models/gpt_neo/modeling_gpt_neo.py b/src/transformers/models/gpt_neo/modeling_gpt_neo.py index 002b6881752c1a..7a5a913292e32b 100755 --- a/src/transformers/models/gpt_neo/modeling_gpt_neo.py +++ b/src/transformers/models/gpt_neo/modeling_gpt_neo.py @@ -39,7 +39,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "GPTNeoConfig" -_TOKENIZER_FOR_DOC = "GPT2Tokenizer" GPT_NEO_PRETRAINED_MODEL_ARCHIVE_LIST = [ "EleutherAI/gpt-neo-1.3B", @@ -414,7 +413,7 @@ def _set_gradient_checkpointing(self, module, value=False): If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. - Indices can be obtained using [`GPTNeoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -496,7 +495,6 @@ def set_input_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(GPT_NEO_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPastAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -713,7 +711,6 @@ def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwarg @add_start_docstrings_to_model_forward(GPT_NEO_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -828,7 +825,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(GPT_NEO_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutputWithPast, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/gpt_neox/modeling_gpt_neox.py b/src/transformers/models/gpt_neox/modeling_gpt_neox.py index c6967a0e7865e2..589eaae7804918 100755 --- a/src/transformers/models/gpt_neox/modeling_gpt_neox.py +++ b/src/transformers/models/gpt_neox/modeling_gpt_neox.py @@ -364,7 +364,7 @@ def forward( input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`GPTNeoXTokenizerFast`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -636,10 +636,10 @@ def forward( Example: ```python - >>> from transformers import GPTNeoXTokenizerFast, GPTNeoXForCausalLM, GPTNeoXConfig + >>> from transformers import AutoTokenizer, GPTNeoXForCausalLM, GPTNeoXConfig >>> import torch - >>> tokenizer = GPTNeoXTokenizerFast.from_pretrained("EleutherAI/gpt-neox-20b") + >>> tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b") >>> config = GPTNeoXConfig.from_pretrained("EleutherAI/gpt-neox-20b") >>> config.is_decoder = True >>> model = GPTNeoXForCausalLM.from_pretrained("EleutherAI/gpt-neox-20b", config=config) diff --git a/src/transformers/models/gpt_neox_japanese/modeling_gpt_neox_japanese.py b/src/transformers/models/gpt_neox_japanese/modeling_gpt_neox_japanese.py index aea950a8a93b7c..76df8beee9cc71 100755 --- a/src/transformers/models/gpt_neox_japanese/modeling_gpt_neox_japanese.py +++ b/src/transformers/models/gpt_neox_japanese/modeling_gpt_neox_japanese.py @@ -33,7 +33,6 @@ _CHECKPOINT_FOR_DOC = "abeja/gpt-neox-japanese-2.7b" _CONFIG_FOR_DOC = "GPTNeoXJapaneseConfig" -_TOKENIZER_FOR_DOC = "GPTNeoXJapaneseTokenizer" GPT_NEOX_JAPANESE_PRETRAINED_MODEL_ARCHIVE_LIST = { "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/config.json", @@ -392,7 +391,7 @@ def forward( input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`GPTNeoXJapaneseTokenizer`]. + Indices can be obtained using [`AutoTokenizer`]. attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: @@ -485,10 +484,10 @@ def forward( Example: ```python - >>> from transformers import GPTNeoXJapaneseTokenizer, GPTNeoXJapaneseModel + >>> from transformers import AutoTokenizer, GPTNeoXJapaneseModel >>> import torch - >>> tokenizer = GPTNeoXJapaneseTokenizer.from_pretrained("abeja/gpt-neox-japanese-2.7b") + >>> tokenizer = AutoTokenizer.from_pretrained("abeja/gpt-neox-japanese-2.7b") >>> model = GPTNeoXJapaneseModel.from_pretrained("abeja/gpt-neox-japanese-2.7b") >>> inputs = tokenizer("日本語のGPT-neoxがHugging Faceで使えます😀", return_tensors="pt") @@ -651,10 +650,10 @@ def forward( Example: ```python - >>> from transformers import GPTNeoXJapaneseTokenizer, GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseConfig + >>> from transformers import AutoTokenizer, GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseConfig >>> import torch - >>> tokenizer = GPTNeoXJapaneseTokenizer.from_pretrained("abeja/gpt-neox-japanese-2.7b") + >>> tokenizer = AutoTokenizer.from_pretrained("abeja/gpt-neox-japanese-2.7b") >>> config = GPTNeoXJapaneseConfig.from_pretrained("abeja/gpt-neox-japanese-2.7b") >>> config.is_decoder = True >>> model = GPTNeoXJapaneseForCausalLM.from_pretrained("abeja/gpt-neox-japanese-2.7b", config=config) diff --git a/src/transformers/models/gptj/modeling_flax_gptj.py b/src/transformers/models/gptj/modeling_flax_gptj.py index a10e9598cec3aa..1f00893b5c873b 100644 --- a/src/transformers/models/gptj/modeling_flax_gptj.py +++ b/src/transformers/models/gptj/modeling_flax_gptj.py @@ -37,7 +37,6 @@ _CHECKPOINT_FOR_DOC = "gptj" _CONFIG_FOR_DOC = "GPTJConfig" -_TOKENIZER_FOR_DOC = "GPTJTokenizer" GPTJ_START_DOCSTRING = r""" @@ -80,7 +79,7 @@ input_ids (`numpy.ndarray` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length`. Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`GPTJTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -622,7 +621,6 @@ class FlaxGPTJModel(FlaxGPTJPreTrainedModel): append_call_sample_docstring( FlaxGPTJModel, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxCausalLMOutput, _CONFIG_FOR_DOC, @@ -715,7 +713,6 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): append_call_sample_docstring( FlaxGPTJForCausalLM, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxCausalLMOutput, _CONFIG_FOR_DOC, diff --git a/src/transformers/models/gptj/modeling_gptj.py b/src/transformers/models/gptj/modeling_gptj.py index bf2c04f7e06961..84282fb0736781 100755 --- a/src/transformers/models/gptj/modeling_gptj.py +++ b/src/transformers/models/gptj/modeling_gptj.py @@ -373,7 +373,7 @@ def _set_gradient_checkpointing(self, module, value=False): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`GPTJTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) diff --git a/src/transformers/models/gptj/modeling_tf_gptj.py b/src/transformers/models/gptj/modeling_tf_gptj.py index 06dee874192448..244d4d3ba951b8 100644 --- a/src/transformers/models/gptj/modeling_tf_gptj.py +++ b/src/transformers/models/gptj/modeling_tf_gptj.py @@ -52,7 +52,6 @@ _CHECKPOINT_FOR_DOC = "EleutherAI/gpt-j-6B" _CONFIG_FOR_DOC = "GPTJConfig" -_TOKENIZER_FOR_DOC = "GPTJTokenizer" GPTJ_PRETRAINED_MODEL_ARCHIVE_LIST = [ "EleutherAI/gpt-j-6B", @@ -597,7 +596,7 @@ def serving(self, inputs): If `past` is used, only input IDs that do not have their past calculated should be passed as `input_ids`. - Indices can be obtained using [`GPTJTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -664,7 +663,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(GPTJ_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutputWithPast, config_class=_CONFIG_FOR_DOC, @@ -768,7 +766,6 @@ def prepare_inputs_for_generation(self, inputs, past_key_values=None, use_cache= @unpack_inputs @add_start_docstrings_to_model_forward(GPTJ_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC, @@ -872,7 +869,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(GPTJ_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFSequenceClassifierOutputWithPast, config_class=_CONFIG_FOR_DOC, @@ -995,7 +991,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(GPTJ_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/groupvit/modeling_groupvit.py b/src/transformers/models/groupvit/modeling_groupvit.py index e96563e3f5db5a..a3402f9a49762b 100644 --- a/src/transformers/models/groupvit/modeling_groupvit.py +++ b/src/transformers/models/groupvit/modeling_groupvit.py @@ -857,7 +857,7 @@ def _set_gradient_checkpointing(self, module, value=False): Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`CLIPImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. @@ -891,7 +891,7 @@ def _set_gradient_checkpointing(self, module, value=False): [What are position IDs?](../glossary#position-ids) pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`CLIPImageProcessor`]. See + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. return_loss (`bool`, *optional*): Whether or not to return the contrastive loss. diff --git a/src/transformers/models/groupvit/modeling_tf_groupvit.py b/src/transformers/models/groupvit/modeling_tf_groupvit.py index a540319ba92e66..006bd868c9b7f1 100644 --- a/src/transformers/models/groupvit/modeling_tf_groupvit.py +++ b/src/transformers/models/groupvit/modeling_tf_groupvit.py @@ -1554,7 +1554,7 @@ class TFGroupViTPreTrainedModel(TFPreTrainedModel): GROUPVIT_VISION_INPUTS_DOCSTRING = r""" Args: pixel_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]`, `Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`CLIPImageProcessor`]. See + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned @@ -1582,7 +1582,7 @@ class TFGroupViTPreTrainedModel(TFPreTrainedModel): [What are input IDs?](../glossary#input-ids) pixel_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` `Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`CLIPImageProcessor`]. See + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. attention_mask (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: diff --git a/src/transformers/models/hubert/modeling_hubert.py b/src/transformers/models/hubert/modeling_hubert.py index 59b199e26d27e9..a96ef5cf5db44a 100755 --- a/src/transformers/models/hubert/modeling_hubert.py +++ b/src/transformers/models/hubert/modeling_hubert.py @@ -914,10 +914,10 @@ def _get_feature_vector_attention_mask(self, feature_vector_length: int, attenti HUBERT_INPUTS_DOCSTRING = r""" Args: input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file - into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install - soundfile*). To prepare the array into *input_values*, the [`Wav2Vec2Processor`] should be used for padding - and conversion into a tensor of type *torch.FloatTensor*. See [`Wav2Vec2Processor.__call__`] for details. + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`: @@ -1036,11 +1036,11 @@ def forward( Example: ```python - >>> from transformers import Wav2Vec2Processor, HubertModel + >>> from transformers import AutoProcessor, HubertModel >>> from datasets import load_dataset >>> import soundfile as sf - >>> processor = Wav2Vec2Processor.from_pretrained("facebook/hubert-large-ls960-ft") + >>> processor = AutoProcessor.from_pretrained("facebook/hubert-large-ls960-ft") >>> model = HubertModel.from_pretrained("facebook/hubert-large-ls960-ft") diff --git a/src/transformers/models/hubert/modeling_tf_hubert.py b/src/transformers/models/hubert/modeling_tf_hubert.py index 4ead1f9a6c9702..df34adc66dbec0 100644 --- a/src/transformers/models/hubert/modeling_tf_hubert.py +++ b/src/transformers/models/hubert/modeling_tf_hubert.py @@ -1371,10 +1371,10 @@ def serving(self, inputs): HUBERT_INPUTS_DOCSTRING = r""" Args: - input_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`): + input_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` `Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -1457,11 +1457,11 @@ def call( Example: ```python - >>> from transformers import Wav2Vec2Processor, TFHubertModel + >>> from transformers import AutoProcessor, TFHubertModel >>> from datasets import load_dataset >>> import soundfile as sf - >>> processor = Wav2Vec2Processor.from_pretrained("facebook/hubert-large-ls960-ft") + >>> processor = AutoProcessor.from_pretrained("facebook/hubert-large-ls960-ft") >>> model = TFHubertModel.from_pretrained("facebook/hubert-large-ls960-ft") @@ -1583,11 +1583,11 @@ def call( ```python >>> import tensorflow as tf - >>> from transformers import Wav2Vec2Processor, TFHubertForCTC + >>> from transformers import AutoProcessor, TFHubertForCTC >>> from datasets import load_dataset >>> import soundfile as sf - >>> processor = Wav2Vec2Processor.from_pretrained("facebook/hubert-large-ls960-ft") + >>> processor = AutoProcessor.from_pretrained("facebook/hubert-large-ls960-ft") >>> model = TFHubertForCTC.from_pretrained("facebook/hubert-large-ls960-ft") diff --git a/src/transformers/models/ibert/modeling_ibert.py b/src/transformers/models/ibert/modeling_ibert.py index 2df68b49e97789..de7f6f4d707c54 100644 --- a/src/transformers/models/ibert/modeling_ibert.py +++ b/src/transformers/models/ibert/modeling_ibert.py @@ -46,7 +46,6 @@ _CHECKPOINT_FOR_DOC = "kssteven/ibert-roberta-base" _CONFIG_FOR_DOC = "IBertConfig" -_TOKENIZER_FOR_DOC = "RobertaTokenizer" IBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ "kssteven/ibert-roberta-base", @@ -682,7 +681,7 @@ def resize_token_embeddings(self, new_num_tokens=None): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`RobertaTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -772,7 +771,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(IBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -876,7 +874,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(IBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -985,7 +982,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(IBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1081,7 +1077,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(IBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1176,7 +1171,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(IBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1276,7 +1270,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(IBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/imagegpt/modeling_imagegpt.py b/src/transformers/models/imagegpt/modeling_imagegpt.py index 737e52ed7e7573..4e52ef5c071d66 100755 --- a/src/transformers/models/imagegpt/modeling_imagegpt.py +++ b/src/transformers/models/imagegpt/modeling_imagegpt.py @@ -41,7 +41,6 @@ _CHECKPOINT_FOR_DOC = "openai/imagegpt-small" _CONFIG_FOR_DOC = "ImageGPTConfig" -_TOKENIZER_FOR_DOC = "ImageGPTTokenizer" IMAGEGPT_PRETRAINED_MODEL_ARCHIVE_LIST = [ "openai/imagegpt-small", @@ -556,8 +555,7 @@ def _set_gradient_checkpointing(self, module, value=False): If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. - Indices can be obtained using [`ImageGPTImageProcessor`]. See [`ImageGPTImageProcessor.__call__`] for - details. + Indices can be obtained using [`AutoImageProcessor`]. See [`ImageGPTImageProcessor.__call__`] for details. past_key_values (`Tuple[Tuple[torch.Tensor]]` of length `config.n_layers`): Contains precomputed hidden-states (key and values in the attention blocks) as computed by the model (see @@ -679,14 +677,14 @@ def forward( Examples: ```python - >>> from transformers import ImageGPTImageProcessor, ImageGPTModel + >>> from transformers import AutoImageProcessor, ImageGPTModel >>> from PIL import Image >>> import requests >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = ImageGPTImageProcessor.from_pretrained("openai/imagegpt-small") + >>> image_processor = AutoImageProcessor.from_pretrained("openai/imagegpt-small") >>> model = ImageGPTModel.from_pretrained("openai/imagegpt-small") >>> inputs = image_processor(images=image, return_tensors="pt") @@ -973,12 +971,12 @@ def forward( Examples: ```python - >>> from transformers import ImageGPTImageProcessor, ImageGPTForCausalImageModeling + >>> from transformers import AutoImageProcessor, ImageGPTForCausalImageModeling >>> import torch >>> import matplotlib.pyplot as plt >>> import numpy as np - >>> image_processor = ImageGPTImageProcessor.from_pretrained("openai/imagegpt-small") + >>> image_processor = AutoImageProcessor.from_pretrained("openai/imagegpt-small") >>> model = ImageGPTForCausalImageModeling.from_pretrained("openai/imagegpt-small") >>> device = torch.device("cuda" if torch.cuda.is_available() else "cpu") >>> model.to(device) @@ -1124,14 +1122,14 @@ def forward( Examples: ```python - >>> from transformers import ImageGPTImageProcessor, ImageGPTForImageClassification + >>> from transformers import AutoImageProcessor, ImageGPTForImageClassification >>> from PIL import Image >>> import requests >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = ImageGPTImageProcessor.from_pretrained("openai/imagegpt-small") + >>> image_processor = AutoImageProcessor.from_pretrained("openai/imagegpt-small") >>> model = ImageGPTForImageClassification.from_pretrained("openai/imagegpt-small") >>> inputs = image_processor(images=image, return_tensors="pt") diff --git a/src/transformers/models/jukebox/modeling_jukebox.py b/src/transformers/models/jukebox/modeling_jukebox.py index 949ddb227336d5..38fed91e1b01bb 100755 --- a/src/transformers/models/jukebox/modeling_jukebox.py +++ b/src/transformers/models/jukebox/modeling_jukebox.py @@ -2501,11 +2501,11 @@ def _sample( Example: ```python - >>> from transformers import JukeboxTokenizer, JukeboxModel, set_seed + >>> from transformers import AutoTokenizer, JukeboxModel, set_seed >>> import torch >>> metas = dict(artist="Zac Brown Band", genres="Country", lyrics="I met a traveller from an antique land") - >>> tokenizer = JukeboxTokenizer.from_pretrained("openai/jukebox-1b-lyrics") + >>> tokenizer = AutoTokenizer.from_pretrained("openai/jukebox-1b-lyrics") >>> model = JukeboxModel.from_pretrained("openai/jukebox-1b-lyrics", min_duration=0).eval() >>> labels = tokenizer(**metas)["input_ids"] @@ -2594,10 +2594,10 @@ def ancestral_sample(self, labels, n_samples=1, **sampling_kwargs) -> List[torch Example: ```python - >>> from transformers import JukeboxTokenizer, JukeboxModel, set_seed + >>> from transformers import AutoTokenizer, JukeboxModel, set_seed >>> model = JukeboxModel.from_pretrained("openai/jukebox-1b-lyrics", min_duration=0).eval() - >>> tokenizer = JukeboxTokenizer.from_pretrained("openai/jukebox-1b-lyrics") + >>> tokenizer = AutoTokenizer.from_pretrained("openai/jukebox-1b-lyrics") >>> lyrics = "Hey, are you awake? Can you talk to me?" >>> artist = "Zac Brown Band" diff --git a/src/transformers/models/layoutlm/modeling_layoutlm.py b/src/transformers/models/layoutlm/modeling_layoutlm.py index 8ff5ff092edd30..3b696d778adac3 100644 --- a/src/transformers/models/layoutlm/modeling_layoutlm.py +++ b/src/transformers/models/layoutlm/modeling_layoutlm.py @@ -661,7 +661,7 @@ def _set_gradient_checkpointing(self, module, value=False): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`LayoutLMTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) diff --git a/src/transformers/models/layoutlm/modeling_tf_layoutlm.py b/src/transformers/models/layoutlm/modeling_tf_layoutlm.py index 8219657af480fb..41267ab1f52939 100644 --- a/src/transformers/models/layoutlm/modeling_tf_layoutlm.py +++ b/src/transformers/models/layoutlm/modeling_tf_layoutlm.py @@ -861,7 +861,7 @@ class TFLayoutLMPreTrainedModel(TFPreTrainedModel): input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`LayoutLMTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) diff --git a/src/transformers/models/layoutlmv2/modeling_layoutlmv2.py b/src/transformers/models/layoutlmv2/modeling_layoutlmv2.py index be31af99d6dfd8..af792a5e82d806 100755 --- a/src/transformers/models/layoutlmv2/modeling_layoutlmv2.py +++ b/src/transformers/models/layoutlmv2/modeling_layoutlmv2.py @@ -52,7 +52,6 @@ _CHECKPOINT_FOR_DOC = "microsoft/layoutlmv2-base-uncased" _CONFIG_FOR_DOC = "LayoutLMv2Config" -_TOKENIZER_FOR_DOC = "LayoutLMv2Tokenizer" LAYOUTLMV2_PRETRAINED_MODEL_ARCHIVE_LIST = [ "microsoft/layoutlmv2-base-uncased", @@ -633,7 +632,7 @@ def synchronize_batch_norm(self): input_ids (`torch.LongTensor` of shape `{0}`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`LayoutLMv2Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -837,14 +836,14 @@ def forward( Examples: ```python - >>> from transformers import LayoutLMv2Processor, LayoutLMv2Model, set_seed + >>> from transformers import AutoProcessor, LayoutLMv2Model, set_seed >>> from PIL import Image >>> import torch >>> from datasets import load_dataset >>> set_seed(88) - >>> processor = LayoutLMv2Processor.from_pretrained("microsoft/layoutlmv2-base-uncased") + >>> processor = AutoProcessor.from_pretrained("microsoft/layoutlmv2-base-uncased") >>> model = LayoutLMv2Model.from_pretrained("microsoft/layoutlmv2-base-uncased") @@ -1008,7 +1007,7 @@ def forward( Example: ```python - >>> from transformers import LayoutLMv2Processor, LayoutLMv2ForSequenceClassification, set_seed + >>> from transformers import AutoProcessor, LayoutLMv2ForSequenceClassification, set_seed >>> from PIL import Image >>> import torch >>> from datasets import load_dataset @@ -1019,7 +1018,7 @@ def forward( >>> data = next(iter(dataset)) >>> image = data["image"].convert("RGB") - >>> processor = LayoutLMv2Processor.from_pretrained("microsoft/layoutlmv2-base-uncased") + >>> processor = AutoProcessor.from_pretrained("microsoft/layoutlmv2-base-uncased") >>> model = LayoutLMv2ForSequenceClassification.from_pretrained( ... "microsoft/layoutlmv2-base-uncased", num_labels=dataset.info.features["label"].num_classes ... ) @@ -1187,7 +1186,7 @@ def forward( Example: ```python - >>> from transformers import LayoutLMv2Processor, LayoutLMv2ForTokenClassification, set_seed + >>> from transformers import AutoProcessor, LayoutLMv2ForTokenClassification, set_seed >>> from PIL import Image >>> from datasets import load_dataset @@ -1197,7 +1196,7 @@ def forward( >>> labels = datasets.features["ner_tags"].feature.names >>> id2label = {v: k for v, k in enumerate(labels)} - >>> processor = LayoutLMv2Processor.from_pretrained("microsoft/layoutlmv2-base-uncased", revision="no_ocr") + >>> processor = AutoProcessor.from_pretrained("microsoft/layoutlmv2-base-uncased", revision="no_ocr") >>> model = LayoutLMv2ForTokenClassification.from_pretrained( ... "microsoft/layoutlmv2-base-uncased", num_labels=len(labels) ... ) @@ -1328,13 +1327,13 @@ def forward( a prediction of what it thinks the answer is (the span of the answer within the texts parsed from the image). ```python - >>> from transformers import LayoutLMv2Processor, LayoutLMv2ForQuestionAnswering, set_seed + >>> from transformers import AutoProcessor, LayoutLMv2ForQuestionAnswering, set_seed >>> import torch >>> from PIL import Image >>> from datasets import load_dataset >>> set_seed(88) - >>> processor = LayoutLMv2Processor.from_pretrained("microsoft/layoutlmv2-base-uncased") + >>> processor = AutoProcessor.from_pretrained("microsoft/layoutlmv2-base-uncased") >>> model = LayoutLMv2ForQuestionAnswering.from_pretrained("microsoft/layoutlmv2-base-uncased") >>> dataset = load_dataset("hf-internal-testing/fixtures_docvqa") diff --git a/src/transformers/models/layoutlmv3/modeling_layoutlmv3.py b/src/transformers/models/layoutlmv3/modeling_layoutlmv3.py index 6f1e37aacc01aa..56e4fa1e68a2b0 100644 --- a/src/transformers/models/layoutlmv3/modeling_layoutlmv3.py +++ b/src/transformers/models/layoutlmv3/modeling_layoutlmv3.py @@ -68,7 +68,7 @@ Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS] token. See `pixel_values` for `patch_sequence_length`. - Indices can be obtained using [`LayoutLMv3Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -141,7 +141,7 @@ input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`LayoutLMv3Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) diff --git a/src/transformers/models/layoutlmv3/modeling_tf_layoutlmv3.py b/src/transformers/models/layoutlmv3/modeling_tf_layoutlmv3.py index 242d8d3983d618..95ef5580b95409 100644 --- a/src/transformers/models/layoutlmv3/modeling_tf_layoutlmv3.py +++ b/src/transformers/models/layoutlmv3/modeling_tf_layoutlmv3.py @@ -1067,7 +1067,7 @@ def serving(self, inputs): Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS] token. See `pixel_values` for `patch_sequence_length`. - Indices can be obtained using [`LayoutLMv3Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) diff --git a/src/transformers/models/led/modeling_led.py b/src/transformers/models/led/modeling_led.py index dff90268cf557b..a4e246f26bbfd4 100755 --- a/src/transformers/models/led/modeling_led.py +++ b/src/transformers/models/led/modeling_led.py @@ -51,7 +51,6 @@ _CHECKPOINT_FOR_DOC = "allenai/led-base-16384" _CONFIG_FOR_DOC = "LEDConfig" -_TOKENIZER_FOR_DOC = "LEDTokenizer" LED_PRETRAINED_MODEL_ARCHIVE_LIST = [ @@ -1507,10 +1506,10 @@ class LEDSeq2SeqQuestionAnsweringModelOutput(ModelOutput): ```python >>> import torch - >>> from transformers import LEDTokenizer, LEDForConditionalGeneration + >>> from transformers import AutoTokenizer, LEDForConditionalGeneration >>> model = LEDForConditionalGeneration.from_pretrained("allenai/led-large-16384-arxiv") - >>> tokenizer = LEDTokenizer.from_pretrained("allenai/led-large-16384-arxiv") + >>> tokenizer = AutoTokenizer.from_pretrained("allenai/led-large-16384-arxiv") >>> ARTICLE_TO_SUMMARIZE = '''Transformers (Vaswani et al., 2017) have achieved state-of-the-art ... results in a wide range of natural language tasks including generative language modeling @@ -1546,7 +1545,7 @@ class LEDSeq2SeqQuestionAnsweringModelOutput(ModelOutput): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`LEDTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1764,7 +1763,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`LEDTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1992,7 +1991,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`LEDTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -2239,7 +2238,6 @@ def get_decoder(self): @add_start_docstrings_to_model_forward(LED_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC, @@ -2413,9 +2411,9 @@ def forward( Conditional generation example: ```python - >>> from transformers import LEDTokenizer, LEDForConditionalGeneration + >>> from transformers import AutoTokenizer, LEDForConditionalGeneration - >>> tokenizer = LEDTokenizer.from_pretrained("allenai/led-base-16384") + >>> tokenizer = AutoTokenizer.from_pretrained("allenai/led-base-16384") >>> TXT = "My friends are but they eat too many carbs." >>> model = LEDForConditionalGeneration.from_pretrained("allenai/led-base-16384") @@ -2551,7 +2549,6 @@ def __init__(self, config: LEDConfig, **kwargs): @add_start_docstrings_to_model_forward(LED_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=Seq2SeqSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -2680,7 +2677,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(LED_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=Seq2SeqQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/led/modeling_tf_led.py b/src/transformers/models/led/modeling_tf_led.py index 0db380831b5e2b..e45d94c742f8e9 100644 --- a/src/transformers/models/led/modeling_tf_led.py +++ b/src/transformers/models/led/modeling_tf_led.py @@ -50,7 +50,6 @@ _CHECKPOINT_FOR_DOC = "allenai/led-base-16384" _CONFIG_FOR_DOC = "LEDConfig" -_TOKENIZER_FOR_DOC = "LEDTokenizer" LARGE_NEGATIVE = -1e8 @@ -1704,7 +1703,7 @@ def call( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`LEDTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1967,7 +1966,7 @@ def call( Args: input_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you - provide it. Indices can be obtained using [`LEDTokenizer`]. See [`PreTrainedTokenizer.encode`] and + provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: @@ -2269,7 +2268,6 @@ def get_decoder(self): @unpack_inputs @add_start_docstrings_to_model_forward(LED_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFLEDSeq2SeqModelOutput, config_class=_CONFIG_FOR_DOC, @@ -2430,11 +2428,11 @@ def call( Examples: ```python - >>> from transformers import LEDTokenizer, TFLEDForConditionalGeneration + >>> from transformers import AutoTokenizer, TFLEDForConditionalGeneration >>> import tensorflow as tf >>> mname = "allenai/led-base-16384" - >>> tokenizer = LEDTokenizer.from_pretrained(mname) + >>> tokenizer = AutoTokenizer.from_pretrained(mname) >>> TXT = "My friends are but they eat too many carbs." >>> model = TFLEDForConditionalGeneration.from_pretrained(mname) >>> batch = tokenizer([TXT], return_tensors="tf") diff --git a/src/transformers/models/levit/modeling_levit.py b/src/transformers/models/levit/modeling_levit.py index d350e83d2cbd48..e45ffa05b15768 100644 --- a/src/transformers/models/levit/modeling_levit.py +++ b/src/transformers/models/levit/modeling_levit.py @@ -523,7 +523,7 @@ def _set_gradient_checkpointing(self, module, value=False): Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + [`LevitImageProcessor.__call__`] for details. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for diff --git a/src/transformers/models/lilt/modeling_lilt.py b/src/transformers/models/lilt/modeling_lilt.py index 6859aff7e632d1..b372ddd9fca545 100644 --- a/src/transformers/models/lilt/modeling_lilt.py +++ b/src/transformers/models/lilt/modeling_lilt.py @@ -641,7 +641,7 @@ def update_keys_to_ignore(self, config, del_keys_to_ignore): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`RobertaTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) diff --git a/src/transformers/models/longformer/modeling_longformer.py b/src/transformers/models/longformer/modeling_longformer.py index 606459067df5f3..7c0c03827edfce 100755 --- a/src/transformers/models/longformer/modeling_longformer.py +++ b/src/transformers/models/longformer/modeling_longformer.py @@ -1469,7 +1469,7 @@ def _set_gradient_checkpointing(self, module, value=False): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`LongformerTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1671,10 +1671,10 @@ def forward( ```python >>> import torch - >>> from transformers import LongformerModel, LongformerTokenizer + >>> from transformers import LongformerModel, AutoTokenizer >>> model = LongformerModel.from_pretrained("allenai/longformer-base-4096") - >>> tokenizer = LongformerTokenizer.from_pretrained("allenai/longformer-base-4096") + >>> tokenizer = AutoTokenizer.from_pretrained("allenai/longformer-base-4096") >>> SAMPLE_TEXT = " ".join(["Hello world! "] * 1000) # long input document >>> input_ids = torch.tensor(tokenizer.encode(SAMPLE_TEXT)).unsqueeze(0) # batch of size 1 @@ -1820,9 +1820,9 @@ def forward( Mask filling example: ```python - >>> from transformers import LongformerTokenizer, LongformerForMaskedLM + >>> from transformers import AutoTokenizer, LongformerForMaskedLM - >>> tokenizer = LongformerTokenizer.from_pretrained("allenai/longformer-base-4096") + >>> tokenizer = AutoTokenizer.from_pretrained("allenai/longformer-base-4096") >>> model = LongformerForMaskedLM.from_pretrained("allenai/longformer-base-4096") ``` @@ -2059,10 +2059,10 @@ def forward( Examples: ```python - >>> from transformers import LongformerTokenizer, LongformerForQuestionAnswering + >>> from transformers import AutoTokenizer, LongformerForQuestionAnswering >>> import torch - >>> tokenizer = LongformerTokenizer.from_pretrained("allenai/longformer-large-4096-finetuned-triviaqa") + >>> tokenizer = AutoTokenizer.from_pretrained("allenai/longformer-large-4096-finetuned-triviaqa") >>> model = LongformerForQuestionAnswering.from_pretrained("allenai/longformer-large-4096-finetuned-triviaqa") >>> question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet" diff --git a/src/transformers/models/longformer/modeling_tf_longformer.py b/src/transformers/models/longformer/modeling_tf_longformer.py index f4fb11c485ce7c..bb282ed81a0b3f 100644 --- a/src/transformers/models/longformer/modeling_tf_longformer.py +++ b/src/transformers/models/longformer/modeling_tf_longformer.py @@ -50,7 +50,6 @@ _CHECKPOINT_FOR_DOC = "allenai/longformer-base-4096" _CONFIG_FOR_DOC = "LongformerConfig" -_TOKENIZER_FOR_DOC = "LongformerTokenizer" LARGE_NEGATIVE = -1e8 @@ -1960,7 +1959,7 @@ def serving(self, inputs): input_ids (`np.ndarray` or `tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`LongformerTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -2119,7 +2118,6 @@ def get_prefix_bias_name(self): @unpack_inputs @add_start_docstrings_to_model_forward(LONGFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="allenai/longformer-base-4096", output_type=TFLongformerMaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -2214,7 +2212,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(LONGFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="allenai/longformer-large-4096-finetuned-triviaqa", output_type=TFLongformerQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, @@ -2380,7 +2377,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(LONGFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFLongformerSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -2504,7 +2500,6 @@ def dummy_inputs(self): LONGFORMER_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFLongformerMultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -2633,7 +2628,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(LONGFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFLongformerTokenClassifierOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/longt5/modeling_flax_longt5.py b/src/transformers/models/longt5/modeling_flax_longt5.py index 6e4558f3ff313c..de3e43f02cc5fd 100644 --- a/src/transformers/models/longt5/modeling_flax_longt5.py +++ b/src/transformers/models/longt5/modeling_flax_longt5.py @@ -52,7 +52,6 @@ _CHECKPOINT_FOR_DOC = "google/long-t5-local-base" _CONFIG_FOR_DOC = "LongT5Config" -_TOKENIZER_FOR_DOC = "T5Tokenizer" remat = nn_partitioning.remat @@ -1541,7 +1540,7 @@ def __call__( Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so you should be able to pad the inputs on both the right and the left. - Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail. To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5 @@ -1568,7 +1567,7 @@ def __call__( decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -1611,7 +1610,7 @@ def __call__( Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so you should be able to pad the inputs on both the right and the left. - Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail. [What are input IDs?](../glossary#input-ids) @@ -1628,7 +1627,7 @@ def __call__( decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -1828,9 +1827,9 @@ def encode( Example: ```python - >>> from transformers import T5Tokenizer, FlaxLongT5ForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxLongT5ForConditionalGeneration - >>> tokenizer = T5Tokenizer.from_pretrained("t5-base") + >>> tokenizer = AutoTokenizer.from_pretrained("t5-base") >>> model = FlaxLongT5ForConditionalGeneration.from_pretrained("google/long-t5-local-base") >>> text = "My friends are cool but they eat too many carbs." @@ -1889,10 +1888,10 @@ def decode( Example: ```python - >>> from transformers import T5Tokenizer, FlaxLongT5ForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxLongT5ForConditionalGeneration >>> import jax.numpy as jnp - >>> tokenizer = T5Tokenizer.from_pretrained("t5-base") + >>> tokenizer = AutoTokenizer.from_pretrained("t5-base") >>> model = FlaxLongT5ForConditionalGeneration.from_pretrained("google/long-t5-local-base") >>> text = "My friends are cool but they eat too many carbs." @@ -2111,9 +2110,7 @@ class FlaxLongT5Model(FlaxLongT5PreTrainedModel): module_class = FlaxLongT5Module -append_call_sample_docstring( - FlaxLongT5Model, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqModelOutput, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxLongT5Model, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqModelOutput, _CONFIG_FOR_DOC) FLAX_LONGT5_MODEL_DOCSTRING = """ Returns: @@ -2121,9 +2118,9 @@ class FlaxLongT5Model(FlaxLongT5PreTrainedModel): Example: ```python - >>> from transformers import T5Tokenizer, FlaxLongT5Model + >>> from transformers import AutoTokenizer, FlaxLongT5Model - >>> tokenizer = T5Tokenizer.from_pretrained("t5-base") + >>> tokenizer = AutoTokenizer.from_pretrained("t5-base") >>> model = FlaxLongT5Model.from_pretrained("google/long-t5-local-base") >>> input_ids = tokenizer( @@ -2279,10 +2276,10 @@ def decode( Example: ```python - >>> from transformers import T5Tokenizer, FlaxLongT5ForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxLongT5ForConditionalGeneration >>> import jax.numpy as jnp - >>> tokenizer = T5Tokenizer.from_pretrained("t5-base") + >>> tokenizer = AutoTokenizer.from_pretrained("t5-base") >>> model = FlaxLongT5ForConditionalGeneration.from_pretrained("google/long-t5-local-base") >>> text = "summarize: My friends are cool but they eat too many carbs." @@ -2428,9 +2425,9 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): Example: ```python - >>> from transformers import T5Tokenizer, FlaxLongT5ForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxLongT5ForConditionalGeneration - >>> tokenizer = T5Tokenizer.from_pretrained("t5-base") + >>> tokenizer = AutoTokenizer.from_pretrained("t5-base") >>> model = FlaxLongT5ForConditionalGeneration.from_pretrained("google/long-t5-local-base") >>> ARTICLE_TO_SUMMARIZE = "summarize: My friends are cool but they eat too many carbs." diff --git a/src/transformers/models/longt5/modeling_longt5.py b/src/transformers/models/longt5/modeling_longt5.py index 196f26f9d38add..2101d247c1fb61 100644 --- a/src/transformers/models/longt5/modeling_longt5.py +++ b/src/transformers/models/longt5/modeling_longt5.py @@ -49,7 +49,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "LongT5Config" -_TOKENIZER_FOR_DOC = "T5Tokenizer" _CHECKPOINT_FOR_DOC = "google/long-t5-local-base" # TODO: Update before the merge @@ -1611,7 +1610,7 @@ def custom_forward(*inputs): Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so you should be able to pad the inputs on both the right and the left. - Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail. [What are input IDs?](../glossary#input-ids) @@ -1628,7 +1627,7 @@ def custom_forward(*inputs): decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -1706,7 +1705,7 @@ def custom_forward(*inputs): Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so you should be able to pad the inputs on both the right and the left. - Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail. To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5 @@ -1827,9 +1826,9 @@ def forward( Example: ```python - >>> from transformers import T5Tokenizer, LongT5Model + >>> from transformers import AutoTokenizer, LongT5Model - >>> tokenizer = T5Tokenizer.from_pretrained("google/long-t5-local-base") + >>> tokenizer = AutoTokenizer.from_pretrained("google/long-t5-local-base") >>> model = LongT5Model.from_pretrained("google/long-t5-local-base") >>> # Let's try a very long encoder input. diff --git a/src/transformers/models/luke/modeling_luke.py b/src/transformers/models/luke/modeling_luke.py index a1f9f3cbd915fe..25427f8a63a3cd 100644 --- a/src/transformers/models/luke/modeling_luke.py +++ b/src/transformers/models/luke/modeling_luke.py @@ -41,7 +41,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "LukeConfig" -_TOKENIZER_FOR_DOC = "LukeTokenizer" _CHECKPOINT_FOR_DOC = "studio-ousia/luke-base" LUKE_PRETRAINED_MODEL_ARCHIVE_LIST = [ @@ -947,7 +946,7 @@ def _set_gradient_checkpointing(self, module, value=False): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`LukeTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -975,7 +974,7 @@ def _set_gradient_checkpointing(self, module, value=False): entity_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`): Indices of entity tokens in the entity vocabulary. - Indices can be obtained using [`LukeTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. entity_attention_mask (`torch.FloatTensor` of shape `(batch_size, entity_length)`, *optional*): @@ -1079,9 +1078,9 @@ def forward( Examples: ```python - >>> from transformers import LukeTokenizer, LukeModel + >>> from transformers import AutoTokenizer, LukeModel - >>> tokenizer = LukeTokenizer.from_pretrained("studio-ousia/luke-base") + >>> tokenizer = AutoTokenizer.from_pretrained("studio-ousia/luke-base") >>> model = LukeModel.from_pretrained("studio-ousia/luke-base") # Compute the contextualized entity representation corresponding to the entity mention "Beyoncé" @@ -1467,9 +1466,9 @@ def forward( Examples: ```python - >>> from transformers import LukeTokenizer, LukeForEntityClassification + >>> from transformers import AutoTokenizer, LukeForEntityClassification - >>> tokenizer = LukeTokenizer.from_pretrained("studio-ousia/luke-large-finetuned-open-entity") + >>> tokenizer = AutoTokenizer.from_pretrained("studio-ousia/luke-large-finetuned-open-entity") >>> model = LukeForEntityClassification.from_pretrained("studio-ousia/luke-large-finetuned-open-entity") >>> text = "Beyoncé lives in Los Angeles." @@ -1580,9 +1579,9 @@ def forward( Examples: ```python - >>> from transformers import LukeTokenizer, LukeForEntityPairClassification + >>> from transformers import AutoTokenizer, LukeForEntityPairClassification - >>> tokenizer = LukeTokenizer.from_pretrained("studio-ousia/luke-large-finetuned-tacred") + >>> tokenizer = AutoTokenizer.from_pretrained("studio-ousia/luke-large-finetuned-tacred") >>> model = LukeForEntityPairClassification.from_pretrained("studio-ousia/luke-large-finetuned-tacred") >>> text = "Beyoncé lives in Los Angeles." @@ -1706,9 +1705,9 @@ def forward( Examples: ```python - >>> from transformers import LukeTokenizer, LukeForEntitySpanClassification + >>> from transformers import AutoTokenizer, LukeForEntitySpanClassification - >>> tokenizer = LukeTokenizer.from_pretrained("studio-ousia/luke-large-finetuned-conll-2003") + >>> tokenizer = AutoTokenizer.from_pretrained("studio-ousia/luke-large-finetuned-conll-2003") >>> model = LukeForEntitySpanClassification.from_pretrained("studio-ousia/luke-large-finetuned-conll-2003") >>> text = "Beyoncé lives in Los Angeles" @@ -1812,7 +1811,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(LUKE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=LukeSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1926,7 +1924,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(LUKE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=LukeTokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -2019,7 +2016,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(LUKE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=LukeQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, @@ -2140,7 +2136,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(LUKE_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=LukeMultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/lxmert/modeling_lxmert.py b/src/transformers/models/lxmert/modeling_lxmert.py index daf4abf67ccdd9..572fa30d8dd172 100644 --- a/src/transformers/models/lxmert/modeling_lxmert.py +++ b/src/transformers/models/lxmert/modeling_lxmert.py @@ -42,7 +42,6 @@ _CHECKPOINT_FOR_DOC = "unc-nlp/lxmert-base-uncased" _CONFIG_FOR_DOC = "LxmertConfig" -_TOKENIZER_FOR_DOC = "LxmertTokenizer" LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ "unc-nlp/lxmert-base-uncased", @@ -827,7 +826,7 @@ def _init_weights(self, module): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`LxmertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -900,7 +899,6 @@ def set_input_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(LXMERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=LxmertModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1386,7 +1384,6 @@ def _get_resized_qa_labels(self, cur_qa_logit_layer, num_labels): @add_start_docstrings_to_model_forward(LXMERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=LxmertForQuestionAnsweringOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/lxmert/modeling_tf_lxmert.py b/src/transformers/models/lxmert/modeling_tf_lxmert.py index 34cea3a6e98350..5510c7ff84525e 100644 --- a/src/transformers/models/lxmert/modeling_tf_lxmert.py +++ b/src/transformers/models/lxmert/modeling_tf_lxmert.py @@ -49,7 +49,6 @@ _CHECKPOINT_FOR_DOC = "unc-nlp/lxmert-base-uncased" _CONFIG_FOR_DOC = "LxmertConfig" -_TOKENIZER_FOR_DOC = "LxmertTokenizer" TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ "unc-nlp/lxmert-base-uncased", @@ -880,7 +879,7 @@ def serving(self, inputs): input_ids (`np.ndarray` or `tf.Tensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`LxmertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -950,7 +949,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(LXMERT_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFLxmertModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/m2m_100/modeling_m2m_100.py b/src/transformers/models/m2m_100/modeling_m2m_100.py index 1a86e3e07d3279..cdd746443949c4 100755 --- a/src/transformers/models/m2m_100/modeling_m2m_100.py +++ b/src/transformers/models/m2m_100/modeling_m2m_100.py @@ -46,7 +46,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "M2M100Config" -_TOKENIZER_FOR_DOC = "M2M100Tokenizer" _CHECKPOINT_FOR_DOC = "facebook/m2m100_418M" @@ -577,10 +576,10 @@ def _set_gradient_checkpointing(self, module, value=False): Translation example: ```python - >>> from transformers import M2M100Tokenizer, M2M100ForConditionalGeneration + >>> from transformers import AutoTokenizer, M2M100ForConditionalGeneration >>> model = M2M100ForConditionalGeneration.from_pretrained("facebook/m2m100_418M") - >>> tokenizer = M2M100Tokenizer.from_pretrained("facebook/m2m100_418M") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/m2m100_418M") >>> text_to_translate = "Life is like a box of chocolates" >>> model_inputs = tokenizer(text_to_translate, return_tensors="pt") @@ -597,7 +596,7 @@ def _set_gradient_checkpointing(self, module, value=False): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`M2M100Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -611,7 +610,7 @@ def _set_gradient_checkpointing(self, module, value=False): decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`M2M100Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -734,7 +733,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`M2M100Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -915,7 +914,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`M2M100Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1172,7 +1171,6 @@ def get_decoder(self): @add_start_docstrings_to_model_forward(M2M_100_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/marian/modeling_flax_marian.py b/src/transformers/models/marian/modeling_flax_marian.py index 064879bc402a4d..db543ef8d9c940 100644 --- a/src/transformers/models/marian/modeling_flax_marian.py +++ b/src/transformers/models/marian/modeling_flax_marian.py @@ -53,7 +53,6 @@ _CHECKPOINT_FOR_DOC = "Helsinki-NLP/opus-mt-en-de" _CONFIG_FOR_DOC = "MarianConfig" -_TOKENIZER_FOR_DOC = "MarianTokenizer" MARIAN_START_DOCSTRING = r""" @@ -96,7 +95,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -110,7 +109,7 @@ decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -147,7 +146,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -176,7 +175,7 @@ decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -981,9 +980,9 @@ def encode( Example: ```python - >>> from transformers import MarianTokenizer, FlaxMarianMTModel + >>> from transformers import AutoTokenizer, FlaxMarianMTModel - >>> tokenizer = MarianTokenizer.from_pretrained("Helsinki-NLP/opus-mt-en-de") + >>> tokenizer = AutoTokenizer.from_pretrained("Helsinki-NLP/opus-mt-en-de") >>> model = FlaxMarianMTModel.from_pretrained("Helsinki-NLP/opus-mt-en-de") >>> text = "My friends are cool but they eat too many carbs." @@ -1049,9 +1048,9 @@ def decode( ```python >>> import jax.numpy as jnp - >>> from transformers import MarianTokenizer, FlaxMarianMTModel + >>> from transformers import AutoTokenizer, FlaxMarianMTModel - >>> tokenizer = MarianTokenizer.from_pretrained("Helsinki-NLP/opus-mt-en-de") + >>> tokenizer = AutoTokenizer.from_pretrained("Helsinki-NLP/opus-mt-en-de") >>> model = FlaxMarianMTModel.from_pretrained("Helsinki-NLP/opus-mt-en-de") >>> text = "My friends are cool but they eat too many carbs." @@ -1210,9 +1209,7 @@ class FlaxMarianModel(FlaxMarianPreTrainedModel): module_class = FlaxMarianModule -append_call_sample_docstring( - FlaxMarianModel, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqModelOutput, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxMarianModel, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqModelOutput, _CONFIG_FOR_DOC) class FlaxMarianMTModule(nn.Module): @@ -1318,10 +1315,10 @@ def decode( ```python >>> import jax.numpy as jnp - >>> from transformers import MarianTokenizer, FlaxMarianMTModel + >>> from transformers import AutoTokenizer, FlaxMarianMTModel >>> model = FlaxMarianMTModel.from_pretrained("Helsinki-NLP/opus-mt-en-de") - >>> tokenizer = MarianTokenizer.from_pretrained("Helsinki-NLP/opus-mt-en-de") + >>> tokenizer = AutoTokenizer.from_pretrained("Helsinki-NLP/opus-mt-en-de") >>> text = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, max_length=64, return_tensors="jax") @@ -1479,10 +1476,10 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): Example: ```python - >>> from transformers import MarianTokenizer, FlaxMarianMTModel + >>> from transformers import AutoTokenizer, FlaxMarianMTModel >>> model = FlaxMarianMTModel.from_pretrained("Helsinki-NLP/opus-mt-en-de") - >>> tokenizer = MarianTokenizer.from_pretrained("Helsinki-NLP/opus-mt-en-de") + >>> tokenizer = AutoTokenizer.from_pretrained("Helsinki-NLP/opus-mt-en-de") >>> text = "My friends are cool but they eat too many carbs." >>> input_ids = tokenizer(text, max_length=64, return_tensors="jax").input_ids diff --git a/src/transformers/models/marian/modeling_marian.py b/src/transformers/models/marian/modeling_marian.py index c408abf805b4d2..3937058a566016 100755 --- a/src/transformers/models/marian/modeling_marian.py +++ b/src/transformers/models/marian/modeling_marian.py @@ -48,7 +48,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "MarianConfig" -_TOKENIZER_FOR_DOC = "MarianTokenizer" _CHECKPOINT_FOR_DOC = "Helsinki-NLP/opus-mt-en-de" @@ -539,14 +538,14 @@ def dummy_inputs(self): Examples: ```python - >>> from transformers import MarianTokenizer, MarianMTModel + >>> from transformers import AutoTokenizer, MarianMTModel >>> src = "fr" # source language >>> trg = "en" # target language >>> model_name = f"Helsinki-NLP/opus-mt-{src}-{trg}" >>> model = MarianMTModel.from_pretrained(model_name) - >>> tokenizer = MarianTokenizer.from_pretrained(model_name) + >>> tokenizer = AutoTokenizer.from_pretrained(model_name) >>> sample_text = "où est l'arrêt de bus ?" >>> batch = tokenizer([sample_text], return_tensors="pt") @@ -563,7 +562,7 @@ def dummy_inputs(self): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -577,7 +576,7 @@ def dummy_inputs(self): decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -705,7 +704,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -897,7 +896,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1203,9 +1202,9 @@ def forward( Example: ```python - >>> from transformers import MarianTokenizer, MarianModel + >>> from transformers import AutoTokenizer, MarianModel - >>> tokenizer = MarianTokenizer.from_pretrained("Helsinki-NLP/opus-mt-en-de") + >>> tokenizer = AutoTokenizer.from_pretrained("Helsinki-NLP/opus-mt-en-de") >>> model = MarianModel.from_pretrained("Helsinki-NLP/opus-mt-en-de") >>> inputs = tokenizer("Studies have been shown that owning a dog is good for you", return_tensors="pt") @@ -1606,7 +1605,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1671,9 +1670,9 @@ def forward( Example: ```python - >>> from transformers import MarianTokenizer, MarianForCausalLM + >>> from transformers import AutoTokenizer, MarianForCausalLM - >>> tokenizer = MarianTokenizer.from_pretrained("Helsinki-NLP/opus-mt-fr-en") + >>> tokenizer = AutoTokenizer.from_pretrained("Helsinki-NLP/opus-mt-fr-en") >>> model = MarianForCausalLM.from_pretrained("Helsinki-NLP/opus-mt-fr-en", add_cross_attention=False) >>> assert model.config.is_decoder, f"{model.__class__} has to be configured as a decoder." >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") diff --git a/src/transformers/models/marian/modeling_tf_marian.py b/src/transformers/models/marian/modeling_tf_marian.py index b93680d84f39f4..17a4f0f5d1b587 100644 --- a/src/transformers/models/marian/modeling_tf_marian.py +++ b/src/transformers/models/marian/modeling_tf_marian.py @@ -54,7 +54,6 @@ _CHECKPOINT_FOR_DOC = "Helsinki-NLP/opus-mt-en-de" _CONFIG_FOR_DOC = "MarianConfig" -_TOKENIZER_FOR_DOC = "MarianTokenizer" LARGE_NEGATIVE = -1e8 @@ -577,7 +576,7 @@ def serving(self, inputs): Examples: ```python - >>> from transformers import MarianTokenizer, TFMarianMTModel + >>> from transformers import AutoTokenizer, TFMarianMTModel >>> from typing import List >>> src = "fr" # source language @@ -586,7 +585,7 @@ def serving(self, inputs): >>> model_name = f"Helsinki-NLP/opus-mt-{src}-{trg}" >>> model = TFMarianMTModel.from_pretrained(model_name) - >>> tokenizer = MarianTokenizer.from_pretrained(model_name) + >>> tokenizer = AutoTokenizer.from_pretrained(model_name) >>> batch = tokenizer([sample_text], return_tensors="tf") >>> gen = model.generate(**batch) >>> tokenizer.batch_decode(gen, skip_special_tokens=True) @@ -599,7 +598,7 @@ def serving(self, inputs): input_ids (`tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -613,7 +612,7 @@ def serving(self, inputs): decoder_input_ids (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -724,7 +723,7 @@ def call( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -902,7 +901,7 @@ def call( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1217,7 +1216,6 @@ def get_decoder(self): @unpack_inputs @add_start_docstrings_to_model_forward(MARIAN_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFSeq2SeqModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/markuplm/modeling_markuplm.py b/src/transformers/models/markuplm/modeling_markuplm.py index d1c7962ef4b405..b9a3b70326eac5 100755 --- a/src/transformers/models/markuplm/modeling_markuplm.py +++ b/src/transformers/models/markuplm/modeling_markuplm.py @@ -52,7 +52,6 @@ _CHECKPOINT_FOR_DOC = "microsoft/markuplm-base" _CONFIG_FOR_DOC = "MarkupLMConfig" -_TOKENIZER_FOR_DOC = "MarkupLMTokenizer" MARKUPLM_PRETRAINED_MODEL_ARCHIVE_LIST = [ "microsoft/markuplm-base", @@ -756,7 +755,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`MarkupLMTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -854,9 +853,9 @@ def forward( Examples: ```python - >>> from transformers import MarkupLMProcessor, MarkupLMModel + >>> from transformers import AutoProcessor, MarkupLMModel - >>> processor = MarkupLMProcessor.from_pretrained("microsoft/markuplm-base") + >>> processor = AutoProcessor.from_pretrained("microsoft/markuplm-base") >>> model = MarkupLMModel.from_pretrained("microsoft/markuplm-base") >>> html_string = " Page Title " @@ -1018,10 +1017,10 @@ def forward( Examples: ```python - >>> from transformers import MarkupLMProcessor, MarkupLMForQuestionAnswering + >>> from transformers import AutoProcessor, MarkupLMForQuestionAnswering >>> import torch - >>> processor = MarkupLMProcessor.from_pretrained("microsoft/markuplm-base-finetuned-websrc") + >>> processor = AutoProcessor.from_pretrained("microsoft/markuplm-base-finetuned-websrc") >>> model = MarkupLMForQuestionAnswering.from_pretrained("microsoft/markuplm-base-finetuned-websrc") >>> html_string = " My name is Niels " diff --git a/src/transformers/models/mask2former/modeling_mask2former.py b/src/transformers/models/mask2former/modeling_mask2former.py index 5e1ab60ad93dd5..abcba3b65a36e0 100644 --- a/src/transformers/models/mask2former/modeling_mask2former.py +++ b/src/transformers/models/mask2former/modeling_mask2former.py @@ -49,7 +49,6 @@ _CONFIG_FOR_DOC = "Mask2FormerConfig" _CHECKPOINT_FOR_DOC = "facebook/mask2former-swin-small-coco-instance" -_IMAGE_PROCESSOR_FOR_DOC = "MaskFormerImageProcessor" MASK2FORMER_PRETRAINED_MODEL_ARCHIVE_LIST = [ "facebook/mask2former-swin-small-coco-instance", diff --git a/src/transformers/models/maskformer/modeling_maskformer.py b/src/transformers/models/maskformer/modeling_maskformer.py index 00a3e53d56206c..1482b76f209cb1 100644 --- a/src/transformers/models/maskformer/modeling_maskformer.py +++ b/src/transformers/models/maskformer/modeling_maskformer.py @@ -1462,7 +1462,7 @@ def forward( Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + [`MaskFormerImageProcessor.__call__`] for details. pixel_mask (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: @@ -1561,12 +1561,12 @@ def forward( Examples: ```python - >>> from transformers import MaskFormerImageProcessor, MaskFormerModel + >>> from transformers import AutoImageProcessor, MaskFormerModel >>> from PIL import Image >>> import requests >>> # load MaskFormer fine-tuned on ADE20k semantic segmentation - >>> image_processor = MaskFormerImageProcessor.from_pretrained("facebook/maskformer-swin-base-ade") + >>> image_processor = AutoImageProcessor.from_pretrained("facebook/maskformer-swin-base-ade") >>> model = MaskFormerModel.from_pretrained("facebook/maskformer-swin-base-ade") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" @@ -1740,12 +1740,12 @@ def forward( Semantic segmentation example: ```python - >>> from transformers import MaskFormerImageProcessor, MaskFormerForInstanceSegmentation + >>> from transformers import AutoImageProcessor, MaskFormerForInstanceSegmentation >>> from PIL import Image >>> import requests >>> # load MaskFormer fine-tuned on ADE20k semantic segmentation - >>> image_processor = MaskFormerImageProcessor.from_pretrained("facebook/maskformer-swin-base-ade") + >>> image_processor = AutoImageProcessor.from_pretrained("facebook/maskformer-swin-base-ade") >>> model = MaskFormerForInstanceSegmentation.from_pretrained("facebook/maskformer-swin-base-ade") >>> url = ( @@ -1773,12 +1773,12 @@ def forward( Panoptic segmentation example: ```python - >>> from transformers import MaskFormerImageProcessor, MaskFormerForInstanceSegmentation + >>> from transformers import AutoImageProcessor, MaskFormerForInstanceSegmentation >>> from PIL import Image >>> import requests >>> # load MaskFormer fine-tuned on COCO panoptic segmentation - >>> image_processor = MaskFormerImageProcessor.from_pretrained("facebook/maskformer-swin-base-coco") + >>> image_processor = AutoImageProcessor.from_pretrained("facebook/maskformer-swin-base-coco") >>> model = MaskFormerForInstanceSegmentation.from_pretrained("facebook/maskformer-swin-base-coco") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" diff --git a/src/transformers/models/mbart/modeling_flax_mbart.py b/src/transformers/models/mbart/modeling_flax_mbart.py index 4d3156b9d15e11..42660e260754c6 100644 --- a/src/transformers/models/mbart/modeling_flax_mbart.py +++ b/src/transformers/models/mbart/modeling_flax_mbart.py @@ -55,7 +55,6 @@ _CHECKPOINT_FOR_DOC = "facebook/mbart-large-cc25" _CONFIG_FOR_DOC = "MBartConfig" -_TOKENIZER_FOR_DOC = "MBartTokenizer" MBART_START_DOCSTRING = r""" @@ -98,7 +97,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -112,7 +111,7 @@ decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -149,7 +148,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -178,7 +177,7 @@ decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -1057,10 +1056,10 @@ def encode( Example: ```python - >>> from transformers import MBartTokenizer, FlaxMBartForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxMBartForConditionalGeneration >>> model = FlaxMBartForConditionalGeneration.from_pretrained("facebook/mbart-large-cc25") - >>> tokenizer = MBartTokenizer.from_pretrained("facebook/mbart-large-cc25") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-cc25") >>> text = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, max_length=1024, return_tensors="jax") @@ -1123,10 +1122,10 @@ def decode( Example: ```python - >>> from transformers import MBartTokenizer, FlaxMBartForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxMBartForConditionalGeneration >>> model = FlaxMBartForConditionalGeneration.from_pretrained("facebook/mbart-large-cc25") - >>> tokenizer = MBartTokenizer.from_pretrained("facebook/mbart-large-cc25") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-cc25") >>> text = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, max_length=1024, return_tensors="jax") @@ -1282,9 +1281,7 @@ class FlaxMBartModel(FlaxMBartPreTrainedModel): module_class = FlaxMBartModule -append_call_sample_docstring( - FlaxMBartModel, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqModelOutput, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxMBartModel, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqModelOutput, _CONFIG_FOR_DOC) # Copied from transformers.models.bart.modeling_flax_bart.FlaxBartForConditionalGenerationModule with Bart->MBart @@ -1390,10 +1387,10 @@ def decode( Example: ```python - >>> from transformers import MBartTokenizer, FlaxMBartForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxMBartForConditionalGeneration >>> model = FlaxMBartForConditionalGeneration.from_pretrained("facebook/mbart-large-cc25") - >>> tokenizer = MBartTokenizer.from_pretrained("facebook/mbart-large-cc25") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-cc25") >>> text = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, max_length=1024, return_tensors="jax") @@ -1546,10 +1543,10 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): Summarization example: ```python - >>> from transformers import MBartTokenizer, FlaxMBartForConditionalGeneration, MBartConfig + >>> from transformers import AutoTokenizer, FlaxMBartForConditionalGeneration, MBartConfig >>> model = FlaxMBartForConditionalGeneration.from_pretrained("facebook/mbart-large-cc25") - >>> tokenizer = MBartTokenizer.from_pretrained("facebook/mbart-large-cc25") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-cc25") >>> ARTICLE_TO_SUMMARIZE = "Meine Freunde sind cool, aber sie essen zu viel Kuchen." >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors="np") @@ -1562,10 +1559,10 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): Mask filling example: ```python - >>> from transformers import MBartTokenizer, FlaxMBartForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxMBartForConditionalGeneration >>> model = FlaxMBartForConditionalGeneration.from_pretrained("facebook/mbart-large-cc25") - >>> tokenizer = MBartTokenizer.from_pretrained("facebook/mbart-large-cc25") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-cc25") >>> # de_DE is the language symbol id for German >>> TXT = " Meine Freunde sind nett aber sie essen zu viel Kuchen. de_DE" @@ -1683,7 +1680,6 @@ class FlaxMBartForSequenceClassification(FlaxMBartPreTrainedModel): append_call_sample_docstring( FlaxMBartForSequenceClassification, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqSequenceClassifierOutput, _CONFIG_FOR_DOC, @@ -1771,7 +1767,6 @@ class FlaxMBartForQuestionAnswering(FlaxMBartPreTrainedModel): append_call_sample_docstring( FlaxMBartForQuestionAnswering, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqQuestionAnsweringModelOutput, _CONFIG_FOR_DOC, diff --git a/src/transformers/models/mbart/modeling_mbart.py b/src/transformers/models/mbart/modeling_mbart.py index 1cf3b00ce328c6..ca440038960791 100755 --- a/src/transformers/models/mbart/modeling_mbart.py +++ b/src/transformers/models/mbart/modeling_mbart.py @@ -549,10 +549,10 @@ def dummy_inputs(self): Translation example: ```python - >>> from transformers import MBartTokenizer, MBartForConditionalGeneration + >>> from transformers import AutoTokenizer, MBartForConditionalGeneration >>> model = MBartForConditionalGeneration.from_pretrained("facebook/mbart-large-en-ro") - >>> tokenizer = MBartTokenizer.from_pretrained("facebook/mbart-large-en-ro") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-en-ro") >>> example_english_phrase = "42 is the answer" >>> inputs = tokenizer(example_english_phrase, return_tensors="pt") @@ -566,10 +566,10 @@ def dummy_inputs(self): Mask filling example: ```python - >>> from transformers import MBartTokenizer, MBartForConditionalGeneration + >>> from transformers import AutoTokenizer, MBartForConditionalGeneration >>> model = MBartForConditionalGeneration.from_pretrained("facebook/mbart-large-cc25") - >>> tokenizer = MBartTokenizer.from_pretrained("facebook/mbart-large-cc25") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-cc25") >>> # de_DE is the language symbol id for German >>> TXT = " Meine Freunde sind nett aber sie essen zu viel Kuchen. de_DE" @@ -592,7 +592,7 @@ def dummy_inputs(self): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -606,7 +606,7 @@ def dummy_inputs(self): decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -740,7 +740,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -941,7 +941,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1752,7 +1752,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1817,9 +1817,9 @@ def forward( Example: ```python - >>> from transformers import MBartTokenizer, MBartForCausalLM + >>> from transformers import AutoTokenizer, MBartForCausalLM - >>> tokenizer = MBartTokenizer.from_pretrained("facebook/mbart-large-cc25") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-cc25") >>> model = MBartForCausalLM.from_pretrained("facebook/mbart-large-cc25", add_cross_attention=False) >>> assert model.config.is_decoder, f"{model.__class__} has to be configured as a decoder." >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") diff --git a/src/transformers/models/mbart/modeling_tf_mbart.py b/src/transformers/models/mbart/modeling_tf_mbart.py index 71e9a66b6d62e2..b222983331fb27 100644 --- a/src/transformers/models/mbart/modeling_tf_mbart.py +++ b/src/transformers/models/mbart/modeling_tf_mbart.py @@ -54,7 +54,6 @@ _CHECKPOINT_FOR_DOC = "facebook/mbart-large-cc25" _CONFIG_FOR_DOC = "MBartConfig" -_TOKENIZER_FOR_DOC = "MBartTokenizer" LARGE_NEGATIVE = -1e8 @@ -542,7 +541,7 @@ def serving(self, inputs): input_ids (`tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -556,7 +555,7 @@ def serving(self, inputs): decoder_input_ids (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -623,10 +622,10 @@ def serving(self, inputs): Summarization example: ```python - >>> from transformers import MBartTokenizer, TFMBartForConditionalGeneration, MBartConfig + >>> from transformers import AutoTokenizer, TFMBartForConditionalGeneration, MBartConfig >>> model = TFMBartForConditionalGeneration.from_pretrained("facebook/mbart-large-cc25") - >>> tokenizer = MBartTokenizer.from_pretrained("facebook/mbart-large-cc25") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-cc25") >>> ARTICLE_TO_SUMMARIZE = "Meine Freunde sind cool, aber sie essen zu viel Kuchen." >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors="tf") @@ -639,10 +638,10 @@ def serving(self, inputs): Mask filling example: ```python - >>> from transformers import MBartTokenizer, TFMBartForConditionalGeneration + >>> from transformers import AutoTokenizer, TFMBartForConditionalGeneration >>> model = MBartForConditionalGeneration.from_pretrained("facebook/mbart-large-cc25") - >>> tokenizer = MBartTokenizer.from_pretrained("facebook/mbart-large-cc25") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-cc25") >>> # de_DE is the language symbol id for German >>> TXT = " Meine Freunde sind nett aber sie essen zu viel Kuchen. de_DE" @@ -709,7 +708,7 @@ def call( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -894,7 +893,7 @@ def call( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1215,7 +1214,6 @@ def get_decoder(self): @unpack_inputs @add_start_docstrings_to_model_forward(MBART_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFSeq2SeqModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/mctct/modeling_mctct.py b/src/transformers/models/mctct/modeling_mctct.py index 43a45f2377250f..3effb52de5335f 100755 --- a/src/transformers/models/mctct/modeling_mctct.py +++ b/src/transformers/models/mctct/modeling_mctct.py @@ -49,7 +49,6 @@ _HIDDEN_STATES_START_POSITION = 1 _CONFIG_FOR_DOC = "MCTCTConfig" -_PROCESSOR_FOR_DOC = "MCTCTProcessor" # Base docstring _CHECKPOINT_FOR_DOC = "speechbrain/m-ctc-t-large" @@ -678,7 +677,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(MCTCT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_PROCESSOR_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC, @@ -749,7 +747,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(MCTCT_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_PROCESSOR_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=CausalLMOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/megatron_bert/modeling_megatron_bert.py b/src/transformers/models/megatron_bert/modeling_megatron_bert.py index ab0c2003670750..636130b6f5a044 100755 --- a/src/transformers/models/megatron_bert/modeling_megatron_bert.py +++ b/src/transformers/models/megatron_bert/modeling_megatron_bert.py @@ -55,7 +55,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "MegatronBertConfig" -_TOKENIZER_FOR_DOC = "BertTokenizer" _CHECKPOINT_FOR_DOC = "nvidia/megatron-bert-cased-345m" MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ @@ -789,7 +788,7 @@ class MegatronBertForPreTrainingOutput(ModelOutput): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -879,7 +878,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(MEGATRON_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1068,10 +1066,10 @@ def forward( Example: ```python - >>> from transformers import BertTokenizer, MegatronBertForPreTraining + >>> from transformers import AutoTokenizer, MegatronBertForPreTraining >>> import torch - >>> tokenizer = BertTokenizer.from_pretrained("nvidia/megatron-bert-cased-345m") + >>> tokenizer = AutoTokenizer.from_pretrained("nvidia/megatron-bert-cased-345m") >>> model = MegatronBertForPreTraining.from_pretrained("nvidia/megatron-bert-cased-345m") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") @@ -1192,10 +1190,10 @@ def forward( Example: ```python - >>> from transformers import BertTokenizer, MegatronBertForCausalLM, MegatronBertConfig + >>> from transformers import AutoTokenizer, MegatronBertForCausalLM, MegatronBertConfig >>> import torch - >>> tokenizer = BertTokenizer.from_pretrained("nvidia/megatron-bert-cased-345m") + >>> tokenizer = AutoTokenizer.from_pretrained("nvidia/megatron-bert-cased-345m") >>> model = MegatronBertForCausalLM.from_pretrained("nvidia/megatron-bert-cased-345m", is_decoder=True) >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") @@ -1295,7 +1293,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(MEGATRON_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1419,10 +1416,10 @@ def forward( Example: ```python - >>> from transformers import BertTokenizer, MegatronBertForNextSentencePrediction + >>> from transformers import AutoTokenizer, MegatronBertForNextSentencePrediction >>> import torch - >>> tokenizer = BertTokenizer.from_pretrained("nvidia/megatron-bert-cased-345m") + >>> tokenizer = AutoTokenizer.from_pretrained("nvidia/megatron-bert-cased-345m") >>> model = MegatronBertForNextSentencePrediction.from_pretrained("nvidia/megatron-bert-cased-345m") >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." @@ -1498,7 +1495,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(MEGATRON_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1597,7 +1593,6 @@ def __init__(self, config): MEGATRON_BERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1693,7 +1688,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(MEGATRON_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1774,7 +1768,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(MEGATRON_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/mobilebert/modeling_mobilebert.py b/src/transformers/models/mobilebert/modeling_mobilebert.py index 99d68eec3f44a1..c74096ae4e3453 100644 --- a/src/transformers/models/mobilebert/modeling_mobilebert.py +++ b/src/transformers/models/mobilebert/modeling_mobilebert.py @@ -984,10 +984,10 @@ def forward( Examples: ```python - >>> from transformers import MobileBertTokenizer, MobileBertForPreTraining + >>> from transformers import AutoTokenizer, MobileBertForPreTraining >>> import torch - >>> tokenizer = MobileBertTokenizer.from_pretrained("google/mobilebert-uncased") + >>> tokenizer = AutoTokenizer.from_pretrained("google/mobilebert-uncased") >>> model = MobileBertForPreTraining.from_pretrained("google/mobilebert-uncased") >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) @@ -1179,10 +1179,10 @@ def forward( Examples: ```python - >>> from transformers import MobileBertTokenizer, MobileBertForNextSentencePrediction + >>> from transformers import AutoTokenizer, MobileBertForNextSentencePrediction >>> import torch - >>> tokenizer = MobileBertTokenizer.from_pretrained("google/mobilebert-uncased") + >>> tokenizer = AutoTokenizer.from_pretrained("google/mobilebert-uncased") >>> model = MobileBertForNextSentencePrediction.from_pretrained("google/mobilebert-uncased") >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." diff --git a/src/transformers/models/mobilebert/modeling_tf_mobilebert.py b/src/transformers/models/mobilebert/modeling_tf_mobilebert.py index 8db13c83da5de9..832a4fa3f52bef 100644 --- a/src/transformers/models/mobilebert/modeling_tf_mobilebert.py +++ b/src/transformers/models/mobilebert/modeling_tf_mobilebert.py @@ -63,7 +63,6 @@ _CHECKPOINT_FOR_DOC = "google/mobilebert-uncased" _CONFIG_FOR_DOC = "MobileBertConfig" -_TOKENIZER_FOR_DOC = "MobileBertTokenizer" # TokenClassification docstring _CHECKPOINT_FOR_TOKEN_CLASSIFICATION = "vumichien/mobilebert-finetuned-ner" @@ -910,7 +909,7 @@ class TFMobileBertForPreTrainingOutput(ModelOutput): input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`MobileBertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -973,7 +972,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(MOBILEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutputWithPooling, config_class=_CONFIG_FOR_DOC, @@ -1064,9 +1062,9 @@ def call( ```python >>> import tensorflow as tf - >>> from transformers import MobileBertTokenizer, TFMobileBertForPreTraining + >>> from transformers import AutoTokenizer, TFMobileBertForPreTraining - >>> tokenizer = MobileBertTokenizer.from_pretrained("google/mobilebert-uncased") + >>> tokenizer = AutoTokenizer.from_pretrained("google/mobilebert-uncased") >>> model = TFMobileBertForPreTraining.from_pretrained("google/mobilebert-uncased") >>> input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 >>> outputs = model(input_ids) @@ -1144,7 +1142,6 @@ def get_prefix_bias_name(self): @unpack_inputs @add_start_docstrings_to_model_forward(MOBILEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1255,9 +1252,9 @@ def call( ```python >>> import tensorflow as tf - >>> from transformers import MobileBertTokenizer, TFMobileBertForNextSentencePrediction + >>> from transformers import AutoTokenizer, TFMobileBertForNextSentencePrediction - >>> tokenizer = MobileBertTokenizer.from_pretrained("google/mobilebert-uncased") + >>> tokenizer = AutoTokenizer.from_pretrained("google/mobilebert-uncased") >>> model = TFMobileBertForNextSentencePrediction.from_pretrained("google/mobilebert-uncased") >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." @@ -1339,7 +1336,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(MOBILEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_SEQUENCE_CLASSIFICATION, output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1433,7 +1429,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(MOBILEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_QA, output_type=TFQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1554,7 +1549,6 @@ def dummy_inputs(self): MOBILEBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1681,7 +1675,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(MOBILEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_TOKEN_CLASSIFICATION, output_type=TFTokenClassifierOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/mobilenet_v1/modeling_mobilenet_v1.py b/src/transformers/models/mobilenet_v1/modeling_mobilenet_v1.py index 73338a479e684c..3963e60f3562bd 100755 --- a/src/transformers/models/mobilenet_v1/modeling_mobilenet_v1.py +++ b/src/transformers/models/mobilenet_v1/modeling_mobilenet_v1.py @@ -284,7 +284,7 @@ def _init_weights(self, module: Union[nn.Linear, nn.Conv2d]) -> None: MOBILENET_V1_INPUTS_DOCSTRING = r""" Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`MobileNetV1ImageProcessor`]. See + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`MobileNetV1ImageProcessor.__call__`] for details. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for diff --git a/src/transformers/models/mobilenet_v2/modeling_mobilenet_v2.py b/src/transformers/models/mobilenet_v2/modeling_mobilenet_v2.py index 284931e794eca5..b76e68f9067ec7 100755 --- a/src/transformers/models/mobilenet_v2/modeling_mobilenet_v2.py +++ b/src/transformers/models/mobilenet_v2/modeling_mobilenet_v2.py @@ -485,7 +485,7 @@ def _init_weights(self, module: Union[nn.Linear, nn.Conv2d]) -> None: MOBILENET_V2_INPUTS_DOCSTRING = r""" Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`MobileNetV2ImageProcessor`]. See + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`MobileNetV2ImageProcessor.__call__`] for details. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for @@ -808,14 +808,14 @@ def forward( Examples: ```python - >>> from transformers import MobileNetV2ImageProcessor, MobileNetV2ForSemanticSegmentation + >>> from transformers import AutoImageProcessor, MobileNetV2ForSemanticSegmentation >>> from PIL import Image >>> import requests >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = MobileNetV2ImageProcessor.from_pretrained("google/deeplabv3_mobilenet_v2_1.0_513") + >>> image_processor = AutoImageProcessor.from_pretrained("google/deeplabv3_mobilenet_v2_1.0_513") >>> model = MobileNetV2ForSemanticSegmentation.from_pretrained("google/deeplabv3_mobilenet_v2_1.0_513") >>> inputs = image_processor(images=image, return_tensors="pt") diff --git a/src/transformers/models/mobilevit/modeling_mobilevit.py b/src/transformers/models/mobilevit/modeling_mobilevit.py index 34910e187b5fa6..3503e86c9c75c2 100755 --- a/src/transformers/models/mobilevit/modeling_mobilevit.py +++ b/src/transformers/models/mobilevit/modeling_mobilevit.py @@ -691,7 +691,7 @@ def _set_gradient_checkpointing(self, module, value=False): MOBILEVIT_INPUTS_DOCSTRING = r""" Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`MobileViTImageProcessor`]. See + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`MobileViTImageProcessor.__call__`] for details. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for @@ -1024,14 +1024,14 @@ def forward( Examples: ```python - >>> from transformers import MobileViTImageProcessor, MobileViTForSemanticSegmentation + >>> from transformers import AutoImageProcessor, MobileViTForSemanticSegmentation >>> from PIL import Image >>> import requests >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = MobileViTImageProcessor.from_pretrained("apple/deeplabv3-mobilevit-small") + >>> image_processor = AutoImageProcessor.from_pretrained("apple/deeplabv3-mobilevit-small") >>> model = MobileViTForSemanticSegmentation.from_pretrained("apple/deeplabv3-mobilevit-small") >>> inputs = image_processor(images=image, return_tensors="pt") diff --git a/src/transformers/models/mobilevit/modeling_tf_mobilevit.py b/src/transformers/models/mobilevit/modeling_tf_mobilevit.py index 61c10c8054d692..e7f44b222b67b0 100644 --- a/src/transformers/models/mobilevit/modeling_tf_mobilevit.py +++ b/src/transformers/models/mobilevit/modeling_tf_mobilevit.py @@ -810,7 +810,7 @@ def serving(self, inputs): MOBILEVIT_INPUTS_DOCSTRING = r""" Args: pixel_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]`, `Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`MobileViTImageProcessor`]. See + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`MobileViTImageProcessor.__call__`] for details. output_hidden_states (`bool`, *optional*): @@ -1100,14 +1100,14 @@ def call( Examples: ```python - >>> from transformers import MobileViTImageProcessor, TFMobileViTForSemanticSegmentation + >>> from transformers import AutoImageProcessor, TFMobileViTForSemanticSegmentation >>> from PIL import Image >>> import requests >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = MobileViTImageProcessor.from_pretrained("apple/deeplabv3-mobilevit-small") + >>> image_processor = AutoImageProcessor.from_pretrained("apple/deeplabv3-mobilevit-small") >>> model = TFMobileViTForSemanticSegmentation.from_pretrained("apple/deeplabv3-mobilevit-small") >>> inputs = image_processor(images=image, return_tensors="tf") diff --git a/src/transformers/models/mpnet/modeling_mpnet.py b/src/transformers/models/mpnet/modeling_mpnet.py index 01d1375ac93498..cd9515aa4c8933 100644 --- a/src/transformers/models/mpnet/modeling_mpnet.py +++ b/src/transformers/models/mpnet/modeling_mpnet.py @@ -43,7 +43,6 @@ _CHECKPOINT_FOR_DOC = "microsoft/mpnet-base" _CONFIG_FOR_DOC = "MPNetConfig" -_TOKENIZER_FOR_DOC = "MPNetTokenizer" MPNET_PRETRAINED_MODEL_ARCHIVE_LIST = [ @@ -439,7 +438,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`MPNetTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -511,7 +510,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(MPNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPooling, config_class=_CONFIG_FOR_DOC, @@ -594,7 +592,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(MPNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -697,7 +694,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(MPNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -792,7 +788,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(MPNET_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -885,7 +880,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(MPNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -984,7 +978,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(MPNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/mpnet/modeling_tf_mpnet.py b/src/transformers/models/mpnet/modeling_tf_mpnet.py index 853c879f60b2ab..fd944ce678129d 100644 --- a/src/transformers/models/mpnet/modeling_tf_mpnet.py +++ b/src/transformers/models/mpnet/modeling_tf_mpnet.py @@ -60,7 +60,6 @@ _CHECKPOINT_FOR_DOC = "microsoft/mpnet-base" _CONFIG_FOR_DOC = "MPNetConfig" -_TOKENIZER_FOR_DOC = "MPNetTokenizer" TF_MPNET_PRETRAINED_MODEL_ARCHIVE_LIST = [ "microsoft/mpnet-base", @@ -632,7 +631,7 @@ def call( input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`MPNetTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -687,7 +686,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(MPNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutput, config_class=_CONFIG_FOR_DOC, @@ -802,7 +800,6 @@ def get_prefix_bias_name(self): @unpack_inputs @add_start_docstrings_to_model_forward(MPNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -907,7 +904,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(MPNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -997,7 +993,6 @@ def dummy_inputs(self): @unpack_inputs @add_start_docstrings_to_model_forward(MPNET_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1108,7 +1103,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(MPNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFTokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1190,7 +1184,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(MPNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/mt5/modeling_flax_mt5.py b/src/transformers/models/mt5/modeling_flax_mt5.py index 4f2fa5b9fb39e5..6b6eaf7fd135df 100644 --- a/src/transformers/models/mt5/modeling_flax_mt5.py +++ b/src/transformers/models/mt5/modeling_flax_mt5.py @@ -24,7 +24,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "T5Config" -_TOKENIZER_FOR_DOC = "T5Tokenizer" # Copied from transformers.models.bart.modeling_flax_bart.shift_tokens_right @@ -48,10 +47,10 @@ class FlaxMT5Model(FlaxT5Model): Examples: ```python - >>> from transformers import FlaxMT5Model, T5Tokenizer + >>> from transformers import FlaxMT5Model, AutoTokenizer >>> model = FlaxMT5Model.from_pretrained("google/mt5-small") - >>> tokenizer = T5Tokenizer.from_pretrained("google/mt5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small") >>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien." >>> summary = "Weiter Verhandlung in Syrien." @@ -74,10 +73,10 @@ class FlaxMT5EncoderModel(FlaxT5EncoderModel): Examples: ```python - >>> from transformers import FlaxT5EncoderModel, T5Tokenizer + >>> from transformers import FlaxT5EncoderModel, AutoTokenizer >>> model = FlaxT5EncoderModel.from_pretrained("google/mt5-small") - >>> tokenizer = T5Tokenizer.from_pretrained("google/mt5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small") >>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien." >>> summary = "Weiter Verhandlung in Syrien." @@ -100,10 +99,10 @@ class FlaxMT5ForConditionalGeneration(FlaxT5ForConditionalGeneration): Examples: ```python - >>> from transformers import FlaxMT5ForConditionalGeneration, T5Tokenizer + >>> from transformers import FlaxMT5ForConditionalGeneration, AutoTokenizer >>> model = FlaxMT5ForConditionalGeneration.from_pretrained("google/mt5-small") - >>> tokenizer = T5Tokenizer.from_pretrained("google/mt5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small") >>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien." >>> summary = "Weiter Verhandlung in Syrien." diff --git a/src/transformers/models/mt5/modeling_mt5.py b/src/transformers/models/mt5/modeling_mt5.py index 50b40e961290d6..1cada1b235edbd 100644 --- a/src/transformers/models/mt5/modeling_mt5.py +++ b/src/transformers/models/mt5/modeling_mt5.py @@ -50,7 +50,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "MT5Config" -_TOKENIZER_FOR_DOC = "MT5Tokenizer" _CHECKPOINT_FOR_DOC = "mt5-small" @@ -1118,7 +1117,7 @@ def custom_forward(*inputs): Indices of input sequence tokens in the vocabulary. MT5 is a model with relative position embeddings so you should be able to pad the inputs on both the right and the left. - Indices can be obtained using [`MT5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail. [What are input IDs?](../glossary#input-ids) @@ -1134,7 +1133,7 @@ def custom_forward(*inputs): decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`MT5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -1211,7 +1210,7 @@ def custom_forward(*inputs): Indices of input sequence tokens in the vocabulary. MT5 is a model with relative position embeddings so you should be able to pad the inputs on both the right and the left. - Indices can be obtained using [`MT5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail. To know more on how to prepare `input_ids` for pretraining take a look a [MT5 Training](./mt5#training). @@ -1260,10 +1259,10 @@ class MT5Model(MT5PreTrainedModel): Examples: ```python - >>> from transformers import MT5Model, MT5Tokenizer + >>> from transformers import MT5Model, AutoTokenizer >>> model = MT5Model.from_pretrained("google/mt5-small") - >>> tokenizer = MT5Tokenizer.from_pretrained("google/mt5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small") >>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien." >>> summary = "Weiter Verhandlung in Syrien." >>> inputs = tokenizer(article, return_tensors="pt") @@ -1389,9 +1388,9 @@ def forward( Example: ```python - >>> from transformers import MT5Tokenizer, MT5Model + >>> from transformers import AutoTokenizer, MT5Model - >>> tokenizer = MT5Tokenizer.from_pretrained("mt5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("mt5-small") >>> model = MT5Model.from_pretrained("mt5-small") >>> input_ids = tokenizer( @@ -1484,10 +1483,10 @@ class MT5ForConditionalGeneration(MT5PreTrainedModel): Examples: ```python - >>> from transformers import MT5ForConditionalGeneration, MT5Tokenizer + >>> from transformers import MT5ForConditionalGeneration, AutoTokenizer >>> model = MT5ForConditionalGeneration.from_pretrained("google/mt5-small") - >>> tokenizer = MT5Tokenizer.from_pretrained("google/mt5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small") >>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien." >>> summary = "Weiter Verhandlung in Syrien." >>> inputs = tokenizer(article, text_target=summary, return_tensors="pt") @@ -1621,9 +1620,9 @@ def forward( Examples: ```python - >>> from transformers import MT5Tokenizer, MT5ForConditionalGeneration + >>> from transformers import AutoTokenizer, MT5ForConditionalGeneration - >>> tokenizer = MT5Tokenizer.from_pretrained("mt5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("mt5-small") >>> model = MT5ForConditionalGeneration.from_pretrained("mt5-small") >>> # training @@ -1810,10 +1809,10 @@ class MT5EncoderModel(MT5PreTrainedModel): Examples: ```python - >>> from transformers import MT5EncoderModel, MT5Tokenizer + >>> from transformers import MT5EncoderModel, AutoTokenizer >>> model = MT5EncoderModel.from_pretrained("google/mt5-small") - >>> tokenizer = MT5Tokenizer.from_pretrained("google/mt5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small") >>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien." >>> input_ids = tokenizer(article, return_tensors="pt").input_ids >>> outputs = model(input_ids) @@ -1909,9 +1908,9 @@ def forward( Example: ```python - >>> from transformers import MT5Tokenizer, MT5EncoderModel + >>> from transformers import AutoTokenizer, MT5EncoderModel - >>> tokenizer = MT5Tokenizer.from_pretrained("mt5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("mt5-small") >>> model = MT5EncoderModel.from_pretrained("mt5-small") >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="pt" diff --git a/src/transformers/models/mt5/modeling_tf_mt5.py b/src/transformers/models/mt5/modeling_tf_mt5.py index 71aa0bb66a7a03..ba7bd33c344747 100644 --- a/src/transformers/models/mt5/modeling_tf_mt5.py +++ b/src/transformers/models/mt5/modeling_tf_mt5.py @@ -22,7 +22,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "T5Config" -_TOKENIZER_FOR_DOC = "T5Tokenizer" class TFMT5Model(TFT5Model): @@ -33,10 +32,10 @@ class TFMT5Model(TFT5Model): Examples: ```python - >>> from transformers import TFMT5Model, T5Tokenizer + >>> from transformers import TFMT5Model, AutoTokenizer >>> model = TFMT5Model.from_pretrained("google/mt5-small") - >>> tokenizer = T5Tokenizer.from_pretrained("google/mt5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small") >>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien." >>> summary = "Weiter Verhandlung in Syrien." >>> inputs = tokenizer(article, return_tensors="tf") @@ -57,10 +56,10 @@ class TFMT5ForConditionalGeneration(TFT5ForConditionalGeneration): Examples: ```python - >>> from transformers import TFMT5ForConditionalGeneration, T5Tokenizer + >>> from transformers import TFMT5ForConditionalGeneration, AutoTokenizer >>> model = TFMT5ForConditionalGeneration.from_pretrained("google/mt5-small") - >>> tokenizer = T5Tokenizer.from_pretrained("google/mt5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small") >>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien." >>> summary = "Weiter Verhandlung in Syrien." >>> inputs = tokenizer(article, text_target=summary, return_tensors="tf") @@ -81,10 +80,10 @@ class TFMT5EncoderModel(TFT5EncoderModel): Examples: ```python - >>> from transformers import TFMT5EncoderModel, T5Tokenizer + >>> from transformers import TFMT5EncoderModel, AutoTokenizer >>> model = TFMT5EncoderModel.from_pretrained("google/mt5-small") - >>> tokenizer = T5Tokenizer.from_pretrained("google/mt5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small") >>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien." >>> input_ids = tokenizer(article, return_tensors="tf").input_ids >>> outputs = model(input_ids) diff --git a/src/transformers/models/mvp/modeling_mvp.py b/src/transformers/models/mvp/modeling_mvp.py index 999d61cda17b7c..8823bf17197c0c 100644 --- a/src/transformers/models/mvp/modeling_mvp.py +++ b/src/transformers/models/mvp/modeling_mvp.py @@ -49,7 +49,6 @@ _CHECKPOINT_FOR_DOC = "RUCAIBox/mvp" _CONFIG_FOR_DOC = "MvpConfig" -_TOKENIZER_FOR_DOC = "MvpTokenizer" # Base model docstring _EXPECTED_OUTPUT_SHAPE = [1, 8, 1024] @@ -601,7 +600,7 @@ def dummy_inputs(self): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`MvpTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -615,7 +614,7 @@ def dummy_inputs(self): decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`MvpTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -697,9 +696,9 @@ def dummy_inputs(self): Fine-tuning a model ```python >>> import torch - >>> from transformers import MvpTokenizer, MvpForConditionalGeneration + >>> from transformers import AutoTokenizer, MvpForConditionalGeneration - >>> tokenizer = MvpTokenizer.from_pretrained("RUCAIBox/mvp") + >>> tokenizer = AutoTokenizer.from_pretrained("RUCAIBox/mvp") >>> model = MvpForConditionalGeneration.from_pretrained("RUCAIBox/mvp") >>> inputs = tokenizer( @@ -727,10 +726,10 @@ def dummy_inputs(self): Fine-tuning a model on `num_labels` classes ```python >>> import torch - >>> from transformers import MvpTokenizer, MvpForSequenceClassification + >>> from transformers import AutoTokenizer, MvpForSequenceClassification >>> num_labels = 2 # for example, this is a binary classification task - >>> tokenizer = MvpTokenizer.from_pretrained("RUCAIBox/mvp") + >>> tokenizer = AutoTokenizer.from_pretrained("RUCAIBox/mvp") >>> model = MvpForSequenceClassification.from_pretrained("RUCAIBox/mvp", num_labels=num_labels) >>> inputs = tokenizer("Classify: Hello, my dog is cute", return_tensors="pt") @@ -756,9 +755,9 @@ def dummy_inputs(self): using `BartForConditionalGeneration` ```python >>> import torch - >>> from transformers import MvpTokenizer, MvpForQuestionAnswering + >>> from transformers import AutoTokenizer, MvpForQuestionAnswering - >>> tokenizer = MvpTokenizer.from_pretrained("RUCAIBox/mvp") + >>> tokenizer = AutoTokenizer.from_pretrained("RUCAIBox/mvp") >>> model = MvpForQuestionAnswering.from_pretrained("RUCAIBox/mvp") >>> inputs = tokenizer( @@ -857,7 +856,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`MvpTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1078,7 +1077,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`MvpTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1331,7 +1330,6 @@ def set_lightweight_tuning(self): @add_start_docstrings_to_model_forward(MVP_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1920,7 +1918,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`MvpTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1985,9 +1983,9 @@ def forward( Example: ```python - >>> from transformers import MvpTokenizer, MvpForCausalLM + >>> from transformers import AutoTokenizer, MvpForCausalLM - >>> tokenizer = MvpTokenizer.from_pretrained("RUCAIBox/mvp") + >>> tokenizer = AutoTokenizer.from_pretrained("RUCAIBox/mvp") >>> model = MvpForCausalLM.from_pretrained("RUCAIBox/mvp", add_cross_attention=False) >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") diff --git a/src/transformers/models/nat/modeling_nat.py b/src/transformers/models/nat/modeling_nat.py index 89e7b185b814e6..c2e445b7ae994b 100644 --- a/src/transformers/models/nat/modeling_nat.py +++ b/src/transformers/models/nat/modeling_nat.py @@ -657,8 +657,8 @@ def _set_gradient_checkpointing(self, module: NatEncoder, value: bool = False) - NAT_INPUTS_DOCSTRING = r""" Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] + for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned diff --git a/src/transformers/models/nezha/modeling_nezha.py b/src/transformers/models/nezha/modeling_nezha.py index 82a0633f98d0db..f7061d6db9928d 100644 --- a/src/transformers/models/nezha/modeling_nezha.py +++ b/src/transformers/models/nezha/modeling_nezha.py @@ -54,7 +54,6 @@ _CHECKPOINT_FOR_DOC = "sijunhe/nezha-cn-base" _CONFIG_FOR_DOC = "NezhaConfig" -_TOKENIZER_FOR_DOC = "BertTokenizer" NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST = [ "sijunhe/nezha-cn-base", @@ -813,7 +812,7 @@ class NezhaForPreTrainingOutput(ModelOutput): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -898,7 +897,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(NEZHA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1089,10 +1087,10 @@ def forward( Example: ```python - >>> from transformers import BertTokenizer, NezhaForPreTraining + >>> from transformers import AutoTokenizer, NezhaForPreTraining >>> import torch - >>> tokenizer = BertTokenizer.from_pretrained("sijunhe/nezha-cn-base") + >>> tokenizer = AutoTokenizer.from_pretrained("sijunhe/nezha-cn-base") >>> model = NezhaForPreTraining.from_pretrained("sijunhe/nezha-cn-base") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") @@ -1167,7 +1165,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(NEZHA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1286,10 +1283,10 @@ def forward( Example: ```python - >>> from transformers import BertTokenizer, NezhaForNextSentencePrediction + >>> from transformers import AutoTokenizer, NezhaForNextSentencePrediction >>> import torch - >>> tokenizer = BertTokenizer.from_pretrained("sijunhe/nezha-cn-base") + >>> tokenizer = AutoTokenizer.from_pretrained("sijunhe/nezha-cn-base") >>> model = NezhaForNextSentencePrediction.from_pretrained("sijunhe/nezha-cn-base") >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." @@ -1369,7 +1366,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(NEZHA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1467,7 +1463,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(NEZHA_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1564,7 +1559,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(NEZHA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1643,7 +1637,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(NEZHA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/nystromformer/modeling_nystromformer.py b/src/transformers/models/nystromformer/modeling_nystromformer.py index 72a0d347983424..4cea21e705befb 100755 --- a/src/transformers/models/nystromformer/modeling_nystromformer.py +++ b/src/transformers/models/nystromformer/modeling_nystromformer.py @@ -42,7 +42,6 @@ _CHECKPOINT_FOR_DOC = "uw-madison/nystromformer-512" _CONFIG_FOR_DOC = "NystromformerConfig" -_TOKENIZER_FOR_DOC = "AutoTokenizer" NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST = [ "uw-madison/nystromformer-512", @@ -574,7 +573,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(NYSTROMFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPastAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -679,7 +677,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(NYSTROMFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -777,7 +774,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(NYSTROMFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -874,7 +870,6 @@ def __init__(self, config): NYSTROMFORMER_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -969,7 +964,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(NYSTROMFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1049,7 +1043,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(NYSTROMFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/oneformer/modeling_oneformer.py b/src/transformers/models/oneformer/modeling_oneformer.py index 9e72003a9f9d5d..a6ec2011ab89c1 100644 --- a/src/transformers/models/oneformer/modeling_oneformer.py +++ b/src/transformers/models/oneformer/modeling_oneformer.py @@ -46,7 +46,6 @@ _CONFIG_FOR_DOC = "OneFormerConfig" _CHECKPOINT_FOR_DOC = "shi-labs/oneformer_ade20k_swin_tiny" -_IMAGE_PROCESSOR_FOR_DOC = "OneFormerImageProcessor" ONEFORMER_PRETRAINED_MODEL_ARCHIVE_LIST = [ "shi-labs/oneformer_ade20k_swin_tiny", @@ -2724,8 +2723,8 @@ def forward(self, inputs: Tensor) -> Tensor: Pixel values. Pixel values can be obtained using [`OneFormerProcessor`]. See [`OneFormerProcessor.__call__`] for details. task_inputs (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Task inputs. Task inputs can be obtained using [`OneFormerImageProcessor`]. See - [`OneFormerProcessor.__call__`] for details. + Task inputs. Task inputs can be obtained using [`AutoImageProcessor`]. See [`OneFormerProcessor.__call__`] + for details. pixel_mask (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: diff --git a/src/transformers/models/openai/modeling_openai.py b/src/transformers/models/openai/modeling_openai.py index 6102ce377af5d4..8ac487aa47da05 100644 --- a/src/transformers/models/openai/modeling_openai.py +++ b/src/transformers/models/openai/modeling_openai.py @@ -45,7 +45,6 @@ _CHECKPOINT_FOR_DOC = "openai-gpt" _CONFIG_FOR_DOC = "OpenAIGPTConfig" -_TOKENIZER_FOR_DOC = "OpenAIGPTTokenizer" OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST = [ "openai-gpt", @@ -350,7 +349,7 @@ class OpenAIGPTDoubleHeadsModelOutput(ModelOutput): input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`OpenAIGPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -427,7 +426,6 @@ def _prune_heads(self, heads_to_prune): @add_start_docstrings_to_model_forward(OPENAI_GPT_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC, @@ -549,7 +547,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(OPENAI_GPT_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=CausalLMOutput, config_class=_CONFIG_FOR_DOC, @@ -676,10 +673,10 @@ def forward( Examples: ```python - >>> from transformers import OpenAIGPTTokenizer, OpenAIGPTDoubleHeadsModel + >>> from transformers import AutoTokenizer, OpenAIGPTDoubleHeadsModel >>> import torch - >>> tokenizer = OpenAIGPTTokenizer.from_pretrained("openai-gpt") + >>> tokenizer = AutoTokenizer.from_pretrained("openai-gpt") >>> model = OpenAIGPTDoubleHeadsModel.from_pretrained("openai-gpt") >>> tokenizer.add_special_tokens( ... {"cls_token": "[CLS]"} @@ -762,7 +759,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(OPENAI_GPT_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/openai/modeling_tf_openai.py b/src/transformers/models/openai/modeling_tf_openai.py index 5144eeecef1845..fbe63a001e3856 100644 --- a/src/transformers/models/openai/modeling_tf_openai.py +++ b/src/transformers/models/openai/modeling_tf_openai.py @@ -51,7 +51,6 @@ _CHECKPOINT_FOR_DOC = "openai-gpt" _CONFIG_FOR_DOC = "OpenAIGPTConfig" -_TOKENIZER_FOR_DOC = "OpenAIGPTTokenizer" TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST = [ "openai-gpt", @@ -465,7 +464,7 @@ class TFOpenAIGPTDoubleHeadsModelOutput(ModelOutput): input_ids (`Numpy array` or `tf.Tensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`OpenAIGPTTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -528,7 +527,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(OPENAI_GPT_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutput, config_class=_CONFIG_FOR_DOC, @@ -592,7 +590,6 @@ def set_output_embeddings(self, value): @unpack_inputs @add_start_docstrings_to_model_forward(OPENAI_GPT_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFCausalLMOutput, config_class=_CONFIG_FOR_DOC, @@ -707,9 +704,9 @@ def call( ```python >>> import tensorflow as tf - >>> from transformers import OpenAIGPTTokenizer, TFOpenAIGPTDoubleHeadsModel + >>> from transformers import AutoTokenizer, TFOpenAIGPTDoubleHeadsModel - >>> tokenizer = OpenAIGPTTokenizer.from_pretrained("openai-gpt") + >>> tokenizer = AutoTokenizer.from_pretrained("openai-gpt") >>> model = TFOpenAIGPTDoubleHeadsModel.from_pretrained("openai-gpt") >>> # Add a [CLS] to the vocabulary (we should train it also!) @@ -820,7 +817,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(OPENAI_GPT_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/opt/modeling_flax_opt.py b/src/transformers/models/opt/modeling_flax_opt.py index 79d1ff7193f33a..db9154b33b277b 100644 --- a/src/transformers/models/opt/modeling_flax_opt.py +++ b/src/transformers/models/opt/modeling_flax_opt.py @@ -37,7 +37,6 @@ _CHECKPOINT_FOR_DOC = "facebook/opt-350m" _CONFIG_FOR_DOC = "OPTConfig" -_TOKENIZER_FOR_DOC = "GPT2Tokenizer" OPT_START_DOCSTRING = r""" @@ -80,7 +79,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`GPT2Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -696,9 +695,7 @@ class FlaxOPTModel(FlaxOPTPreTrainedModel): module_class = FlaxOPTModule -append_call_sample_docstring( - FlaxOPTModel, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutput, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxOPTModel, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutput, _CONFIG_FOR_DOC) @add_start_docstrings( @@ -799,7 +796,6 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): append_call_sample_docstring( FlaxOPTForCausalLM, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutput, _CONFIG_FOR_DOC, diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 3aebb95fcf6a59..0007c0c5fd68ce 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -43,7 +43,6 @@ _CHECKPOINT_FOR_DOC = "facebook/opt-350m" _CONFIG_FOR_DOC = "OPTConfig" -_TOKENIZER_FOR_DOC = "GPT2Tokenizer" # Base model docstring _EXPECTED_OUTPUT_SHAPE = [1, 8, 1024] @@ -421,7 +420,7 @@ def _set_gradient_checkpointing(self, module, value=False): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`GPT2Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -433,7 +432,7 @@ def _set_gradient_checkpointing(self, module, value=False): [What are attention masks?](../glossary#attention-mask) - Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see @@ -567,7 +566,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -755,7 +754,6 @@ def get_decoder(self): @add_start_docstrings_to_model_forward(OPT_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPast, config_class=_CONFIG_FOR_DOC, @@ -856,7 +854,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -910,10 +908,10 @@ def forward( Example: ```python - >>> from transformers import GPT2Tokenizer, OPTForCausalLM + >>> from transformers import AutoTokenizer, OPTForCausalLM >>> model = OPTForCausalLM.from_pretrained("facebook/opt-350m") - >>> tokenizer = GPT2Tokenizer.from_pretrained("facebook/opt-350m") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/opt-350m") >>> prompt = "Hey, are you consciours? Can you talk to me?" >>> inputs = tokenizer(prompt, return_tensors="pt") @@ -1020,7 +1018,6 @@ def __init__(self, config: OPTConfig): @add_start_docstrings_to_model_forward(OPT_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_SEQUENCE_CLASSIFICATION, output_type=SequenceClassifierOutputWithPast, config_class=_CONFIG_FOR_DOC, @@ -1171,11 +1168,11 @@ def forward( Example: ```python - >>> from transformers import GPT2Tokenizer, OPTForQuestionAnswering + >>> from transformers import AutoTokenizer, OPTForQuestionAnswering >>> import torch >>> torch.manual_seed(4) # doctest: +IGNORE_RESULT - >>> tokenizer = GPT2Tokenizer.from_pretrained("facebook/opt-350m") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/opt-350m") >>> # note: we are loading a OPTForQuestionAnswering from the hub here, >>> # so the head will be randomly initialized, hence the predictions will be random diff --git a/src/transformers/models/opt/modeling_tf_opt.py b/src/transformers/models/opt/modeling_tf_opt.py index 3a7bb3cf845266..c9f4c087b387a6 100644 --- a/src/transformers/models/opt/modeling_tf_opt.py +++ b/src/transformers/models/opt/modeling_tf_opt.py @@ -48,7 +48,6 @@ _CHECKPOINT_FOR_DOC = "facebook/opt-350m" _CONFIG_FOR_DOC = "OPTConfig" -_TOKENIZER_FOR_DOC = "GPT2Tokenizer" # Base model docstring _EXPECTED_OUTPUT_SHAPE = [1, 8, 1024] @@ -441,7 +440,7 @@ def serving(self, inputs): input_ids (`tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -570,7 +569,7 @@ def call( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -799,7 +798,6 @@ def set_input_embeddings(self, new_embeddings): @unpack_inputs @add_start_docstrings_to_model_forward(OPT_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutputWithPast, config_class=_CONFIG_FOR_DOC, @@ -898,7 +896,6 @@ def prepare_inputs_for_generation(self, inputs, past_key_values=None, use_cache= @unpack_inputs @replace_return_docstrings(output_type=TFCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC, @@ -926,7 +923,7 @@ def call( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) diff --git a/src/transformers/models/owlvit/modeling_owlvit.py b/src/transformers/models/owlvit/modeling_owlvit.py index cf7b853b6f91d7..a33a5f7f892565 100644 --- a/src/transformers/models/owlvit/modeling_owlvit.py +++ b/src/transformers/models/owlvit/modeling_owlvit.py @@ -594,7 +594,7 @@ def _set_gradient_checkpointing(self, module, value=False): OWLVIT_TEXT_INPUTS_DOCSTRING = r""" Args: input_ids (`torch.LongTensor` of shape `(batch_size * num_max_text_queries, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`CLIPTokenizer`]. See + Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.Tensor` of shape `(batch_size, num_max_text_queries, sequence_length)`, *optional*): @@ -629,7 +629,7 @@ def _set_gradient_checkpointing(self, module, value=False): OWLVIT_INPUTS_DOCSTRING = r""" Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`CLIPTokenizer`]. See + Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): @@ -656,7 +656,7 @@ def _set_gradient_checkpointing(self, module, value=False): pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. input_ids (`torch.LongTensor` of shape `(batch_size * num_max_text_queries, sequence_length)`, *optional*): - Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`CLIPTokenizer`]. See + Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids). attention_mask (`torch.Tensor` of shape `(batch_size, num_max_text_queries, sequence_length)`, *optional*): @@ -894,10 +894,10 @@ def forward( Examples: ```python - >>> from transformers import OwlViTProcessor, OwlViTTextModel + >>> from transformers import AutoProcessor, OwlViTTextModel >>> model = OwlViTTextModel.from_pretrained("google/owlvit-base-patch32") - >>> processor = OwlViTProcessor.from_pretrained("google/owlvit-base-patch32") + >>> processor = AutoProcessor.from_pretrained("google/owlvit-base-patch32") >>> inputs = processor( ... text=[["a photo of a cat", "a photo of a dog"], ["photo of a astranaut"]], return_tensors="pt" ... ) @@ -1003,10 +1003,10 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import OwlViTProcessor, OwlViTVisionModel + >>> from transformers import AutoProcessor, OwlViTVisionModel >>> model = OwlViTVisionModel.from_pretrained("google/owlvit-base-patch32") - >>> processor = OwlViTProcessor.from_pretrained("google/owlvit-base-patch32") + >>> processor = AutoProcessor.from_pretrained("google/owlvit-base-patch32") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1076,10 +1076,10 @@ def get_text_features( Examples: ```python - >>> from transformers import OwlViTProcessor, OwlViTModel + >>> from transformers import AutoProcessor, OwlViTModel >>> model = OwlViTModel.from_pretrained("google/owlvit-base-patch32") - >>> processor = OwlViTProcessor.from_pretrained("google/owlvit-base-patch32") + >>> processor = AutoProcessor.from_pretrained("google/owlvit-base-patch32") >>> inputs = processor( ... text=[["a photo of a cat", "a photo of a dog"], ["photo of a astranaut"]], return_tensors="pt" ... ) @@ -1112,10 +1112,10 @@ def get_image_features( ```python >>> from PIL import Image >>> import requests - >>> from transformers import OwlViTProcessor, OwlViTModel + >>> from transformers import AutoProcessor, OwlViTModel >>> model = OwlViTModel.from_pretrained("google/owlvit-base-patch32") - >>> processor = OwlViTProcessor.from_pretrained("google/owlvit-base-patch32") + >>> processor = AutoProcessor.from_pretrained("google/owlvit-base-patch32") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(images=image, return_tensors="pt") @@ -1160,10 +1160,10 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import OwlViTProcessor, OwlViTModel + >>> from transformers import AutoProcessor, OwlViTModel >>> model = OwlViTModel.from_pretrained("google/owlvit-base-patch32") - >>> processor = OwlViTProcessor.from_pretrained("google/owlvit-base-patch32") + >>> processor = AutoProcessor.from_pretrained("google/owlvit-base-patch32") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(text=[["a photo of a cat", "a photo of a dog"]], images=image, return_tensors="pt") @@ -1535,9 +1535,9 @@ def image_guided_detection( >>> import requests >>> from PIL import Image >>> import torch - >>> from transformers import OwlViTProcessor, OwlViTForObjectDetection + >>> from transformers import AutoProcessor, OwlViTForObjectDetection - >>> processor = OwlViTProcessor.from_pretrained("google/owlvit-base-patch16") + >>> processor = AutoProcessor.from_pretrained("google/owlvit-base-patch16") >>> model = OwlViTForObjectDetection.from_pretrained("google/owlvit-base-patch16") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1631,9 +1631,9 @@ def forward( >>> import requests >>> from PIL import Image >>> import torch - >>> from transformers import OwlViTProcessor, OwlViTForObjectDetection + >>> from transformers import AutoProcessor, OwlViTForObjectDetection - >>> processor = OwlViTProcessor.from_pretrained("google/owlvit-base-patch32") + >>> processor = AutoProcessor.from_pretrained("google/owlvit-base-patch32") >>> model = OwlViTForObjectDetection.from_pretrained("google/owlvit-base-patch32") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" diff --git a/src/transformers/models/pegasus/modeling_flax_pegasus.py b/src/transformers/models/pegasus/modeling_flax_pegasus.py index 3eddce4dd49827..75d38d59ef2214 100644 --- a/src/transformers/models/pegasus/modeling_flax_pegasus.py +++ b/src/transformers/models/pegasus/modeling_flax_pegasus.py @@ -55,7 +55,6 @@ _CHECKPOINT_FOR_DOC = "google/pegasus-large" _CONFIG_FOR_DOC = "PegasusConfig" -_TOKENIZER_FOR_DOC = "PegasusTokenizer" PEGASUS_START_DOCSTRING = r""" This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the @@ -97,7 +96,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -111,7 +110,7 @@ decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -144,7 +143,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -173,7 +172,7 @@ decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -1004,10 +1003,10 @@ def encode( Example: ```python - >>> from transformers import PegasusTokenizer, FlaxPegasusForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxPegasusForConditionalGeneration >>> model = FlaxPegasusForConditionalGeneration.from_pretrained("google/pegasus-large") - >>> tokenizer = PegasusTokenizer.from_pretrained("google/pegasus-large") + >>> tokenizer = AutoTokenizer.from_pretrained("google/pegasus-large") >>> text = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, max_length=1024, return_tensors="np") @@ -1071,10 +1070,10 @@ def decode( ```python >>> import jax.numpy as jnp - >>> from transformers import PegasusTokenizer, FlaxPegasusForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxPegasusForConditionalGeneration >>> model = FlaxPegasusForConditionalGeneration.from_pretrained("google/pegasus-large") - >>> tokenizer = PegasusTokenizer.from_pretrained("google/pegasus-large") + >>> tokenizer = AutoTokenizer.from_pretrained("google/pegasus-large") >>> text = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, max_length=1024, return_tensors="np") @@ -1232,9 +1231,7 @@ class FlaxPegasusModel(FlaxPegasusPreTrainedModel): module_class = FlaxPegasusModule -append_call_sample_docstring( - FlaxPegasusModel, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqModelOutput, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxPegasusModel, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqModelOutput, _CONFIG_FOR_DOC) # Copied from transformers.models.bart.modeling_flax_bart.FlaxBartForConditionalGenerationModule with Bart->Pegasus @@ -1341,10 +1338,10 @@ def decode( ```python >>> import jax.numpy as jnp - >>> from transformers import PegasusTokenizer, FlaxPegasusForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxPegasusForConditionalGeneration >>> model = FlaxPegasusForConditionalGeneration.from_pretrained("google/pegasus-large") - >>> tokenizer = PegasusTokenizer.from_pretrained("google/pegasus-large") + >>> tokenizer = AutoTokenizer.from_pretrained("google/pegasus-large") >>> text = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer(text, max_length=1024, return_tensors="np") @@ -1496,30 +1493,38 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): Summarization example: - >>> from transformers import PegasusTokenizer, FlaxPegasusForConditionalGeneration + ```pyton + >>> from transformers import AutoTokenizer, FlaxPegasusForConditionalGeneration - >>> model = FlaxPegasusForConditionalGeneration.from_pretrained('google/pegasus-large') >>> tokenizer = - PegasusTokenizer.from_pretrained('google/pegasus-large') + >>> model = FlaxPegasusForConditionalGeneration.from_pretrained('google/pegasus-large') + >>> tokenizer = AutoTokenizer.from_pretrained('google/pegasus-large') - >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs." >>> inputs = - tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='np') + >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs." + >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='np') - >>> # Generate Summary >>> summary_ids = model.generate(inputs['input_ids']).sequences >>> - print(tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)) + >>> # Generate Summary + >>> summary_ids = model.generate(inputs['input_ids']).sequences + >>> print(tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)) + ``` Mask filling example: - >>> from transformers import PegasusTokenizer, FlaxPegasusForConditionalGeneration >>> tokenizer = - PegasusTokenizer.from_pretrained('google/pegasus-large') >>> TXT = "My friends are but they eat too many - carbs." + ```python + >>> from transformers import AutoTokenizer, FlaxPegasusForConditionalGeneration + + >>> tokenizer = AutoTokenizer.from_pretrained("google/pegasus-large") + >>> TXT = "My friends are but they eat too many carbs." - >>> model = FlaxPegasusForConditionalGeneration.from_pretrained('google/pegasus-large') >>> input_ids = - tokenizer([TXT], return_tensors='np')['input_ids'] >>> logits = model(input_ids).logits + >>> model = FlaxPegasusForConditionalGeneration.from_pretrained("google/pegasus-large") + >>> input_ids = tokenizer([TXT], return_tensors="np")["input_ids"] + >>> logits = model(input_ids).logits - >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item() >>> probs = - jax.nn.softmax(logits[0, masked_index], axis=0) >>> values, predictions = jax.lax.top_k(probs) + >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item() + >>> probs = jax.nn.softmax(logits[0, masked_index], axis=0) + >>> values, predictions = jax.lax.top_k(probs) - >>> tokenizer.decode(predictions).split() + >>> tokenizer.decode(predictions).split() + ``` """ overwrite_call_docstring( diff --git a/src/transformers/models/pegasus/modeling_pegasus.py b/src/transformers/models/pegasus/modeling_pegasus.py index 2b88944d2854aa..18e17ab5732de7 100755 --- a/src/transformers/models/pegasus/modeling_pegasus.py +++ b/src/transformers/models/pegasus/modeling_pegasus.py @@ -48,7 +48,6 @@ _CHECKPOINT_FOR_DOC = "google/pegasus-large" _CONFIG_FOR_DOC = "PegasusConfig" -_TOKENIZER_FOR_DOC = "PegasusTokenizer" PEGASUS_PRETRAINED_MODEL_ARCHIVE_LIST = [ @@ -525,10 +524,10 @@ def _set_gradient_checkpointing(self, module, value=False): Summarization example: ```python - >>> from transformers import PegasusTokenizer, PegasusForConditionalGeneration + >>> from transformers import AutoTokenizer, PegasusForConditionalGeneration >>> model = PegasusForConditionalGeneration.from_pretrained("google/pegasus-xsum") - >>> tokenizer = PegasusTokenizer.from_pretrained("google/pegasus-xsum") + >>> tokenizer = AutoTokenizer.from_pretrained("google/pegasus-xsum") >>> ARTICLE_TO_SUMMARIZE = ( ... "PG&E stated it scheduled the blackouts in response to forecasts for high winds " @@ -550,7 +549,7 @@ def _set_gradient_checkpointing(self, module, value=False): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -564,7 +563,7 @@ def _set_gradient_checkpointing(self, module, value=False): decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -717,7 +716,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -946,7 +945,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1224,9 +1223,9 @@ def forward( Example: ```python - >>> from transformers import PegasusTokenizer, PegasusModel + >>> from transformers import AutoTokenizer, PegasusModel - >>> tokenizer = PegasusTokenizer.from_pretrained("google/pegasus-large") + >>> tokenizer = AutoTokenizer.from_pretrained("google/pegasus-large") >>> model = PegasusModel.from_pretrained("google/pegasus-large") >>> inputs = tokenizer("Studies have been shown that owning a dog is good for you", return_tensors="pt") @@ -1585,7 +1584,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1650,9 +1649,9 @@ def forward( Example: ```python - >>> from transformers import PegasusTokenizer, PegasusForCausalLM + >>> from transformers import AutoTokenizer, PegasusForCausalLM - >>> tokenizer = PegasusTokenizer.from_pretrained("google/pegasus-large") + >>> tokenizer = AutoTokenizer.from_pretrained("google/pegasus-large") >>> model = PegasusForCausalLM.from_pretrained("google/pegasus-large", add_cross_attention=False) >>> assert model.config.is_decoder, f"{model.__class__} has to be configured as a decoder." >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") diff --git a/src/transformers/models/pegasus/modeling_tf_pegasus.py b/src/transformers/models/pegasus/modeling_tf_pegasus.py index 6ba4597d1ca657..a8c27b6497b909 100644 --- a/src/transformers/models/pegasus/modeling_tf_pegasus.py +++ b/src/transformers/models/pegasus/modeling_tf_pegasus.py @@ -55,7 +55,6 @@ _CHECKPOINT_FOR_DOC = "google/pegasus-large" _CONFIG_FOR_DOC = "PegasusConfig" -_TOKENIZER_FOR_DOC = "PegasusTokenizer" LARGE_NEGATIVE = -1e8 @@ -576,10 +575,10 @@ def serving(self, inputs): Summarization example: ```python - >>> from transformers import PegasusTokenizer, TFPegasusForConditionalGeneration + >>> from transformers import AutoTokenizer, TFPegasusForConditionalGeneration >>> model = TFPegasusForConditionalGeneration.from_pretrained("google/pegasus-xsum") - >>> tokenizer = PegasusTokenizer.from_pretrained("google/pegasus-xsum") + >>> tokenizer = AutoTokenizer.from_pretrained("google/pegasus-xsum") >>> ARTICLE_TO_SUMMARIZE = ( ... "PG&E stated it scheduled the blackouts in response to forecasts for high winds " @@ -599,7 +598,7 @@ def serving(self, inputs): input_ids (`tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -613,7 +612,7 @@ def serving(self, inputs): decoder_input_ids (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -728,7 +727,7 @@ def call( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -909,7 +908,7 @@ def call( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1226,7 +1225,6 @@ def get_decoder(self): @unpack_inputs @add_start_docstrings_to_model_forward(PEGASUS_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFSeq2SeqModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/pegasus_x/modeling_pegasus_x.py b/src/transformers/models/pegasus_x/modeling_pegasus_x.py index 7ed712d26f8f58..e33cf1cd3fdee1 100755 --- a/src/transformers/models/pegasus_x/modeling_pegasus_x.py +++ b/src/transformers/models/pegasus_x/modeling_pegasus_x.py @@ -47,7 +47,6 @@ _CHECKPOINT_FOR_DOC = "google/pegasus-x-base" _CONFIG_FOR_DOC = "PegasusXConfig" -_TOKENIZER_FOR_DOC = "PegasusTokenizer" PEGASUS_X_PRETRAINED_MODEL_ARCHIVE_LIST = [ @@ -804,10 +803,10 @@ def _set_gradient_checkpointing(self, module, value=False): Summarization example: ```python - >>> from transformers import PegasusTokenizer, PegasusXForConditionalGeneration + >>> from transformers import AutoTokenizer, PegasusXForConditionalGeneration >>> model = PegasusXForConditionalGeneration.from_pretrained("google/pegasus-x-base") - >>> tokenizer = PegasusTokenizer.from_pretrained("google/pegasus-x-large") + >>> tokenizer = AutoTokenizer.from_pretrained("google/pegasus-x-large") >>> ARTICLE_TO_SUMMARIZE = ( ... "PG&E stated it scheduled the blackouts in response to forecasts for high winds " @@ -829,7 +828,7 @@ def _set_gradient_checkpointing(self, module, value=False): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -845,7 +844,7 @@ def _set_gradient_checkpointing(self, module, value=False): decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -978,7 +977,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1206,7 +1205,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1456,9 +1455,9 @@ def forward( Example: ```python - >>> from transformers import PegasusTokenizer, PegasusModel + >>> from transformers import AutoTokenizer, PegasusModel - >>> tokenizer = PegasusTokenizer.from_pretrained("google/pegasus-x-large") + >>> tokenizer = AutoTokenizer.from_pretrained("google/pegasus-x-large") >>> model = PegasusModel.from_pretrained("google/pegasus-x-large") >>> inputs = tokenizer("Studies have been shown that owning a dog is good for you", return_tensors="pt") diff --git a/src/transformers/models/perceiver/modeling_perceiver.py b/src/transformers/models/perceiver/modeling_perceiver.py index 02961b8b617a41..f29ad80aba1791 100755 --- a/src/transformers/models/perceiver/modeling_perceiver.py +++ b/src/transformers/models/perceiver/modeling_perceiver.py @@ -50,7 +50,6 @@ _CHECKPOINT_FOR_DOC = "deepmind/language-perceiver" _CONFIG_FOR_DOC = "PerceiverConfig" -_TOKENIZER_FOR_DOC = "PerceiverTokenizer" PERCEIVER_PRETRAINED_MODEL_ARCHIVE_LIST = [ "deepmind/language-perceiver", @@ -768,7 +767,7 @@ def forward( Examples: ```python - >>> from transformers import PerceiverConfig, PerceiverTokenizer, PerceiverImageProcessor, PerceiverModel + >>> from transformers import PerceiverConfig, AutoTokenizer, PerceiverImageProcessor, PerceiverModel >>> from transformers.models.perceiver.modeling_perceiver import ( ... PerceiverTextPreprocessor, ... PerceiverImagePreprocessor, @@ -794,7 +793,7 @@ def forward( >>> model = PerceiverModel(config, input_preprocessor=preprocessor, decoder=decoder) >>> # you can then do a forward pass as follows: - >>> tokenizer = PerceiverTokenizer() + >>> tokenizer = AutoTokenizer() >>> text = "hello world" >>> inputs = tokenizer(text, return_tensors="pt").input_ids @@ -1007,10 +1006,10 @@ def forward( Examples: ```python - >>> from transformers import PerceiverTokenizer, PerceiverForMaskedLM + >>> from transformers import AutoTokenizer, PerceiverForMaskedLM >>> import torch - >>> tokenizer = PerceiverTokenizer.from_pretrained("deepmind/language-perceiver") + >>> tokenizer = AutoTokenizer.from_pretrained("deepmind/language-perceiver") >>> model = PerceiverForMaskedLM.from_pretrained("deepmind/language-perceiver") >>> # training @@ -1131,9 +1130,9 @@ def forward( Examples: ```python - >>> from transformers import PerceiverTokenizer, PerceiverForSequenceClassification + >>> from transformers import AutoTokenizer, PerceiverForSequenceClassification - >>> tokenizer = PerceiverTokenizer.from_pretrained("deepmind/language-perceiver") + >>> tokenizer = AutoTokenizer.from_pretrained("deepmind/language-perceiver") >>> model = PerceiverForSequenceClassification.from_pretrained("deepmind/language-perceiver") >>> text = "hello world" @@ -1266,14 +1265,14 @@ def forward( Examples: ```python - >>> from transformers import PerceiverImageProcessor, PerceiverForImageClassificationLearned + >>> from transformers import AutoImageProcessor, PerceiverForImageClassificationLearned >>> from PIL import Image >>> import requests >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = PerceiverImageProcessor.from_pretrained("deepmind/vision-perceiver-learned") + >>> image_processor = AutoImageProcessor.from_pretrained("deepmind/vision-perceiver-learned") >>> model = PerceiverForImageClassificationLearned.from_pretrained("deepmind/vision-perceiver-learned") >>> inputs = image_processor(images=image, return_tensors="pt").pixel_values @@ -1407,14 +1406,14 @@ def forward( Examples: ```python - >>> from transformers import PerceiverImageProcessor, PerceiverForImageClassificationFourier + >>> from transformers import AutoImageProcessor, PerceiverForImageClassificationFourier >>> from PIL import Image >>> import requests >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = PerceiverImageProcessor.from_pretrained("deepmind/vision-perceiver-fourier") + >>> image_processor = AutoImageProcessor.from_pretrained("deepmind/vision-perceiver-fourier") >>> model = PerceiverForImageClassificationFourier.from_pretrained("deepmind/vision-perceiver-fourier") >>> inputs = image_processor(images=image, return_tensors="pt").pixel_values @@ -1548,14 +1547,14 @@ def forward( Examples: ```python - >>> from transformers import PerceiverImageProcessor, PerceiverForImageClassificationConvProcessing + >>> from transformers import AutoImageProcessor, PerceiverForImageClassificationConvProcessing >>> from PIL import Image >>> import requests >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = PerceiverImageProcessor.from_pretrained("deepmind/vision-perceiver-conv") + >>> image_processor = AutoImageProcessor.from_pretrained("deepmind/vision-perceiver-conv") >>> model = PerceiverForImageClassificationConvProcessing.from_pretrained("deepmind/vision-perceiver-conv") >>> inputs = image_processor(images=image, return_tensors="pt").pixel_values diff --git a/src/transformers/models/plbart/modeling_plbart.py b/src/transformers/models/plbart/modeling_plbart.py index 98017c48c31ba6..d0828941ffe75e 100644 --- a/src/transformers/models/plbart/modeling_plbart.py +++ b/src/transformers/models/plbart/modeling_plbart.py @@ -541,10 +541,10 @@ def _set_gradient_checkpointing(self, module, value=False): Mask-filling example: ```python - >>> from transformers import PLBartTokenizer, PLBartForConditionalGeneration + >>> from transformers import AutoTokenizer, PLBartForConditionalGeneration >>> model = PLBartForConditionalGeneration.from_pretrained("uclanlp/plbart-base") - >>> tokenizer = PLBartTokenizer.from_pretrained("uclanlp/plbart-base") + >>> tokenizer = AutoTokenizer.from_pretrained("uclanlp/plbart-base") >>> # en_XX is the language symbol id for English >>> TXT = " Is 0 the Fibonacci number ? en_XX" @@ -566,7 +566,7 @@ def _set_gradient_checkpointing(self, module, value=False): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`PLBartTokenizer`] or [`PLBartMultiTokenizer`] depending on the checkpoint. + Indices can be obtained using [`AutoTokenizer`] or [`PLBartMultiTokenizer`] depending on the checkpoint. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -580,7 +580,7 @@ def _set_gradient_checkpointing(self, module, value=False): decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`PLBartTokenizer`] or [`PLBartMultiTokenizer`] depending on the checkpoint. + Indices can be obtained using [`AutoTokenizer`] or [`PLBartMultiTokenizer`] depending on the checkpoint. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -719,7 +719,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`PLBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -918,7 +918,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`PLBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1602,7 +1602,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`PLBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1667,9 +1667,9 @@ def forward( Example: ```python - >>> from transformers import PLBartTokenizer, PLBartForCausalLM + >>> from transformers import AutoTokenizer, PLBartForCausalLM - >>> tokenizer = PLBartTokenizer.from_pretrained("uclanlp/plbart-base") + >>> tokenizer = AutoTokenizer.from_pretrained("uclanlp/plbart-base") >>> model = PLBartForCausalLM.from_pretrained("uclanlp/plbart-base", add_cross_attention=False) >>> assert model.config.is_decoder, f"{model.__class__} has to be configured as a decoder." >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") diff --git a/src/transformers/models/poolformer/modeling_poolformer.py b/src/transformers/models/poolformer/modeling_poolformer.py index c987c150d2303c..688a9239f0a0d5 100755 --- a/src/transformers/models/poolformer/modeling_poolformer.py +++ b/src/transformers/models/poolformer/modeling_poolformer.py @@ -301,7 +301,7 @@ def _set_gradient_checkpointing(self, module, value=False): POOLFORMER_INPUTS_DOCSTRING = r""" Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`PoolFormerImageProcessor`]. See + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`PoolFormerImageProcessor.__call__`] for details. """ diff --git a/src/transformers/models/prophetnet/modeling_prophetnet.py b/src/transformers/models/prophetnet/modeling_prophetnet.py index 231145ae7c2461..baf7e1dc4ebb4c 100644 --- a/src/transformers/models/prophetnet/modeling_prophetnet.py +++ b/src/transformers/models/prophetnet/modeling_prophetnet.py @@ -41,7 +41,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "ProphenetConfig" -_TOKENIZER_FOR_DOC = "ProphetNetTokenizer" _CHECKPOINT_FOR_DOC = "microsoft/prophetnet-large-uncased" PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST = [ @@ -75,7 +74,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`ProphetNetTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -89,7 +88,7 @@ decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`ProphetNetTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -148,7 +147,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`ProphetNetTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1306,10 +1305,10 @@ def forward( Example: ```python - >>> from transformers import ProphetNetTokenizer, ProphetNetEncoder + >>> from transformers import AutoTokenizer, ProphetNetEncoder >>> import torch - >>> tokenizer = ProphetNetTokenizer.from_pretrained("microsoft/prophetnet-large-uncased") + >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased") >>> model = ProphetNetEncoder.from_pretrained("patrickvonplaten/prophetnet-large-uncased-standalone") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) @@ -1483,10 +1482,10 @@ def forward( Example: ```python - >>> from transformers import ProphetNetTokenizer, ProphetNetDecoder + >>> from transformers import AutoTokenizer, ProphetNetDecoder >>> import torch - >>> tokenizer = ProphetNetTokenizer.from_pretrained("microsoft/prophetnet-large-uncased") + >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased") >>> model = ProphetNetDecoder.from_pretrained("microsoft/prophetnet-large-uncased", add_cross_attention=False) >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) @@ -1829,9 +1828,9 @@ def forward( Example: ```python - >>> from transformers import ProphetNetTokenizer, ProphetNetModel + >>> from transformers import AutoTokenizer, ProphetNetModel - >>> tokenizer = ProphetNetTokenizer.from_pretrained("microsoft/prophetnet-large-uncased") + >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased") >>> model = ProphetNetModel.from_pretrained("microsoft/prophetnet-large-uncased") >>> input_ids = tokenizer( @@ -1957,9 +1956,9 @@ def forward( Example: ```python - >>> from transformers import ProphetNetTokenizer, ProphetNetForConditionalGeneration + >>> from transformers import AutoTokenizer, ProphetNetForConditionalGeneration - >>> tokenizer = ProphetNetTokenizer.from_pretrained("microsoft/prophetnet-large-uncased") + >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased") >>> model = ProphetNetForConditionalGeneration.from_pretrained("microsoft/prophetnet-large-uncased") >>> input_ids = tokenizer( @@ -2205,10 +2204,10 @@ def forward( Example: ```python - >>> from transformers import ProphetNetTokenizer, ProphetNetForCausalLM + >>> from transformers import AutoTokenizer, ProphetNetForCausalLM >>> import torch - >>> tokenizer = ProphetNetTokenizer.from_pretrained("microsoft/prophetnet-large-uncased") + >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased") >>> model = ProphetNetForCausalLM.from_pretrained("microsoft/prophetnet-large-uncased") >>> assert model.config.is_decoder, f"{model.__class__} has to be configured as a decoder." >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") @@ -2217,11 +2216,11 @@ def forward( >>> logits = outputs.logits >>> # Model can also be used with EncoderDecoder framework - >>> from transformers import BertTokenizer, EncoderDecoderModel, ProphetNetTokenizer + >>> from transformers import BertTokenizer, EncoderDecoderModel, AutoTokenizer >>> import torch >>> tokenizer_enc = BertTokenizer.from_pretrained("bert-large-uncased") - >>> tokenizer_dec = ProphetNetTokenizer.from_pretrained("microsoft/prophetnet-large-uncased") + >>> tokenizer_dec = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased") >>> model = EncoderDecoderModel.from_encoder_decoder_pretrained( ... "bert-large-uncased", "microsoft/prophetnet-large-uncased" ... ) diff --git a/src/transformers/models/qdqbert/modeling_qdqbert.py b/src/transformers/models/qdqbert/modeling_qdqbert.py index 67aba873939a4a..40e581d4f6da12 100755 --- a/src/transformers/models/qdqbert/modeling_qdqbert.py +++ b/src/transformers/models/qdqbert/modeling_qdqbert.py @@ -68,7 +68,6 @@ _CHECKPOINT_FOR_DOC = "bert-base-uncased" _CONFIG_FOR_DOC = "QDQBertConfig" -_TOKENIZER_FOR_DOC = "BertTokenizer" QDQBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ "bert-base-uncased", @@ -784,7 +783,7 @@ def _set_gradient_checkpointing(self, module, value=False): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -875,7 +874,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(QDQBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1085,10 +1083,10 @@ def forward( Example: ```python - >>> from transformers import BertTokenizer, QDQBertLMHeadModel, QDQBertConfig + >>> from transformers import AutoTokenizer, QDQBertLMHeadModel, QDQBertConfig >>> import torch - >>> tokenizer = BertTokenizer.from_pretrained("bert-base-cased") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") >>> config = QDQBertConfig.from_pretrained("bert-base-cased") >>> config.is_decoder = True >>> model = QDQBertLMHeadModel.from_pretrained("bert-base-cased", config=config) @@ -1196,7 +1194,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(QDQBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1320,10 +1317,10 @@ def forward( Example: ```python - >>> from transformers import BertTokenizer, QDQBertForNextSentencePrediction + >>> from transformers import AutoTokenizer, QDQBertForNextSentencePrediction >>> import torch - >>> tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") >>> model = QDQBertForNextSentencePrediction.from_pretrained("bert-base-uncased") >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." @@ -1399,7 +1396,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(QDQBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1496,7 +1492,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(QDQBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1592,7 +1587,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(QDQBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1673,7 +1667,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(QDQBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/rag/modeling_rag.py b/src/transformers/models/rag/modeling_rag.py index ecf664b9041b74..c4186638337426 100644 --- a/src/transformers/models/rag/modeling_rag.py +++ b/src/transformers/models/rag/modeling_rag.py @@ -559,10 +559,10 @@ def forward( Example: ```python - >>> from transformers import RagTokenizer, RagRetriever, RagModel + >>> from transformers import AutoTokenizer, RagRetriever, RagModel >>> import torch - >>> tokenizer = RagTokenizer.from_pretrained("facebook/rag-token-base") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/rag-token-base") >>> retriever = RagRetriever.from_pretrained( ... "facebook/rag-token-base", index_name="exact", use_dummy_dataset=True ... ) @@ -806,10 +806,10 @@ def forward( Example: ```python - >>> from transformers import RagTokenizer, RagRetriever, RagSequenceForGeneration + >>> from transformers import AutoTokenizer, RagRetriever, RagSequenceForGeneration >>> import torch - >>> tokenizer = RagTokenizer.from_pretrained("facebook/rag-sequence-nq") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/rag-sequence-nq") >>> retriever = RagRetriever.from_pretrained( ... "facebook/rag-sequence-nq", index_name="exact", use_dummy_dataset=True ... ) @@ -1274,10 +1274,10 @@ def forward( Example: ```python - >>> from transformers import RagTokenizer, RagRetriever, RagTokenForGeneration + >>> from transformers import AutoTokenizer, RagRetriever, RagTokenForGeneration >>> import torch - >>> tokenizer = RagTokenizer.from_pretrained("facebook/rag-token-nq") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/rag-token-nq") >>> retriever = RagRetriever.from_pretrained( ... "facebook/rag-token-nq", index_name="exact", use_dummy_dataset=True ... ) diff --git a/src/transformers/models/rag/modeling_tf_rag.py b/src/transformers/models/rag/modeling_tf_rag.py index aa8a8da90fdf4d..81c9d94c1a4589 100644 --- a/src/transformers/models/rag/modeling_tf_rag.py +++ b/src/transformers/models/rag/modeling_tf_rag.py @@ -569,10 +569,10 @@ def call( Example: ```python - >>> from transformers import RagTokenizer, RagRetriever, TFRagModel + >>> from transformers import AutoTokenizer, RagRetriever, TFRagModel >>> import torch - >>> tokenizer = RagTokenizer.from_pretrained("facebook/rag-token-base") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/rag-token-base") >>> retriever = RagRetriever.from_pretrained( ... "facebook/rag-token-base", index_name="exact", use_dummy_dataset=True ... ) @@ -884,9 +884,9 @@ def call( ```python >>> import tensorflow as tf - >>> from transformers import RagTokenizer, RagRetriever, TFRagTokenForGeneration + >>> from transformers import AutoTokenizer, RagRetriever, TFRagTokenForGeneration - >>> tokenizer = RagTokenizer.from_pretrained("facebook/rag-token-nq") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/rag-token-nq") >>> retriever = RagRetriever.from_pretrained( ... "facebook/rag-token-nq", index_name="exact", use_dummy_dataset=True ... ) @@ -1384,9 +1384,9 @@ def call( Example: ```python - >>> from transformers import RagTokenizer, RagRetriever, TFRagSequenceForGeneration + >>> from transformers import AutoTokenizer, RagRetriever, TFRagSequenceForGeneration - >>> tokenizer = RagTokenizer.from_pretrained("facebook/rag-sequence-nq") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/rag-sequence-nq") >>> retriever = RagRetriever.from_pretrained( ... "facebook/rag-sequence-nq", index_name="exact", use_dummy_dataset=True ... ) diff --git a/src/transformers/models/realm/modeling_realm.py b/src/transformers/models/realm/modeling_realm.py index da4eaf0f118705..1231cea1b9b66b 100644 --- a/src/transformers/models/realm/modeling_realm.py +++ b/src/transformers/models/realm/modeling_realm.py @@ -41,7 +41,6 @@ _ENCODER_CHECKPOINT_FOR_DOC = "google/realm-cc-news-pretrained-encoder" _SCORER_CHECKPOINT_FOR_DOC = "google/realm-cc-news-pretrained-scorer" _CONFIG_FOR_DOC = "RealmConfig" -_TOKENIZER_FOR_DOC = "RealmTokenizer" REALM_PRETRAINED_MODEL_ARCHIVE_LIST = [ "google/realm-cc-news-pretrained-embedder", @@ -914,7 +913,7 @@ def mask_to_score(mask, dtype=torch.float32): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`RealmTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1182,10 +1181,10 @@ def forward( Example: ```python - >>> from transformers import RealmTokenizer, RealmEmbedder + >>> from transformers import AutoTokenizer, RealmEmbedder >>> import torch - >>> tokenizer = RealmTokenizer.from_pretrained("google/realm-cc-news-pretrained-embedder") + >>> tokenizer = AutoTokenizer.from_pretrained("google/realm-cc-news-pretrained-embedder") >>> model = RealmEmbedder.from_pretrained("google/realm-cc-news-pretrained-embedder") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") @@ -1266,7 +1265,7 @@ def forward( candidate_input_ids (`torch.LongTensor` of shape `(batch_size, num_candidates, sequence_length)`): Indices of candidate input sequence tokens in the vocabulary. - Indices can be obtained using [`RealmTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1296,9 +1295,9 @@ def forward( ```python >>> import torch - >>> from transformers import RealmTokenizer, RealmScorer + >>> from transformers import AutoTokenizer, RealmScorer - >>> tokenizer = RealmTokenizer.from_pretrained("google/realm-cc-news-pretrained-scorer") + >>> tokenizer = AutoTokenizer.from_pretrained("google/realm-cc-news-pretrained-scorer") >>> model = RealmScorer.from_pretrained("google/realm-cc-news-pretrained-scorer", num_candidates=2) >>> # batch_size = 2, num_candidates = 2 @@ -1439,9 +1438,9 @@ def forward( ```python >>> import torch - >>> from transformers import RealmTokenizer, RealmKnowledgeAugEncoder + >>> from transformers import AutoTokenizer, RealmKnowledgeAugEncoder - >>> tokenizer = RealmTokenizer.from_pretrained("google/realm-cc-news-pretrained-encoder") + >>> tokenizer = AutoTokenizer.from_pretrained("google/realm-cc-news-pretrained-encoder") >>> model = RealmKnowledgeAugEncoder.from_pretrained( ... "google/realm-cc-news-pretrained-encoder", num_candidates=2 ... ) @@ -1701,7 +1700,7 @@ def mask_to_score(mask, dtype=torch.float32): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`RealmTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1783,10 +1782,10 @@ def forward( ```python >>> import torch - >>> from transformers import RealmForOpenQA, RealmRetriever, RealmTokenizer + >>> from transformers import RealmForOpenQA, RealmRetriever, AutoTokenizer >>> retriever = RealmRetriever.from_pretrained("google/realm-orqa-nq-openqa") - >>> tokenizer = RealmTokenizer.from_pretrained("google/realm-orqa-nq-openqa") + >>> tokenizer = AutoTokenizer.from_pretrained("google/realm-orqa-nq-openqa") >>> model = RealmForOpenQA.from_pretrained("google/realm-orqa-nq-openqa", retriever=retriever) >>> question = "Who is the pioneer in modern computer science?" diff --git a/src/transformers/models/reformer/modeling_reformer.py b/src/transformers/models/reformer/modeling_reformer.py index 39e26241a334d5..af76a55b1feb01 100755 --- a/src/transformers/models/reformer/modeling_reformer.py +++ b/src/transformers/models/reformer/modeling_reformer.py @@ -1922,7 +1922,7 @@ class ReformerModelWithLMHeadOutput(ModelOutput): a multiple of the relevant model's chunk lengths (lsh's, local's or both). During evaluation, the indices are automatically padded to be a multiple of the chunk length. - Indices can be obtained using [`ReformerTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) diff --git a/src/transformers/models/regnet/modeling_regnet.py b/src/transformers/models/regnet/modeling_regnet.py index f35a917fc3e7fc..07ef29fd33320b 100644 --- a/src/transformers/models/regnet/modeling_regnet.py +++ b/src/transformers/models/regnet/modeling_regnet.py @@ -313,7 +313,7 @@ def _set_gradient_checkpointing(self, module, value=False): Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + [`ConvNextImageProcessor.__call__`] for details. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for diff --git a/src/transformers/models/regnet/modeling_tf_regnet.py b/src/transformers/models/regnet/modeling_tf_regnet.py index c87c5739ebff5f..b1759d71b0a8db 100644 --- a/src/transformers/models/regnet/modeling_tf_regnet.py +++ b/src/transformers/models/regnet/modeling_tf_regnet.py @@ -389,7 +389,7 @@ def serving(self, inputs): Args: pixel_values (`tf.Tensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + [`ConveNextImageProcessor.__call__`] for details. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. diff --git a/src/transformers/models/rembert/modeling_rembert.py b/src/transformers/models/rembert/modeling_rembert.py index 20c3675108c7f7..c94c3a491eefec 100755 --- a/src/transformers/models/rembert/modeling_rembert.py +++ b/src/transformers/models/rembert/modeling_rembert.py @@ -50,7 +50,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "RemBertConfig" -_TOKENIZER_FOR_DOC = "RemBertTokenizer" _CHECKPOINT_FOR_DOC = "google/rembert" REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ @@ -695,7 +694,7 @@ def _set_gradient_checkpointing(self, module, value=False): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`RemBertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -785,7 +784,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(REMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="google/rembert", output_type=BaseModelOutputWithPastAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -938,7 +936,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(REMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="google/rembert", output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1086,10 +1083,10 @@ def forward( Example: ```python - >>> from transformers import RemBertTokenizer, RemBertForCausalLM, RemBertConfig + >>> from transformers import AutoTokenizer, RemBertForCausalLM, RemBertConfig >>> import torch - >>> tokenizer = RemBertTokenizer.from_pretrained("google/rembert") + >>> tokenizer = AutoTokenizer.from_pretrained("google/rembert") >>> config = RemBertConfig.from_pretrained("google/rembert") >>> config.is_decoder = True >>> model = RemBertForCausalLM.from_pretrained("google/rembert", config=config) @@ -1183,7 +1180,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(REMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="google/rembert", output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1280,7 +1276,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(REMBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="google/rembert", output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1373,7 +1368,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(REMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="google/rembert", output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1452,7 +1446,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(REMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="google/rembert", output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/rembert/modeling_tf_rembert.py b/src/transformers/models/rembert/modeling_tf_rembert.py index f55f7964577557..46bcfa3458a792 100644 --- a/src/transformers/models/rembert/modeling_tf_rembert.py +++ b/src/transformers/models/rembert/modeling_tf_rembert.py @@ -60,7 +60,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "RemBertConfig" -_TOKENIZER_FOR_DOC = "RemBertTokenizer" TF_REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ "google/rembert", @@ -887,7 +886,7 @@ def dummy_inputs(self): input_ids (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -951,7 +950,6 @@ def __init__(self, config: RemBertConfig, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(REMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="google/rembert", output_type=TFBaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1054,7 +1052,6 @@ def get_lm_head(self) -> tf.keras.layers.Layer: @unpack_inputs @add_start_docstrings_to_model_forward(REMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="google/rembert", output_type=TFMaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1144,7 +1141,6 @@ def prepare_inputs_for_generation(self, input_ids, past_key_values=None, attenti @unpack_inputs @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="google/rembert", output_type=TFCausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1267,7 +1263,6 @@ def __init__(self, config: RemBertConfig, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(REMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="google/rembert", output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1357,7 +1352,6 @@ def dummy_inputs(self) -> Dict[str, tf.Tensor]: @unpack_inputs @add_start_docstrings_to_model_forward(REMBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="google/rembert", output_type=TFMultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1476,7 +1470,6 @@ def __init__(self, config: RemBertConfig, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(REMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="google/rembert", output_type=TFTokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1555,7 +1548,6 @@ def __init__(self, config: RemBertConfig, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(REMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="google/rembert", output_type=TFQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/resnet/modeling_resnet.py b/src/transformers/models/resnet/modeling_resnet.py index 4c07453861e549..ce11c3d329898f 100644 --- a/src/transformers/models/resnet/modeling_resnet.py +++ b/src/transformers/models/resnet/modeling_resnet.py @@ -285,7 +285,7 @@ def _set_gradient_checkpointing(self, module, value=False): Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + [`ConvNextImageProcessor.__call__`] for details. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for diff --git a/src/transformers/models/resnet/modeling_tf_resnet.py b/src/transformers/models/resnet/modeling_tf_resnet.py index 61db2aacef5ff7..9f02b46f8baffc 100644 --- a/src/transformers/models/resnet/modeling_tf_resnet.py +++ b/src/transformers/models/resnet/modeling_tf_resnet.py @@ -313,7 +313,7 @@ def serving(self, inputs): Args: pixel_values (`tf.Tensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + [`ConvNextImageProcessor.__call__`] for details. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for diff --git a/src/transformers/models/retribert/modeling_retribert.py b/src/transformers/models/retribert/modeling_retribert.py index 03ffc92ba659d4..240d9476e70b01 100644 --- a/src/transformers/models/retribert/modeling_retribert.py +++ b/src/transformers/models/retribert/modeling_retribert.py @@ -186,7 +186,7 @@ def forward( input_ids_query (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary for the queries in a batch. - Indices can be obtained using [`RetriBertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) diff --git a/src/transformers/models/roberta/modeling_flax_roberta.py b/src/transformers/models/roberta/modeling_flax_roberta.py index 56ba89b01ec3ca..bfb6ea365adcf6 100644 --- a/src/transformers/models/roberta/modeling_flax_roberta.py +++ b/src/transformers/models/roberta/modeling_flax_roberta.py @@ -46,7 +46,6 @@ _CHECKPOINT_FOR_DOC = "roberta-base" _CONFIG_FOR_DOC = "RobertaConfig" -_TOKENIZER_FOR_DOC = "RobertaTokenizer" remat = nn_partitioning.remat @@ -102,7 +101,7 @@ def create_position_ids_from_input_ids(input_ids, padding_idx): input_ids (`numpy.ndarray` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -993,9 +992,7 @@ class FlaxRobertaModel(FlaxRobertaPreTrainedModel): module_class = FlaxRobertaModule -append_call_sample_docstring( - FlaxRobertaModel, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutputWithPooling, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxRobertaModel, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutputWithPooling, _CONFIG_FOR_DOC) class FlaxRobertaForMaskedLMModule(nn.Module): @@ -1063,7 +1060,6 @@ class FlaxRobertaForMaskedLM(FlaxRobertaPreTrainedModel): append_call_sample_docstring( FlaxRobertaForMaskedLM, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutputWithPooling, _CONFIG_FOR_DOC, @@ -1136,7 +1132,6 @@ class FlaxRobertaForSequenceClassification(FlaxRobertaPreTrainedModel): append_call_sample_docstring( FlaxRobertaForSequenceClassification, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSequenceClassifierOutput, _CONFIG_FOR_DOC, @@ -1221,7 +1216,6 @@ class FlaxRobertaForMultipleChoice(FlaxRobertaPreTrainedModel): ) append_call_sample_docstring( FlaxRobertaForMultipleChoice, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxMultipleChoiceModelOutput, _CONFIG_FOR_DOC, @@ -1301,7 +1295,6 @@ class FlaxRobertaForTokenClassification(FlaxRobertaPreTrainedModel): append_call_sample_docstring( FlaxRobertaForTokenClassification, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxTokenClassifierOutput, _CONFIG_FOR_DOC, @@ -1379,7 +1372,6 @@ class FlaxRobertaForQuestionAnswering(FlaxRobertaPreTrainedModel): append_call_sample_docstring( FlaxRobertaForQuestionAnswering, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxQuestionAnsweringModelOutput, _CONFIG_FOR_DOC, @@ -1490,7 +1482,6 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): append_call_sample_docstring( FlaxRobertaForCausalLM, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxCausalLMOutputWithCrossAttentions, _CONFIG_FOR_DOC, diff --git a/src/transformers/models/roberta/modeling_roberta.py b/src/transformers/models/roberta/modeling_roberta.py index 5381721f984157..884773e641de1a 100644 --- a/src/transformers/models/roberta/modeling_roberta.py +++ b/src/transformers/models/roberta/modeling_roberta.py @@ -50,7 +50,6 @@ _CHECKPOINT_FOR_DOC = "roberta-base" _CONFIG_FOR_DOC = "RobertaConfig" -_TOKENIZER_FOR_DOC = "RobertaTokenizer" ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST = [ "roberta-base", @@ -645,7 +644,7 @@ def update_keys_to_ignore(self, config, del_keys_to_ignore): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`RobertaTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -742,7 +741,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1064,7 +1062,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1184,7 +1181,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="cardiffnlp/twitter-roberta-base-emotion", output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1283,7 +1279,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1381,7 +1376,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="Jean-Baptiste/roberta-large-ner-english", output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1486,7 +1480,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="deepset/roberta-base-squad2", output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/roberta/modeling_tf_roberta.py b/src/transformers/models/roberta/modeling_tf_roberta.py index 6714e4f7db7469..c38de45e5629e9 100644 --- a/src/transformers/models/roberta/modeling_tf_roberta.py +++ b/src/transformers/models/roberta/modeling_tf_roberta.py @@ -62,7 +62,6 @@ _CHECKPOINT_FOR_DOC = "roberta-base" _CONFIG_FOR_DOC = "RobertaConfig" -_TOKENIZER_FOR_DOC = "RobertaTokenizer" TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST = [ "roberta-base", @@ -865,7 +864,7 @@ def serving(self, inputs): input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`RobertaTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -928,7 +927,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1084,7 +1082,6 @@ def get_prefix_bias_name(self): @unpack_inputs @add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1185,7 +1182,6 @@ def prepare_inputs_for_generation(self, input_ids, past_key_values=None, attenti @unpack_inputs @add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFCausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1335,7 +1331,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="cardiffnlp/twitter-roberta-base-emotion", output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1432,7 +1427,6 @@ def dummy_inputs(self): @unpack_inputs @add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1547,7 +1541,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="ydshieh/roberta-large-ner-english", output_type=TFTokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1633,7 +1626,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="ydshieh/roberta-base-squad2", output_type=TFQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/roberta_prelayernorm/modeling_flax_roberta_prelayernorm.py b/src/transformers/models/roberta_prelayernorm/modeling_flax_roberta_prelayernorm.py index a01dfc520a6d2a..dc9ef03afdc7e8 100644 --- a/src/transformers/models/roberta_prelayernorm/modeling_flax_roberta_prelayernorm.py +++ b/src/transformers/models/roberta_prelayernorm/modeling_flax_roberta_prelayernorm.py @@ -47,7 +47,6 @@ _CHECKPOINT_FOR_DOC = "andreasmadsen/efficient_mlm_m0.40" _CONFIG_FOR_DOC = "RobertaPreLayerNormConfig" -_TOKENIZER_FOR_DOC = "RobertaTokenizer" remat = nn_partitioning.remat @@ -104,7 +103,7 @@ def create_position_ids_from_input_ids(input_ids, padding_idx): input_ids (`numpy.ndarray` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1000,7 +999,6 @@ class FlaxRobertaPreLayerNormModel(FlaxRobertaPreLayerNormPreTrainedModel): append_call_sample_docstring( FlaxRobertaPreLayerNormModel, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutputWithPooling, _CONFIG_FOR_DOC, @@ -1078,7 +1076,6 @@ class FlaxRobertaPreLayerNormForMaskedLM(FlaxRobertaPreLayerNormPreTrainedModel) append_call_sample_docstring( FlaxRobertaPreLayerNormForMaskedLM, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutputWithPooling, _CONFIG_FOR_DOC, @@ -1153,7 +1150,6 @@ class FlaxRobertaPreLayerNormForSequenceClassification(FlaxRobertaPreLayerNormPr append_call_sample_docstring( FlaxRobertaPreLayerNormForSequenceClassification, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSequenceClassifierOutput, _CONFIG_FOR_DOC, @@ -1240,7 +1236,6 @@ class FlaxRobertaPreLayerNormForMultipleChoice(FlaxRobertaPreLayerNormPreTrained ) append_call_sample_docstring( FlaxRobertaPreLayerNormForMultipleChoice, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxMultipleChoiceModelOutput, _CONFIG_FOR_DOC, @@ -1321,7 +1316,6 @@ class FlaxRobertaPreLayerNormForTokenClassification(FlaxRobertaPreLayerNormPreTr append_call_sample_docstring( FlaxRobertaPreLayerNormForTokenClassification, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxTokenClassifierOutput, _CONFIG_FOR_DOC, @@ -1400,7 +1394,6 @@ class FlaxRobertaPreLayerNormForQuestionAnswering(FlaxRobertaPreLayerNormPreTrai append_call_sample_docstring( FlaxRobertaPreLayerNormForQuestionAnswering, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxQuestionAnsweringModelOutput, _CONFIG_FOR_DOC, @@ -1515,7 +1508,6 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): append_call_sample_docstring( FlaxRobertaPreLayerNormForCausalLM, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxCausalLMOutputWithCrossAttentions, _CONFIG_FOR_DOC, diff --git a/src/transformers/models/roberta_prelayernorm/modeling_roberta_prelayernorm.py b/src/transformers/models/roberta_prelayernorm/modeling_roberta_prelayernorm.py index 7efb2c55691f26..2ea6e5da546da9 100644 --- a/src/transformers/models/roberta_prelayernorm/modeling_roberta_prelayernorm.py +++ b/src/transformers/models/roberta_prelayernorm/modeling_roberta_prelayernorm.py @@ -50,7 +50,6 @@ _CHECKPOINT_FOR_DOC = "andreasmadsen/efficient_mlm_m0.40" _CONFIG_FOR_DOC = "RobertaPreLayerNormConfig" -_TOKENIZER_FOR_DOC = "RobertaTokenizer" ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST = [ "andreasmadsen/efficient_mlm_m0.15", @@ -648,7 +647,7 @@ def update_keys_to_ignore(self, config, del_keys_to_ignore): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`RobertaTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -745,7 +744,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(ROBERTA_PRELAYERNORM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1074,7 +1072,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(ROBERTA_PRELAYERNORM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1193,7 +1190,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(ROBERTA_PRELAYERNORM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1294,7 +1290,6 @@ def __init__(self, config): ROBERTA_PRELAYERNORM_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1392,7 +1387,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(ROBERTA_PRELAYERNORM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1497,7 +1491,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(ROBERTA_PRELAYERNORM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/roberta_prelayernorm/modeling_tf_roberta_prelayernorm.py b/src/transformers/models/roberta_prelayernorm/modeling_tf_roberta_prelayernorm.py index 7f6be10cdb79fc..59e35a55abd04e 100644 --- a/src/transformers/models/roberta_prelayernorm/modeling_tf_roberta_prelayernorm.py +++ b/src/transformers/models/roberta_prelayernorm/modeling_tf_roberta_prelayernorm.py @@ -62,7 +62,6 @@ _CHECKPOINT_FOR_DOC = "andreasmadsen/efficient_mlm_m0.40" _CONFIG_FOR_DOC = "RobertaPreLayerNormConfig" -_TOKENIZER_FOR_DOC = "RobertaTokenizer" TF_ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST = [ "andreasmadsen/efficient_mlm_m0.15", @@ -866,7 +865,7 @@ def serving(self, inputs): input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`RobertaTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -930,7 +929,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(ROBERTA_PRELAYERNORM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1092,7 +1090,6 @@ def get_prefix_bias_name(self): @unpack_inputs @add_start_docstrings_to_model_forward(ROBERTA_PRELAYERNORM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1200,7 +1197,6 @@ def prepare_inputs_for_generation(self, input_ids, past_key_values=None, attenti @unpack_inputs @add_start_docstrings_to_model_forward(ROBERTA_PRELAYERNORM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFCausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1355,7 +1351,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(ROBERTA_PRELAYERNORM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1456,7 +1451,6 @@ def dummy_inputs(self): ROBERTA_PRELAYERNORM_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1573,7 +1567,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(ROBERTA_PRELAYERNORM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFTokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1662,7 +1655,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(ROBERTA_PRELAYERNORM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/roc_bert/modeling_roc_bert.py b/src/transformers/models/roc_bert/modeling_roc_bert.py index a6f79a3f1179a6..11b72170c44981 100644 --- a/src/transformers/models/roc_bert/modeling_roc_bert.py +++ b/src/transformers/models/roc_bert/modeling_roc_bert.py @@ -50,7 +50,6 @@ _CHECKPOINT_FOR_DOC = "weiweishi/roc-bert-base-zh" _CONFIG_FOR_DOC = "RoCBertConfig" -_TOKENIZER_FOR_DOC = "RoCBertTokenizer" # Base model docstring _EXPECTED_OUTPUT_SHAPE = [1, 8, 768] @@ -816,21 +815,21 @@ def _set_gradient_checkpointing(self, module, value=False): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`RoCBertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) input_shape_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the shape vocabulary. - Indices can be obtained using [`RoCBertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input_shape_ids) input_pronunciation_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the pronunciation vocabulary. - Indices can be obtained using [`RoCBertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input_pronunciation_ids) @@ -936,7 +935,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(ROC_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1163,10 +1161,10 @@ def forward( Example: ```python - >>> from transformers import RoCBertTokenizer, RoCBertForPreTraining + >>> from transformers import AutoTokenizer, RoCBertForPreTraining >>> import torch - >>> tokenizer = RoCBertTokenizer.from_pretrained("weiweishi/roc-bert-base-zh") + >>> tokenizer = AutoTokenizer.from_pretrained("weiweishi/roc-bert-base-zh") >>> model = RoCBertForPreTraining.from_pretrained("weiweishi/roc-bert-base-zh") >>> inputs = tokenizer("你好，很高兴认识你", return_tensors="pt") @@ -1320,10 +1318,10 @@ def forward( Example: ```python - >>> from transformers import RoCBertTokenizer, RoCBertForMaskedLM + >>> from transformers import AutoTokenizer, RoCBertForMaskedLM >>> import torch - >>> tokenizer = RoCBertTokenizer.from_pretrained("weiweishi/roc-bert-base-zh") + >>> tokenizer = AutoTokenizer.from_pretrained("weiweishi/roc-bert-base-zh") >>> model = RoCBertForMaskedLM.from_pretrained("weiweishi/roc-bert-base-zh") >>> inputs = tokenizer("法国是首都[MASK].", return_tensors="pt") @@ -1488,10 +1486,10 @@ def forward( Example: ```python - >>> from transformers import RoCBertTokenizer, RoCBertForCausalLM, RoCBertConfig + >>> from transformers import AutoTokenizer, RoCBertForCausalLM, RoCBertConfig >>> import torch - >>> tokenizer = RoCBertTokenizer.from_pretrained("weiweishi/roc-bert-base-zh") + >>> tokenizer = AutoTokenizer.from_pretrained("weiweishi/roc-bert-base-zh") >>> config = RoCBertConfig.from_pretrained("weiweishi/roc-bert-base-zh") >>> config.is_decoder = True >>> model = RoCBertForCausalLM.from_pretrained("weiweishi/roc-bert-base-zh", config=config) @@ -1609,7 +1607,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(ROC_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_SEQUENCE_CLASSIFICATION, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1716,7 +1713,6 @@ def __init__(self, config): ROC_BERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1823,7 +1819,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(ROC_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_TOKEN_CLASSIFICATION, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1908,7 +1903,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(ROC_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_QA, output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/roformer/modeling_flax_roformer.py b/src/transformers/models/roformer/modeling_flax_roformer.py index 13dd8548e40f90..d18640d29b5760 100644 --- a/src/transformers/models/roformer/modeling_flax_roformer.py +++ b/src/transformers/models/roformer/modeling_flax_roformer.py @@ -43,7 +43,6 @@ _CHECKPOINT_FOR_DOC = "junnyu/roformer_chinese_base" _CONFIG_FOR_DOC = "RoFormerConfig" -_TOKENIZER_FOR_DOC = "RoFormerTokenizer" FLAX_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST = [ "junnyu/roformer_chinese_small", @@ -95,7 +94,7 @@ input_ids (`numpy.ndarray` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`RoFormerTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -750,9 +749,7 @@ class FlaxRoFormerModel(FlaxRoFormerPreTrainedModel): module_class = FlaxRoFormerModule -append_call_sample_docstring( - FlaxRoFormerModel, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutput, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxRoFormerModel, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutput, _CONFIG_FOR_DOC) class FlaxRoFormerForMaskedLMModule(nn.Module): @@ -812,7 +809,6 @@ class FlaxRoFormerForMaskedLM(FlaxRoFormerPreTrainedModel): append_call_sample_docstring( FlaxRoFormerForMaskedLM, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxMaskedLMOutput, _CONFIG_FOR_DOC, @@ -877,7 +873,6 @@ class FlaxRoFormerForSequenceClassification(FlaxRoFormerPreTrainedModel): append_call_sample_docstring( FlaxRoFormerForSequenceClassification, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSequenceClassifierOutput, _CONFIG_FOR_DOC, @@ -956,7 +951,6 @@ class FlaxRoFormerForMultipleChoice(FlaxRoFormerPreTrainedModel): ) append_call_sample_docstring( FlaxRoFormerForMultipleChoice, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxMultipleChoiceModelOutput, _CONFIG_FOR_DOC, @@ -1022,7 +1016,6 @@ class FlaxRoFormerForTokenClassification(FlaxRoFormerPreTrainedModel): append_call_sample_docstring( FlaxRoFormerForTokenClassification, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxTokenClassifierOutput, _CONFIG_FOR_DOC, @@ -1091,7 +1084,6 @@ class FlaxRoFormerForQuestionAnswering(FlaxRoFormerPreTrainedModel): append_call_sample_docstring( FlaxRoFormerForQuestionAnswering, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxQuestionAnsweringModelOutput, _CONFIG_FOR_DOC, diff --git a/src/transformers/models/roformer/modeling_roformer.py b/src/transformers/models/roformer/modeling_roformer.py index 791e13762f834b..33a85deac9ba76 100644 --- a/src/transformers/models/roformer/modeling_roformer.py +++ b/src/transformers/models/roformer/modeling_roformer.py @@ -51,7 +51,6 @@ _CHECKPOINT_FOR_DOC = "junnyu/roformer_chinese_base" _CONFIG_FOR_DOC = "RoFormerConfig" -_TOKENIZER_FOR_DOC = "RoFormerTokenizer" ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST = [ "junnyu/roformer_chinese_small", @@ -742,7 +741,7 @@ def _set_gradient_checkpointing(self, module, value=False): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`RoFormerTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -828,7 +827,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(ROFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPastAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -979,7 +977,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(ROFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1124,10 +1121,10 @@ def forward( Example: ```python - >>> from transformers import RoFormerTokenizer, RoFormerForCausalLM, RoFormerConfig + >>> from transformers import AutoTokenizer, RoFormerForCausalLM, RoFormerConfig >>> import torch - >>> tokenizer = RoFormerTokenizer.from_pretrained("junnyu/roformer_chinese_base") + >>> tokenizer = AutoTokenizer.from_pretrained("junnyu/roformer_chinese_base") >>> config = RoFormerConfig.from_pretrained("junnyu/roformer_chinese_base") >>> config.is_decoder = True >>> model = RoFormerForCausalLM.from_pretrained("junnyu/roformer_chinese_base", config=config) @@ -1240,7 +1237,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(ROFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1335,7 +1331,6 @@ def __init__(self, config): ROFORMER_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1426,7 +1421,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(ROFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1504,7 +1498,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(ROFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/roformer/modeling_tf_roformer.py b/src/transformers/models/roformer/modeling_tf_roformer.py index 71fc8381363196..018595d64baa27 100644 --- a/src/transformers/models/roformer/modeling_tf_roformer.py +++ b/src/transformers/models/roformer/modeling_tf_roformer.py @@ -61,7 +61,6 @@ _CHECKPOINT_FOR_DOC = "junnyu/roformer_chinese_base" _CONFIG_FOR_DOC = "RoFormerConfig" -_TOKENIZER_FOR_DOC = "RoFormerTokenizer" TF_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST = [ "junnyu/roformer_chinese_small", @@ -755,7 +754,7 @@ class TFRoFormerPreTrainedModel(TFPreTrainedModel): input_ids (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`RoFormerTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -814,7 +813,6 @@ def __init__(self, config: RoFormerConfig, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(ROFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutputWithPooling, config_class=_CONFIG_FOR_DOC, @@ -872,7 +870,6 @@ def get_lm_head(self) -> tf.keras.layers.Layer: @unpack_inputs @add_start_docstrings_to_model_forward(ROFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -947,7 +944,6 @@ def get_lm_head(self) -> tf.keras.layers.Layer: @unpack_inputs @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFCausalLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1057,7 +1053,6 @@ def __init__(self, config: RoFormerConfig, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(ROFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1147,7 +1142,6 @@ def dummy_inputs(self) -> Dict[str, tf.Tensor]: ROFORMER_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1260,7 +1254,6 @@ def __init__(self, config: RoFormerConfig, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(ROFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFTokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1337,7 +1330,6 @@ def __init__(self, config: RoFormerConfig, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(ROFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/segformer/modeling_segformer.py b/src/transformers/models/segformer/modeling_segformer.py index ae2b5cd65f7769..6701f66f9af4a9 100755 --- a/src/transformers/models/segformer/modeling_segformer.py +++ b/src/transformers/models/segformer/modeling_segformer.py @@ -490,7 +490,7 @@ def _init_weights(self, module): Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`SegformerImageProcessor`]. See [`SegformerImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`SegformerImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned @@ -769,11 +769,11 @@ def forward( Examples: ```python - >>> from transformers import SegformerImageProcessor, SegformerForSemanticSegmentation + >>> from transformers import AutoImageProcessor, SegformerForSemanticSegmentation >>> from PIL import Image >>> import requests - >>> image_processor = SegformerImageProcessor.from_pretrained("nvidia/segformer-b0-finetuned-ade-512-512") + >>> image_processor = AutoImageProcessor.from_pretrained("nvidia/segformer-b0-finetuned-ade-512-512") >>> model = SegformerForSemanticSegmentation.from_pretrained("nvidia/segformer-b0-finetuned-ade-512-512") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" diff --git a/src/transformers/models/segformer/modeling_tf_segformer.py b/src/transformers/models/segformer/modeling_tf_segformer.py index 78789900b15d36..4ac9a87406497e 100644 --- a/src/transformers/models/segformer/modeling_tf_segformer.py +++ b/src/transformers/models/segformer/modeling_tf_segformer.py @@ -568,7 +568,7 @@ def serving(self, inputs): Args: pixel_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + [`SegformerImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned @@ -832,14 +832,14 @@ def call( Examples: ```python - >>> from transformers import SegformerImageProcessor, TFSegformerForSemanticSegmentation + >>> from transformers import AutoImageProcessor, TFSegformerForSemanticSegmentation >>> from PIL import Image >>> import requests >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = SegformerImageProcessor.from_pretrained("nvidia/segformer-b0-finetuned-ade-512-512") + >>> image_processor = AutoImageProcessor.from_pretrained("nvidia/segformer-b0-finetuned-ade-512-512") >>> model = TFSegformerForSemanticSegmentation.from_pretrained("nvidia/segformer-b0-finetuned-ade-512-512") >>> inputs = image_processor(images=image, return_tensors="tf") diff --git a/src/transformers/models/sew/modeling_sew.py b/src/transformers/models/sew/modeling_sew.py index e358c75bb55708..3f308891ec2b49 100644 --- a/src/transformers/models/sew/modeling_sew.py +++ b/src/transformers/models/sew/modeling_sew.py @@ -813,10 +813,10 @@ def _get_feature_vector_attention_mask(self, feature_vector_length: int, attenti SEW_INPUTS_DOCSTRING = r""" Args: input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file - into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install - soundfile*). To prepare the array into *input_values*, the [`Wav2Vec2Processor`] should be used for padding - and conversion into a tensor of type *torch.FloatTensor*. See [`Wav2Vec2Processor.__call__`] for details. + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`: diff --git a/src/transformers/models/sew_d/modeling_sew_d.py b/src/transformers/models/sew_d/modeling_sew_d.py index 64c7b006363d1e..b25d7be0c7ebed 100644 --- a/src/transformers/models/sew_d/modeling_sew_d.py +++ b/src/transformers/models/sew_d/modeling_sew_d.py @@ -1353,10 +1353,10 @@ def _set_gradient_checkpointing(self, module, value=False): SEWD_INPUTS_DOCSTRING = r""" Args: input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file - into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install - soundfile*). To prepare the array into *input_values*, the [`Wav2Vec2Processor`] should be used for padding - and conversion into a tensor of type *torch.FloatTensor*. See [`Wav2Vec2Processor.__call__`] for details. + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`: diff --git a/src/transformers/models/speech_encoder_decoder/modeling_flax_speech_encoder_decoder.py b/src/transformers/models/speech_encoder_decoder/modeling_flax_speech_encoder_decoder.py index cd304fa0c0a890..8ecf0967635708 100644 --- a/src/transformers/models/speech_encoder_decoder/modeling_flax_speech_encoder_decoder.py +++ b/src/transformers/models/speech_encoder_decoder/modeling_flax_speech_encoder_decoder.py @@ -85,11 +85,11 @@ SPEECH_ENCODER_DECODER_INPUTS_DOCSTRING = r""" Args: inputs (`jnp.ndarray` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, feature_dim)`, *optional*): - Float values of input raw speech waveform or speech features. Values can be obtained by loading a *.flac* - or *.wav* audio file into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile - library (*pip install soundfile*). To prepare the array into *inputs*, either the [`Wav2Vec2Processor`] or + Float values of input raw speech waveform or speech features. Values can be obtained by loading a `.flac` + or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile + library (`pip install soundfile`). To prepare the array into `inputs`, either the [`Wav2Vec2Processor`] or [`Speech2TextProcessor`] should be used for padding and conversion into a tensor of type - *torch.FloatTensor*. + `torch.FloatTensor`. attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: @@ -682,12 +682,12 @@ def __call__( Examples: ```python - >>> from transformers import FlaxSpeechEncoderDecoderModel, BartTokenizer + >>> from transformers import FlaxSpeechEncoderDecoderModel, AutoTokenizer >>> # load a fine-tuned wav2vec2-2-bart model >>> model = FlaxSpeechEncoderDecoderModel.from_pretrained("patrickvonplaten/wav2vec2-2-bart-large") >>> # load output tokenizer - >>> tokenizer_output = BartTokenizer.from_pretrained("facebook/bart-large") + >>> tokenizer_output = AutoTokenizer.from_pretrained("facebook/bart-large") >>> inputs = jnp.ones((2, 5000), dtype=jnp.float32) diff --git a/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py b/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py index 79ad51479d42d9..dab31b947051be 100644 --- a/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py +++ b/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py @@ -70,11 +70,11 @@ SPEECH_ENCODER_DECODER_INPUTS_DOCSTRING = r""" Args: inputs (`torch.FloatTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, feature_dim)`, *optional*): - Float values of input raw speech waveform or speech features. Values can be obtained by loading a *.flac* - or *.wav* audio file into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile - library (*pip install soundfile*). To prepare the array into *inputs*, either the [`Wav2Vec2Processor`] or + Float values of input raw speech waveform or speech features. Values can be obtained by loading a `.flac` + or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile + library (`pip install soundfile`). To prepare the array into `inputs`, either the [`Wav2Vec2Processor`] or [`Speech2TextProcessor`] should be used for padding and conversion into a tensor of type - *torch.FloatTensor*. + `torch.FloatTensor`. attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: @@ -465,11 +465,11 @@ def forward( Examples: ```python - >>> from transformers import SpeechEncoderDecoderModel, Wav2Vec2Processor + >>> from transformers import SpeechEncoderDecoderModel, AutoProcessor >>> from datasets import load_dataset >>> import torch - >>> processor = Wav2Vec2Processor.from_pretrained("facebook/wav2vec2-xls-r-300m-en-to-15") + >>> processor = AutoProcessor.from_pretrained("facebook/wav2vec2-xls-r-300m-en-to-15") >>> model = SpeechEncoderDecoderModel.from_pretrained("facebook/wav2vec2-xls-r-300m-en-to-15") >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") diff --git a/src/transformers/models/speech_to_text/modeling_speech_to_text.py b/src/transformers/models/speech_to_text/modeling_speech_to_text.py index 24329734f4525d..e1c8d467e3160f 100755 --- a/src/transformers/models/speech_to_text/modeling_speech_to_text.py +++ b/src/transformers/models/speech_to_text/modeling_speech_to_text.py @@ -612,8 +612,8 @@ def _get_feature_vector_attention_mask(self, feature_vector_length, attention_ma Float values of fbank features extracted from the raw speech waveform. Raw speech waveform can be obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the - [`Speech2TextFeatureExtractor`] should be used for extracting the fbank features, padding and conversion - into a tensor of type `torch.FloatTensor`. See [`~Speech2TextFeatureExtractor.__call__`] + [`AutoFeatureExtractor`] should be used for extracting the fbank features, padding and conversion into a + tensor of type `torch.FloatTensor`. See [`~Speech2TextFeatureExtractor.__call__`] attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`: @@ -742,7 +742,7 @@ def forward( Float values of fbank features extracted from the raw speech waveform. Raw speech waveform can be obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into - `input_features`, the [`Speech2TextFeatureExtractor`] should be used for extracting the fbank features, + `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the fbank features, padding and conversion into a tensor of type `torch.FloatTensor`. See [`~Speech2TextFeatureExtractor.__call__`] attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): @@ -1172,11 +1172,11 @@ def forward( ```python >>> import torch - >>> from transformers import Speech2TextModel, Speech2TextFeatureExtractor + >>> from transformers import Speech2TextModel, AutoFeatureExtractor >>> from datasets import load_dataset >>> model = Speech2TextModel.from_pretrained("facebook/s2t-small-librispeech-asr") - >>> feature_extractor = Speech2TextFeatureExtractor.from_pretrained("facebook/s2t-small-librispeech-asr") + >>> feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/s2t-small-librispeech-asr") >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") >>> inputs = feature_extractor( ... ds[0]["audio"]["array"], sampling_rate=ds[0]["audio"]["sampling_rate"], return_tensors="pt" diff --git a/src/transformers/models/speech_to_text/modeling_tf_speech_to_text.py b/src/transformers/models/speech_to_text/modeling_tf_speech_to_text.py index b82c77905197af..1994417d7b71a5 100755 --- a/src/transformers/models/speech_to_text/modeling_tf_speech_to_text.py +++ b/src/transformers/models/speech_to_text/modeling_tf_speech_to_text.py @@ -656,8 +656,8 @@ def serving(self, inputs): Float values of fbank features extracted from the raw speech waveform. Raw speech waveform can be obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the - [`Speech2TextFeatureExtractor`] should be used for extracting the fbank features, padding and conversion - into a tensor of floats. See [`~Speech2TextFeatureExtractor.__call__`] + [`AutoFeatureExtractor`] should be used for extracting the fbank features, padding and conversion into a + tensor of floats. See [`~Speech2TextFeatureExtractor.__call__`] attention_mask (`tf.Tensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: @@ -812,7 +812,7 @@ def call( Float values of fbank features extracted from the raw speech waveform. Raw speech waveform can be obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into - `input_features`, the [`Speech2TextFeatureExtractor`] should be used for extracting the fbank features, + `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the fbank features, padding and conversion into a tensor of floats. See [`~Speech2TextFeatureExtractor.__call__`] attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: diff --git a/src/transformers/models/splinter/modeling_splinter.py b/src/transformers/models/splinter/modeling_splinter.py index 914f4784146b82..954e0aa356d524 100755 --- a/src/transformers/models/splinter/modeling_splinter.py +++ b/src/transformers/models/splinter/modeling_splinter.py @@ -36,7 +36,6 @@ _CHECKPOINT_FOR_DOC = "tau/splinter-base" _CONFIG_FOR_DOC = "SplinterConfig" -_TOKENIZER_FOR_DOC = "SplinterTokenizer" SPLINTER_PRETRAINED_MODEL_ARCHIVE_LIST = [ "tau/splinter-base", @@ -564,7 +563,7 @@ def _set_gradient_checkpointing(self, module, value=False): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`SplinterTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -646,7 +645,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(SPLINTER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPastAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -848,7 +846,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(SPLINTER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/squeezebert/modeling_squeezebert.py b/src/transformers/models/squeezebert/modeling_squeezebert.py index ffe43013ef8d58..a74ad5897b2adb 100644 --- a/src/transformers/models/squeezebert/modeling_squeezebert.py +++ b/src/transformers/models/squeezebert/modeling_squeezebert.py @@ -41,7 +41,6 @@ _CHECKPOINT_FOR_DOC = "squeezebert/squeezebert-uncased" _CONFIG_FOR_DOC = "SqueezeBertConfig" -_TOKENIZER_FOR_DOC = "SqueezeBertTokenizer" SQUEEZEBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ "squeezebert/squeezebert-uncased", @@ -497,7 +496,7 @@ def _init_weights(self, module): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`SqueezeBertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -573,7 +572,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(SQUEEZEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPooling, config_class=_CONFIG_FOR_DOC, @@ -671,7 +669,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(SQUEEZEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -751,7 +748,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(SQUEEZEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -851,7 +847,6 @@ def __init__(self, config): SQUEEZEBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -944,7 +939,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(SQUEEZEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1022,7 +1016,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(SQUEEZEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/swin/modeling_swin.py b/src/transformers/models/swin/modeling_swin.py index e6680ad5d15119..7b48cdc84d2ff2 100644 --- a/src/transformers/models/swin/modeling_swin.py +++ b/src/transformers/models/swin/modeling_swin.py @@ -910,8 +910,8 @@ def _set_gradient_checkpointing(self, module, value=False): SWIN_INPUTS_DOCSTRING = r""" Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] + for details. head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: diff --git a/src/transformers/models/swin/modeling_tf_swin.py b/src/transformers/models/swin/modeling_tf_swin.py index 21c8cbba7afca2..46155a7d73ad20 100644 --- a/src/transformers/models/swin/modeling_tf_swin.py +++ b/src/transformers/models/swin/modeling_tf_swin.py @@ -984,8 +984,8 @@ def serving(self, inputs): SWIN_INPUTS_DOCSTRING = r""" Args: pixel_values (`tf.Tensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] + for details. head_mask (`tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: diff --git a/src/transformers/models/swin2sr/modeling_swin2sr.py b/src/transformers/models/swin2sr/modeling_swin2sr.py index 091c0bc8dbad81..256b8decda7ecf 100644 --- a/src/transformers/models/swin2sr/modeling_swin2sr.py +++ b/src/transformers/models/swin2sr/modeling_swin2sr.py @@ -823,7 +823,7 @@ def _set_gradient_checkpointing(self, module, value=False): Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + [`Swin2SRImageProcessor.__call__`] for details. head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: diff --git a/src/transformers/models/swinv2/modeling_swinv2.py b/src/transformers/models/swinv2/modeling_swinv2.py index 9639f99a6d3ec9..e46decde4ca038 100644 --- a/src/transformers/models/swinv2/modeling_swinv2.py +++ b/src/transformers/models/swinv2/modeling_swinv2.py @@ -989,8 +989,8 @@ def _set_gradient_checkpointing(self, module, value=False): SWINV2_INPUTS_DOCSTRING = r""" Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] + for details. head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: diff --git a/src/transformers/models/switch_transformers/modeling_switch_transformers.py b/src/transformers/models/switch_transformers/modeling_switch_transformers.py index 4cae9762e0c8a8..42aae230142eb5 100644 --- a/src/transformers/models/switch_transformers/modeling_switch_transformers.py +++ b/src/transformers/models/switch_transformers/modeling_switch_transformers.py @@ -49,7 +49,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "SwitchTransformersConfig" -_TOKENIZER_FOR_DOC = "T5Tokenizer" _CHECKPOINT_FOR_DOC = "google/switch-base-8" #################################################### @@ -1186,7 +1185,7 @@ def custom_forward(*inputs): Indices of input sequence tokens in the vocabulary. SWITCH_TRANSFORMERS is a model with relative position embeddings so you should be able to pad the inputs on both the right and the left. - Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail. [What are input IDs?](../glossary#input-ids) @@ -1203,7 +1202,7 @@ def custom_forward(*inputs): decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -1284,7 +1283,7 @@ def custom_forward(*inputs): Indices of input sequence tokens in the vocabulary. SWITCH_TRANSFORMERS is a model with relative position embeddings so you should be able to pad the inputs on both the right and the left. - Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail. To know more on how to prepare `input_ids` for pretraining take a look a [SWITCH_TRANSFORMERS @@ -1405,9 +1404,9 @@ def forward( Example: ```python - >>> from transformers import T5Tokenizer, SwitchTransformersModel + >>> from transformers import AutoTokenizer, SwitchTransformersModel - >>> tokenizer = T5Tokenizer.from_pretrained("google/switch-base-8") + >>> tokenizer = AutoTokenizer.from_pretrained("google/switch-base-8") >>> model = SwitchTransformersModel.from_pretrained("google/switch-base-8") >>> input_ids = tokenizer( @@ -1589,9 +1588,9 @@ def forward( Examples: ```python - >>> from transformers import T5Tokenizer, SwitchTransformersForConditionalGeneration + >>> from transformers import AutoTokenizer, SwitchTransformersForConditionalGeneration - >>> tokenizer = T5Tokenizer.from_pretrained("google/switch-base-8") + >>> tokenizer = AutoTokenizer.from_pretrained("google/switch-base-8") >>> model = SwitchTransformersForConditionalGeneration.from_pretrained("google/switch-base-8") >>> # training @@ -1871,9 +1870,9 @@ def forward( Example: ```python - >>> from transformers import T5Tokenizer, SwitchTransformersEncoderModel + >>> from transformers import AutoTokenizer, SwitchTransformersEncoderModel - >>> tokenizer = T5Tokenizer.from_pretrained("google/switch-base-8") + >>> tokenizer = AutoTokenizer.from_pretrained("google/switch-base-8") >>> model = SwitchTransformersEncoderModel.from_pretrained("google/switch-base-8") >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="pt" diff --git a/src/transformers/models/t5/modeling_flax_t5.py b/src/transformers/models/t5/modeling_flax_t5.py index 1e93fb32357255..1006458fbd5a92 100644 --- a/src/transformers/models/t5/modeling_flax_t5.py +++ b/src/transformers/models/t5/modeling_flax_t5.py @@ -52,7 +52,6 @@ _CHECKPOINT_FOR_DOC = "t5-small" _CONFIG_FOR_DOC = "T5Config" -_TOKENIZER_FOR_DOC = "T5Tokenizer" remat = nn_partitioning.remat @@ -805,7 +804,7 @@ def __call__( Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you should be able to pad the inputs on both the right and the left. - Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail. To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training). @@ -831,7 +830,7 @@ def __call__( decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -874,7 +873,7 @@ def __call__( Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you should be able to pad the inputs on both the right and the left. - Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail. [What are input IDs?](../glossary#input-ids) @@ -890,7 +889,7 @@ def __call__( decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -1090,9 +1089,9 @@ def encode( Example: ```python - >>> from transformers import T5Tokenizer, FlaxT5ForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxT5ForConditionalGeneration - >>> tokenizer = T5Tokenizer.from_pretrained("t5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("t5-small") >>> model = FlaxT5ForConditionalGeneration.from_pretrained("t5-small") >>> text = "My friends are cool but they eat too many carbs." @@ -1151,10 +1150,10 @@ def decode( Example: ```python - >>> from transformers import T5Tokenizer, FlaxT5ForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxT5ForConditionalGeneration >>> import jax.numpy as jnp - >>> tokenizer = T5Tokenizer.from_pretrained("t5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("t5-small") >>> model = FlaxT5ForConditionalGeneration.from_pretrained("t5-small") >>> text = "My friends are cool but they eat too many carbs." @@ -1370,9 +1369,7 @@ class FlaxT5Model(FlaxT5PreTrainedModel): module_class = FlaxT5Module -append_call_sample_docstring( - FlaxT5Model, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqModelOutput, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxT5Model, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqModelOutput, _CONFIG_FOR_DOC) FLAX_T5_MODEL_DOCSTRING = """ Returns: @@ -1380,9 +1377,9 @@ class FlaxT5Model(FlaxT5PreTrainedModel): Example: ```python - >>> from transformers import T5Tokenizer, FlaxT5Model + >>> from transformers import AutoTokenizer, FlaxT5Model - >>> tokenizer = T5Tokenizer.from_pretrained("t5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("t5-small") >>> model = FlaxT5Model.from_pretrained("t5-small") >>> input_ids = tokenizer( @@ -1632,10 +1629,10 @@ def decode( Example: ```python - >>> from transformers import T5Tokenizer, FlaxT5ForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxT5ForConditionalGeneration >>> import jax.numpy as jnp - >>> tokenizer = T5Tokenizer.from_pretrained("t5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("t5-small") >>> model = FlaxT5ForConditionalGeneration.from_pretrained("t5-small") >>> text = "summarize: My friends are cool but they eat too many carbs." @@ -1781,9 +1778,9 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): Example: ```python - >>> from transformers import T5Tokenizer, FlaxT5ForConditionalGeneration + >>> from transformers import AutoTokenizer, FlaxT5ForConditionalGeneration - >>> tokenizer = T5Tokenizer.from_pretrained("t5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("t5-small") >>> model = FlaxT5ForConditionalGeneration.from_pretrained("t5-small") >>> ARTICLE_TO_SUMMARIZE = "summarize: My friends are cool but they eat too many carbs." diff --git a/src/transformers/models/t5/modeling_t5.py b/src/transformers/models/t5/modeling_t5.py index 0a624b93c2a8b3..592f33cf22b8da 100644 --- a/src/transformers/models/t5/modeling_t5.py +++ b/src/transformers/models/t5/modeling_t5.py @@ -51,7 +51,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "T5Config" -_TOKENIZER_FOR_DOC = "T5Tokenizer" _CHECKPOINT_FOR_DOC = "t5-small" #################################################### @@ -1147,7 +1146,7 @@ def custom_forward(*inputs): Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you should be able to pad the inputs on both the right and the left. - Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail. [What are input IDs?](../glossary#input-ids) @@ -1163,7 +1162,7 @@ def custom_forward(*inputs): decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -1240,7 +1239,7 @@ def custom_forward(*inputs): Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you should be able to pad the inputs on both the right and the left. - Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail. To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training). @@ -1386,9 +1385,9 @@ def forward( Example: ```python - >>> from transformers import T5Tokenizer, T5Model + >>> from transformers import AutoTokenizer, T5Model - >>> tokenizer = T5Tokenizer.from_pretrained("t5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("t5-small") >>> model = T5Model.from_pretrained("t5-small") >>> input_ids = tokenizer( @@ -1589,9 +1588,9 @@ def forward( Examples: ```python - >>> from transformers import T5Tokenizer, T5ForConditionalGeneration + >>> from transformers import AutoTokenizer, T5ForConditionalGeneration - >>> tokenizer = T5Tokenizer.from_pretrained("t5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("t5-small") >>> model = T5ForConditionalGeneration.from_pretrained("t5-small") >>> # training @@ -1844,9 +1843,9 @@ def forward( Example: ```python - >>> from transformers import T5Tokenizer, T5EncoderModel + >>> from transformers import AutoTokenizer, T5EncoderModel - >>> tokenizer = T5Tokenizer.from_pretrained("t5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("t5-small") >>> model = T5EncoderModel.from_pretrained("t5-small") >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="pt" diff --git a/src/transformers/models/t5/modeling_tf_t5.py b/src/transformers/models/t5/modeling_tf_t5.py index 039dcb132a9c4f..0e420a85d4deca 100644 --- a/src/transformers/models/t5/modeling_tf_t5.py +++ b/src/transformers/models/t5/modeling_tf_t5.py @@ -56,7 +56,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "T5Config" -_TOKENIZER_FOR_DOC = "T5Tokenizer" TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST = [ "t5-small", @@ -1081,7 +1080,7 @@ def _shift_right(self, input_ids): Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you should be able to pad the inputs on the right or the left. - Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. To know more on how to prepare `inputs` for pre-training take a look at [T5 Training](./t5#training). @@ -1182,9 +1181,9 @@ def call( Examples: ```python - >>> from transformers import T5Tokenizer, TFT5Model + >>> from transformers import AutoTokenizer, TFT5Model - >>> tokenizer = T5Tokenizer.from_pretrained("t5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("t5-small") >>> model = TFT5Model.from_pretrained("t5-small") >>> input_ids = tokenizer( @@ -1366,9 +1365,9 @@ def call( Examples: ```python - >>> from transformers import T5Tokenizer, TFT5ForConditionalGeneration + >>> from transformers import AutoTokenizer, TFT5ForConditionalGeneration - >>> tokenizer = T5Tokenizer.from_pretrained("t5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("t5-small") >>> model = TFT5ForConditionalGeneration.from_pretrained("t5-small") >>> # training @@ -1576,9 +1575,9 @@ def call( Examples: ```python - >>> from transformers import T5Tokenizer, TFT5EncoderModel + >>> from transformers import AutoTokenizer, TFT5EncoderModel - >>> tokenizer = T5Tokenizer.from_pretrained("t5-small") + >>> tokenizer = AutoTokenizer.from_pretrained("t5-small") >>> model = TFT5EncoderModel.from_pretrained("t5-small") >>> input_ids = tokenizer( diff --git a/src/transformers/models/tapas/modeling_tapas.py b/src/transformers/models/tapas/modeling_tapas.py index 5b88269788fb59..55243f01fcdaba 100644 --- a/src/transformers/models/tapas/modeling_tapas.py +++ b/src/transformers/models/tapas/modeling_tapas.py @@ -43,8 +43,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "TapasConfig" -_TOKENIZER_FOR_DOC = "TapasTokenizer" -_TOKENIZER_FOR_DOC = "google/tapas-base" _CHECKPOINT_FOR_DOC = "google/tapas-base" TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST = [ @@ -793,7 +791,7 @@ def _set_gradient_checkpointing(self, module, value=False): TAPAS_INPUTS_DOCSTRING = r""" Args: input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`TapasTokenizer`]. See + Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -805,7 +803,7 @@ def _set_gradient_checkpointing(self, module, value=False): [What are attention masks?](../glossary#attention-mask) token_type_ids (`torch.LongTensor` of shape `({0}, 7)`, *optional*): - Token indices that encode tabular structure. Indices can be obtained using [`TapasTokenizer`]. See this + Token indices that encode tabular structure. Indices can be obtained using [`AutoTokenizer`]. See this class for more info. [What are token type IDs?](../glossary#token-type-ids) @@ -896,10 +894,10 @@ def forward( Examples: ```python - >>> from transformers import TapasTokenizer, TapasModel + >>> from transformers import AutoTokenizer, TapasModel >>> import pandas as pd - >>> tokenizer = TapasTokenizer.from_pretrained("google/tapas-base") + >>> tokenizer = AutoTokenizer.from_pretrained("google/tapas-base") >>> model = TapasModel.from_pretrained("google/tapas-base") >>> data = { @@ -1038,10 +1036,10 @@ def forward( Examples: ```python - >>> from transformers import TapasTokenizer, TapasForMaskedLM + >>> from transformers import AutoTokenizer, TapasForMaskedLM >>> import pandas as pd - >>> tokenizer = TapasTokenizer.from_pretrained("google/tapas-base") + >>> tokenizer = AutoTokenizer.from_pretrained("google/tapas-base") >>> model = TapasForMaskedLM.from_pretrained("google/tapas-base") >>> data = { @@ -1166,7 +1164,7 @@ def forward( padding are 0. labels (`torch.LongTensor` of shape `(batch_size, seq_length)`, *optional*): Labels per token for computing the hierarchical cell selection loss. This encodes the positions of the - answer appearing in the table. Can be obtained using [`TapasTokenizer`]. + answer appearing in the table. Can be obtained using [`AutoTokenizer`]. - 1 for tokens that are **part of the answer**, - 0 for tokens that are **not part of the answer**. @@ -1180,10 +1178,10 @@ def forward( required in case of weak supervision (WTQ) to calculate the aggregate mask and regression loss. numeric_values (`torch.FloatTensor` of shape `(batch_size, seq_length)`, *optional*): Numeric values of every token, NaN for tokens which are not numeric values. Can be obtained using - [`TapasTokenizer`]. Only required in case of weak supervision for aggregation (WTQ) to calculate the + [`AutoTokenizer`]. Only required in case of weak supervision for aggregation (WTQ) to calculate the regression loss. numeric_values_scale (`torch.FloatTensor` of shape `(batch_size, seq_length)`, *optional*): - Scale of the numeric values of every token. Can be obtained using [`TapasTokenizer`]. Only required in case + Scale of the numeric values of every token. Can be obtained using [`AutoTokenizer`]. Only required in case of weak supervision for aggregation (WTQ) to calculate the regression loss. Returns: @@ -1191,10 +1189,10 @@ def forward( Examples: ```python - >>> from transformers import TapasTokenizer, TapasForQuestionAnswering + >>> from transformers import AutoTokenizer, TapasForQuestionAnswering >>> import pandas as pd - >>> tokenizer = TapasTokenizer.from_pretrained("google/tapas-base-finetuned-wtq") + >>> tokenizer = AutoTokenizer.from_pretrained("google/tapas-base-finetuned-wtq") >>> model = TapasForQuestionAnswering.from_pretrained("google/tapas-base-finetuned-wtq") >>> data = { @@ -1489,11 +1487,11 @@ def forward( Examples: ```python - >>> from transformers import TapasTokenizer, TapasForSequenceClassification + >>> from transformers import AutoTokenizer, TapasForSequenceClassification >>> import torch >>> import pandas as pd - >>> tokenizer = TapasTokenizer.from_pretrained("google/tapas-base-finetuned-tabfact") + >>> tokenizer = AutoTokenizer.from_pretrained("google/tapas-base-finetuned-tabfact") >>> model = TapasForSequenceClassification.from_pretrained("google/tapas-base-finetuned-tabfact") >>> data = { diff --git a/src/transformers/models/tapas/modeling_tf_tapas.py b/src/transformers/models/tapas/modeling_tf_tapas.py index 1a8602b22b0eec..1d25e04dd80df8 100644 --- a/src/transformers/models/tapas/modeling_tf_tapas.py +++ b/src/transformers/models/tapas/modeling_tf_tapas.py @@ -69,7 +69,6 @@ ) _CONFIG_FOR_DOC = "TapasConfig" -_TOKENIZER_FOR_DOC = "TapasTokenizer" _CHECKPOINT_FOR_DOC = "google/tapas-base" TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST = [ @@ -932,7 +931,7 @@ def serving(self, inputs): input_ids (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`TapasTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -944,7 +943,7 @@ def serving(self, inputs): [What are attention masks?](../glossary#attention-mask) token_type_ids (`np.ndarray` or `tf.Tensor` of shape `({0}, 7)`, *optional*): - Token indices that encode tabular structure. Indices can be obtained using [`TapasTokenizer`]. See this + Token indices that encode tabular structure. Indices can be obtained using [`AutoTokenizer`]. See this class for more info. [What are token type IDs?](../glossary#token-type-ids) @@ -1013,10 +1012,10 @@ def call( Examples: ```python - >>> from transformers import TapasTokenizer, TapasModel + >>> from transformers import AutoTokenizer, TapasModel >>> import pandas as pd - >>> tokenizer = TapasTokenizer.from_pretrained("google/tapas-base") + >>> tokenizer = AutoTokenizer.from_pretrained("google/tapas-base") >>> model = TapasModel.from_pretrained("google/tapas-base") >>> data = { @@ -1104,10 +1103,10 @@ def call( Examples: ```python - >>> from transformers import TapasTokenizer, TapasForMaskedLM + >>> from transformers import AutoTokenizer, TapasForMaskedLM >>> import pandas as pd - >>> tokenizer = TapasTokenizer.from_pretrained("google/tapas-base") + >>> tokenizer = AutoTokenizer.from_pretrained("google/tapas-base") >>> model = TapasForMaskedLM.from_pretrained("google/tapas-base") >>> data = { @@ -1316,7 +1315,7 @@ def call( padding are 0. labels (`tf.Tensor` of shape `(batch_size, seq_length)`, *optional*): Labels per token for computing the hierarchical cell selection loss. This encodes the positions of the - answer appearing in the table. Can be obtained using [`TapasTokenizer`]. + answer appearing in the table. Can be obtained using [`AutoTokenizer`]. - 1 for tokens that are **part of the answer**, - 0 for tokens that are **not part of the answer**. @@ -1330,10 +1329,10 @@ def call( required in case of weak supervision (WTQ) to calculate the aggregate mask and regression loss. numeric_values (`tf.Tensor` of shape `(batch_size, seq_length)`, *optional*): Numeric values of every token, NaN for tokens which are not numeric values. Can be obtained using - [`TapasTokenizer`]. Only required in case of weak supervision for aggregation (WTQ) to calculate the + [`AutoTokenizer`]. Only required in case of weak supervision for aggregation (WTQ) to calculate the regression loss. numeric_values_scale (`tf.Tensor` of shape `(batch_size, seq_length)`, *optional*): - Scale of the numeric values of every token. Can be obtained using [`TapasTokenizer`]. Only required in case + Scale of the numeric values of every token. Can be obtained using [`AutoTokenizer`]. Only required in case of weak supervision for aggregation (WTQ) to calculate the regression loss. Returns: @@ -1341,10 +1340,10 @@ def call( Examples: ```python - >>> from transformers import TapasTokenizer, TapasForQuestionAnswering + >>> from transformers import AutoTokenizer, TapasForQuestionAnswering >>> import pandas as pd - >>> tokenizer = TapasTokenizer.from_pretrained("google/tapas-base-finetuned-wtq") + >>> tokenizer = AutoTokenizer.from_pretrained("google/tapas-base-finetuned-wtq") >>> model = TapasForQuestionAnswering.from_pretrained("google/tapas-base-finetuned-wtq") >>> data = { @@ -1642,11 +1641,11 @@ def call( Examples: ```python - >>> from transformers import TapasTokenizer, TapasForSequenceClassification + >>> from transformers import AutoTokenizer, TapasForSequenceClassification >>> import tensorflow as tf >>> import pandas as pd - >>> tokenizer = TapasTokenizer.from_pretrained("google/tapas-base-finetuned-tabfact") + >>> tokenizer = AutoTokenizer.from_pretrained("google/tapas-base-finetuned-tabfact") >>> model = TapasForSequenceClassification.from_pretrained("google/tapas-base-finetuned-tabfact") >>> data = { diff --git a/src/transformers/models/timesformer/modeling_timesformer.py b/src/transformers/models/timesformer/modeling_timesformer.py index 03fa4251a8aec9..611d854262fd93 100644 --- a/src/transformers/models/timesformer/modeling_timesformer.py +++ b/src/transformers/models/timesformer/modeling_timesformer.py @@ -515,7 +515,7 @@ def _set_gradient_checkpointing(self, module, value=False): TIMESFORMER_INPUTS_DOCSTRING = r""" Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_frames, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`VideoMAEFeatureExtractor`]. See + Pixel values. Pixel values can be obtained using [`AutoFeatureExtractor`]. See [`VideoMAEFeatureExtractor.__call__`] for details. output_attentions (`bool`, *optional*): @@ -575,7 +575,7 @@ def forward( >>> from decord import VideoReader, cpu >>> import numpy as np - >>> from transformers import TimeSformerFeatureExtractor, TimesformerModel + >>> from transformers import AutoFeatureExtractor, TimesformerModel >>> from huggingface_hub import hf_hub_download @@ -599,7 +599,7 @@ def forward( >>> indices = sample_frame_indices(clip_len=8, frame_sample_rate=4, seg_len=len(videoreader)) >>> video = videoreader.get_batch(indices).asnumpy() - >>> feature_extractor = VideoMAEFeatureExtractor.from_pretrained("MCG-NJU/videomae-base") + >>> feature_extractor = AutoFeatureExtractor.from_pretrained("MCG-NJU/videomae-base") >>> model = TimesformerModel.from_pretrained("facebook/timesformer-base-finetuned-k400") >>> # prepare video for the model @@ -682,7 +682,7 @@ def forward( >>> import torch >>> import numpy as np - >>> from transformers import VideoMAEFeatureExtractor, TimesformerForVideoClassification + >>> from transformers import AutoFeatureExtractor, TimesformerForVideoClassification >>> from huggingface_hub import hf_hub_download >>> np.random.seed(0) @@ -708,7 +708,7 @@ def forward( >>> indices = sample_frame_indices(clip_len=8, frame_sample_rate=4, seg_len=len(videoreader)) >>> video = videoreader.get_batch(indices).asnumpy() - >>> feature_extractor = VideoMAEFeatureExtractor.from_pretrained("MCG-NJU/videomae-base-finetuned-kinetics") + >>> feature_extractor = AutoFeatureExtractor.from_pretrained("MCG-NJU/videomae-base-finetuned-kinetics") >>> model = TimesformerForVideoClassification.from_pretrained("facebook/timesformer-base-finetuned-k400") >>> inputs = feature_extractor(list(video), return_tensors="pt") diff --git a/src/transformers/models/transfo_xl/modeling_tf_transfo_xl.py b/src/transformers/models/transfo_xl/modeling_tf_transfo_xl.py index ce3f95df5e52e0..f2c4653b7298dd 100644 --- a/src/transformers/models/transfo_xl/modeling_tf_transfo_xl.py +++ b/src/transformers/models/transfo_xl/modeling_tf_transfo_xl.py @@ -47,7 +47,6 @@ _CHECKPOINT_FOR_DOC = "transfo-xl-wt103" _CONFIG_FOR_DOC = "TransfoXLConfig" -_TOKENIZER_FOR_DOC = "TransfoXLTokenizer" TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST = [ "transfo-xl-wt103", @@ -888,7 +887,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(TRANSFO_XL_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFTransfoXLModelOutput, config_class=_CONFIG_FOR_DOC, @@ -968,7 +966,6 @@ def init_mems(self, bsz): @unpack_inputs @add_start_docstrings_to_model_forward(TRANSFO_XL_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFTransfoXLLMHeadModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1073,7 +1070,6 @@ def get_output_embeddings(self): @unpack_inputs @add_start_docstrings_to_model_forward(TRANSFO_XL_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFTransfoXLSequenceClassifierOutputWithPast, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/transfo_xl/modeling_transfo_xl.py b/src/transformers/models/transfo_xl/modeling_transfo_xl.py index 1750ccc64b53d1..fb0157dd6f1b09 100644 --- a/src/transformers/models/transfo_xl/modeling_transfo_xl.py +++ b/src/transformers/models/transfo_xl/modeling_transfo_xl.py @@ -41,7 +41,6 @@ _CHECKPOINT_FOR_DOC = "transfo-xl-wt103" _CONFIG_FOR_DOC = "TransfoXLConfig" -_TOKENIZER_FOR_DOC = "TransfoXLTokenizer" TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST = [ "transfo-xl-wt103", @@ -730,7 +729,7 @@ def logits(self): input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`TransfoXLTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -869,7 +868,6 @@ def _update_mems(self, hids, mems, mlen, qlen): @add_start_docstrings_to_model_forward(TRANSFO_XL_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TransfoXLModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1063,7 +1061,6 @@ def init_mems(self, bsz): @add_start_docstrings_to_model_forward(TRANSFO_XL_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TransfoXLLMHeadModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1207,7 +1204,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(TRANSFO_XL_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TransfoXLSequenceClassifierOutputWithPast, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/trocr/modeling_trocr.py b/src/transformers/models/trocr/modeling_trocr.py index 3f3ed27b2bc01d..df7f6c569f474e 100644 --- a/src/transformers/models/trocr/modeling_trocr.py +++ b/src/transformers/models/trocr/modeling_trocr.py @@ -34,7 +34,6 @@ logger = logging.get_logger(__name__) _CONFIG_FOR_DOC = "TrOCRConfig" -_TOKENIZER_FOR_DOC = "TrOCRTokenizer" _CHECKPOINT_FOR_DOC = "microsoft/trocr-base-handwritten" @@ -558,7 +557,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`TrOCRTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -840,7 +839,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`TrOCRTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) diff --git a/src/transformers/models/unispeech/modeling_unispeech.py b/src/transformers/models/unispeech/modeling_unispeech.py index 11d7a30ad9c8a8..af7dc87e673ed9 100755 --- a/src/transformers/models/unispeech/modeling_unispeech.py +++ b/src/transformers/models/unispeech/modeling_unispeech.py @@ -1030,11 +1030,10 @@ def _set_gradient_checkpointing(self, module, value=False): UNISPEECH_INPUTS_DOCSTRING = r""" Args: input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file - into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install - soundfile*). To prepare the array into *input_values*, the [`UniSpeechProcessor`] should be used for - padding and conversion into a tensor of type *torch.FloatTensor*. See [`UniSpeechProcessor.__call__`] for - details. + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`: diff --git a/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py b/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py index 045da6231b5b65..7ce1b4465b0ddb 100755 --- a/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py +++ b/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py @@ -1044,11 +1044,10 @@ def _set_gradient_checkpointing(self, module, value=False): UNISPEECH_SAT_INPUTS_DOCSTRING = r""" Args: input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file - into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install - soundfile*). To prepare the array into *input_values*, the [`UniSpeechSatProcessor`] should be used for - padding and conversion into a tensor of type *torch.FloatTensor*. See [`UniSpeechSatProcessor.__call__`] - for details. + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`: diff --git a/src/transformers/models/upernet/modeling_upernet.py b/src/transformers/models/upernet/modeling_upernet.py index 56190d050389e6..79a76f8cd99aab 100644 --- a/src/transformers/models/upernet/modeling_upernet.py +++ b/src/transformers/models/upernet/modeling_upernet.py @@ -326,7 +326,7 @@ def _set_gradient_checkpointing(self, module, value=False): Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`AutoImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`SegformerImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers in case the backbone has them. See `attentions` under returned tensors for more detail. diff --git a/src/transformers/models/van/modeling_van.py b/src/transformers/models/van/modeling_van.py index 7cedfaf3eae7cb..313950d7559a58 100644 --- a/src/transformers/models/van/modeling_van.py +++ b/src/transformers/models/van/modeling_van.py @@ -407,7 +407,7 @@ def _set_gradient_checkpointing(self, module, value=False): Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + [`ConvNextImageProcessor.__call__`] for details. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all stages. See `hidden_states` under returned tensors for diff --git a/src/transformers/models/videomae/modeling_videomae.py b/src/transformers/models/videomae/modeling_videomae.py index 13e89929816585..8831f65a49b19c 100644 --- a/src/transformers/models/videomae/modeling_videomae.py +++ b/src/transformers/models/videomae/modeling_videomae.py @@ -510,7 +510,7 @@ def _set_gradient_checkpointing(self, module, value=False): VIDEOMAE_INPUTS_DOCSTRING = r""" Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_frames, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`VideoMAEImageProcessor`]. See + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`VideoMAEImageProcessor.__call__`] for details. head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): @@ -581,7 +581,7 @@ def forward( >>> from decord import VideoReader, cpu >>> import numpy as np - >>> from transformers import VideoMAEImageProcessor, VideoMAEModel + >>> from transformers import AutoImageProcessor, VideoMAEModel >>> from huggingface_hub import hf_hub_download @@ -605,7 +605,7 @@ def forward( >>> indices = sample_frame_indices(clip_len=16, frame_sample_rate=4, seg_len=len(videoreader)) >>> video = videoreader.get_batch(indices).asnumpy() - >>> image_processor = VideoMAEImageProcessor.from_pretrained("MCG-NJU/videomae-base") + >>> image_processor = AutoImageProcessor.from_pretrained("MCG-NJU/videomae-base") >>> model = VideoMAEModel.from_pretrained("MCG-NJU/videomae-base") >>> # prepare video for the model @@ -765,14 +765,14 @@ def forward( Examples: ```python - >>> from transformers import VideoMAEImageProcessor, VideoMAEForPreTraining + >>> from transformers import AutoImageProcessor, VideoMAEForPreTraining >>> import numpy as np >>> import torch >>> num_frames = 16 >>> video = list(np.random.randn(16, 3, 224, 224)) - >>> image_processor = VideoMAEImageProcessor.from_pretrained("MCG-NJU/videomae-base") + >>> image_processor = AutoImageProcessor.from_pretrained("MCG-NJU/videomae-base") >>> model = VideoMAEForPreTraining.from_pretrained("MCG-NJU/videomae-base") >>> pixel_values = image_processor(video, return_tensors="pt").pixel_values @@ -950,7 +950,7 @@ def forward( >>> import torch >>> import numpy as np - >>> from transformers import VideoMAEImageProcessor, VideoMAEForVideoClassification + >>> from transformers import AutoImageProcessor, VideoMAEForVideoClassification >>> from huggingface_hub import hf_hub_download >>> np.random.seed(0) @@ -976,7 +976,7 @@ def forward( >>> indices = sample_frame_indices(clip_len=16, frame_sample_rate=4, seg_len=len(videoreader)) >>> video = videoreader.get_batch(indices).asnumpy() - >>> image_processor = VideoMAEImageProcessor.from_pretrained("MCG-NJU/videomae-base-finetuned-kinetics") + >>> image_processor = AutoImageProcessor.from_pretrained("MCG-NJU/videomae-base-finetuned-kinetics") >>> model = VideoMAEForVideoClassification.from_pretrained("MCG-NJU/videomae-base-finetuned-kinetics") >>> inputs = image_processor(list(video), return_tensors="pt") diff --git a/src/transformers/models/vilt/modeling_vilt.py b/src/transformers/models/vilt/modeling_vilt.py index 8bf7847f5254b6..0be05ea4aa1e93 100755 --- a/src/transformers/models/vilt/modeling_vilt.py +++ b/src/transformers/models/vilt/modeling_vilt.py @@ -635,7 +635,7 @@ def _set_gradient_checkpointing(self, module, value=False): [What are token type IDs?](../glossary#token-type-ids) pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`ViltImageProcessor`]. See + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViltImageProcessor.__call__`] for details. pixel_mask (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): @@ -690,7 +690,7 @@ def _set_gradient_checkpointing(self, module, value=False): [What are token type IDs?](../glossary#token-type-ids) pixel_values (`torch.FloatTensor` of shape `(batch_size, num_images, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`ViltImageProcessor`]. See + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViltImageProcessor.__call__`] for details. pixel_mask (`torch.LongTensor` of shape `(batch_size, num_images, height, width)`, *optional*): diff --git a/src/transformers/models/vision_encoder_decoder/modeling_flax_vision_encoder_decoder.py b/src/transformers/models/vision_encoder_decoder/modeling_flax_vision_encoder_decoder.py index 5f9edbe7f93043..0bdc6366867c2c 100644 --- a/src/transformers/models/vision_encoder_decoder/modeling_flax_vision_encoder_decoder.py +++ b/src/transformers/models/vision_encoder_decoder/modeling_flax_vision_encoder_decoder.py @@ -87,7 +87,7 @@ Args: pixel_values (`jnp.ndarray` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using the vision model's image processor. For example, using - [`ViTImageProcessor`]. See [`ViTImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] for details. decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. @@ -115,7 +115,7 @@ Args: pixel_values (`jnp.ndarray` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using the vision model's image processor. For example, using - [`ViTImageProcessor`]. See [`ViTImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. @@ -409,14 +409,14 @@ def encode( Example: ```python - >>> from transformers import ViTImageProcessor, FlaxVisionEncoderDecoderModel + >>> from transformers import AutoImageProcessor, FlaxVisionEncoderDecoderModel >>> from PIL import Image >>> import requests >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = ViTImageProcessor.from_pretrained("google/vit-base-patch16-224-in21k") + >>> image_processor = AutoImageProcessor.from_pretrained("google/vit-base-patch16-224-in21k") >>> # initialize a vit-gpt2 from pretrained ViT and GPT2 models. Note that the cross-attention layers will be randomly initialized >>> model = FlaxVisionEncoderDecoderModel.from_encoder_decoder_pretrained( @@ -487,7 +487,7 @@ def decode( Example: ```python - >>> from transformers import ViTImageProcessor, FlaxVisionEncoderDecoderModel + >>> from transformers import AutoImageProcessor, FlaxVisionEncoderDecoderModel >>> import jax.numpy as jnp >>> from PIL import Image >>> import requests @@ -495,7 +495,7 @@ def decode( >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = ViTImageProcessor.from_pretrained("google/vit-base-patch16-224-in21k") + >>> image_processor = AutoImageProcessor.from_pretrained("google/vit-base-patch16-224-in21k") >>> # initialize a vit-gpt2 from pretrained ViT and GPT2 models. Note that the cross-attention layers will be randomly initialized >>> model = FlaxVisionEncoderDecoderModel.from_encoder_decoder_pretrained( @@ -617,17 +617,17 @@ def __call__( Examples: ```python - >>> from transformers import FlaxVisionEncoderDecoderModel, ViTImageProcessor, GPT2Tokenizer + >>> from transformers import FlaxVisionEncoderDecoderModel, AutoImageProcessor, AutoTokenizer >>> from PIL import Image >>> import requests >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = ViTImageProcessor.from_pretrained("google/vit-base-patch16-224-in21k") + >>> image_processor = AutoImageProcessor.from_pretrained("google/vit-base-patch16-224-in21k") >>> # load output tokenizer - >>> tokenizer_output = GPT2Tokenizer.from_pretrained("gpt2") + >>> tokenizer_output = AutoTokenizer.from_pretrained("gpt2") >>> # initialize a vit-gpt2 from pretrained ViT and GPT2 models. Note that the cross-attention layers will be randomly initialized >>> model = FlaxVisionEncoderDecoderModel.from_encoder_decoder_pretrained( diff --git a/src/transformers/models/vision_encoder_decoder/modeling_tf_vision_encoder_decoder.py b/src/transformers/models/vision_encoder_decoder/modeling_tf_vision_encoder_decoder.py index 50564de22abd6e..88d9b83bea6bab 100644 --- a/src/transformers/models/vision_encoder_decoder/modeling_tf_vision_encoder_decoder.py +++ b/src/transformers/models/vision_encoder_decoder/modeling_tf_vision_encoder_decoder.py @@ -88,7 +88,7 @@ Args: pixel_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using the vision's model's image processor. For example, using - [`ViTImageProcessor`]. See [`ViTImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] for details. decoder_input_ids (`np.ndarray` or `tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. @@ -299,12 +299,12 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): Example: ```python - >>> from transformers import TFVisionEncoderDecoderModel, ViTImageProcessor, GPT2Tokenizer + >>> from transformers import TFVisionEncoderDecoderModel, AutoImageProcessor, AutoTokenizer >>> from PIL import Image >>> import requests - >>> image_processor = ViTImageProcessor.from_pretrained("ydshieh/vit-gpt2-coco-en") - >>> decoder_tokenizer = GPT2Tokenizer.from_pretrained("ydshieh/vit-gpt2-coco-en") + >>> image_processor = AutoImageProcessor.from_pretrained("ydshieh/vit-gpt2-coco-en") + >>> decoder_tokenizer = AutoTokenizer.from_pretrained("ydshieh/vit-gpt2-coco-en") >>> model = TFVisionEncoderDecoderModel.from_pretrained("ydshieh/vit-gpt2-coco-en") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" diff --git a/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py b/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py index e6c7658da41901..91c08901ce0014 100644 --- a/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py +++ b/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py @@ -93,7 +93,7 @@ def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using an image processor (e.g. if you use ViT as the encoder, - you should use [`ViTImageProcessor`]). See [`ViTImageProcessor.__call__`] for details. + you should use [`AutoImageProcessor`]). See [`ViTImageProcessor.__call__`] for details. decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. @@ -248,12 +248,12 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): Example: ```python - >>> from transformers import VisionEncoderDecoderModel, ViTImageProcessor, GPT2Tokenizer + >>> from transformers import VisionEncoderDecoderModel, AutoImageProcessor, AutoTokenizer >>> from PIL import Image >>> import requests - >>> image_processor = ViTImageProcessor.from_pretrained("ydshieh/vit-gpt2-coco-en") - >>> decoder_tokenizer = GPT2Tokenizer.from_pretrained("ydshieh/vit-gpt2-coco-en") + >>> image_processor = AutoImageProcessor.from_pretrained("ydshieh/vit-gpt2-coco-en") + >>> decoder_tokenizer = AutoTokenizer.from_pretrained("ydshieh/vit-gpt2-coco-en") >>> model = VisionEncoderDecoderModel.from_pretrained("ydshieh/vit-gpt2-coco-en") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" @@ -539,12 +539,12 @@ def forward( Examples: ```python - >>> from transformers import TrOCRProcessor, VisionEncoderDecoderModel + >>> from transformers import AutoProcessor, VisionEncoderDecoderModel >>> import requests >>> from PIL import Image >>> import torch - >>> processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-handwritten") + >>> processor = AutoProcessor.from_pretrained("microsoft/trocr-base-handwritten") >>> model = VisionEncoderDecoderModel.from_pretrained("microsoft/trocr-base-handwritten") >>> # load image from the IAM dataset diff --git a/src/transformers/models/vision_text_dual_encoder/modeling_flax_vision_text_dual_encoder.py b/src/transformers/models/vision_text_dual_encoder/modeling_flax_vision_text_dual_encoder.py index 3990b8f3d01c65..4b163965486308 100644 --- a/src/transformers/models/vision_text_dual_encoder/modeling_flax_vision_text_dual_encoder.py +++ b/src/transformers/models/vision_text_dual_encoder/modeling_flax_vision_text_dual_encoder.py @@ -88,7 +88,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -106,7 +106,7 @@ [What are position IDs?](../glossary#position-ids) pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - an image processor (e.g. if you use ViT as the encoder, you should use [`ViTImageProcessor`]). See + an image processor (e.g. if you use ViT as the encoder, you should use [`AutoImageProcessor`]). See [`ViTImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned @@ -556,12 +556,12 @@ def from_vision_text_pretrained( >>> from transformers import ( ... FlaxVisionTextDualEncoderModel, ... VisionTextDualEncoderProcessor, - ... ViTImageProcessor, - ... BertTokenizer, + ... AutoImageProcessor, + ... AutoTokenizer, ... ) - >>> tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") - >>> image_processor = ViTImageProcesor.from_pretrained("google/vit-base-patch16-224") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") + >>> image_processor = AutoImageProcesor.from_pretrained("google/vit-base-patch16-224") >>> processor = VisionTextDualEncoderProcessor(image_processor, tokenizer) >>> model = FlaxVisionTextDualEncoderModel.from_vision_text_pretrained( ... "google/vit-base-patch16-224", "bert-base-uncased" diff --git a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py index 80bba55d3f2c72..d9c075f6d9f34a 100755 --- a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py +++ b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py @@ -96,7 +96,7 @@ Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`CLIPImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. @@ -113,7 +113,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -131,7 +131,7 @@ [What are position IDs?](../glossary#position-ids) pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - an image processor (e.g. if you use ViT as the encoder, you should use [`ViTImageProcessor`]). See + an image processor (e.g. if you use ViT as the encoder, you should use [`AutoImageProcessor`]). See [`ViTImageProcessor.__call__`] for details. return_loss (`bool`, *optional*): Whether or not to return the contrastive loss. @@ -316,12 +316,12 @@ def forward( >>> from transformers import ( ... VisionTextDualEncoderModel, ... VisionTextDualEncoderProcessor, - ... ViTImageProcessor, - ... BertTokenizer, + ... AutoImageProcessor, + ... Autookenizer, ... ) - >>> tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") - >>> image_processor = ViTImageProcessor.from_pretrained("google/vit-base-patch16-224") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") + >>> image_processor = AutoImageProcessor.from_pretrained("google/vit-base-patch16-224") >>> processor = VisionTextDualEncoderProcessor(image_processor, tokenizer) >>> model = VisionTextDualEncoderModel.from_vision_text_pretrained( ... "google/vit-base-patch16-224", "bert-base-uncased" diff --git a/src/transformers/models/visual_bert/modeling_visual_bert.py b/src/transformers/models/visual_bert/modeling_visual_bert.py index 91d44a7143ef80..d9250d73b17086 100755 --- a/src/transformers/models/visual_bert/modeling_visual_bert.py +++ b/src/transformers/models/visual_bert/modeling_visual_bert.py @@ -605,7 +605,7 @@ class VisualBertForPreTrainingOutput(ModelOutput): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -739,10 +739,10 @@ def forward( ```python # Assumption: *get_visual_embeddings(image)* gets the visual embeddings of the image. - from transformers import BertTokenizer, VisualBertModel + from transformers import AutoTokenizer, VisualBertModel import torch - tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") + tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") model = VisualBertModel.from_pretrained("uclanlp/visualbert-vqa-coco-pre") inputs = tokenizer("The capital of France is Paris.", return_tensors="pt") @@ -926,9 +926,9 @@ def forward( ```python # Assumption: *get_visual_embeddings(image)* gets the visual embeddings of the image in the batch. - from transformers import BertTokenizer, VisualBertForPreTraining + from transformers import AutoTokenizer, VisualBertForPreTraining - tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") + tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") model = VisualBertForPreTraining.from_pretrained("uclanlp/visualbert-vqa-coco-pre") inputs = tokenizer("The capital of France is [MASK].", return_tensors="pt") @@ -1065,10 +1065,10 @@ def forward( ```python # Assumption: *get_visual_embeddings(image)* gets the visual embeddings of the image in the batch. - from transformers import BertTokenizer, VisualBertForMultipleChoice + from transformers import AutoTokenizer, VisualBertForMultipleChoice import torch - tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") + tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") model = VisualBertForMultipleChoice.from_pretrained("uclanlp/visualbert-vcr") prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." @@ -1216,10 +1216,10 @@ def forward( ```python # Assumption: *get_visual_embeddings(image)* gets the visual embeddings of the image in the batch. - from transformers import BertTokenizer, VisualBertForQuestionAnswering + from transformers import AutoTokenizer, VisualBertForQuestionAnswering import torch - tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") + tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") model = VisualBertForQuestionAnswering.from_pretrained("uclanlp/visualbert-vqa") text = "Who is eating the apple?" @@ -1342,10 +1342,10 @@ def forward( ```python # Assumption: *get_visual_embeddings(image)* gets the visual embeddings of the image in the batch. - from transformers import BertTokenizer, VisualBertForVisualReasoning + from transformers import AutoTokenizer, VisualBertForVisualReasoning import torch - tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") + tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") model = VisualBertForVisualReasoning.from_pretrained("uclanlp/visualbert-nlvr2") text = "Who is eating the apple?" @@ -1508,10 +1508,10 @@ def forward( ```python # Assumption: *get_visual_embeddings(image)* gets the visual embeddings of the image in the batch. - from transformers import BertTokenizer, VisualBertForRegionToPhraseAlignment + from transformers import AutoTokenizer, VisualBertForRegionToPhraseAlignment import torch - tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") + tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") model = VisualBertForRegionToPhraseAlignment.from_pretrained("uclanlp/visualbert-vqa-coco-pre") text = "Who is eating the apple?" diff --git a/src/transformers/models/vit/modeling_flax_vit.py b/src/transformers/models/vit/modeling_flax_vit.py index 0ba305e936f159..ff9230777bbb6b 100644 --- a/src/transformers/models/vit/modeling_flax_vit.py +++ b/src/transformers/models/vit/modeling_flax_vit.py @@ -70,7 +70,7 @@ VIT_INPUTS_DOCSTRING = r""" Args: pixel_values (`numpy.ndarray` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`ViTImageProcessor`]. See [`ViTImageProcessor.__call__`] + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): @@ -565,14 +565,14 @@ class FlaxViTModel(FlaxViTPreTrainedModel): Examples: ```python - >>> from transformers import ViTImageProcessor, FlaxViTModel + >>> from transformers import AutoImageProcessor, FlaxViTModel >>> from PIL import Image >>> import requests >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = ViTImageProcessor.from_pretrained("google/vit-base-patch16-224-in21k") + >>> image_processor = AutoImageProcessor.from_pretrained("google/vit-base-patch16-224-in21k") >>> model = FlaxViTModel.from_pretrained("google/vit-base-patch16-224-in21k") >>> inputs = image_processor(images=image, return_tensors="np") @@ -648,7 +648,7 @@ class FlaxViTForImageClassification(FlaxViTPreTrainedModel): Example: ```python - >>> from transformers import ViTImageProcessor, FlaxViTForImageClassification + >>> from transformers import AutoImageProcessor, FlaxViTForImageClassification >>> from PIL import Image >>> import jax >>> import requests @@ -656,7 +656,7 @@ class FlaxViTForImageClassification(FlaxViTPreTrainedModel): >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = ViTImageProcessor.from_pretrained("google/vit-base-patch16-224") + >>> image_processor = AutoImageProcessor.from_pretrained("google/vit-base-patch16-224") >>> model = FlaxViTForImageClassification.from_pretrained("google/vit-base-patch16-224") >>> inputs = image_processor(images=image, return_tensors="np") diff --git a/src/transformers/models/vit/modeling_tf_vit.py b/src/transformers/models/vit/modeling_tf_vit.py index 613bd6add174e1..acc4f68bf38efb 100644 --- a/src/transformers/models/vit/modeling_tf_vit.py +++ b/src/transformers/models/vit/modeling_tf_vit.py @@ -628,7 +628,7 @@ def serving(self, inputs): VIT_INPUTS_DOCSTRING = r""" Args: pixel_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`ViTImageProcessor`]. See [`ViTImageProcessor.__call__`] + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] for details. head_mask (`np.ndarray` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): diff --git a/src/transformers/models/vit/modeling_vit.py b/src/transformers/models/vit/modeling_vit.py index f5eae22a04215a..79f61a392b7375 100644 --- a/src/transformers/models/vit/modeling_vit.py +++ b/src/transformers/models/vit/modeling_vit.py @@ -482,7 +482,7 @@ def _set_gradient_checkpointing(self, module: ViTEncoder, value: bool = False) - VIT_INPUTS_DOCSTRING = r""" Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`ViTImageProcessor`]. See [`ViTImageProcessor.__call__`] + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] for details. head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): @@ -664,7 +664,7 @@ def forward( Examples: ```python - >>> from transformers import ViTImageProcessor, ViTForMaskedImageModeling + >>> from transformers import AutoImageProcessor, ViTForMaskedImageModeling >>> import torch >>> from PIL import Image >>> import requests @@ -672,7 +672,7 @@ def forward( >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) - >>> image_processor = ViTImageProcessor.from_pretrained("google/vit-base-patch16-224-in21k") + >>> image_processor = AutoImageProcessor.from_pretrained("google/vit-base-patch16-224-in21k") >>> model = ViTForMaskedImageModeling.from_pretrained("google/vit-base-patch16-224-in21k") >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2 diff --git a/src/transformers/models/vit_hybrid/modeling_vit_hybrid.py b/src/transformers/models/vit_hybrid/modeling_vit_hybrid.py index 00a205fdc02b3d..650e065cf3caaf 100644 --- a/src/transformers/models/vit_hybrid/modeling_vit_hybrid.py +++ b/src/transformers/models/vit_hybrid/modeling_vit_hybrid.py @@ -508,7 +508,7 @@ def _set_gradient_checkpointing(self, module: ViTHybridEncoder, value: bool = Fa Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + [`ViTHybridImageProcessor.__call__`] for details. head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: diff --git a/src/transformers/models/vit_mae/modeling_tf_vit_mae.py b/src/transformers/models/vit_mae/modeling_tf_vit_mae.py index ef5de2545787b9..afb40478ccdae1 100644 --- a/src/transformers/models/vit_mae/modeling_tf_vit_mae.py +++ b/src/transformers/models/vit_mae/modeling_tf_vit_mae.py @@ -770,8 +770,8 @@ def serving(self, inputs): VIT_MAE_INPUTS_DOCSTRING = r""" Args: pixel_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] + for details. head_mask (`np.ndarray` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: diff --git a/src/transformers/models/vit_mae/modeling_vit_mae.py b/src/transformers/models/vit_mae/modeling_vit_mae.py index 39be66e691a2e5..119b7ff3783444 100755 --- a/src/transformers/models/vit_mae/modeling_vit_mae.py +++ b/src/transformers/models/vit_mae/modeling_vit_mae.py @@ -612,8 +612,8 @@ def _set_gradient_checkpointing(self, module, value=False): VIT_MAE_INPUTS_DOCSTRING = r""" Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] + for details. head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: diff --git a/src/transformers/models/vit_msn/modeling_vit_msn.py b/src/transformers/models/vit_msn/modeling_vit_msn.py index 54be1afcc8e955..61faa34eb978f4 100644 --- a/src/transformers/models/vit_msn/modeling_vit_msn.py +++ b/src/transformers/models/vit_msn/modeling_vit_msn.py @@ -464,8 +464,8 @@ def _set_gradient_checkpointing(self, module: ViTMSNEncoder, value: bool = False VIT_MSN_INPUTS_DOCSTRING = r""" Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] + for details. head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: diff --git a/src/transformers/models/wav2vec2/modeling_flax_wav2vec2.py b/src/transformers/models/wav2vec2/modeling_flax_wav2vec2.py index 0d718b8c1fdfb9..364d1590efa9d7 100644 --- a/src/transformers/models/wav2vec2/modeling_flax_wav2vec2.py +++ b/src/transformers/models/wav2vec2/modeling_flax_wav2vec2.py @@ -255,10 +255,10 @@ def _sample_negative_indices(features_shape: Tuple, num_negatives: int, attentio WAV_2_VEC_2_INPUTS_DOCSTRING = r""" Args: input_values (`jnp.ndarray` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file - into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install - soundfile*). To prepare the array into *input_values*, the [`Wav2Vec2Processor`] should be used for padding - and conversion into a tensor of type *jnp.ndarray*. See [`Wav2Vec2Processor.__call__`] for details. + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `jnp.ndarray`. See [`Wav2Vec2Processor.__call__`] for details. attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`: @@ -1065,11 +1065,11 @@ class FlaxWav2Vec2Model(FlaxWav2Vec2PreTrainedModel): Example: ```python - >>> from transformers import Wav2Vec2Processor, FlaxWav2Vec2Model + >>> from transformers import AutoProcessor, FlaxWav2Vec2Model >>> from datasets import load_dataset >>> import soundfile as sf - >>> processor = Wav2Vec2Processor.from_pretrained("facebook/wav2vec2-large-lv60") + >>> processor = AutoProcessor.from_pretrained("facebook/wav2vec2-large-lv60") >>> model = FlaxWav2Vec2Model.from_pretrained("facebook/wav2vec2-large-lv60") @@ -1184,11 +1184,11 @@ class FlaxWav2Vec2ForCTC(FlaxWav2Vec2PreTrainedModel): ```python >>> import jax.numpy as jnp - >>> from transformers import Wav2Vec2Processor, FlaxWav2Vec2ForCTC + >>> from transformers import AutoProcessor, FlaxWav2Vec2ForCTC >>> from datasets import load_dataset >>> import soundfile as sf - >>> processor = Wav2Vec2Processor.from_pretrained("facebook/wav2vec2-large-960h-lv60") + >>> processor = AutoProcessor.from_pretrained("facebook/wav2vec2-large-960h-lv60") >>> model = FlaxWav2Vec2ForCTC.from_pretrained("facebook/wav2vec2-large-960h-lv60") @@ -1384,12 +1384,12 @@ def __call__( >>> import optax >>> import numpy as np >>> import jax.numpy as jnp - >>> from transformers import Wav2Vec2FeatureExtractor, FlaxWav2Vec2ForPreTraining + >>> from transformers import AutoFeatureExtractor, FlaxWav2Vec2ForPreTraining >>> from transformers.models.wav2vec2.modeling_flax_wav2vec2 import _compute_mask_indices >>> from datasets import load_dataset >>> import soundfile as sf - >>> feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained("facebook/wav2vec2-large-lv60") + >>> feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-large-lv60") >>> model = FlaxWav2Vec2ForPreTraining.from_pretrained("facebook/wav2vec2-large-lv60") diff --git a/src/transformers/models/wav2vec2/modeling_tf_wav2vec2.py b/src/transformers/models/wav2vec2/modeling_tf_wav2vec2.py index 055043e75b6eba..dfaf53099c0f5a 100644 --- a/src/transformers/models/wav2vec2/modeling_tf_wav2vec2.py +++ b/src/transformers/models/wav2vec2/modeling_tf_wav2vec2.py @@ -50,7 +50,6 @@ _CHECKPOINT_FOR_DOC = "facebook/wav2vec2-base-960h" _CONFIG_FOR_DOC = "Wav2Vec2Config" -_TOKENIZER_FOR_DOC = "Wav2Vec2Tokenizer" TF_WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST = [ "facebook/wav2vec2-base-960h", @@ -1404,10 +1403,10 @@ def serving(self, inputs): WAV_2_VEC_2_INPUTS_DOCSTRING = r""" Args: - input_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`): + input_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` `Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -1490,11 +1489,11 @@ def call( Example: ```python - >>> from transformers import Wav2Vec2Processor, TFWav2Vec2Model + >>> from transformers import AutoProcessor, TFWav2Vec2Model >>> from datasets import load_dataset >>> import soundfile as sf - >>> processor = Wav2Vec2Processor.from_pretrained("facebook/wav2vec2-base-960h") + >>> processor = AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h") >>> model = TFWav2Vec2Model.from_pretrained("facebook/wav2vec2-base-960h") @@ -1621,11 +1620,11 @@ def call( ```python >>> import tensorflow as tf - >>> from transformers import Wav2Vec2Processor, TFWav2Vec2ForCTC + >>> from transformers import AutoProcessor, TFWav2Vec2ForCTC >>> from datasets import load_dataset >>> import soundfile as sf - >>> processor = Wav2Vec2Processor.from_pretrained("facebook/wav2vec2-base-960h") + >>> processor = AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h") >>> model = TFWav2Vec2ForCTC.from_pretrained("facebook/wav2vec2-base-960h") diff --git a/src/transformers/models/wav2vec2/modeling_wav2vec2.py b/src/transformers/models/wav2vec2/modeling_wav2vec2.py index 2a88147e3219bf..c364d52807c615 100755 --- a/src/transformers/models/wav2vec2/modeling_wav2vec2.py +++ b/src/transformers/models/wav2vec2/modeling_wav2vec2.py @@ -1150,10 +1150,10 @@ def _set_gradient_checkpointing(self, module, value=False): WAV_2_VEC_2_INPUTS_DOCSTRING = r""" Args: input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file - into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install - soundfile*). To prepare the array into *input_values*, the [`Wav2Vec2Processor`] should be used for padding - and conversion into a tensor of type *torch.FloatTensor*. See [`Wav2Vec2Processor.__call__`] for details. + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`: diff --git a/src/transformers/models/wav2vec2_conformer/modeling_wav2vec2_conformer.py b/src/transformers/models/wav2vec2_conformer/modeling_wav2vec2_conformer.py index 5649f4c81ed36a..c1dc7ae94a77cc 100644 --- a/src/transformers/models/wav2vec2_conformer/modeling_wav2vec2_conformer.py +++ b/src/transformers/models/wav2vec2_conformer/modeling_wav2vec2_conformer.py @@ -1194,10 +1194,10 @@ def _set_gradient_checkpointing(self, module, value=False): WAV2VEC2_CONFORMER_INPUTS_DOCSTRING = r""" Args: input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file - into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install - soundfile*). To prepare the array into *input_values*, the [`Wav2Vec2Processor`] should be used for padding - and conversion into a tensor of type *torch.FloatTensor*. See [`Wav2Vec2Processor.__call__`] for details. + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`: diff --git a/src/transformers/models/wavlm/modeling_wavlm.py b/src/transformers/models/wavlm/modeling_wavlm.py index 30cec09efac5b8..e4813447168888 100755 --- a/src/transformers/models/wavlm/modeling_wavlm.py +++ b/src/transformers/models/wavlm/modeling_wavlm.py @@ -1078,10 +1078,10 @@ def _set_gradient_checkpointing(self, module, value=False): WAVLM_INPUTS_DOCSTRING = r""" Args: input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file - into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install - soundfile*). To prepare the array into *input_values*, the [`WavLMProcessor`] should be used for padding - and conversion into a tensor of type *torch.FloatTensor*. See [`WavLMProcessor.__call__`] for details. + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`: diff --git a/src/transformers/models/whisper/modeling_tf_whisper.py b/src/transformers/models/whisper/modeling_tf_whisper.py index 7a76d42fd526b6..e8148366ac882c 100644 --- a/src/transformers/models/whisper/modeling_tf_whisper.py +++ b/src/transformers/models/whisper/modeling_tf_whisper.py @@ -520,7 +520,7 @@ def serving(self, inputs): Float values of fbank features extracted from the raw speech waveform. Raw speech waveform can be obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the - [`WhisperFeatureExtractor`] should be used for extracting the fbank features, padding and conversion into a + [`AutoFeatureExtractor`] should be used for extracting the fbank features, padding and conversion into a tensor of type `tf.Tensor`. See [`~WhisperFeatureExtractor.__call__`] decoder_input_ids (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. @@ -644,7 +644,7 @@ def call( Float values of fbank features extracted from the raw speech waveform. Raw speech waveform can be obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into - `input_features`, the [`WhisperFeatureExtractor`] should be used for extracting the fbank features, + `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the fbank features, padding and conversion into a tensor of type `tf.Tensor`. See [`~WhisperFeatureExtractor.__call__`] head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: @@ -1026,11 +1026,11 @@ def call( ```python >>> import tensorflow as tf - >>> from transformers import TFWhisperModel, WhisperFeatureExtractor + >>> from transformers import TFWhisperModel, AutoFeatureExtractor >>> from datasets import load_dataset >>> model = TFWhisperModel.from_pretrained("openai/whisper-base") - >>> feature_extractor = WhisperFeatureExtractor.from_pretrained("openai/whisper-base") + >>> feature_extractor = AutoFeatureExtractor.from_pretrained("openai/whisper-base") >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") >>> inputs = feature_extractor(ds[0]["audio"]["array"], return_tensors="tf") >>> input_features = inputs.input_features @@ -1151,11 +1151,11 @@ def call( ```python >>> import tensorflow as tf - >>> from transformers import TFWhisperModel, WhisperFeatureExtractor + >>> from transformers import TFWhisperModel, AutoFeatureExtractor >>> from datasets import load_dataset >>> model = TFWhisperModel.from_pretrained("openai/whisper-base") - >>> feature_extractor = WhisperFeatureExtractor.from_pretrained("openai/whisper-base") + >>> feature_extractor = AutoFeatureExtractor.from_pretrained("openai/whisper-base") >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") >>> inputs = feature_extractor(ds[0]["audio"]["array"], return_tensors="tf") >>> input_features = inputs.input_features @@ -1272,10 +1272,10 @@ def call( ```python >>> import tensorflow as tf - >>> from transformers import WhisperProcessor, TFWhisperForConditionalGeneration + >>> from transformers import AutoProcessor, TFWhisperForConditionalGeneration >>> from datasets import load_dataset - >>> processor = WhisperProcessor.from_pretrained("openai/whisper-tiny.en") + >>> processor = AutoProcessor.from_pretrained("openai/whisper-tiny.en") >>> model = TFWhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny.en") >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") diff --git a/src/transformers/models/whisper/modeling_whisper.py b/src/transformers/models/whisper/modeling_whisper.py index 4ae94fb399c514..fb03b63de1f8e9 100644 --- a/src/transformers/models/whisper/modeling_whisper.py +++ b/src/transformers/models/whisper/modeling_whisper.py @@ -501,7 +501,7 @@ def _get_feat_extract_output_lengths(self, input_lengths: torch.LongTensor): Float values mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the - [`WhisperFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a + [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. @@ -626,9 +626,8 @@ def forward( Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into - `input_features`, the [`WhisperFeatureExtractor`] should be used for extracting the mel features, - padding and conversion into a tensor of type `torch.FloatTensor`. See - [`~WhisperFeatureExtractor.__call__`] + `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding + and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] attention_mask (`torch.Tensor`)`, *optional*): Whisper does not support masking of the `input_features`, this argument is preserved for compatibility, but it is not used. By default the silence in the input log mel spectrogram are ignored. @@ -1026,11 +1025,11 @@ def forward( Example: ```python >>> import torch - >>> from transformers import WhisperFeatureExtractor, WhisperModel + >>> from transformers import AutoFeatureExtractor, WhisperModel >>> from datasets import load_dataset >>> model = WhisperModel.from_pretrained("openai/whisper-base") - >>> feature_extractor = WhisperFeatureExtractor.from_pretrained("openai/whisper-base") + >>> feature_extractor = AutoFeatureExtractor.from_pretrained("openai/whisper-base") >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") >>> inputs = feature_extractor(ds[0]["audio"]["array"], return_tensors="pt") >>> input_features = inputs.input_features @@ -1169,10 +1168,10 @@ def forward( ```python >>> import torch - >>> from transformers import WhisperProcessor, WhisperForConditionalGeneration + >>> from transformers import AutoProcessor, WhisperForConditionalGeneration >>> from datasets import load_dataset - >>> processor = WhisperProcessor.from_pretrained("openai/whisper-tiny.en") + >>> processor = AutoProcessor.from_pretrained("openai/whisper-tiny.en") >>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny.en") >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") diff --git a/src/transformers/models/x_clip/modeling_x_clip.py b/src/transformers/models/x_clip/modeling_x_clip.py index d70eb436a9784a..4d3e066bd6422a 100644 --- a/src/transformers/models/x_clip/modeling_x_clip.py +++ b/src/transformers/models/x_clip/modeling_x_clip.py @@ -554,7 +554,7 @@ def _set_gradient_checkpointing(self, module, value=False): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -584,7 +584,7 @@ def _set_gradient_checkpointing(self, module, value=False): Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`CLIPImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. @@ -601,7 +601,7 @@ def _set_gradient_checkpointing(self, module, value=False): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -619,7 +619,7 @@ def _set_gradient_checkpointing(self, module, value=False): [What are position IDs?](../glossary#position-ids) pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`CLIPImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. return_loss (`bool`, *optional*): Whether or not to return the contrastive loss. output_attentions (`bool`, *optional*): @@ -854,10 +854,10 @@ def forward( Examples: ```python - >>> from transformers import CLIPTokenizer, XCLIPTextModel + >>> from transformers import AutoTokenizer, XCLIPTextModel >>> model = XCLIPTextModel.from_pretrained("microsoft/xclip-base-patch32") - >>> tokenizer = CLIPTokenizer.from_pretrained("microsoft/xclip-base-patch32") + >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/xclip-base-patch32") >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") diff --git a/src/transformers/models/xglm/modeling_flax_xglm.py b/src/transformers/models/xglm/modeling_flax_xglm.py index 0b3cd6d73e1a69..f6ae740624094f 100644 --- a/src/transformers/models/xglm/modeling_flax_xglm.py +++ b/src/transformers/models/xglm/modeling_flax_xglm.py @@ -45,7 +45,6 @@ _CHECKPOINT_FOR_DOC = "facebook/xglm-564M" _CONFIG_FOR_DOC = "XGLMConfig" -_TOKENIZER_FOR_DOC = "XGLMTokenizer" XGLM_START_DOCSTRING = r""" This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the @@ -87,7 +86,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`~XGLMTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -706,7 +705,6 @@ class FlaxXGLMModel(FlaxXGLMPreTrainedModel): append_call_sample_docstring( FlaxXGLMModel, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutputWithPastAndCrossAttentions, _CONFIG_FOR_DOC, @@ -811,7 +809,6 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): append_call_sample_docstring( FlaxXGLMForCausalLM, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxCausalLMOutputWithCrossAttentions, _CONFIG_FOR_DOC, diff --git a/src/transformers/models/xglm/modeling_tf_xglm.py b/src/transformers/models/xglm/modeling_tf_xglm.py index 4ca15c78c832f8..2efcd4b1e32703 100644 --- a/src/transformers/models/xglm/modeling_tf_xglm.py +++ b/src/transformers/models/xglm/modeling_tf_xglm.py @@ -51,7 +51,6 @@ _CHECKPOINT_FOR_DOC = "facebook/xglm-564M" _CONFIG_FOR_DOC = "XGLMConfig" -_TOKENIZER_FOR_DOC = "XGLMTokenizer" TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST = [ @@ -703,7 +702,7 @@ def serving(self, inputs): input_ids (`tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`XGLMTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -789,7 +788,6 @@ def __init__( @unpack_inputs @add_start_docstrings_to_model_forward(XGLM_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutputWithPastAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -906,7 +904,6 @@ def prepare_inputs_for_generation(self, inputs, past_key_values=None, use_cache= @add_start_docstrings_to_model_forward(XGLM_INPUTS_DOCSTRING) @replace_return_docstrings(output_type=TFCausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFCausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/xglm/modeling_xglm.py b/src/transformers/models/xglm/modeling_xglm.py index 64386e2946ff41..fd3571ab225d7e 100755 --- a/src/transformers/models/xglm/modeling_xglm.py +++ b/src/transformers/models/xglm/modeling_xglm.py @@ -35,7 +35,6 @@ _CHECKPOINT_FOR_DOC = "facebook/xglm-564M" _CONFIG_FOR_DOC = "XGLMConfig" -_TOKENIZER_FOR_DOC = "XGLMTokenizer" XGLM_PRETRAINED_MODEL_ARCHIVE_LIST = [ @@ -65,7 +64,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`XGLMTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -592,7 +591,6 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em @add_start_docstrings_to_model_forward(XGLM_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPastAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -618,7 +616,7 @@ def forward( Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`~XGLMTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -854,7 +852,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(XGLM_INPUTS_DOCSTRING) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/xlm/modeling_tf_xlm.py b/src/transformers/models/xlm/modeling_tf_xlm.py index f910ff2fdead59..f77111cee450a8 100644 --- a/src/transformers/models/xlm/modeling_tf_xlm.py +++ b/src/transformers/models/xlm/modeling_tf_xlm.py @@ -61,7 +61,6 @@ _CHECKPOINT_FOR_DOC = "xlm-mlm-en-2048" _CONFIG_FOR_DOC = "XLMConfig" -_TOKENIZER_FOR_DOC = "XLMTokenizer" TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST = [ "xlm-mlm-en-2048", @@ -619,7 +618,7 @@ class TFXLMWithLMHeadModelOutput(ModelOutput): input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) @@ -701,7 +700,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(XLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutput, config_class=_CONFIG_FOR_DOC, @@ -838,7 +836,6 @@ def prepare_inputs_for_generation(self, inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(XLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFXLMWithLMHeadModelOutput, config_class=_CONFIG_FOR_DOC, @@ -910,7 +907,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(XLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1017,7 +1013,6 @@ def dummy_inputs(self): @unpack_inputs @add_start_docstrings_to_model_forward(XLM_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1141,7 +1136,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(XLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFTokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1226,7 +1220,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(XLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/xlm/modeling_xlm.py b/src/transformers/models/xlm/modeling_xlm.py index 00014048933b4b..d054f535a42c4e 100755 --- a/src/transformers/models/xlm/modeling_xlm.py +++ b/src/transformers/models/xlm/modeling_xlm.py @@ -52,7 +52,6 @@ _CHECKPOINT_FOR_DOC = "xlm-mlm-en-2048" _CONFIG_FOR_DOC = "XLMConfig" -_TOKENIZER_FOR_DOC = "XLMTokenizer" XLM_PRETRAINED_MODEL_ARCHIVE_LIST = [ "xlm-mlm-en-2048", @@ -324,7 +323,7 @@ class XLMForQuestionAnsweringOutput(ModelOutput): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`XLMTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -481,7 +480,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(XLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC, @@ -704,7 +702,6 @@ def prepare_inputs_for_generation(self, input_ids, **kwargs): @add_start_docstrings_to_model_forward(XLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -784,7 +781,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(XLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -885,7 +881,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(XLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1033,10 +1028,10 @@ def forward( Example: ```python - >>> from transformers import XLMTokenizer, XLMForQuestionAnswering + >>> from transformers import AutoTokenizer, XLMForQuestionAnswering >>> import torch - >>> tokenizer = XLMTokenizer.from_pretrained("xlm-mlm-en-2048") + >>> tokenizer = AutoTokenizer.from_pretrained("xlm-mlm-en-2048") >>> model = XLMForQuestionAnswering.from_pretrained("xlm-mlm-en-2048") >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze( @@ -1113,7 +1108,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(XLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1197,7 +1191,6 @@ def __init__(self, config, *inputs, **kwargs): @add_start_docstrings_to_model_forward(XLM_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/xlm_prophetnet/modeling_xlm_prophetnet.py b/src/transformers/models/xlm_prophetnet/modeling_xlm_prophetnet.py index 57a32d257708c8..c8b6fee9facd99 100644 --- a/src/transformers/models/xlm_prophetnet/modeling_xlm_prophetnet.py +++ b/src/transformers/models/xlm_prophetnet/modeling_xlm_prophetnet.py @@ -43,7 +43,6 @@ _CONFIG_FOR_DOC = "XLMProphetNetConfig" -_TOKENIZER_FOR_DOC = "XLMProphetNetTokenizer" XLM_PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST = [ "microsoft/xprophetnet-large-wiki100-cased", @@ -77,7 +76,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`XLMProphetNetTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -91,7 +90,7 @@ decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. - Indices can be obtained using [`XLMProphetNetTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) @@ -152,7 +151,7 @@ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`XLMProphetNetTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1326,10 +1325,10 @@ def forward( Example: ```python - >>> from transformers import XLMProphetNetTokenizer, XLMProphetNetEncoder + >>> from transformers import AutoTokenizer, XLMProphetNetEncoder >>> import torch - >>> tokenizer = XLMProphetNetTokenizer.from_pretrained("patrickvonplaten/xprophetnet-large-uncased-standalone") + >>> tokenizer = AutoTokenizer.from_pretrained("patrickvonplaten/xprophetnet-large-uncased-standalone") >>> model = XLMProphetNetEncoder.from_pretrained("patrickvonplaten/prophetnet-large-uncased-standalone") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) @@ -1504,10 +1503,10 @@ def forward( Example: ```python - >>> from transformers import XLMProphetNetTokenizer, XLMProphetNetDecoder + >>> from transformers import AutoTokenizer, XLMProphetNetDecoder >>> import torch - >>> tokenizer = XLMProphetNetTokenizer.from_pretrained("patrickvonplaten/xprophetnet-large-uncased-standalone") + >>> tokenizer = AutoTokenizer.from_pretrained("patrickvonplaten/xprophetnet-large-uncased-standalone") >>> model = XLMProphetNetDecoder.from_pretrained( ... "patrickvonplaten/xprophetnet-large-uncased-standalone", add_cross_attention=False ... ) @@ -1853,9 +1852,9 @@ def forward( Example: ```python - >>> from transformers import XLMProphetNetTokenizer, XLMProphetNetModel + >>> from transformers import AutoTokenizer, XLMProphetNetModel - >>> tokenizer = XLMProphetNetTokenizer.from_pretrained("patrickvonplaten/xprophetnet-large-uncased-standalone") + >>> tokenizer = AutoTokenizer.from_pretrained("patrickvonplaten/xprophetnet-large-uncased-standalone") >>> model = XLMProphetNetModel.from_pretrained("patrickvonplaten/xprophetnet-large-uncased-standalone") >>> input_ids = tokenizer( @@ -1982,9 +1981,9 @@ def forward( Example: ```python - >>> from transformers import XLMProphetNetTokenizer, XLMProphetNetForConditionalGeneration + >>> from transformers import AutoTokenizer, XLMProphetNetForConditionalGeneration - >>> tokenizer = XLMProphetNetTokenizer.from_pretrained("patrickvonplaten/xprophetnet-large-uncased-standalone") + >>> tokenizer = AutoTokenizer.from_pretrained("patrickvonplaten/xprophetnet-large-uncased-standalone") >>> model = XLMProphetNetForConditionalGeneration.from_pretrained( ... "patrickvonplaten/xprophetnet-large-uncased-standalone" ... ) @@ -2233,10 +2232,10 @@ def forward( Example: ```python - >>> from transformers import XLMProphetNetTokenizer, XLMProphetNetForCausalLM + >>> from transformers import AutoTokenizer, XLMProphetNetForCausalLM >>> import torch - >>> tokenizer = XLMProphetNetTokenizer.from_pretrained("patrickvonplaten/xprophetnet-large-uncased-standalone") + >>> tokenizer = AutoTokenizer.from_pretrained("patrickvonplaten/xprophetnet-large-uncased-standalone") >>> model = XLMProphetNetForCausalLM.from_pretrained("patrickvonplaten/xprophetnet-large-uncased-standalone") >>> assert model.config.is_decoder, f"{model.__class__} has to be configured as a decoder." >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") @@ -2245,13 +2244,11 @@ def forward( >>> logits = outputs.logits >>> # Model can also be used with EncoderDecoder framework - >>> from transformers import BertTokenizer, EncoderDecoderModel, XLMProphetNetTokenizer + >>> from transformers import BertTokenizer, EncoderDecoderModel, AutoTokenizer >>> import torch >>> tokenizer_enc = BertTokenizer.from_pretrained("bert-large-uncased") - >>> tokenizer_dec = XLMProphetNetTokenizer.from_pretrained( - ... "patrickvonplaten/xprophetnet-large-uncased-standalone" - ... ) + >>> tokenizer_dec = AutoTokenizer.from_pretrained("patrickvonplaten/xprophetnet-large-uncased-standalone") >>> model = EncoderDecoderModel.from_encoder_decoder_pretrained( ... "bert-large-uncased", "patrickvonplaten/xprophetnet-large-uncased-standalone" ... ) diff --git a/src/transformers/models/xlm_roberta/modeling_flax_xlm_roberta.py b/src/transformers/models/xlm_roberta/modeling_flax_xlm_roberta.py index fdb58bce499037..9ae7a9b8b8c9c1 100644 --- a/src/transformers/models/xlm_roberta/modeling_flax_xlm_roberta.py +++ b/src/transformers/models/xlm_roberta/modeling_flax_xlm_roberta.py @@ -49,7 +49,6 @@ _CHECKPOINT_FOR_DOC = "xlm-roberta-base" _CONFIG_FOR_DOC = "XLMRobertaConfig" -_TOKENIZER_FOR_DOC = "XLMRobertaTokenizer" remat = nn_partitioning.remat @@ -112,7 +111,7 @@ def create_position_ids_from_input_ids(input_ids, padding_idx): input_ids (`numpy.ndarray` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1006,9 +1005,7 @@ class FlaxXLMRobertaModel(FlaxXLMRobertaPreTrainedModel): module_class = FlaxXLMRobertaModule -append_call_sample_docstring( - FlaxXLMRobertaModel, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutputWithPooling, _CONFIG_FOR_DOC -) +append_call_sample_docstring(FlaxXLMRobertaModel, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutputWithPooling, _CONFIG_FOR_DOC) # Copied from transformers.models.roberta.modeling_flax_roberta.FlaxRobertaForMaskedLMModule with Roberta->XLMRoberta @@ -1077,7 +1074,6 @@ class FlaxXLMRobertaForMaskedLM(FlaxXLMRobertaPreTrainedModel): append_call_sample_docstring( FlaxXLMRobertaForMaskedLM, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxBaseModelOutputWithPooling, _CONFIG_FOR_DOC, @@ -1151,7 +1147,6 @@ class FlaxXLMRobertaForSequenceClassification(FlaxXLMRobertaPreTrainedModel): append_call_sample_docstring( FlaxXLMRobertaForSequenceClassification, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSequenceClassifierOutput, _CONFIG_FOR_DOC, @@ -1236,7 +1231,6 @@ class FlaxXLMRobertaForMultipleChoice(FlaxXLMRobertaPreTrainedModel): ) append_call_sample_docstring( FlaxXLMRobertaForMultipleChoice, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxMultipleChoiceModelOutput, _CONFIG_FOR_DOC, @@ -1316,7 +1310,6 @@ class FlaxXLMRobertaForTokenClassification(FlaxXLMRobertaPreTrainedModel): append_call_sample_docstring( FlaxXLMRobertaForTokenClassification, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxTokenClassifierOutput, _CONFIG_FOR_DOC, @@ -1394,7 +1387,6 @@ class FlaxXLMRobertaForQuestionAnswering(FlaxXLMRobertaPreTrainedModel): append_call_sample_docstring( FlaxXLMRobertaForQuestionAnswering, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxQuestionAnsweringModelOutput, _CONFIG_FOR_DOC, @@ -1507,7 +1499,6 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): append_call_sample_docstring( FlaxXLMRobertaForCausalLM, - _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxCausalLMOutputWithCrossAttentions, _CONFIG_FOR_DOC, diff --git a/src/transformers/models/xlm_roberta/modeling_tf_xlm_roberta.py b/src/transformers/models/xlm_roberta/modeling_tf_xlm_roberta.py index a0cecea96302d0..5377bfdec19e30 100644 --- a/src/transformers/models/xlm_roberta/modeling_tf_xlm_roberta.py +++ b/src/transformers/models/xlm_roberta/modeling_tf_xlm_roberta.py @@ -64,7 +64,6 @@ _CHECKPOINT_FOR_DOC = "xlm-roberta-base" _CONFIG_FOR_DOC = "XLMRobertaConfig" -_TOKENIZER_FOR_DOC = "XLMRobertaTokenizer" TF_XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST = [ "xlm-roberta-base", @@ -119,8 +118,8 @@ XLM_ROBERTA_INPUTS_DOCSTRING = r""" Args: input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`XLMRobertaTokenizer`]. - See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input + Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See + [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: @@ -922,7 +921,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(XLM_ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFBaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1080,7 +1078,6 @@ def get_prefix_bias_name(self): @unpack_inputs @add_start_docstrings_to_model_forward(XLM_ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1186,7 +1183,6 @@ def prepare_inputs_for_generation(self, input_ids, past_key_values=None, attenti @unpack_inputs @add_start_docstrings_to_model_forward(XLM_ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFCausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1338,7 +1334,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(XLM_ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="cardiffnlp/twitter-roberta-base-emotion", output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1438,7 +1433,6 @@ def dummy_inputs(self): XLM_ROBERTA_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFMultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1554,7 +1548,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(XLM_ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="ydshieh/roberta-large-ner-english", output_type=TFTokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1641,7 +1634,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(XLM_ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="ydshieh/roberta-base-squad2", output_type=TFQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/xlm_roberta/modeling_xlm_roberta.py b/src/transformers/models/xlm_roberta/modeling_xlm_roberta.py index 774b446a960ddd..1807cc67d524b7 100644 --- a/src/transformers/models/xlm_roberta/modeling_xlm_roberta.py +++ b/src/transformers/models/xlm_roberta/modeling_xlm_roberta.py @@ -50,7 +50,6 @@ _CHECKPOINT_FOR_DOC = "xlm-roberta-base" _CONFIG_FOR_DOC = "XLMRobertaConfig" -_TOKENIZER_FOR_DOC = "XLMRobertaTokenizer" XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST = [ "xlm-roberta-base", @@ -647,7 +646,7 @@ def update_keys_to_ignore(self, config, del_keys_to_ignore): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`XLMRobertaTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -744,7 +743,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(XLM_ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -1072,7 +1070,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(XLM_ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1194,7 +1191,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(XLM_ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="cardiffnlp/twitter-roberta-base-emotion", output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1296,7 +1292,6 @@ def __init__(self, config): XLM_ROBERTA_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1395,7 +1390,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(XLM_ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="Jean-Baptiste/roberta-large-ner-english", output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1502,7 +1496,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(XLM_ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint="deepset/roberta-base-squad2", output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/xlm_roberta_xl/modeling_xlm_roberta_xl.py b/src/transformers/models/xlm_roberta_xl/modeling_xlm_roberta_xl.py index 7097add2dad57c..a54d6835b92846 100644 --- a/src/transformers/models/xlm_roberta_xl/modeling_xlm_roberta_xl.py +++ b/src/transformers/models/xlm_roberta_xl/modeling_xlm_roberta_xl.py @@ -49,7 +49,6 @@ _CHECKPOINT_FOR_DOC = "xlm-roberta-xlarge" _CONFIG_FOR_DOC = "XLMRobertaXLConfig" -_TOKENIZER_FOR_DOC = "XLMRobertaTokenizer" XLM_ROBERTA_XL_PRETRAINED_MODEL_ARCHIVE_LIST = [ "facebook/xlm-roberta-xl", @@ -626,7 +625,7 @@ def update_keys_to_ignore(self, config, del_keys_to_ignore): XLM_ROBERTA_XL_INPUTS_DOCSTRING = r""" Args: input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`RobertaTokenizer`]. See + Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): @@ -711,7 +710,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(XLM_ROBERTA_XL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -923,10 +921,10 @@ def forward( Example: ```python - >>> from transformers import RobertaTokenizer, RobertaForCausalLM, RobertaConfig + >>> from transformers import AutoTokenizer, RobertaForCausalLM, RobertaConfig >>> import torch - >>> tokenizer = RobertaTokenizer.from_pretrained("roberta-base") + >>> tokenizer = AutoTokenizer.from_pretrained("roberta-base") >>> config = RobertaConfig.from_pretrained("roberta-base") >>> config.is_decoder = True >>> model = RobertaForCausalLM.from_pretrained("roberta-base", config=config) @@ -1031,7 +1029,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(XLM_ROBERTA_XL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -1144,7 +1141,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(XLM_ROBERTA_XL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1242,7 +1238,6 @@ def __init__(self, config): XLM_ROBERTA_XL_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1339,7 +1334,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(XLM_ROBERTA_XL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1450,7 +1444,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(XLM_ROBERTA_XL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/xlnet/modeling_tf_xlnet.py b/src/transformers/models/xlnet/modeling_tf_xlnet.py index a838e61f3d6036..42ec90efc4d46a 100644 --- a/src/transformers/models/xlnet/modeling_tf_xlnet.py +++ b/src/transformers/models/xlnet/modeling_tf_xlnet.py @@ -56,7 +56,6 @@ _CHECKPOINT_FOR_DOC = "xlnet-base-cased" _CONFIG_FOR_DOC = "XLNetConfig" -_TOKENIZER_FOR_DOC = "XLNetTokenizer" TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST = [ "xlnet-base-cased", @@ -1065,7 +1064,7 @@ class TFXLNetForQuestionAnsweringSimpleOutput(ModelOutput): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`XLNetTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1145,7 +1144,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(XLNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFXLNetModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1289,9 +1287,9 @@ def call( ```python >>> import tensorflow as tf >>> import numpy as np - >>> from transformers import XLNetTokenizer, TFXLNetLMHeadModel + >>> from transformers import AutoTokenizer, TFXLNetLMHeadModel - >>> tokenizer = XLNetTokenizer.from_pretrained("xlnet-large-cased") + >>> tokenizer = AutoTokenizer.from_pretrained("xlnet-large-cased") >>> model = TFXLNetLMHeadModel.from_pretrained("xlnet-large-cased") >>> # We show how to setup inputs to predict a next token using a bi-directional context. @@ -1385,7 +1383,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(XLNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFXLNetForSequenceClassificationOutput, config_class=_CONFIG_FOR_DOC, @@ -1491,7 +1488,6 @@ def dummy_inputs(self): @unpack_inputs @add_start_docstrings_to_model_forward(XLNET_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFXLNetForMultipleChoiceOutput, config_class=_CONFIG_FOR_DOC, @@ -1612,7 +1608,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(XLNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFXLNetForTokenClassificationOutput, config_class=_CONFIG_FOR_DOC, @@ -1698,7 +1693,6 @@ def __init__(self, config, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(XLNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TFXLNetForQuestionAnsweringSimpleOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/models/xlnet/modeling_xlnet.py b/src/transformers/models/xlnet/modeling_xlnet.py index b1ac4c75b9b3f7..07362fe29c53f7 100755 --- a/src/transformers/models/xlnet/modeling_xlnet.py +++ b/src/transformers/models/xlnet/modeling_xlnet.py @@ -42,7 +42,6 @@ _CHECKPOINT_FOR_DOC = "xlnet-base-cased" _CONFIG_FOR_DOC = "XLNetConfig" -_TOKENIZER_FOR_DOC = "XLNetTokenizer" XLNET_PRETRAINED_MODEL_ARCHIVE_LIST = [ "xlnet-base-cased", @@ -857,7 +856,7 @@ class XLNetForQuestionAnsweringOutput(ModelOutput): input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`XLNetTokenizer`]. See [`PreTrainedTokenizer.encode`] and + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) @@ -1060,7 +1059,6 @@ def relative_positional_encoding(self, qlen, klen, bsz=None): @add_start_docstrings_to_model_forward(XLNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=XLNetModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1394,10 +1392,10 @@ def forward( Examples: ```python - >>> from transformers import XLNetTokenizer, XLNetLMHeadModel + >>> from transformers import AutoTokenizer, XLNetLMHeadModel >>> import torch - >>> tokenizer = XLNetTokenizer.from_pretrained("xlnet-large-cased") + >>> tokenizer = AutoTokenizer.from_pretrained("xlnet-large-cased") >>> model = XLNetLMHeadModel.from_pretrained("xlnet-large-cased") >>> # We show how to setup inputs to predict a next token using a bi-directional context. @@ -1516,7 +1514,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(XLNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=XLNetForSequenceClassificationOutput, config_class=_CONFIG_FOR_DOC, @@ -1624,7 +1621,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(XLNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=XLNetForTokenClassificationOutput, config_class=_CONFIG_FOR_DOC, @@ -1712,7 +1708,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(XLNET_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=XLNetForMultipleChoiceOutput, config_class=_CONFIG_FOR_DOC, @@ -1816,7 +1811,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(XLNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=XLNetForQuestionAnsweringSimpleOutput, config_class=_CONFIG_FOR_DOC, @@ -1975,10 +1969,10 @@ def forward( Example: ```python - >>> from transformers import XLNetTokenizer, XLNetForQuestionAnswering + >>> from transformers import AutoTokenizer, XLNetForQuestionAnswering >>> import torch - >>> tokenizer = XLNetTokenizer.from_pretrained("xlnet-base-cased") + >>> tokenizer = AutoTokenizer.from_pretrained("xlnet-base-cased") >>> model = XLNetForQuestionAnswering.from_pretrained("xlnet-base-cased") >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze( diff --git a/src/transformers/models/yolos/modeling_yolos.py b/src/transformers/models/yolos/modeling_yolos.py index cf1fee555887f8..1921b87a9fcb9c 100755 --- a/src/transformers/models/yolos/modeling_yolos.py +++ b/src/transformers/models/yolos/modeling_yolos.py @@ -82,7 +82,7 @@ class YolosObjectDetectionOutput(ModelOutput): pred_boxes (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)`): Normalized boxes coordinates for all queries, represented as (center_x, center_y, width, height). These values are normalized in [0, 1], relative to the size of each individual image in the batch (disregarding - possible padding). You can use [`~DetrImageProcessor.post_process`] to retrieve the unnormalized bounding + possible padding). You can use [`~YolosImageProcessor.post_process`] to retrieve the unnormalized bounding boxes. auxiliary_outputs (`list[Dict]`, *optional*): Optional, only returned when auxilary losses are activated (i.e. `config.auxiliary_loss` is set to `True`) @@ -573,7 +573,7 @@ def _set_gradient_checkpointing(self, module: YolosEncoder, value: bool = False) Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. + [`YolosImageProcessor.__call__`] for details. head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: diff --git a/src/transformers/models/yoso/modeling_yoso.py b/src/transformers/models/yoso/modeling_yoso.py index 9795824f85dfa5..df1ec304b952e2 100644 --- a/src/transformers/models/yoso/modeling_yoso.py +++ b/src/transformers/models/yoso/modeling_yoso.py @@ -43,7 +43,6 @@ _CHECKPOINT_FOR_DOC = "uw-madison/yoso-4096" _CONFIG_FOR_DOC = "YosoConfig" -_TOKENIZER_FOR_DOC = "AutoTokenizer" YOSO_PRETRAINED_MODEL_ARCHIVE_LIST = [ "uw-madison/yoso-4096", @@ -766,7 +765,6 @@ class PreTrainedModel @add_start_docstrings_to_model_forward(YOSO_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=BaseModelOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC, @@ -875,7 +873,6 @@ def set_output_embeddings(self, new_embeddings): @add_start_docstrings_to_model_forward(YOSO_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC, @@ -971,7 +968,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(YOSO_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1064,7 +1060,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(YOSO_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC, @@ -1157,7 +1152,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(YOSO_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC, @@ -1244,7 +1238,6 @@ def __init__(self, config): @add_start_docstrings_to_model_forward(YOSO_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC, diff --git a/src/transformers/utils/doc.py b/src/transformers/utils/doc.py index e37ad3fff249f6..2e6264c508907a 100644 --- a/src/transformers/utils/doc.py +++ b/src/transformers/utils/doc.py @@ -607,10 +607,10 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): Example: ```python - >>> from transformers import {processor_class}, {model_class} + >>> from transformers import AutoTokenizer, {model_class} >>> import tensorflow as tf - >>> tokenizer = {processor_class}.from_pretrained("{checkpoint}") + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") >>> model = {model_class}.from_pretrained("{checkpoint}") >>> inputs = tokenizer( @@ -640,10 +640,10 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): Example: ```python - >>> from transformers import {processor_class}, {model_class} + >>> from transformers import AutoTokenizer, {model_class} >>> import tensorflow as tf - >>> tokenizer = {processor_class}.from_pretrained("{checkpoint}") + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") >>> model = {model_class}.from_pretrained("{checkpoint}") >>> question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet" @@ -675,10 +675,10 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): Example: ```python - >>> from transformers import {processor_class}, {model_class} + >>> from transformers import AutoTokenizer, {model_class} >>> import tensorflow as tf - >>> tokenizer = {processor_class}.from_pretrained("{checkpoint}") + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") >>> model = {model_class}.from_pretrained("{checkpoint}") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="tf") @@ -706,10 +706,10 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): Example: ```python - >>> from transformers import {processor_class}, {model_class} + >>> from transformers import AutoTokenizer, {model_class} >>> import tensorflow as tf - >>> tokenizer = {processor_class}.from_pretrained("{checkpoint}") + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") >>> model = {model_class}.from_pretrained("{checkpoint}") >>> inputs = tokenizer("The capital of France is {mask}.", return_tensors="tf") @@ -739,10 +739,10 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): Example: ```python - >>> from transformers import {processor_class}, {model_class} + >>> from transformers import AutoTokenizer, {model_class} >>> import tensorflow as tf - >>> tokenizer = {processor_class}.from_pretrained("{checkpoint}") + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") >>> model = {model_class}.from_pretrained("{checkpoint}") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="tf") @@ -756,10 +756,10 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): Example: ```python - >>> from transformers import {processor_class}, {model_class} + >>> from transformers import AutoTokenizer, {model_class} >>> import tensorflow as tf - >>> tokenizer = {processor_class}.from_pretrained("{checkpoint}") + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") >>> model = {model_class}.from_pretrained("{checkpoint}") >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." @@ -779,10 +779,10 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): Example: ```python - >>> from transformers import {processor_class}, {model_class} + >>> from transformers import AutoTokenizer, {model_class} >>> import tensorflow as tf - >>> tokenizer = {processor_class}.from_pretrained("{checkpoint}") + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") >>> model = {model_class}.from_pretrained("{checkpoint}") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="tf") @@ -795,14 +795,14 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): Example: ```python - >>> from transformers import {processor_class}, {model_class} + >>> from transformers import AutoProcessor, {model_class} >>> from datasets import load_dataset >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_demo", "clean", split="validation") >>> dataset = dataset.sort("id") >>> sampling_rate = dataset.features["audio"].sampling_rate - >>> processor = {processor_class}.from_pretrained("{checkpoint}") + >>> processor = AutoProcessor.from_pretrained("{checkpoint}") >>> model = {model_class}.from_pretrained("{checkpoint}") >>> # audio file is decoded on the fly @@ -819,7 +819,7 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): Example: ```python - >>> from transformers import {processor_class}, {model_class} + >>> from transformers import AutoProcessor, {model_class} >>> from datasets import load_dataset >>> import tensorflow as tf @@ -827,7 +827,7 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): >>> dataset = dataset.sort("id") >>> sampling_rate = dataset.features["audio"].sampling_rate - >>> processor = {processor_class}.from_pretrained("{checkpoint}") + >>> processor = AutoProcessor.from_pretrained("{checkpoint}") >>> model = {model_class}.from_pretrained("{checkpoint}") >>> # audio file is decoded on the fly @@ -855,13 +855,13 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): Example: ```python - >>> from transformers import {processor_class}, {model_class} + >>> from transformers import AutoImageProcessor, {model_class} >>> from datasets import load_dataset >>> dataset = load_dataset("huggingface/cats-image") >>> image = dataset["test"]["image"][0] - >>> image_processor = {processor_class}.from_pretrained("{checkpoint}") + >>> image_processor = AutoImageProcessor.from_pretrained("{checkpoint}") >>> model = {model_class}.from_pretrained("{checkpoint}") >>> inputs = image_processor(image, return_tensors="tf") @@ -877,14 +877,14 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): Example: ```python - >>> from transformers import {processor_class}, {model_class} + >>> from transformers import AutoImageProcessor, {model_class} >>> import tensorflow as tf >>> from datasets import load_dataset >>> dataset = load_dataset("huggingface/cats-image") >>> image = dataset["test"]["image"][0] - >>> image_processor = {processor_class}.from_pretrained("{checkpoint}") + >>> image_processor = AutoImageProcessor.from_pretrained("{checkpoint}") >>> model = {model_class}.from_pretrained("{checkpoint}") >>> inputs = image_processor(image, return_tensors="tf") @@ -916,9 +916,9 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): Example: ```python - >>> from transformers import {processor_class}, {model_class} + >>> from transformers import AutoTokenizer, {model_class} - >>> tokenizer = {processor_class}.from_pretrained("{checkpoint}") + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") >>> model = {model_class}.from_pretrained("{checkpoint}") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="jax") @@ -932,9 +932,9 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): Example: ```python - >>> from transformers import {processor_class}, {model_class} + >>> from transformers import AutoTokenizer, {model_class} - >>> tokenizer = {processor_class}.from_pretrained("{checkpoint}") + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") >>> model = {model_class}.from_pretrained("{checkpoint}") >>> question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet" @@ -950,9 +950,9 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): Example: ```python - >>> from transformers import {processor_class}, {model_class} + >>> from transformers import AutoTokenizer, {model_class} - >>> tokenizer = {processor_class}.from_pretrained("{checkpoint}") + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") >>> model = {model_class}.from_pretrained("{checkpoint}") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="jax") @@ -966,9 +966,9 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): Example: ```python - >>> from transformers import {processor_class}, {model_class} + >>> from transformers import AutoTokenizer, {model_class} - >>> tokenizer = {processor_class}.from_pretrained("{checkpoint}") + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") >>> model = {model_class}.from_pretrained("{checkpoint}") >>> inputs = tokenizer("The capital of France is {mask}.", return_tensors="jax") @@ -982,9 +982,9 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): Example: ```python - >>> from transformers import {processor_class}, {model_class} + >>> from transformers import AutoTokenizer, {model_class} - >>> tokenizer = {processor_class}.from_pretrained("{checkpoint}") + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") >>> model = {model_class}.from_pretrained("{checkpoint}") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="jax") @@ -998,9 +998,9 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): Example: ```python - >>> from transformers import {processor_class}, {model_class} + >>> from transformers import AutoTokenizer, {model_class} - >>> tokenizer = {processor_class}.from_pretrained("{checkpoint}") + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") >>> model = {model_class}.from_pretrained("{checkpoint}") >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." @@ -1018,9 +1018,9 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): Example: ```python - >>> from transformers import {processor_class}, {model_class} + >>> from transformers import AutoTokenizer, {model_class} - >>> tokenizer = {processor_class}.from_pretrained("{checkpoint}") + >>> tokenizer = AutoTokenizer.from_pretrained("{checkpoint}") >>> model = {model_class}.from_pretrained("{checkpoint}") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="np")