Stop tracking of 2/3 of target models due to CI limitations

forge/test/models/pytorch/audio/stereo/test_stereo.py

@@ -21,6 +21,9 @@
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants)
 def test_stereo(record_forge_property, variant):
+    if variant != "facebook/musicgen-small":
+        pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(framework=Framework.PYTORCH, model="stereo", variant=variant)
 

forge/test/models/pytorch/audio/whisper/test_whisper_0.py

@@ -97,6 +97,9 @@ def forward(self, decoder_input_ids, encoder_hidden_states):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_whisper(record_forge_property, variant):
+    if variant != "openai/whisper-tiny":
+        pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(framework=Framework.PYTORCH, model="whisper", variant=variant)
 

forge/test/models/pytorch/audio/whisper/test_whisper_3.py

@@ -40,6 +40,8 @@ def forward(self, decoder_input_ids, encoder_hidden_states):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", ["openai/whisper-large-v3-turbo"])
 def test_whisper_large_v3_speech_translation(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(
         framework=Framework.PYTORCH, model="whisper", variant=variant, task=Task.SPEECH_TRANSLATE

forge/test/models/pytorch/multimodal/vilt/test_vilt.py

@@ -105,6 +105,8 @@ def generate_model_vilt_maskedlm_hf_pytorch(variant):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_vilt_maskedlm_hf_pytorch(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(
         framework=Framework.PYTORCH, model="vilt", variant=variant, task=Task.MASKED_LM, source=Source.HUGGINGFACE

forge/test/models/pytorch/text/albert/test_albert.py

@@ -23,6 +23,9 @@
 @pytest.mark.parametrize("variant", variants, ids=variants)
 @pytest.mark.parametrize("size", sizes, ids=sizes)
 def test_albert_masked_lm_pytorch(record_forge_property, size, variant):
+    if size != "base" and variant != "v1":
+        pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(
         framework=Framework.PYTORCH, model="albert", variant=f"{size}_{variant}", task=Task.MASKED_LM
@@ -65,6 +68,9 @@ def test_albert_masked_lm_pytorch(record_forge_property, size, variant):
 @pytest.mark.parametrize("variant", variants, ids=variants)
 @pytest.mark.parametrize("size", sizes, ids=sizes)
 def test_albert_token_classification_pytorch(record_forge_property, size, variant):
+    if size != "base" and variant != "v1":
+        pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(
         framework=Framework.PYTORCH, model="albert", variant=f"{size}_{variant}", task=Task.TOKEN_CLASSIFICATION

forge/test/models/pytorch/text/bert/test_bert.py

@@ -89,6 +89,8 @@ def generate_model_bert_qa_hf_pytorch(variant):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", ["bert-large-cased-whole-word-masking-finetuned-squad"])
 def test_bert_question_answering_pytorch(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(framework=Framework.PYTORCH, model="bert", variant=variant, task=Task.QA)
 
@@ -127,6 +129,8 @@ def generate_model_bert_seqcls_hf_pytorch(variant):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", ["textattack/bert-base-uncased-SST-2"])
 def test_bert_sequence_classification_pytorch(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(
         framework=Framework.PYTORCH, model="bert", variant=variant, task=Task.SEQUENCE_CLASSIFICATION
@@ -173,6 +177,8 @@ def generate_model_bert_tkcls_hf_pytorch(variant):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", ["dbmdz/bert-large-cased-finetuned-conll03-english"])
 def test_bert_token_classification_pytorch(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(
         framework=Framework.PYTORCH, model="bert", variant=variant, task=Task.TOKEN_CLASSIFICATION

forge/test/models/pytorch/text/codegen/test_codegen.py

@@ -15,14 +15,17 @@
 
 variants = [
     "Salesforce/codegen-350M-mono",
-    # "Salesforce/codegen-350M-multi", # Currently not supported
-    # "Salesforce/codegen-350M-nl", # Currently not supported
+    "Salesforce/codegen-350M-multi",
+    "Salesforce/codegen-350M-nl",
 ]
 
 
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_codegen(record_forge_property, variant):
+    if variant != "Salesforce/codegen-350M-mono":
+        pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(framework=Framework.PYTORCH, model="codegen", variant=variant, task=Task.CAUSAL_LM)
 

forge/test/models/pytorch/text/distilbert/test_distilbert.py

@@ -22,6 +22,9 @@
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_distilbert_masked_lm_pytorch(record_forge_property, variant):
+    if variant != "distilbert-base-uncased":
+        pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(
         framework=Framework.PYTORCH, model="distilbert", variant=variant, task=Task.MASKED_LM
@@ -62,6 +65,8 @@ def test_distilbert_masked_lm_pytorch(record_forge_property, variant):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", ["distilbert-base-cased-distilled-squad"])
 def test_distilbert_question_answering_pytorch(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(framework=Framework.PYTORCH, model="distilbert", variant=variant, task=Task.QA)
 
@@ -106,6 +111,8 @@ def test_distilbert_question_answering_pytorch(record_forge_property, variant):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", ["distilbert-base-uncased-finetuned-sst-2-english"])
 def test_distilbert_sequence_classification_pytorch(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(
         framework=Framework.PYTORCH, model="distilbert", variant=variant, task=Task.SEQUENCE_CLASSIFICATION
@@ -142,6 +149,8 @@ def test_distilbert_sequence_classification_pytorch(record_forge_property, varia
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", ["Davlan/distilbert-base-multilingual-cased-ner-hrl"])
 def test_distilbert_token_classification_pytorch(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(
         framework=Framework.PYTORCH, model="distilbert", variant=variant, task=Task.TOKEN_CLASSIFICATION

forge/test/models/pytorch/text/dpr/test_dpr.py

@@ -24,6 +24,9 @@
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_dpr_context_encoder_pytorch(record_forge_property, variant):
+    if variant != "facebook/dpr-ctx_encoder-single-nq-base":
+        pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(framework=Framework.PYTORCH, model="dpr", variant=variant, suffix="context_encoder")
 
@@ -63,6 +66,8 @@ def test_dpr_context_encoder_pytorch(record_forge_property, variant):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_dpr_question_encoder_pytorch(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(
         framework=Framework.PYTORCH, model="dpr", variant=variant, suffix="question_encoder"
@@ -109,6 +114,8 @@ def test_dpr_question_encoder_pytorch(record_forge_property, variant):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_dpr_reader_pytorch(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(framework=Framework.PYTORCH, model="dpr", variant=variant, suffix="reader")
 

forge/test/models/pytorch/text/gptneo/test_gptneo.py

@@ -26,6 +26,9 @@
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_gptneo_causal_lm(record_forge_property, variant):
+    if variant != "EleutherAI/gpt-neo-125M":
+        pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(framework=Framework.PYTORCH, model="gptneo", variant=variant, task=Task.CAUSAL_LM)
 
@@ -84,6 +87,8 @@ def forward(self, input_ids, attention_mask):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_gptneo_sequence_classification(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(
         framework=Framework.PYTORCH, model="gptneo", variant=variant, task=Task.SEQUENCE_CLASSIFICATION

forge/test/models/pytorch/text/llama/test_llama3.py

@@ -131,6 +131,8 @@ def _update_causal_mask(
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_llama3_causal_lm(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(framework=Framework.PYTORCH, model="llama3", variant=variant, task=Task.CAUSAL_LM)
 
@@ -175,6 +177,8 @@ def test_llama3_causal_lm(record_forge_property, variant):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_llama3_sequence_classification(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(
         framework=Framework.PYTORCH, model="llama3", variant=variant, task=Task.SEQUENCE_CLASSIFICATION

forge/test/models/pytorch/text/opt/test_opt.py

@@ -22,6 +22,9 @@
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_opt_causal_lm(record_forge_property, variant):
+    if variant != "facebook/opt-125m":
+        pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(framework=Framework.PYTORCH, model="opt", variant=variant, task=Task.CAUSAL_LM)
 
@@ -68,6 +71,8 @@ def test_opt_causal_lm(record_forge_property, variant):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_opt_qa(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(framework=Framework.PYTORCH, model="opt", variant=variant, task=Task.QA)
 
@@ -110,6 +115,8 @@ def test_opt_qa(record_forge_property, variant):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_opt_sequence_classification(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(
         framework=Framework.PYTORCH, model="opt", variant=variant, task=Task.SEQUENCE_CLASSIFICATION

forge/test/models/pytorch/text/phi2/test_phi2.py

@@ -22,6 +22,9 @@
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_phi2_clm(record_forge_property, variant):
+    if variant != "microsoft/phi-2":
+        pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(framework=Framework.PYTORCH, model="phi2", variant=variant, task=Task.CAUSAL_LM)
 
@@ -68,6 +71,8 @@ def test_phi2_clm(record_forge_property, variant):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants)
 def test_phi2_token_classification(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(
         framework=Framework.PYTORCH, model="phi2", variant=variant, task=Task.TOKEN_CLASSIFICATION
@@ -106,6 +111,8 @@ def test_phi2_token_classification(record_forge_property, variant):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants)
 def test_phi2_sequence_classification(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(
         framework=Framework.PYTORCH, model="phi2", variant=variant, task=Task.SEQUENCE_CLASSIFICATION

forge/test/models/pytorch/text/phi3/test_phi3.py

@@ -68,6 +68,8 @@ def test_phi3_causal_lm(record_forge_property, variant):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants)
 def test_phi3_token_classification(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(
         framework=Framework.PYTORCH, model="phi3", variant=variant, task=Task.TOKEN_CLASSIFICATION
@@ -107,6 +109,8 @@ def test_phi3_token_classification(record_forge_property, variant):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants)
 def test_phi3_sequence_classification(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(
         framework=Framework.PYTORCH, model="phi3", variant=variant, task=Task.SEQUENCE_CLASSIFICATION

forge/test/models/pytorch/text/qwen/test_qwen.py

@@ -68,6 +68,8 @@ def parse_chat_completion(text: str):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", ["Qwen/Qwen1.5-0.5B-Chat"])
 def test_qwen1_5_chat(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(framework=Framework.PYTORCH, model="qwen1.5", variant=variant)
 

forge/test/models/pytorch/text/qwen/test_qwen_coder.py

@@ -24,6 +24,9 @@
 @pytest.mark.parametrize("variant", variants, ids=variants)
 @pytest.mark.nightly
 def test_qwen_clm(record_forge_property, variant):
+    if variant != "Qwen/Qwen2.5-Coder-0.5B":
+        pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(
         framework=Framework.PYTORCH, model="qwen_coder", variant=variant, task=Task.CAUSAL_LM

forge/test/models/pytorch/text/qwen/test_qwen_v2.py

@@ -25,6 +25,9 @@
 @pytest.mark.parametrize("variant", variants, ids=variants)
 @pytest.mark.nightly
 def test_qwen_clm(record_forge_property, variant):
+    if variant != "Qwen/Qwen2.5-0.5B":
+        pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(framework=Framework.PYTORCH, model="qwen_v2", variant=variant, task=Task.CAUSAL_LM)
 

forge/test/models/pytorch/text/roberta/test_roberta.py

@@ -53,6 +53,8 @@ def test_roberta_masked_lm(record_forge_property, variant):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", ["cardiffnlp/twitter-roberta-base-sentiment"])
 def test_roberta_sentiment_pytorch(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(
         framework=Framework.PYTORCH, model="roberta", variant=variant, task=Task.SEQUENCE_CLASSIFICATION

forge/test/models/pytorch/text/t5/test_t5.py

@@ -101,6 +101,9 @@ def forward(self, decoder_input_ids, encoder_outputs):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants)
 def test_t5_generation(record_forge_property, variant):
+    if variant != "t5-small":
+        pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(framework=Framework.PYTORCH, model="t5", variant=variant, task=Task.TEXT_GENERATION)
 

forge/test/models/pytorch/text/xglm/test_xglm.py

@@ -16,6 +16,9 @@
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_xglm_causal_lm(record_forge_property, variant):
+    if variant != "facebook/xglm-564M":
+        pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(framework=Framework.PYTORCH, model="xglm", variant=variant, task=Task.CAUSAL_LM)
 

forge/test/models/pytorch/timeseries/nbeats/test_nbeats.py

@@ -50,6 +50,8 @@ def test_nbeats_with_seasonality_basis(record_forge_property, variant):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", ["generic_basis"])
 def test_nbeats_with_generic_basis(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(framework=Framework.PYTORCH, model="nbeats", variant=variant)
 
@@ -73,6 +75,8 @@ def test_nbeats_with_generic_basis(record_forge_property, variant):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", ["trend_basis"])
 def test_nbeats_with_trend_basis(record_forge_property, variant):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(framework=Framework.PYTORCH, model="nbeats", variant=variant)
 

forge/test/models/pytorch/vision/alexnet/test_alexnet.py

@@ -60,6 +60,8 @@ def test_alexnet_torchhub(record_forge_property):
 
 @pytest.mark.nightly
 def test_alexnet_osmr(record_forge_property):
+    pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(framework=Framework.PYTORCH, model="alexnet", source=Source.OSMR)
 

forge/test/models/pytorch/vision/bts/test_bts.py

@@ -27,6 +27,9 @@
 @pytest.mark.parametrize("variant", variants, ids=variants)
 @pytest.mark.nightly
 def test_bts_pytorch(record_forge_property, variant):
+    if variant != "densenet161_bts":
+        pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(framework=Framework.PYTORCH, model="bts", variant=variant)
 

forge/test/models/pytorch/vision/ddrnet/test_ddrnet.py

@@ -26,6 +26,9 @@
 @pytest.mark.parametrize("variant", variants)
 @pytest.mark.nightly
 def test_ddrnet_pytorch(record_forge_property, variant):
+    if variant != "ddrnet23s":
+        pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(framework=Framework.PYTORCH, model="ddrnet", variant=variant)
 

forge/test/models/pytorch/vision/deit/test_deit.py

@@ -41,6 +41,9 @@ def generate_model_deit_imgcls_hf_pytorch(variant):
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_deit_imgcls_hf_pytorch(record_forge_property, variant):
+    if variant != "facebook/deit-base-patch16-224":
+        pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+
     # Build Module Name
     module_name = build_module_name(
         framework=Framework.PYTORCH, model="deit", variant=variant, task=Task.IMAGE_CLASSIFICATION