#7527: Disable old ViT tests

tenstorrent · Apr 18, 2024 · ae149f4 · ae149f4
1 parent 8906448
commit ae149f4
Show file tree

Hide file tree

Showing 12 changed files with 37 additions and 62 deletions.
diff --git a/models/experimental/vit/tests/test_perf_accuracy_vit.py b/models/experimental/vit/tests/test_perf_accuracy_vit.py
@@ -120,6 +120,7 @@ def run_perf_vit(
     logger.info(f"Accuracy for {batch_size}x{iterations} inputs: {accuracy}")
 
 
+@pytest.mark.skip(reason="#7527: Test needs review")
 @pytest.mark.models_performance_bare_metal
 @pytest.mark.parametrize(
     "expected_inference_time, expected_compile_time,iterations",
@@ -151,6 +152,7 @@ def test_perf_bare_metal(
     )
 
 
+@pytest.mark.skip(reason="#7527: Test needs review")
 @pytest.mark.models_performance_virtual_machine
 @pytest.mark.parametrize(
     "expected_inference_time, expected_compile_time,iterations",

diff --git a/models/experimental/vit/tests/test_perf_vit.py b/models/experimental/vit/tests/test_perf_vit.py
@@ -80,6 +80,7 @@ def run_perf_vit(expected_inference_time, expected_compile_time, hf_cat_image_sa
     logger.info(f"vit compile time: {compile_time}")
 
 
+@pytest.mark.skip(reason="#7527: Test needs review")
 @pytest.mark.models_performance_bare_metal
 @pytest.mark.parametrize(
     "expected_inference_time, expected_compile_time",
@@ -105,6 +106,7 @@ def test_perf_bare_metal(
     )
 
 
+@pytest.mark.skip(reason="#7527: Test needs review")
 @pytest.mark.models_performance_virtual_machine
 @pytest.mark.parametrize(
     "expected_inference_time, expected_compile_time",

diff --git a/models/experimental/vit/tests/test_vit_attention.py b/models/experimental/vit/tests/test_vit_attention.py
@@ -18,24 +18,21 @@
 )
 
 
+@pytest.mark.skip(reason="#7527: Test needs review")
 def test_vit_attention(device, pcc=0.99):
     hidden_state_shape = (1, 1, 197, 768)
     head_mask = None
     output_attentions = False
     hidden_state = torch.randn(hidden_state_shape)
     with torch.no_grad():
-        HF_model = HF_ViTForImageClassication.from_pretrained(
-            "google/vit-base-patch16-224"
-        )
+        HF_model = HF_ViTForImageClassication.from_pretrained("google/vit-base-patch16-224")
 
         state_dict = HF_model.state_dict()
         reference = HF_model.vit.encoder.layer[5].attention
 
         config = HF_model.config
         HF_output = reference(hidden_state.squeeze(0), head_mask, output_attentions)[0]
-        tt_hidden_state = torch_to_tt_tensor_rm(
-            hidden_state, device, put_on_device=False
-        )
+        tt_hidden_state = torch_to_tt_tensor_rm(hidden_state, device, put_on_device=False)
         tt_layer = TtViTAttention(
             config,
             base_address="vit.encoder.layer.5.attention",

diff --git a/models/experimental/vit/tests/test_vit_embeddings.py b/models/experimental/vit/tests/test_vit_embeddings.py
@@ -11,13 +11,12 @@
 from models.experimental.vit.tt.modeling_vit import ViTEmbeddings
 
 
+@pytest.mark.skip(reason="#7527: Test needs review")
 def test_vit_embeddings(imagenet_sample_input, pcc=0.99):
     image = imagenet_sample_input
 
     with torch.no_grad():
-        HF_model = HF_ViTForImageClassication.from_pretrained(
-            "google/vit-base-patch16-224"
-        )
+        HF_model = HF_ViTForImageClassication.from_pretrained("google/vit-base-patch16-224")
 
         state_dict = HF_model.state_dict()
         reference = HF_model.vit.embeddings
@@ -26,9 +25,7 @@ def test_vit_embeddings(imagenet_sample_input, pcc=0.99):
         HF_output = reference(image, None, None)
 
         tt_image = image
-        tt_layer = ViTEmbeddings(
-            config, base_address="vit.embeddings", state_dict=state_dict
-        )
+        tt_layer = ViTEmbeddings(config, base_address="vit.embeddings", state_dict=state_dict)
 
         tt_output = tt_layer(tt_image, None, None)
         pcc_passing, _ = comp_pcc(HF_output, tt_output, pcc)

diff --git a/models/experimental/vit/tests/test_vit_encoder.py b/models/experimental/vit/tests/test_vit_encoder.py
@@ -19,8 +19,7 @@
 )
 
 
-
-
+@pytest.mark.skip(reason="#7527: Test needs review")
 def test_vit_encoder(device, hf_cat_image_sample_input, pcc=0.92):
     image = hf_cat_image_sample_input
 
@@ -30,12 +29,8 @@ def test_vit_encoder(device, hf_cat_image_sample_input, pcc=0.92):
     return_dict = True
 
     with torch.no_grad():
-        HF_model = HF_ViTForImageClassication.from_pretrained(
-            "google/vit-base-patch16-224"
-        )
-        image_processor = HF_AutoImageProcessor.from_pretrained(
-            "google/vit-base-patch16-224"
-        )
+        HF_model = HF_ViTForImageClassication.from_pretrained("google/vit-base-patch16-224")
+        image_processor = HF_AutoImageProcessor.from_pretrained("google/vit-base-patch16-224")
         inputs = image_processor(image, return_tensors="pt")["pixel_values"]
 
         state_dict = HF_model.state_dict()
@@ -52,12 +47,8 @@ def test_vit_encoder(device, hf_cat_image_sample_input, pcc=0.92):
             return_dict,
         )[0]
 
-        tt_embedding_output = torch_to_tt_tensor_rm(
-            embedding_output, device, put_on_device=False
-        )
-        tt_layer = TtViTEncoder(
-            config, base_address="vit.encoder", state_dict=state_dict, device=device
-        )
+        tt_embedding_output = torch_to_tt_tensor_rm(embedding_output, device, put_on_device=False)
+        tt_layer = TtViTEncoder(config, base_address="vit.encoder", state_dict=state_dict, device=device)
 
         tt_output = tt_layer(
             tt_embedding_output,

diff --git a/models/experimental/vit/tests/test_vit_image_classification.py b/models/experimental/vit/tests/test_vit_image_classification.py
@@ -23,6 +23,7 @@
 pytest.skip(allow_module_level=True, reason="https://github.com/tenstorrent/tt-metal/issues/7508")
 
 
+@pytest.mark.skip(reason="#7527: Test needs review")
 def test_vit_image_classification(device, hf_cat_image_sample_input, pcc=0.95):
     image = hf_cat_image_sample_input
 

diff --git a/models/experimental/vit/tests/test_vit_intermediate.py b/models/experimental/vit/tests/test_vit_intermediate.py
@@ -17,24 +17,22 @@
     comp_allclose_and_pcc,
 )
 
+
+@pytest.mark.skip(reason="#7527: Test needs review")
 def test_vit_intermediate(device, pcc=0.99):
     hidden_state_shape = (1, 1, 197, 768)
 
     hidden_state = torch.randn(hidden_state_shape)
     with torch.no_grad():
-        HF_model = HF_ViTForImageClassication.from_pretrained(
-            "google/vit-base-patch16-224"
-        )
+        HF_model = HF_ViTForImageClassication.from_pretrained("google/vit-base-patch16-224")
 
         state_dict = HF_model.state_dict()
         reference = HF_model.vit.encoder.layer[5].intermediate
 
         config = HF_model.config
         HF_output = reference(hidden_state.squeeze(0))
 
-        tt_hidden_state = torch_to_tt_tensor_rm(
-            hidden_state, device, put_on_device=False
-        )
+        tt_hidden_state = torch_to_tt_tensor_rm(hidden_state, device, put_on_device=False)
         tt_layer = TtViTIntermediate(
             config,
             base_address="vit.encoder.layer.5.intermediate",

diff --git a/models/experimental/vit/tests/test_vit_layer.py b/models/experimental/vit/tests/test_vit_layer.py
@@ -18,25 +18,22 @@
 )
 
 
+@pytest.mark.skip(reason="#7527: Test needs review")
 def test_vit_layer(device, pcc=0.99):
     hidden_state_shape = (1, 1, 197, 768)
     head_mask = None
     output_attentions = False
     hidden_state = torch.randn(hidden_state_shape)
     with torch.no_grad():
-        HF_model = HF_ViTForImageClassication.from_pretrained(
-            "google/vit-base-patch16-224"
-        )
+        HF_model = HF_ViTForImageClassication.from_pretrained("google/vit-base-patch16-224")
 
         state_dict = HF_model.state_dict()
         reference = HF_model.vit.encoder.layer[5]
 
         config = HF_model.config
         HF_output = reference(hidden_state.squeeze(0), head_mask, output_attentions)[0]
 
-        tt_hidden_state = torch_to_tt_tensor_rm(
-            hidden_state, device, put_on_device=False
-        )
+        tt_hidden_state = torch_to_tt_tensor_rm(hidden_state, device, put_on_device=False)
         tt_layer = TtViTLayer(
             config,
             base_address="vit.encoder.layer.5",

diff --git a/models/experimental/vit/tests/test_vit_model.py b/models/experimental/vit/tests/test_vit_model.py
@@ -17,6 +17,8 @@
     comp_allclose_and_pcc,
 )
 
+
+@pytest.mark.skip(reason="#7527: Test needs review")
 def test_vit_model(device, imagenet_sample_input, pcc=0.95):
     image = imagenet_sample_input
     head_mask = None
@@ -25,9 +27,7 @@ def test_vit_model(device, imagenet_sample_input, pcc=0.95):
     interpolate_pos_encoding = None
     return_dict = None
     with torch.no_grad():
-        HF_model = HF_ViTForImageClassication.from_pretrained(
-            "google/vit-base-patch16-224"
-        )
+        HF_model = HF_ViTForImageClassication.from_pretrained("google/vit-base-patch16-224")
 
         state_dict = HF_model.state_dict()
 

diff --git a/models/experimental/vit/tests/test_vit_output.py b/models/experimental/vit/tests/test_vit_output.py
@@ -17,27 +17,23 @@
     comp_allclose_and_pcc,
 )
 
+
+@pytest.mark.skip(reason="#7527: Test needs review")
 def test_vit_output(device, pcc=0.99):
     hidden_state_shape = (1, 1, 197, 3072)
     input_tensor_shape = (1, 1, 197, 768)
     hidden_state = torch.randn(hidden_state_shape)
     input_tensor = torch.randn(input_tensor_shape)
     with torch.no_grad():
-        HF_model = HF_ViTForImageClassication.from_pretrained(
-            "google/vit-base-patch16-224"
-        )
+        HF_model = HF_ViTForImageClassication.from_pretrained("google/vit-base-patch16-224")
 
         state_dict = HF_model.state_dict()
         reference = HF_model.vit.encoder.layer[11].output
         config = HF_model.config
         HF_output = reference(hidden_state, input_tensor)
 
-        tt_hidden_state = torch_to_tt_tensor_rm(
-            hidden_state, device, put_on_device=False
-        )
-        tt_input_tensor = torch_to_tt_tensor_rm(
-            input_tensor, device, put_on_device=False
-        )
+        tt_hidden_state = torch_to_tt_tensor_rm(hidden_state, device, put_on_device=False)
+        tt_input_tensor = torch_to_tt_tensor_rm(input_tensor, device, put_on_device=False)
         tt_layer = TtViTOutput(
             config,
             base_address="vit.encoder.layer.11.output",

diff --git a/models/experimental/vit/tests/test_vit_selfattention.py b/models/experimental/vit/tests/test_vit_selfattention.py
@@ -18,25 +18,22 @@
 )
 
 
+@pytest.mark.skip(reason="#7527: Test needs review")
 def test_vit_selfattention(device, pcc=0.99):
     hidden_state_shape = (1, 1, 197, 768)
     head_mask = None
     output_attentions = False
     hidden_state = torch.randn(hidden_state_shape)
     with torch.no_grad():
-        HF_model = HF_ViTForImageClassication.from_pretrained(
-            "google/vit-base-patch16-224"
-        )
+        HF_model = HF_ViTForImageClassication.from_pretrained("google/vit-base-patch16-224")
 
         state_dict = HF_model.state_dict()
         reference = HF_model.vit.encoder.layer[5].attention.attention
 
         config = HF_model.config
         HF_output = reference(hidden_state.squeeze(0), head_mask, output_attentions)[0]
 
-        tt_hidden_state = torch_to_tt_tensor_rm(
-            hidden_state, device, put_on_device=False
-        )
+        tt_hidden_state = torch_to_tt_tensor_rm(hidden_state, device, put_on_device=False)
         tt_layer = TtViTSelfAttention(
             config,
             base_address="vit.encoder.layer.5.attention.attention",

diff --git a/models/experimental/vit/tests/test_vit_selfoutput.py b/models/experimental/vit/tests/test_vit_selfoutput.py
@@ -18,24 +18,21 @@
 )
 
 
+@pytest.mark.skip(reason="#7527: Test needs review")
 def test_vit_selfoutput(device, pcc=0.99):
     hidden_state_shape = (1, 1, 197, 768)
     hidden_state = torch.randn(hidden_state_shape)
 
     with torch.no_grad():
-        HF_model = HF_ViTForImageClassication.from_pretrained(
-            "google/vit-base-patch16-224"
-        )
+        HF_model = HF_ViTForImageClassication.from_pretrained("google/vit-base-patch16-224")
 
         state_dict = HF_model.state_dict()
 
         reference = HF_model.vit.encoder.layer[0].attention.output
         config = HF_model.config
         HF_output = reference(hidden_state, None)
 
-        tt_hidden_state = torch_to_tt_tensor_rm(
-            hidden_state, device, put_on_device=False
-        )
+        tt_hidden_state = torch_to_tt_tensor_rm(hidden_state, device, put_on_device=False)
         tt_layer = TtViTSelfOutput(
             config,
             base_address="vit.encoder.layer.0.attention.output",