Add changes for e2e inference for passing test cases

forge/test/models/pytorch/multimodal/vilt/test_vilt.py

@@ -56,6 +56,7 @@ def generate_model_vilt_question_answering_hf_pytorch(variant):
 
 
 @pytest.mark.nightly
+@pytest.mark.push
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_vilt_question_answering_hf_pytorch(record_forge_property, variant):
     # Build Module Name

forge/test/models/pytorch/vision/dla/test_dla.py

@@ -1,6 +1,8 @@
 # SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC
 
 # SPDX-License-Identifier: Apache-2.0
+import urllib
+
 import pytest
 import requests
 import torchvision.transforms as transforms
@@ -43,6 +45,8 @@
 def test_dla_pytorch(record_forge_property, variant):
     if variant != "dla34":
         pytest.skip("Skipping due to the current CI/CD pipeline limitations")
+    else:
+        pytest.mark.push
 
     # Build Module Name
     module_name = build_module_name(
@@ -79,3 +83,14 @@ def test_dla_pytorch(record_forge_property, variant):
 
     # Model Verification
     verify(inputs, framework_model, compiled_model)
+
+    # Inference
+    output = compiled_model(*inputs)
+
+    # Post processing
+    url = "https://raw.githubusercontent.com/pytorch/hub/master/imagenet_classes.txt"
+    image_classes = urllib.request.urlopen(url)
+    categories = [s.decode("utf-8").strip() for s in image_classes.readlines()]
+    predicted_value = output[0].argmax(-1)
+    predicted_label = categories[predicted_value]
+    print(f"Predicted Label: {predicted_label}")

forge/test/models/pytorch/vision/ghostnet/test_ghostnet.py

@@ -5,6 +5,7 @@
 
 import pytest
 import timm
+import torch
 from PIL import Image
 from timm.data import resolve_data_config
 from timm.data.transforms_factory import create_transform
@@ -18,6 +19,7 @@
 variants = ["ghostnet_100"]
 
 
+@pytest.mark.push
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", variants, ids=variants)
 def test_ghostnet_timm(record_forge_property, variant):
@@ -55,3 +57,18 @@ def test_ghostnet_timm(record_forge_property, variant):
 
     # Model Verification
     verify(inputs, framework_model, compiled_model)
+
+    # Inference
+    output = compiled_model(*inputs)
+
+    # Post processing
+    probabilities = torch.nn.functional.softmax(output[0][0], dim=0)
+
+    url = "https://raw.githubusercontent.com/pytorch/hub/master/imagenet_classes.txt"
+    urllib.request.urlretrieve(url, "imagenet_classes.txt")
+
+    with open("imagenet_classes.txt", "r") as f:
+        categories = [s.strip() for s in f.readlines()]
+    top5_prob, top5_catid = torch.topk(probabilities, 5)
+    for i in range(top5_prob.size(0)):
+        print(categories[top5_catid[i]], top5_prob[i].item())

forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v1.py

@@ -1,10 +1,13 @@
 # SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC
 
 # SPDX-License-Identifier: Apache-2.0
+import urllib
+
 import pytest
 import requests
 import torch
 from PIL import Image
+from torchvision import transforms
 from transformers import AutoImageProcessor, AutoModelForImageClassification
 
 import forge
@@ -16,16 +19,33 @@
 
 
 def generate_model_mobilenetV1_base_custom_pytorch():
-    # Create Forge module from PyTorch model
-    model = MobileNetV1(9)
 
-    input_shape = (1, 3, 64, 64)
-    image_tensor = torch.rand(*input_shape)
+    # Create model
+    model = MobileNetV1(9)
+    model.eval()
+
+    # Load data sample
+    url, filename = ("https://github.com/pytorch/hub/raw/master/images/dog.jpg", "dog.jpg")
+    urllib.request.urlretrieve(url, filename)
+
+    # Preprocessing
+    input_image = Image.open(filename)
+    preprocess = transforms.Compose(
+        [
+            transforms.Resize(256),
+            transforms.CenterCrop(224),
+            transforms.ToTensor(),
+            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+        ]
+    )
+    input_tensor = preprocess(input_image)
+    input_batch = input_tensor.unsqueeze(0)
 
-    return model, [image_tensor], {}
+    return model, [input_batch]
 
 
 @pytest.mark.nightly
+@pytest.mark.push
 def test_mobilenetv1_basic(record_forge_property):
     # Build Module Name
     module_name = build_module_name(
@@ -39,14 +59,29 @@ def test_mobilenetv1_basic(record_forge_property):
     # Record Forge Property
     record_forge_property("model_name", module_name)
 
-    framework_model, inputs, _ = generate_model_mobilenetV1_base_custom_pytorch()
+    framework_model, inputs = generate_model_mobilenetV1_base_custom_pytorch()
 
     # Forge compile framework model
     compiled_model = forge.compile(framework_model, sample_inputs=inputs, module_name=module_name)
 
-    # Model Verification
+    #  Model Verification
     verify(inputs, framework_model, compiled_model)
 
+    # Inference
+    output = compiled_model(*inputs)
+
+    # Post processing
+    probabilities = torch.nn.functional.softmax(output[0][0], dim=0)
+
+    url = "https://raw.githubusercontent.com/pytorch/hub/master/imagenet_classes.txt"
+    urllib.request.urlretrieve(url, "imagenet_classes.txt")
+
+    with open("imagenet_classes.txt", "r") as f:
+        categories = [s.strip() for s in f.readlines()]
+    top5_prob, top5_catid = torch.topk(probabilities, 5)
+    for i in range(top5_prob.size(0)):
+        print(categories[top5_catid[i]], top5_prob[i].item())
+
 
 def generate_model_mobilenetv1_imgcls_hf_pytorch(variant):
     # Create Forge module from PyTorch model

forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v2.py

@@ -11,6 +11,7 @@
 from PIL import Image
 from timm.data import resolve_data_config
 from timm.data.transforms_factory import create_transform
+from torchvision import transforms
 from transformers import (
     AutoImageProcessor,
     AutoModelForImageClassification,
@@ -26,19 +27,30 @@
 
 def generate_model_mobilenetV2_imgcls_torchhub_pytorch():
     model = download_model(torch.hub.load, "pytorch/vision:v0.10.0", "mobilenet_v2", pretrained=True)
+    model.eval()
+
+    # Load data sample
+    url, filename = ("https://github.com/pytorch/hub/raw/master/images/dog.jpg", "dog.jpg")
+    urllib.request.urlretrieve(url, filename)
+
+    # Preprocessing
+    input_image = Image.open(filename)
+    preprocess = transforms.Compose(
+        [
+            transforms.Resize(256),
+            transforms.CenterCrop(224),
+            transforms.ToTensor(),
+            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+        ]
+    )
+    input_tensor = preprocess(input_image)
+    input_batch = input_tensor.unsqueeze(0)
 
-    # Image preprocessing
-    url = "http://images.cocodataset.org/val2017/000000039769.jpg"
-    image = Image.open(requests.get(url, stream=True).raw)
-    # TODO : Choose image preprocessor from torchvision,
-    # to make a compatible postprocessing of the predicted class
-    preprocessor = download_model(AutoImageProcessor.from_pretrained, "google/mobilenet_v2_1.0_224")
-    image_tensor = preprocessor(images=image, return_tensors="pt").pixel_values
-
-    return model, [image_tensor], {}
+    return model, [input_batch]
 
 
 @pytest.mark.nightly
+@pytest.mark.push
 def test_mobilenetv2_basic(record_forge_property):
     # Build Module Name
     module_name = build_module_name(
@@ -52,14 +64,29 @@ def test_mobilenetv2_basic(record_forge_property):
     # Record Forge Property
     record_forge_property("model_name", module_name)
 
-    framework_model, inputs, _ = generate_model_mobilenetV2_imgcls_torchhub_pytorch()
+    framework_model, inputs = generate_model_mobilenetV2_imgcls_torchhub_pytorch()
 
     # Forge compile framework model
     compiled_model = forge.compile(framework_model, sample_inputs=inputs, module_name=module_name)
 
     # Model Verification
     verify(inputs, framework_model, compiled_model)
 
+    # Inference
+    output = compiled_model(*inputs)
+
+    # Post processing
+    probabilities = torch.nn.functional.softmax(output[0][0], dim=0)
+
+    url = "https://raw.githubusercontent.com/pytorch/hub/master/imagenet_classes.txt"
+    urllib.request.urlretrieve(url, "imagenet_classes.txt")
+
+    with open("imagenet_classes.txt", "r") as f:
+        categories = [s.strip() for s in f.readlines()]
+    top5_prob, top5_catid = torch.topk(probabilities, 5)
+    for i in range(top5_prob.size(0)):
+        print(categories[top5_catid[i]], top5_prob[i].item())
+
 
 def generate_model_mobilenetV2I96_imgcls_hf_pytorch(variant):
     preprocessor = download_model(AutoImageProcessor.from_pretrained, variant)

forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v3.py

@@ -4,14 +4,13 @@
 import urllib
 
 import pytest
-import requests
 import timm
 import torch
 from loguru import logger
 from PIL import Image
 from timm.data import resolve_data_config
 from timm.data.transforms_factory import create_transform
-from transformers import AutoImageProcessor
+from torchvision import transforms
 
 import forge
 from forge.verify.verify import verify
@@ -22,22 +21,36 @@
 
 def generate_model_mobilenetV3_imgcls_torchhub_pytorch(variant):
     model = download_model(torch.hub.load, "pytorch/vision:v0.10.0", variant, pretrained=True)
+    model.eval()
+
+    # Load data sample
+    url, filename = ("https://github.com/pytorch/hub/raw/master/images/dog.jpg", "dog.jpg")
+    urllib.request.urlretrieve(url, filename)
+
+    # Preprocessing
+    input_image = Image.open(filename)
+    preprocess = transforms.Compose(
+        [
+            transforms.Resize(256),
+            transforms.CenterCrop(224),
+            transforms.ToTensor(),
+            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+        ]
+    )
+    input_tensor = preprocess(input_image)
+    input_batch = input_tensor.unsqueeze(0)
 
-    # Run inference on Tenstorrent device
-    url = "http://images.cocodataset.org/val2017/000000039769.jpg"
-    image = Image.open(requests.get(url, stream=True).raw)
-    # TODO : Choose image preprocessor from torchvision, to make a compatible postprocessing of the predicted class
-    preprocessor = AutoImageProcessor.from_pretrained("google/mobilenet_v2_1.0_224")
-    image_tensor = preprocessor(images=image, return_tensors="pt").pixel_values
-
-    return model, [image_tensor], {}
+    return model, [input_batch]
 
 
-variants = ["mobilenet_v3_large", "mobilenet_v3_small"]
+params = [
+    pytest.param("mobilenet_v3_large", marks=[pytest.mark.push]),
+    pytest.param("mobilenet_v3_small"),
+]
 
 
 @pytest.mark.nightly
-@pytest.mark.parametrize("variant", variants, ids=variants)
+@pytest.mark.parametrize("variant", params)
 def test_mobilenetv3_basic(record_forge_property, variant):
     if variant != "mobilenet_v3_large":
         pytest.skip("Skipping due to the current CI/CD pipeline limitations")
@@ -54,7 +67,7 @@ def test_mobilenetv3_basic(record_forge_property, variant):
     # Record Forge Property
     record_forge_property("model_name", module_name)
 
-    framework_model, inputs, _ = generate_model_mobilenetV3_imgcls_torchhub_pytorch(
+    framework_model, inputs = generate_model_mobilenetV3_imgcls_torchhub_pytorch(
         variant,
     )
 
@@ -64,6 +77,21 @@ def test_mobilenetv3_basic(record_forge_property, variant):
     # Model Verification
     verify(inputs, framework_model, compiled_model)
 
+    # Inference
+    output = compiled_model(*inputs)
+
+    # Post processing
+    probabilities = torch.nn.functional.softmax(output[0][0], dim=0)
+
+    url = "https://raw.githubusercontent.com/pytorch/hub/master/imagenet_classes.txt"
+    urllib.request.urlretrieve(url, "imagenet_classes.txt")
+
+    with open("imagenet_classes.txt", "r") as f:
+        categories = [s.strip() for s in f.readlines()]
+    top5_prob, top5_catid = torch.topk(probabilities, 5)
+    for i in range(top5_prob.size(0)):
+        print(categories[top5_catid[i]], top5_prob[i].item())
+
 
 def generate_model_mobilenetV3_imgcls_timm_pytorch(variant):
     # Both options are good

forge/test/models/pytorch/vision/resnext/test_resnext.py

@@ -1,6 +1,8 @@
 # SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC
 
 # SPDX-License-Identifier: Apache-2.0
+import urllib
+
 import pytest
 import torch
 from pytorchcv.model_provider import get_model as ptcv_get_model
@@ -13,6 +15,7 @@
 from test.utils import download_model
 
 
+@pytest.mark.push
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", ["resnext50_32x4d"])
 def test_resnext_50_torchhub_pytorch(record_forge_property, variant):
@@ -41,7 +44,23 @@ def test_resnext_50_torchhub_pytorch(record_forge_property, variant):
     # Model Verification
     verify(inputs, framework_model, compiled_model)
 
+    # Inference
+    output = compiled_model(*inputs)
+
+    # Post processing
+    probabilities = torch.nn.functional.softmax(output[0][0], dim=0)
+
+    url = "https://raw.githubusercontent.com/pytorch/hub/master/imagenet_classes.txt"
+    urllib.request.urlretrieve(url, "imagenet_classes.txt")
+
+    with open("imagenet_classes.txt", "r") as f:
+        categories = [s.strip() for s in f.readlines()]
+    top5_prob, top5_catid = torch.topk(probabilities, 5)
+    for i in range(top5_prob.size(0)):
+        print(categories[top5_catid[i]], top5_prob[i].item())
+
 
+@pytest.mark.push
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", ["resnext101_32x8d"])
 def test_resnext_101_torchhub_pytorch(record_forge_property, variant):
@@ -70,6 +89,21 @@ def test_resnext_101_torchhub_pytorch(record_forge_property, variant):
     # Model Verification
     verify(inputs, framework_model, compiled_model)
 
+    # Inference
+    output = compiled_model(*inputs)
+
+    # Post processing
+    probabilities = torch.nn.functional.softmax(output[0][0], dim=0)
+
+    url = "https://raw.githubusercontent.com/pytorch/hub/master/imagenet_classes.txt"
+    urllib.request.urlretrieve(url, "imagenet_classes.txt")
+
+    with open("imagenet_classes.txt", "r") as f:
+        categories = [s.strip() for s in f.readlines()]
+    top5_prob, top5_catid = torch.topk(probabilities, 5)
+    for i in range(top5_prob.size(0)):
+        print(categories[top5_catid[i]], top5_prob[i].item())
+
 
 @pytest.mark.nightly
 @pytest.mark.parametrize("variant", ["resnext101_32x8d_wsl"])