From 451a8990153e12068be07b1de2fce3f9349472c5 Mon Sep 17 00:00:00 2001 From: Nikola Vukobrat Date: Wed, 29 Jan 2025 16:16:24 +0000 Subject: [PATCH] Demo version of ResNet 50 Created demo version of ResNet 50 that runs on a few images and outputs a table for CPU comparison (predicted class and confidence score) Additionally, unified existing ResNet tests into a single file. Fix #1132 --- .../test/mlir/resnet/test_resnet_inference.py | 29 ---- .../mlir/resnet/test_resnet_unique_ops.py | 145 ------------------ .../pytorch/vision/resnet/test_resnet.py | 125 +++++++++------ pytest.ini | 4 - 4 files changed, 78 insertions(+), 225 deletions(-) delete mode 100644 forge/test/mlir/resnet/test_resnet_inference.py delete mode 100644 forge/test/mlir/resnet/test_resnet_unique_ops.py diff --git a/forge/test/mlir/resnet/test_resnet_inference.py b/forge/test/mlir/resnet/test_resnet_inference.py deleted file mode 100644 index 1086318b2..000000000 --- a/forge/test/mlir/resnet/test_resnet_inference.py +++ /dev/null @@ -1,29 +0,0 @@ -# SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC - -# SPDX-License-Identifier: Apache-2.0 - -import pytest -import torch -from torchvision.models.resnet import resnet50 - -import forge -from forge.verify.verify import verify - - -@pytest.mark.push -def test_resnet_inference(): - # Compiler configurations - compiler_cfg = forge.config._get_global_compiler_config() - compiler_cfg.enable_tvm_cpu_fallback = False - - # Load ResNet50 model - framework_model = resnet50() - framework_model.eval() - - input_image = torch.rand(1, 3, 224, 224) - inputs = [input_image] - - # Compile the model - compiled_model = forge.compile(framework_model, input_image) - - verify(inputs, framework_model, compiled_model) diff --git a/forge/test/mlir/resnet/test_resnet_unique_ops.py b/forge/test/mlir/resnet/test_resnet_unique_ops.py deleted file mode 100644 index a0e4030ea..000000000 --- a/forge/test/mlir/resnet/test_resnet_unique_ops.py +++ /dev/null @@ -1,145 +0,0 @@ -# SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC - -# SPDX-License-Identifier: Apache-2.0 - -import os - -import pytest -import torch -from torch import nn - -import forge -from forge.tensor import to_forge_tensors -from tvm.relay.op.transform import squeeze -from forge.verify.verify import verify -from forge.verify.config import VerifyConfig - - -@pytest.mark.parametrize( - "input_shape, kernel_size, stride_size, padding, ceil_mode", - [ - pytest.param( - (1, 64, 112, 112), # Input shape - 3, # kernel size - 2, # stride size - (1, 1, 1, 1), # padding - False, # ceil mode - ), - ], -) -@pytest.mark.push -def test_maxpool2d_resnet(input_shape, kernel_size, stride_size, padding, ceil_mode): - class maxpool2d(nn.Module): - def __init__(self): - super().__init__() - self.padding = padding - self.maxpool2d = nn.MaxPool2d( - kernel_size=kernel_size, stride=stride_size, padding=0, dilation=1, ceil_mode=ceil_mode - ) - - def forward(self, x): - if padding != 0: - x = nn.functional.pad(x, self.padding, mode="constant", value=0) - return self.maxpool2d(x) - - inputs = [torch.rand(input_shape).to(dtype=torch.bfloat16)] - - framework_model = maxpool2d().to(dtype=torch.bfloat16) - compiled_model = forge.compile(framework_model, sample_inputs=inputs) - - verify(inputs, framework_model, compiled_model) - - -@pytest.mark.push -def test_avg_pool2d_resnet(): - class AvgPool2d(nn.Module): - def __init__(self): - super().__init__() - - def forward(self, x): - return torch.nn.functional.avg_pool2d( - x, kernel_size=[7, 7], stride=[7, 7], padding=(0, 0), ceil_mode=False, count_include_pad=True - ) - - inputs = [torch.rand(1, 2048, 7, 7)] - - framework_model = AvgPool2d() - compiled_model = forge.compile(framework_model, sample_inputs=inputs) - - verify(inputs, framework_model, compiled_model) - - -@pytest.mark.parametrize( - "outer_dim_x, outer_dim_y, inner_dim", - [ - pytest.param( - 1, # Outer dimension x - 1000, # Outer dimension y - 2048, # Inner dimension - ), - ], -) -@pytest.mark.push -def test_matmul_resnet(outer_dim_x, outer_dim_y, inner_dim): - class Matmul(nn.Module): - def __init__(self): - super().__init__() - - def forward(self, x, y): - return torch.matmul(x, y) - - inputs = [ - torch.rand(outer_dim_x, inner_dim), - torch.rand(inner_dim, outer_dim_y), - ] - - framework_model = Matmul() - compiled_model = forge.compile(framework_model, sample_inputs=inputs) - - verify(inputs, framework_model, compiled_model) - - -@pytest.mark.parametrize( - "input_shape, dim", - [ - pytest.param( - (64,), # Input shape - 1, # Dimension to unsqueeze - ), - pytest.param( - (128,), - 1, - ), - pytest.param( - (256,), - 1, - ), - pytest.param( - (512,), - 1, - ), - pytest.param( - (1024,), - 1, - ), - pytest.param( - (2048,), - 1, - ), - ], -) -@pytest.mark.push -def test_unsqueeze_resnet(input_shape, dim): - class Unsqueeze(nn.Module): - def __init__(self): - super().__init__() - - def forward(self, a): - return torch.unsqueeze(a, dim) - - inputs = [torch.rand(*input_shape)] - - framework_model = Unsqueeze() - compiled_model = forge.compile(framework_model, sample_inputs=inputs) - - verify(inputs, framework_model, compiled_model) diff --git a/forge/test/models/pytorch/vision/resnet/test_resnet.py b/forge/test/models/pytorch/vision/resnet/test_resnet.py index 36e62dae7..bad8689f2 100644 --- a/forge/test/models/pytorch/vision/resnet/test_resnet.py +++ b/forge/test/models/pytorch/vision/resnet/test_resnet.py @@ -1,18 +1,19 @@ # SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC # SPDX-License-Identifier: Apache-2.0 +import random + import pytest -import requests import timm import torch from datasets import load_dataset -from loguru import logger -from PIL import Image -from timm.data import resolve_data_config -from timm.data.transforms_factory import create_transform +from tabulate import tabulate +from torchvision.models.resnet import resnet50 from transformers import AutoImageProcessor, ResNetForImageClassification import forge +from forge.verify.config import VerifyConfig +from forge.verify.value_checkers import AutomaticValueChecker from forge.verify.verify import verify from test.models.utils import Framework, Source, Task, build_module_name @@ -27,7 +28,9 @@ @pytest.mark.nightly @pytest.mark.parametrize("variant", variants, ids=variants) def test_resnet_hf(variant, record_forge_property): - # Record model properties + random.seed(0) + + # Record model details module_name = build_module_name( framework=Framework.PYTORCH, model="resnet", @@ -37,69 +40,97 @@ def test_resnet_hf(variant, record_forge_property): ) record_forge_property("model_name", module_name) - # Load dataset - dataset = load_dataset("huggingface/cats-image") - image = dataset["test"]["image"][0] + # Load tiny dataset + dataset = load_dataset("zh-plus/tiny-imagenet") + images = random.sample(dataset["valid"]["image"], 10) - # Load Torch model, preprocess image, and label dictionary - processor = download_model(AutoImageProcessor.from_pretrained, variant) + # Load framework model framework_model = download_model(ResNetForImageClassification.from_pretrained, variant, return_dict=False) - label_dict = framework_model.config.id2label - inputs = processor(image, return_tensors="pt") - inputs = inputs["pixel_values"] + # Compile model + input_sample = [torch.rand(1, 3, 224, 224)] + compiled_model = forge.compile(framework_model, input_sample) - compiled_model = forge.compile(framework_model, inputs) + # Verify data on sample input + verify(input_sample, framework_model, compiled_model, VerifyConfig(value_checker=AutomaticValueChecker(pcc=0.95))) - cpu_logits = framework_model(inputs)[0] - cpu_pred = label_dict[cpu_logits.argmax(-1).item()] + # Run model on sample data and print results + run_and_print_results(framework_model, compiled_model, images) - tt_logits = compiled_model(inputs)[0] - tt_pred = label_dict[tt_logits.argmax(-1).item()] - assert cpu_pred == tt_pred, f"Inference mismatch: CPU prediction: {cpu_pred}, TT prediction: {tt_pred}" +def run_and_print_results(framework_model, compiled_model, inputs): + """ + Runs inference using both a framework model and a compiled model on a list of input images, + then prints the results in a formatted table. - verify([inputs], framework_model, compiled_model) + Args: + framework_model: The original framework-based model. + compiled_model: The compiled version of the model. + inputs: A list of images to process and classify. + """ + label_dict = framework_model.config.id2label + processor = AutoImageProcessor.from_pretrained("microsoft/resnet-50") + results = [] + for i, image in enumerate(inputs): + processed_inputs = processor(image, return_tensors="pt")["pixel_values"] -def generate_model_resnet_imgcls_timm_pytorch(variant): - # Load ResNet50 feature extractor and model from TIMM - model = download_model(timm.create_model, variant, pretrained=True) - config = resolve_data_config({}, model=model) - transform = create_transform(**config) + cpu_logits = framework_model(processed_inputs)[0] + cpu_conf, cpu_idx = cpu_logits.softmax(-1).max(-1) + cpu_pred = label_dict.get(cpu_idx.item(), "Unknown") - # Load data sample - try: - url = "https://images.rawpixel.com/image_1300/cHJpdmF0ZS9sci9pbWFnZXMvd2Vic2l0ZS8yMDIyLTA1L3BkMTA2LTA0Ny1jaGltXzEuanBn.jpg" - image = Image.open(requests.get(url, stream=True).raw).convert("RGB") - except: - logger.warning( - "Failed to download the image file, replacing input with random tensor. Please check if the URL is up to date" - ) - image = torch.rand(1, 3, 256, 256) + tt_logits = compiled_model(processed_inputs)[0] + tt_conf, tt_idx = tt_logits.softmax(-1).max(-1) + tt_pred = label_dict.get(tt_idx.item(), "Unknown") - # Data preprocessing - pixel_values = transform(image).unsqueeze(0) + results.append([i + 1, cpu_pred, cpu_conf.item(), tt_pred, tt_conf.item()]) - return model, [pixel_values], {} + print( + tabulate( + results, + headers=["Example", "CPU Prediction", "CPU Confidence", "Compiled Prediction", "Compiled Confidence"], + tablefmt="grid", + ) + ) @pytest.mark.nightly def test_resnet_timm(record_forge_property): - pytest.skip("Skipping due to the current CI/CD pipeline limitations") - - # Build Module Name + # Record model details module_name = build_module_name( framework=Framework.PYTORCH, model="resnet", source=Source.TIMM, variant="50", task=Task.IMAGE_CLASSIFICATION ) + record_forge_property("model_name", module_name) + + # Load framework model + framework_model = download_model(timm.create_model, "resnet50", pretrained=True) - # Record Forge Property + # Compile model + input_sample = [torch.rand(1, 3, 224, 224)] + compiled_model = forge.compile(framework_model, sample_inputs=input_sample, module_name=module_name) + + # Verify data on sample input + verify(input_sample, framework_model, compiled_model, VerifyConfig(value_checker=AutomaticValueChecker(pcc=0.95))) + + +@pytest.mark.nightly +def test_resnet_torchvision(record_forge_property): + # Record model details + module_name = build_module_name( + framework=Framework.PYTORCH, + model="resnet", + source=Source.TORCHVISION, + variant="50", + task=Task.IMAGE_CLASSIFICATION, + ) record_forge_property("model_name", module_name) - framework_model, inputs, _ = generate_model_resnet_imgcls_timm_pytorch("resnet50") + # Load framework model + framework_model = resnet50() - # Forge compile framework model - compiled_model = forge.compile(framework_model, sample_inputs=inputs, module_name=module_name) + # Compile model + input_sample = [torch.rand(1, 3, 224, 224)] + compiled_model = forge.compile(framework_model, input_sample) - # Model Verification - verify(inputs, framework_model, compiled_model) + # Verify data on sample input + verify(input_sample, framework_model, compiled_model, VerifyConfig(value_checker=AutomaticValueChecker(pcc=0.95))) diff --git a/pytest.ini b/pytest.ini index 5ebc30cf5..353a8db54 100644 --- a/pytest.ini +++ b/pytest.ini @@ -44,10 +44,6 @@ testpaths = forge/test/mlir/llama/test_llama_inference.py::test_llama_inference forge/test/mlir/llama/tests - # Resnet - forge/test/mlir/resnet/test_resnet_inference.py::test_resnet_inference - forge/test/mlir/resnet/test_resnet_unique_ops.py - # Benchmark # MNIST Linear forge/test/benchmark/benchmark/models/mnist_linear.py::test_mnist_linear