Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Porting to pytest #3996

Merged
merged 19 commits into from
Jun 10, 2021
Merged

Porting to pytest #3996

Changes from 13 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
56beb8f
Porting to pytest
tanvimoharir Jun 7, 2021
fe2c62e
Merge branch 'pytorch:master' into port-test-transform-to-pytest
tanvimoharir Jun 7, 2021
867e614
Lint fixes
tanvimoharir Jun 7, 2021
2a7bd30
Merge branch 'port-test-transform-to-pytest' of https://github.com/ta…
tanvimoharir Jun 7, 2021
c292f35
Using decorator for cpu_only
tanvimoharir Jun 8, 2021
1710947
Update T.AutoAugmentPolicy call
tanvimoharir Jun 8, 2021
1f7a6dd
Merge branch 'master' into port-test-transform-to-pytest
tanvimoharir Jun 8, 2021
bb2184f
Adding missing paranthesis
tanvimoharir Jun 9, 2021
43b1e33
Changes for failed tests
tanvimoharir Jun 9, 2021
fa8feba
Merge branch 'master' into port-test-transform-to-pytest
tanvimoharir Jun 9, 2021
5c5773b
Lint fixes
tanvimoharir Jun 9, 2021
df62652
Changes
tanvimoharir Jun 9, 2021
5accfe5
Merge branch 'master' into port-test-transform-to-pytest
tanvimoharir Jun 10, 2021
ed48ad2
Changes from review comments
tanvimoharir Jun 10, 2021
1c1e5b1
Adding 'return' to fix failure
tanvimoharir Jun 10, 2021
0c88667
Merge branch 'master' into port-test-transform-to-pytest
tanvimoharir Jun 10, 2021
28b2d9b
remove parametrization from _save tests
NicolasHug Jun 10, 2021
a6d5392
Merge branch 'master' of github.com:pytorch/vision into port-test-tra…
NicolasHug Jun 10, 2021
f3fdb20
Merge branch 'master' into port-test-transform-to-pytest
NicolasHug Jun 10, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
217 changes: 99 additions & 118 deletions test/test_transforms_tensor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
from torchvision.transforms import InterpolationMode

import numpy as np
import pytest

import unittest
import pytest
from typing import Sequence

from common_utils import (
Expand Down Expand Up @@ -97,121 +97,23 @@ def _test_op(func, method, device, fn_kwargs=None, meth_kwargs=None, test_exact_
_test_class_op(method, device, meth_kwargs, test_exact_match=test_exact_match, **match_kwargs)


class Tester(unittest.TestCase):

def setUp(self):
self.device = "cpu"

def test_random_horizontal_flip(self):
_test_op(F.hflip, T.RandomHorizontalFlip, device=self.device)

def test_random_vertical_flip(self):
_test_op(F.vflip, T.RandomVerticalFlip, device=self.device)

def test_random_invert(self):
_test_op(F.invert, T.RandomInvert, device=self.device)

def test_random_posterize(self):
fn_kwargs = meth_kwargs = {"bits": 4}
_test_op(
F.posterize, T.RandomPosterize, device=self.device, fn_kwargs=fn_kwargs,
meth_kwargs=meth_kwargs
)

def test_random_solarize(self):
fn_kwargs = meth_kwargs = {"threshold": 192.0}
_test_op(
F.solarize, T.RandomSolarize, device=self.device, fn_kwargs=fn_kwargs,
meth_kwargs=meth_kwargs
)

def test_random_adjust_sharpness(self):
fn_kwargs = meth_kwargs = {"sharpness_factor": 2.0}
_test_op(
F.adjust_sharpness, T.RandomAdjustSharpness, device=self.device, fn_kwargs=fn_kwargs,
meth_kwargs=meth_kwargs
)

def test_random_autocontrast(self):
# We check the max abs difference because on some (very rare) pixels, the actual value may be different
# between PIL and tensors due to floating approximations.
_test_op(
F.autocontrast, T.RandomAutocontrast, device=self.device, test_exact_match=False,
agg_method='max', tol=(1 + 1e-5), allowed_percentage_diff=.05
)

def test_random_equalize(self):
_test_op(F.equalize, T.RandomEqualize, device=self.device)

def test_random_erasing(self):
img = torch.rand(3, 60, 60)

# Test Set 0: invalid value
random_erasing = T.RandomErasing(value=(0.1, 0.2, 0.3, 0.4), p=1.0)
with self.assertRaises(ValueError, msg="If value is a sequence, it should have either a single value or 3"):
random_erasing(img)

tensor, _ = _create_data(24, 32, channels=3, device=self.device)
batch_tensors = torch.rand(4, 3, 44, 56, device=self.device)

test_configs = [
{"value": 0.2},
{"value": "random"},
{"value": (0.2, 0.2, 0.2)},
{"value": "random", "ratio": (0.1, 0.2)},
]

for config in test_configs:
fn = T.RandomErasing(**config)
scripted_fn = torch.jit.script(fn)
_test_transform_vs_scripted(fn, scripted_fn, tensor)
_test_transform_vs_scripted_on_batch(fn, scripted_fn, batch_tensors)

with get_tmp_dir() as tmp_dir:
scripted_fn.save(os.path.join(tmp_dir, "t_random_erasing.pt"))

def test_convert_image_dtype(self):
tensor, _ = _create_data(26, 34, device=self.device)
batch_tensors = torch.rand(4, 3, 44, 56, device=self.device)

for in_dtype in int_dtypes() + float_dtypes():
in_tensor = tensor.to(in_dtype)
in_batch_tensors = batch_tensors.to(in_dtype)
for out_dtype in int_dtypes() + float_dtypes():

fn = T.ConvertImageDtype(dtype=out_dtype)
scripted_fn = torch.jit.script(fn)

if (in_dtype == torch.float32 and out_dtype in (torch.int32, torch.int64)) or \
(in_dtype == torch.float64 and out_dtype == torch.int64):
with self.assertRaisesRegex(RuntimeError, r"cannot be performed safely"):
_test_transform_vs_scripted(fn, scripted_fn, in_tensor)
with self.assertRaisesRegex(RuntimeError, r"cannot be performed safely"):
_test_transform_vs_scripted_on_batch(fn, scripted_fn, in_batch_tensors)
continue

_test_transform_vs_scripted(fn, scripted_fn, in_tensor)
_test_transform_vs_scripted_on_batch(fn, scripted_fn, in_batch_tensors)

with get_tmp_dir() as tmp_dir:
scripted_fn.save(os.path.join(tmp_dir, "t_convert_dtype.pt"))

def test_autoaugment(self):
tensor = torch.randint(0, 256, size=(3, 44, 56), dtype=torch.uint8, device=self.device)
batch_tensors = torch.randint(0, 256, size=(4, 3, 44, 56), dtype=torch.uint8, device=self.device)

s_transform = None
for policy in T.AutoAugmentPolicy:
for fill in [None, 85, (10, -10, 10), 0.7, [0.0, 0.0, 0.0], [1, ], 1]:
transform = T.AutoAugment(policy=policy, fill=fill)
s_transform = torch.jit.script(transform)
for _ in range(25):
_test_transform_vs_scripted(transform, s_transform, tensor)
_test_transform_vs_scripted_on_batch(transform, s_transform, batch_tensors)

if s_transform is not None:
with get_tmp_dir() as tmp_dir:
s_transform.save(os.path.join(tmp_dir, "t_autoaugment.pt"))
@cpu_only
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This test is actually called twice: with self.device = 'cpu' and with self.device = 'cuda' from the CUDATester class.

So instead of using the @cpu_only decorator, we should parametrize with a new

@pytest.mark.parametrize('device', cpu_and_gpu())

and replace self.device by device.

You'll need to remove device from the parametrization below as well :)

@pytest.mark.parametrize(
'func,method,device,fn_kwargs,match_kwargs', [
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
'func,method,device,fn_kwargs,match_kwargs', [
'func, method, device, fn_kwargs, match_kwargs', [

(F.hflip, T.RandomHorizontalFlip, "cpu", None, {}),
(F.vflip, T.RandomVerticalFlip, "cpu", None, {}),
(F.invert, T.RandomInvert, "cpu", None, {}),
(F.posterize, T.RandomPosterize, "cpu", {"bits": 4}, {}),
(F.solarize, T.RandomSolarize, "cpu", {"threshold": 192.0}, {}),
(F.adjust_sharpness, T.RandomAdjustSharpness, "cpu", {"sharpness_factor": 2.0}, {}),
(F.autocontrast, T.RandomAutocontrast, "cpu", None, {'test_exact_match': False,
'agg_method': 'max', 'tol': (1 + 1e-5),
'allowed_percentage_diff': .05}),
(F.equalize, T.RandomEqualize, "cpu", None, {})
]
)
def test_random(func, method, device, fn_kwargs, match_kwargs):
_test_op(func, method, device, fn_kwargs, fn_kwargs, **match_kwargs)


@pytest.mark.parametrize('device', cpu_and_gpu())
Expand Down Expand Up @@ -481,7 +383,7 @@ def test_resized_crop_save(self):


@unittest.skipIf(not torch.cuda.is_available(), reason="Skip if no CUDA device")
class CUDATester(Tester):
class CUDATester(unittest.TestCase):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we should be able to remove this class now


def setUp(self):
torch.set_deterministic(False)
Expand Down Expand Up @@ -608,6 +510,86 @@ def test_to_grayscale(device, Klass, meth_kwargs):
)


@cpu_only
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

here as well we should parametrize with cpu_and_gpu() instead.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In fact we should do it in all of the other tests here that were using self.device

Copy link
Contributor Author

@tanvimoharir tanvimoharir Jun 10, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So the self.device had a value of 'cpu' which is why I thought using cpu_only() instead of cpu_and_gpu()

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, but as I mentioned above all these tests were called twice: once as part of Tester with 'cpu', and once as part of CudaTester with 'cuda'. Which is why we need to parametrize over 'device' with cpu_and_gpu() now :)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay, thanks for clarifying :)

@pytest.mark.xfail()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why mark xfail? This should probably be removed

@pytest.mark.parametrize(
'in_dtype,out_dtype', [
(int_dtypes() + float_dtypes(), int_dtypes() + float_dtypes())
]
)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

there are 2 nested for loops here so we should not parametrize over tuples, instead we should have 2 separate paraemtrization to have a cross-produce:

@pytest.mark.parametrize('in_dtype', int_dtypes() + float_dtypes())
@pytest.mark.parametrize('out_dtype', int_dtypes() + float_dtypes())

def test_convert_image_dtype(in_dtype, out_dtype):
tensor, _ = _create_data(26, 34, device="cpu")
batch_tensors = torch.rand(4, 3, 44, 56, device="cpu")

in_tensor = tensor.to(in_dtype)
in_batch_tensors = batch_tensors.to(in_dtype)

fn = T.ConvertImageDtype(dtype=out_dtype)
scripted_fn = torch.jit.script(fn)

if (in_dtype == torch.float32 and out_dtype in (torch.int32, torch.int64)) or \
(in_dtype == torch.float64 and out_dtype == torch.int64):
with pytest.raises(RuntimeError, match=r"cannot be performed safely"):
_test_transform_vs_scripted(fn, scripted_fn, in_tensor)
with pytest.raises(RuntimeError, match=r"cannot be performed safely"):
_test_transform_vs_scripted_on_batch(fn, scripted_fn, in_batch_tensors)
NicolasHug marked this conversation as resolved.
Show resolved Hide resolved

_test_transform_vs_scripted(fn, scripted_fn, in_tensor)
_test_transform_vs_scripted_on_batch(fn, scripted_fn, in_batch_tensors)

with get_tmp_dir() as tmp_dir:
scripted_fn.save(os.path.join(tmp_dir, "t_convert_dtype.pt"))
Comment on lines +530 to +531
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this can take a bit of time, especially when the test iss heavily parametrized. Here and in the rest of the test, let's extract the saving part into separate tests. Here we could name it test_convert_image_dtype_save()



@cpu_only
@pytest.mark.parametrize('policy', [policy for policy in T.AutoAugmentPolicy])
@pytest.mark.parametrize('fill', [None, 85, (10, -10, 10), 0.7, [0.0, 0.0, 0.0], [1, ], 1])
def test_autoaugment(policy, fill):
tensor = torch.randint(0, 256, size=(3, 44, 56), dtype=torch.uint8, device="cpu")
batch_tensors = torch.randint(0, 256, size=(4, 3, 44, 56), dtype=torch.uint8, device="cpu")

s_transform = None
transform = T.AutoAugment(policy=policy, fill=fill)
s_transform = torch.jit.script(transform)
for _ in range(25):
_test_transform_vs_scripted(transform, s_transform, tensor)
_test_transform_vs_scripted_on_batch(transform, s_transform, batch_tensors)

if s_transform is not None:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

let's extract this out as well in another test without parametrization

with get_tmp_dir() as tmp_dir:
s_transform.save(os.path.join(tmp_dir, "t_autoaugment.pt"))


@cpu_only
@pytest.mark.parametrize(
'config', [
{"value": 0.2},
{"value": "random"},
{"value": (0.2, 0.2, 0.2)},
{"value": "random", "ratio": (0.1, 0.2)}
]
)
def test_random_erasing(config):
tensor, _ = _create_data(24, 32, channels=3, device="cpu")
batch_tensors = torch.rand(4, 3, 44, 56, device="cpu")

fn = T.RandomErasing(**config)
scripted_fn = torch.jit.script(fn)
_test_transform_vs_scripted(fn, scripted_fn, tensor)
_test_transform_vs_scripted_on_batch(fn, scripted_fn, batch_tensors)

with get_tmp_dir() as tmp_dir:
scripted_fn.save(os.path.join(tmp_dir, "t_random_erasing.pt"))
Comment on lines +578 to +579
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

here as well



def test_random_erasing_with_invalid_data():
img = torch.rand(3, 60, 60)
# Test Set 0: invalid value
random_erasing = T.RandomErasing(value=(0.1, 0.2, 0.3, 0.4), p=1.0)
with pytest.raises(ValueError, match="If value is a sequence, it should have either a single value or 3"):
random_erasing(img)


@pytest.mark.parametrize('device', cpu_and_gpu())
def test_normalize(device):
fn = T.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
Expand Down Expand Up @@ -659,7 +641,6 @@ def test_linear_transformation(device):
def test_compose(device):
tensor, _ = _create_data(26, 34, device=device)
tensor = tensor.to(dtype=torch.float32) / 255.0

transforms = T.Compose([
T.CenterCrop(10),
T.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
Expand Down