From cf513797f4051e9b3596491ca04255f690af2438 Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Tue, 17 Aug 2021 15:38:00 -0400 Subject: [PATCH 01/31] ops.masks_to_bounding_boxes --- torchvision/ops/__init__.py | 3 ++- torchvision/ops/_masks_to_bounding_boxes.py | 26 +++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 torchvision/ops/_masks_to_bounding_boxes.py diff --git a/torchvision/ops/__init__.py b/torchvision/ops/__init__.py index 0ec189dbc2a..5fa1aeec51e 100644 --- a/torchvision/ops/__init__.py +++ b/torchvision/ops/__init__.py @@ -8,6 +8,7 @@ from .poolers import MultiScaleRoIAlign from .feature_pyramid_network import FeaturePyramidNetwork from .focal_loss import sigmoid_focal_loss +from ._masks_to_bounding_boxes import masks_to_bounding_boxes from ._register_onnx_ops import _register_custom_op @@ -20,5 +21,5 @@ 'box_area', 'box_iou', 'generalized_box_iou', 'roi_align', 'RoIAlign', 'roi_pool', 'RoIPool', 'ps_roi_align', 'PSRoIAlign', 'ps_roi_pool', 'PSRoIPool', 'MultiScaleRoIAlign', 'FeaturePyramidNetwork', - 'sigmoid_focal_loss' + 'sigmoid_focal_loss', 'masks_to_bounding_boxes' ] diff --git a/torchvision/ops/_masks_to_bounding_boxes.py b/torchvision/ops/_masks_to_bounding_boxes.py new file mode 100644 index 00000000000..84bc57f7af4 --- /dev/null +++ b/torchvision/ops/_masks_to_bounding_boxes.py @@ -0,0 +1,26 @@ +import torch + + +def masks_to_bounding_boxes(masks: torch.Tensor) -> torch.Tensor: + """Compute the bounding boxes around the provided masks + The masks should be in format [N, H, W] where N is the number of masks, (H, W) are the spatial dimensions. + Returns a [N, 4] tensors, with the boxes in xyxy format + """ + if masks.numel() == 0: + return torch.zeros((0, 4), device=masks.device) + + h, w = masks.shape[-2:] + + y = torch.arange(0, h, dtype=torch.float) + x = torch.arange(0, w, dtype=torch.float) + y, x = torch.meshgrid(y, x) + + x_mask = masks * x.unsqueeze(0) + x_max = x_mask.flatten(1).max(-1)[0] + x_min = x_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + + y_mask = masks * y.unsqueeze(0) + y_max = y_mask.flatten(1).max(-1)[0] + y_min = y_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + + return torch.stack([x_min, y_min, x_max, y_max], 1) From c67e03504292a84357513153bc3ec7fd87a0b6c0 Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Wed, 18 Aug 2021 13:11:56 -0400 Subject: [PATCH 02/31] test fixtures --- test/assets/labeled_image.png | Bin 0 -> 896 bytes test/assets/masks.tiff | Bin 0 -> 352484 bytes test/test_masks_to_bounding_boxes.py | 31 +++++++++++++++++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 test/assets/labeled_image.png create mode 100644 test/assets/masks.tiff create mode 100644 test/test_masks_to_bounding_boxes.py diff --git a/test/assets/labeled_image.png b/test/assets/labeled_image.png new file mode 100644 index 0000000000000000000000000000000000000000..9d163243773a824c6990aafd43acfddefc631a01 GIT binary patch literal 896 zcmeAS@N?(olHy`uVBq!ia0vp^4?vg$NHEyDnc6ZiFuQxYIEGZ*dOPQ0(H#XI*WAPx z|LgONeojPx{l46&`im-LN`Il#hu+p@9KP zJdn2%SzT~pO)I-?hUABgtOcy>{nKh(O7^rK|I?r^nRQ<;vXXJvwu755H@hp_-MiTD zY$Mhp;KZ?Twe`&W-Iw3+emHs0+M4Ol`HM+eE|!HQLh21k|1Ly`UNje|EMtG_G-df) z$LW(c&oMGx@Z)RLx%IIx99K6y`kC}X&&q4vv@zrz}L#Cc`f6?a#dcwV*J#om1DY(a4EEFWXhZ5taLv+Xx)_H)lXvOme`qWOkn zhkR?FoHqF8c0VK{S&(bRdzUp0aHpMT^o#vBmmdThOneY5@l zfB$ZsH=Z)*v@Ta$)rZqM_wScn+T~jFGO%##{LJ-TE9~z7m@vzV|J^UPYYc2hSIxHg zac#j~^@MJ5*WI?^02D=;fNyJa6*;wURo81PzbVw=Q|Rav1=qo+M>UK}jn)Gfq!i_M2ibQcS1o30X1y&Ap7l?e_z&eEoms@56oUj4T2U47iCsx(ZI9klXW%)w0_E Uf=>in0Vr#Ey85}Sb4q9e0Nuuo5dZ)H literal 0 HcmV?d00001 diff --git a/test/assets/masks.tiff b/test/assets/masks.tiff new file mode 100644 index 0000000000000000000000000000000000000000..7a8efc6dd0ee958b3ebf2733a0bfb32d39449478 GIT binary patch literal 352484 zcmeI&&#Gis6$ju`_x8QXAOw1l1d~a~D2NCKL2%$apidy}pFtV}4gxw6@d11R(TPJJ z!6%51;5yZd~*>UKZguDfr>?{02}?~I@Qb9_JF@7->CuUYecbLI!H)_gq- zA8zgr_kOe=|7Ses(`N4v#`E79Kl|_a9$wrHfBfWkzxnl7Uwrc8kAME7?|uB)XFuHi z^5Re1FZZu)rS1FoU%q_#^2^_T@%^v<&@VJ2FjHXDzGj|vecRh`yYcmvIHRHm>9!xt zIPc1~9*tKf;E8lqT6iX2g%+NTS0G^Vd_lbI4GiNY4_G|kgm|eNSUlgPc!?V*-%jAl zez!cm9R1V3=}*hkuk?I+KA--nn|`C`)AKp~X?gmU)t}GtZ+a%Yp5A@28z-bo{J#$) z(}aMFVXkpd^?Y#y*HB;b&Y*)!UbV_u)$^TyLRTSQ_|h{ST==H7jysWXt#X#X_h`W7 zuUYZL2MJfK<#Dd=5;WXFX;m+-$7uN=WFhzhw9_O+^bgEAICbZSf#$XmmaE*2Xn7l zWq%y&uws?^=3aWJJ|4`yYL)$Qtiy^`>YID%q560*_o`L)$FUA8R;h39rHAU{!Q87> z*&oL`tXQSK*_R&5j|Q`ETIYZ$*5T{xlsEg4tt^ewm|{Zdz-Y2i{=tb5MMfa{jE-0dd?cx<~VDWtWcvTyCh*vaV@qA6Zk_{}LuZvf(fyMK^7Oz?ZAH*vb zuz0>B@k%wYc)p|Y3N^5Jz7z4vG_ZKSlksXa@JzfC0gLB58?QhEi{}gCrEg&Id||xY z4V(}!b-?2JCdEtKz~cEP#!K74;`t`WOWVL~|N23_1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlya0LR-+Se7-vup`%xZQZ!<5xcILAvb+E7!E(^+)5?33wu2m4GMX)d+Yd zUWI^XqHz8i?2Ff=IEchG2r#?#m^zW&kxl8Wze0n}Fd0w7+ zt>@G8`P9$cCHGZ-K8FbbFY&s%wbk>*4ZOORTvkxnz{`%izSTj%>nr%uip~bS^ssAO ze2_K>@~+5wB_8{?I3VDWsrcy$}Nk5~35%&UAw10LcPeL{=p zYvPq`VDWriym}3MEnc}Fp|9sF7Vts5VozxCd`IGyYGCnvN8{CL;E8x;euSRNS0v!c zctxJj;`z?RE78E>`Oe16-@qVV`X8ZDzT^SJc*&p8;`t`TOWnZY`6k6n-N38;i&ljZ zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBng zVuAZBu5E$R?(b^{ES_(SSGs}4^X=jlZea0z`*>9wc!*atVDWrSypjzpp0A5nuz|(% zy%w)p10TdI7O;4}Bk@W#uz0?s@d`Dtc)kBctVTkI~%V+ z1B>Si;-zn3@qA&t+zp%%FLl7;`6k6n+`!`bCdNzKz~cEP$IIEk3*w~=SUlfF@e($$ zc)kncrE6gEd>6+{*T4n-v6>?Y5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FpSCymiT| z-#jL459;Qgt7a50YR`gp_%$n-s zVTwe3k#VSIMZY!0tf@X8rbyHm8HZ|C^jlNRn(E_WibQ>paj0fRzct0IsXiX2NYob@ zhiX>zTT{%M>f>RGM17HQsAfgKHN~u{J|3n>)E60tYF6}HQ_Py`<6(+KeUWjfW<|d> z#jL459;Qgt7a50YR`gp_%$n-sVTwe3k#VSIMZY!0tf@X8rbyHm8HZ|C^jlNRn(E_W zibQ>paj0fRzct0IsXiX2NYob@hiX>zTT{%M>f>RGM17HQsAfgKHN~u{J|3n>)E60t zYF6}HQ_Py`<6(+KeUWjfW<|d>#jL459;Qgt7a50YR`gp_%$n-sVTwe3k#VSIMZY!0 ztf@X8rbyHm8HZ|C^jlNRn(E_WibQ>paj0fRzct0IsXiX2NYob@hiX>zTT{%M>f>RG zM17HQsAfgKHN~u{J|3n>)E60tYF6}HQ_Py`<6(+KeUWjfW<|d>#jL459;Qgt7a50Y zR`gp_%$n-sVTwe3k#VSIMZY!0tf@X8rbyHm8HZ|C^jlNRn(E_WibQ>paj0fRzct0I zsXiX2NYob@hiX>zTT{%M>f>RGM17HQsAfgK)rFOn z(~NI6x4X~Z9zVa`kGJdYoALAg`r%*0@Lu!z`^`BYyjt`1FnqYV+nxEB7k9%)`|;n$ z@u$t+e~k0(J?(p;D*wD$tjeGM;?b)7+09~Ae*1V;KEGY8%D?{d(JcSHT$Mlh)uUDU Rw`Yr4j^8-lch2^^=zrO2=;8nX literal 0 HcmV?d00001 diff --git a/test/test_masks_to_bounding_boxes.py b/test/test_masks_to_bounding_boxes.py new file mode 100644 index 00000000000..20db6b7e252 --- /dev/null +++ b/test/test_masks_to_bounding_boxes.py @@ -0,0 +1,31 @@ +import os.path + +import PIL.Image +import numpy +import pytest +import torch + +ASSETS_DIRECTORY = os.path.join(os.path.dirname(os.path.abspath(__file__)), "assets") + + +@pytest.fixture +def labeled_image() -> torch.Tensor: + with PIL.Image.open(os.path.join(ASSETS_DIRECTORY, "labeled_image.png")) as image: + return torch.tensor(numpy.array(image, numpy.int)) + + +@pytest.fixture +def masks() -> torch.Tensor: + with PIL.Image.open(os.path.join(ASSETS_DIRECTORY, "masks.tiff")) as image: + frames = numpy.zeros((image.n_frames, image.height, image.width), numpy.int) + + for index in range(image.n_frames): + image.seek(index) + + frames[index] = numpy.array(image) + + return torch.tensor(frames) + + +def test_masks_to_bounding_boxes(masks): + pass From 3830dd11ae16b0085c7abcd885fc69695a2f63fb Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Wed, 18 Aug 2021 13:18:46 -0400 Subject: [PATCH 03/31] unit test --- test/test_masks_to_bounding_boxes.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/test/test_masks_to_bounding_boxes.py b/test/test_masks_to_bounding_boxes.py index 20db6b7e252..2e7374fb5c6 100644 --- a/test/test_masks_to_bounding_boxes.py +++ b/test/test_masks_to_bounding_boxes.py @@ -5,6 +5,8 @@ import pytest import torch +import torchvision.ops + ASSETS_DIRECTORY = os.path.join(os.path.dirname(os.path.abspath(__file__)), "assets") @@ -28,4 +30,14 @@ def masks() -> torch.Tensor: def test_masks_to_bounding_boxes(masks): - pass + expected = torch.tensor( + [[ 127., 2., 165., 40.], + [ 4., 100., 88., 184.], + [ 168., 189., 294., 300.], + [ 556., 272., 700., 416.], + [ 800., 560., 990., 725.], + [ 294., 828., 594., 1092.], + [ 756., 1036., 1064., 1491.]] + ) + + torch.testing.assert_close(torchvision.ops.masks_to_bounding_boxes(masks), expected) From f777416df8d4c6412214eb83fff82e28d1ac6b7c Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Wed, 18 Aug 2021 13:29:45 -0400 Subject: [PATCH 04/31] ignore lint e201 and e202 for in-lined matrix --- test/test_masks_to_bounding_boxes.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test_masks_to_bounding_boxes.py b/test/test_masks_to_bounding_boxes.py index 2e7374fb5c6..3e2fa6d74e5 100644 --- a/test/test_masks_to_bounding_boxes.py +++ b/test/test_masks_to_bounding_boxes.py @@ -31,13 +31,13 @@ def masks() -> torch.Tensor: def test_masks_to_bounding_boxes(masks): expected = torch.tensor( - [[ 127., 2., 165., 40.], - [ 4., 100., 88., 184.], - [ 168., 189., 294., 300.], - [ 556., 272., 700., 416.], - [ 800., 560., 990., 725.], - [ 294., 828., 594., 1092.], - [ 756., 1036., 1064., 1491.]] + [[ 127., 2., 165., 40. ], # noqa: E201, E202 + [ 4., 100., 88., 184. ], # noqa: E201, E202 + [ 168., 189., 294., 300. ], # noqa: E201, E202 + [ 556., 272., 700., 416. ], # noqa: E201, E202 + [ 800., 560., 990., 725. ], # noqa: E201, E202 + [ 294., 828., 594., 1092. ], # noqa: E201, E202 + [ 756., 1036., 1064., 1491. ]] # noqa: E201, E202 ) torch.testing.assert_close(torchvision.ops.masks_to_bounding_boxes(masks), expected) From cd46aa7dc52f1a2ebccad9306d2b96e3a0db4992 Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Wed, 18 Aug 2021 13:32:59 -0400 Subject: [PATCH 05/31] ignore e121 and e241 linting rules for in-lined matrix --- test/test_masks_to_bounding_boxes.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test_masks_to_bounding_boxes.py b/test/test_masks_to_bounding_boxes.py index 3e2fa6d74e5..6752058c6a4 100644 --- a/test/test_masks_to_bounding_boxes.py +++ b/test/test_masks_to_bounding_boxes.py @@ -31,13 +31,13 @@ def masks() -> torch.Tensor: def test_masks_to_bounding_boxes(masks): expected = torch.tensor( - [[ 127., 2., 165., 40. ], # noqa: E201, E202 - [ 4., 100., 88., 184. ], # noqa: E201, E202 - [ 168., 189., 294., 300. ], # noqa: E201, E202 - [ 556., 272., 700., 416. ], # noqa: E201, E202 - [ 800., 560., 990., 725. ], # noqa: E201, E202 - [ 294., 828., 594., 1092. ], # noqa: E201, E202 - [ 756., 1036., 1064., 1491. ]] # noqa: E201, E202 + [[ 127., 2., 165., 40. ], # noqa: E121, E201, E202, E241 + [ 4., 100., 88., 184. ], # noqa: E201, E202, E241 + [ 168., 189., 294., 300. ], # noqa: E201, E202, E241 + [ 556., 272., 700., 416. ], # noqa: E201, E202, E241 + [ 800., 560., 990., 725. ], # noqa: E201, E202, E241 + [ 294., 828., 594., 1092. ], # noqa: E201, E202, E241 + [ 756., 1036., 1064., 1491. ]] # noqa: E201, E202, E241 ) torch.testing.assert_close(torchvision.ops.masks_to_bounding_boxes(masks), expected) From 712131ece751f13b7df2d88cfa6dec84c2f4300a Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Wed, 18 Aug 2021 14:38:27 -0400 Subject: [PATCH 06/31] draft gallery example text --- gallery/plot_repurposing_annotations.py | 73 +++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 gallery/plot_repurposing_annotations.py diff --git a/gallery/plot_repurposing_annotations.py b/gallery/plot_repurposing_annotations.py new file mode 100644 index 00000000000..f196657d97a --- /dev/null +++ b/gallery/plot_repurposing_annotations.py @@ -0,0 +1,73 @@ +""" +======================= +Repurposing annotations +======================= + +The following example illustrates the operations available in :ref:`the torchvision.ops module ` for repurposing +object localization annotations for different tasks (e.g. transforming masks used by instance and panoptic +segmentation methods into bounding boxes used by object detection methods). +""" + +from PIL import Image +from pathlib import Path +import matplotlib.pyplot as plt +import numpy as np + +import torch +import torchvision.transforms as T + + +plt.rcParams["savefig.bbox"] = 'tight' +orig_img = Image.open(Path('assets') / 'astronaut.jpg') +# if you change the seed, make sure that the randomly-applied transforms +# properly show that the image can be both transformed and *not* transformed! +torch.manual_seed(0) + + +def plot(imgs, with_orig=True, row_title=None, **imshow_kwargs): + if not isinstance(imgs[0], list): + # Make a 2d grid even if there's just 1 row + imgs = [imgs] + + num_rows = len(imgs) + num_cols = len(imgs[0]) + with_orig + fig, axs = plt.subplots(nrows=num_rows, ncols=num_cols, squeeze=False) + for row_idx, row in enumerate(imgs): + row = [orig_img] + row if with_orig else row + for col_idx, img in enumerate(row): + ax = axs[row_idx, col_idx] + ax.imshow(np.asarray(img), **imshow_kwargs) + ax.set(xticklabels=[], yticklabels=[], xticks=[], yticks=[]) + + if with_orig: + axs[0, 0].set(title='Original image') + axs[0, 0].title.set_size(8) + if row_title is not None: + for row_idx in range(num_rows): + axs[row_idx, 0].set(ylabel=row_title[row_idx]) + + plt.tight_layout() + +#################################### +# Masks +# -------------------------------------- +# In tasks like instance and panoptic segmentation, masks are commonly defined, and are defined by this package, +# as a multi-dimensional array (e.g. a NumPy array or a PyTorch tensor) with the following shape: +# +# (objects, height, width) +# +# Where objects is the number of annotated objects in the image. Each (height, width) object corresponds to exactly +# one object. For example, if your input image has the dimensions 224 x 224 and has four annotated objects the shape +# of your masks annotation has the following shape: +# +# (4, 224, 224). +# +# A nice property of masks is that they can be easily repurposed to be used in methods to solve a variety of object +# localization tasks. +# +# Masks to bounding boxes +# ~~~~~~~~~~~~~~~~~~~~~~~ +# For example, the masks to bounding_boxes operation can be used to transform masks into bounding boxes that can be +# used in methods like Faster RCNN and YOLO. +padded_imgs = [T.Pad(padding=padding)(orig_img) for padding in (3, 10, 30, 50)] +plot(padded_imgs) From b6f5c42b20e9a744808948ae611130106b191791 Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Tue, 31 Aug 2021 16:42:53 -0400 Subject: [PATCH 07/31] removed type annotations from pytest fixtures --- test/test_masks_to_bounding_boxes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_masks_to_bounding_boxes.py b/test/test_masks_to_bounding_boxes.py index 6752058c6a4..d1957aaa205 100644 --- a/test/test_masks_to_bounding_boxes.py +++ b/test/test_masks_to_bounding_boxes.py @@ -11,13 +11,13 @@ @pytest.fixture -def labeled_image() -> torch.Tensor: +def labeled_image(): with PIL.Image.open(os.path.join(ASSETS_DIRECTORY, "labeled_image.png")) as image: return torch.tensor(numpy.array(image, numpy.int)) @pytest.fixture -def masks() -> torch.Tensor: +def masks(): with PIL.Image.open(os.path.join(ASSETS_DIRECTORY, "masks.tiff")) as image: frames = numpy.zeros((image.n_frames, image.height, image.width), numpy.int) From b555c681bef8041df14cbdd2cfb20a1832a02337 Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Tue, 31 Aug 2021 16:48:33 -0400 Subject: [PATCH 08/31] inlined fixture --- test/test_masks_to_bounding_boxes.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/test/test_masks_to_bounding_boxes.py b/test/test_masks_to_bounding_boxes.py index d1957aaa205..2d26e74bff1 100644 --- a/test/test_masks_to_bounding_boxes.py +++ b/test/test_masks_to_bounding_boxes.py @@ -10,14 +10,7 @@ ASSETS_DIRECTORY = os.path.join(os.path.dirname(os.path.abspath(__file__)), "assets") -@pytest.fixture -def labeled_image(): - with PIL.Image.open(os.path.join(ASSETS_DIRECTORY, "labeled_image.png")) as image: - return torch.tensor(numpy.array(image, numpy.int)) - - -@pytest.fixture -def masks(): +def test_masks_to_bounding_boxes(): with PIL.Image.open(os.path.join(ASSETS_DIRECTORY, "masks.tiff")) as image: frames = numpy.zeros((image.n_frames, image.height, image.width), numpy.int) @@ -26,10 +19,8 @@ def masks(): frames[index] = numpy.array(image) - return torch.tensor(frames) - + masks = torch.tensor(frames) -def test_masks_to_bounding_boxes(masks): expected = torch.tensor( [[ 127., 2., 165., 40. ], # noqa: E121, E201, E202, E241 [ 4., 100., 88., 184. ], # noqa: E201, E202, E241 From fc26f3ad2c949a0808f1525087da662d39d6d011 Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Tue, 31 Aug 2021 17:02:01 -0400 Subject: [PATCH 09/31] renamed masks_to_bounding_boxes to masks_to_boxes --- ...test_masks_to_bounding_boxes.py => test_masks_to_boxes.py} | 4 ++-- torchvision/ops/__init__.py | 4 ++-- .../ops/{_masks_to_bounding_boxes.py => _masks_to_boxes.py} | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename test/{test_masks_to_bounding_boxes.py => test_masks_to_boxes.py} (89%) rename torchvision/ops/{_masks_to_bounding_boxes.py => _masks_to_boxes.py} (92%) diff --git a/test/test_masks_to_bounding_boxes.py b/test/test_masks_to_boxes.py similarity index 89% rename from test/test_masks_to_bounding_boxes.py rename to test/test_masks_to_boxes.py index 2d26e74bff1..e192e772255 100644 --- a/test/test_masks_to_bounding_boxes.py +++ b/test/test_masks_to_boxes.py @@ -10,7 +10,7 @@ ASSETS_DIRECTORY = os.path.join(os.path.dirname(os.path.abspath(__file__)), "assets") -def test_masks_to_bounding_boxes(): +def test_masks_to_boxes(): with PIL.Image.open(os.path.join(ASSETS_DIRECTORY, "masks.tiff")) as image: frames = numpy.zeros((image.n_frames, image.height, image.width), numpy.int) @@ -31,4 +31,4 @@ def test_masks_to_bounding_boxes(): [ 756., 1036., 1064., 1491. ]] # noqa: E201, E202, E241 ) - torch.testing.assert_close(torchvision.ops.masks_to_bounding_boxes(masks), expected) + torch.testing.assert_close(torchvision.ops.masks_to_boxes(masks), expected) diff --git a/torchvision/ops/__init__.py b/torchvision/ops/__init__.py index 5fa1aeec51e..9b221bc3cd5 100644 --- a/torchvision/ops/__init__.py +++ b/torchvision/ops/__init__.py @@ -8,7 +8,7 @@ from .poolers import MultiScaleRoIAlign from .feature_pyramid_network import FeaturePyramidNetwork from .focal_loss import sigmoid_focal_loss -from ._masks_to_bounding_boxes import masks_to_bounding_boxes +from ._masks_to_boxes import masks_to_boxes from ._register_onnx_ops import _register_custom_op @@ -21,5 +21,5 @@ 'box_area', 'box_iou', 'generalized_box_iou', 'roi_align', 'RoIAlign', 'roi_pool', 'RoIPool', 'ps_roi_align', 'PSRoIAlign', 'ps_roi_pool', 'PSRoIPool', 'MultiScaleRoIAlign', 'FeaturePyramidNetwork', - 'sigmoid_focal_loss', 'masks_to_bounding_boxes' + 'sigmoid_focal_loss', 'masks_to_boxes' ] diff --git a/torchvision/ops/_masks_to_bounding_boxes.py b/torchvision/ops/_masks_to_boxes.py similarity index 92% rename from torchvision/ops/_masks_to_bounding_boxes.py rename to torchvision/ops/_masks_to_boxes.py index 84bc57f7af4..78676208b48 100644 --- a/torchvision/ops/_masks_to_bounding_boxes.py +++ b/torchvision/ops/_masks_to_boxes.py @@ -1,7 +1,7 @@ import torch -def masks_to_bounding_boxes(masks: torch.Tensor) -> torch.Tensor: +def masks_to_boxes(masks: torch.Tensor) -> torch.Tensor: """Compute the bounding boxes around the provided masks The masks should be in format [N, H, W] where N is the number of masks, (H, W) are the spatial dimensions. Returns a [N, 4] tensors, with the boxes in xyxy format From c4d30453df039aa2af74bf3714e44a0966f0ad8e Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Tue, 31 Aug 2021 18:35:32 -0400 Subject: [PATCH 10/31] reformat inline array --- test/test_masks_to_boxes.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test_masks_to_boxes.py b/test/test_masks_to_boxes.py index e192e772255..ab7c8aa8f7d 100644 --- a/test/test_masks_to_boxes.py +++ b/test/test_masks_to_boxes.py @@ -22,13 +22,13 @@ def test_masks_to_boxes(): masks = torch.tensor(frames) expected = torch.tensor( - [[ 127., 2., 165., 40. ], # noqa: E121, E201, E202, E241 - [ 4., 100., 88., 184. ], # noqa: E201, E202, E241 - [ 168., 189., 294., 300. ], # noqa: E201, E202, E241 - [ 556., 272., 700., 416. ], # noqa: E201, E202, E241 - [ 800., 560., 990., 725. ], # noqa: E201, E202, E241 - [ 294., 828., 594., 1092. ], # noqa: E201, E202, E241 - [ 756., 1036., 1064., 1491. ]] # noqa: E201, E202, E241 + [[127, 2, 165, 40], + [4, 100, 88, 184], + [168, 189, 294, 300], + [556, 272, 700, 416], + [800, 560, 990, 725], + [294, 828, 594, 1092], + [756, 1036, 1064, 1491]] ) torch.testing.assert_close(torchvision.ops.masks_to_boxes(masks), expected) From 45899519487c36eb1f40ade468e4746ea71e0044 Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Tue, 31 Aug 2021 18:58:00 -0400 Subject: [PATCH 11/31] import cleanup --- test/test_masks_to_boxes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_masks_to_boxes.py b/test/test_masks_to_boxes.py index ab7c8aa8f7d..de4bd45ed3b 100644 --- a/test/test_masks_to_boxes.py +++ b/test/test_masks_to_boxes.py @@ -2,7 +2,6 @@ import PIL.Image import numpy -import pytest import torch import torchvision.ops From 6b19d67a8fdc1955ad6d97a60572941d48a455b8 Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Wed, 1 Sep 2021 16:29:45 -0400 Subject: [PATCH 12/31] moved masks_to_boxes into boxes module --- test/test_masks_to_boxes.py | 3 ++- torchvision/ops/__init__.py | 6 +++--- torchvision/ops/_masks_to_boxes.py | 26 ----------------------- torchvision/ops/boxes.py | 34 ++++++++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 30 deletions(-) delete mode 100644 torchvision/ops/_masks_to_boxes.py diff --git a/test/test_masks_to_boxes.py b/test/test_masks_to_boxes.py index de4bd45ed3b..8c3be468d06 100644 --- a/test/test_masks_to_boxes.py +++ b/test/test_masks_to_boxes.py @@ -5,6 +5,7 @@ import torch import torchvision.ops +import torchvision.ops.boxes ASSETS_DIRECTORY = os.path.join(os.path.dirname(os.path.abspath(__file__)), "assets") @@ -30,4 +31,4 @@ def test_masks_to_boxes(): [756, 1036, 1064, 1491]] ) - torch.testing.assert_close(torchvision.ops.masks_to_boxes(masks), expected) + torch.testing.assert_close(torchvision.ops.boxes.masks_to_boxes(masks), expected) diff --git a/torchvision/ops/__init__.py b/torchvision/ops/__init__.py index 9b221bc3cd5..09938c35b9a 100644 --- a/torchvision/ops/__init__.py +++ b/torchvision/ops/__init__.py @@ -1,4 +1,5 @@ -from .boxes import nms, batched_nms, remove_small_boxes, clip_boxes_to_image, box_area, box_iou, generalized_box_iou +from .boxes import nms, batched_nms, remove_small_boxes, clip_boxes_to_image, box_area, box_iou, generalized_box_iou, \ + masks_to_boxes from .boxes import box_convert from .deform_conv import deform_conv2d, DeformConv2d from .roi_align import roi_align, RoIAlign @@ -8,7 +9,6 @@ from .poolers import MultiScaleRoIAlign from .feature_pyramid_network import FeaturePyramidNetwork from .focal_loss import sigmoid_focal_loss -from ._masks_to_boxes import masks_to_boxes from ._register_onnx_ops import _register_custom_op @@ -21,5 +21,5 @@ 'box_area', 'box_iou', 'generalized_box_iou', 'roi_align', 'RoIAlign', 'roi_pool', 'RoIPool', 'ps_roi_align', 'PSRoIAlign', 'ps_roi_pool', 'PSRoIPool', 'MultiScaleRoIAlign', 'FeaturePyramidNetwork', - 'sigmoid_focal_loss', 'masks_to_boxes' + 'sigmoid_focal_loss' ] diff --git a/torchvision/ops/_masks_to_boxes.py b/torchvision/ops/_masks_to_boxes.py deleted file mode 100644 index 78676208b48..00000000000 --- a/torchvision/ops/_masks_to_boxes.py +++ /dev/null @@ -1,26 +0,0 @@ -import torch - - -def masks_to_boxes(masks: torch.Tensor) -> torch.Tensor: - """Compute the bounding boxes around the provided masks - The masks should be in format [N, H, W] where N is the number of masks, (H, W) are the spatial dimensions. - Returns a [N, 4] tensors, with the boxes in xyxy format - """ - if masks.numel() == 0: - return torch.zeros((0, 4), device=masks.device) - - h, w = masks.shape[-2:] - - y = torch.arange(0, h, dtype=torch.float) - x = torch.arange(0, w, dtype=torch.float) - y, x = torch.meshgrid(y, x) - - x_mask = masks * x.unsqueeze(0) - x_max = x_mask.flatten(1).max(-1)[0] - x_min = x_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] - - y_mask = masks * y.unsqueeze(0) - y_max = y_mask.flatten(1).max(-1)[0] - y_min = y_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] - - return torch.stack([x_min, y_min, x_max, y_max], 1) diff --git a/torchvision/ops/boxes.py b/torchvision/ops/boxes.py index c1f176f4da9..3cf7a788b6c 100644 --- a/torchvision/ops/boxes.py +++ b/torchvision/ops/boxes.py @@ -297,3 +297,37 @@ def generalized_box_iou(boxes1: Tensor, boxes2: Tensor) -> Tensor: areai = whi[:, :, 0] * whi[:, :, 1] return iou - (areai - union) / areai + + +def masks_to_boxes(masks: torch.Tensor) -> torch.Tensor: + """ + Compute the bounding boxes around the provided masks + + Returns a [N, 4] tensors, with the boxes in xyxy format + + Args: + masks (Tensor[N, H, W]): masks to transform where N is the number of + masks and (H, W) are the spatial dimensions. + + Returns: + Tensor: int64 tensor with the indices of the elements that have been kept + by NMS, sorted in decreasing order of scores + """ + if masks.numel() == 0: + return torch.zeros((0, 4), device=masks.device) + + h, w = masks.shape[-2:] + + y = torch.arange(0, h, dtype=torch.float) + x = torch.arange(0, w, dtype=torch.float) + y, x = torch.meshgrid(y, x) + + x_mask = masks * x.unsqueeze(0) + x_max = x_mask.flatten(1).max(-1)[0] + x_min = x_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + + y_mask = masks * y.unsqueeze(0) + y_max = y_mask.flatten(1).max(-1)[0] + y_min = y_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + + return torch.stack([x_min, y_min, x_max, y_max], 1) From c6c89ec95c22c3243689c8f2e4e93f1fd6ae918c Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Wed, 1 Sep 2021 16:32:13 -0400 Subject: [PATCH 13/31] docstring cleanup --- torchvision/ops/boxes.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/torchvision/ops/boxes.py b/torchvision/ops/boxes.py index 3cf7a788b6c..13d2a9530cf 100644 --- a/torchvision/ops/boxes.py +++ b/torchvision/ops/boxes.py @@ -303,15 +303,14 @@ def masks_to_boxes(masks: torch.Tensor) -> torch.Tensor: """ Compute the bounding boxes around the provided masks - Returns a [N, 4] tensors, with the boxes in xyxy format + Returns a [N, 4] tensors, with the boxes in (X, Y, X, Y) format. Args: masks (Tensor[N, H, W]): masks to transform where N is the number of masks and (H, W) are the spatial dimensions. Returns: - Tensor: int64 tensor with the indices of the elements that have been kept - by NMS, sorted in decreasing order of scores + Tensor[N, 4]: bounding boxes """ if masks.numel() == 0: return torch.zeros((0, 4), device=masks.device) From 16a99a93e47901bbdf07b12fd39953c8d6348c72 Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Wed, 15 Sep 2021 15:02:42 -0400 Subject: [PATCH 14/31] updated docstring --- torchvision/ops/boxes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torchvision/ops/boxes.py b/torchvision/ops/boxes.py index 13d2a9530cf..cd122513609 100644 --- a/torchvision/ops/boxes.py +++ b/torchvision/ops/boxes.py @@ -303,7 +303,8 @@ def masks_to_boxes(masks: torch.Tensor) -> torch.Tensor: """ Compute the bounding boxes around the provided masks - Returns a [N, 4] tensors, with the boxes in (X, Y, X, Y) format. + Returns a [N, 4] tensor. Both sets of boxes are expected to be in ``(x1, y1, x2, y2)`` format with + ``0 <= x1 < x2`` and ``0 <= y1 < y2. Args: masks (Tensor[N, H, W]): masks to transform where N is the number of From 7115320aa7733fde7d408b0bca60ffbaf5c58bfd Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Wed, 15 Sep 2021 15:11:43 -0400 Subject: [PATCH 15/31] fix formatting issue --- gallery/plot_repurposing_annotations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallery/plot_repurposing_annotations.py b/gallery/plot_repurposing_annotations.py index f196657d97a..6249630df7a 100644 --- a/gallery/plot_repurposing_annotations.py +++ b/gallery/plot_repurposing_annotations.py @@ -16,7 +16,6 @@ import torch import torchvision.transforms as T - plt.rcParams["savefig.bbox"] = 'tight' orig_img = Image.open(Path('assets') / 'astronaut.jpg') # if you change the seed, make sure that the randomly-applied transforms @@ -48,6 +47,7 @@ def plot(imgs, with_orig=True, row_title=None, **imshow_kwargs): plt.tight_layout() + #################################### # Masks # -------------------------------------- From 0a23bcf60a44d71e3a5e504fa636816aa1bd83a0 Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Fri, 17 Sep 2021 10:12:22 -0400 Subject: [PATCH 16/31] gallery example --- docs/requirements.txt | 7 ++- gallery/plot_repurposing_annotations.py | 73 ++++++++++++------------- torchvision/ops/boxes.py | 22 ++++---- 3 files changed, 48 insertions(+), 54 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 44132ef3375..ce60904ced0 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1,7 @@ -sphinx==3.5.4 -sphinx-gallery>=0.9.0 -sphinx-copybutton>=0.3.1 matplotlib numpy +scikit-image>=0.18.3 +sphinx-copybutton>=0.3.1 +sphinx-gallery>=0.9.0 +sphinx==3.5.4 -e git+git://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme diff --git a/gallery/plot_repurposing_annotations.py b/gallery/plot_repurposing_annotations.py index 6249630df7a..bd3350b320a 100644 --- a/gallery/plot_repurposing_annotations.py +++ b/gallery/plot_repurposing_annotations.py @@ -8,44 +8,13 @@ segmentation methods into bounding boxes used by object detection methods). """ -from PIL import Image -from pathlib import Path -import matplotlib.pyplot as plt -import numpy as np - +import matplotlib.patches +import matplotlib.pyplot +import numpy +import skimage.draw +import skimage.measure import torch -import torchvision.transforms as T - -plt.rcParams["savefig.bbox"] = 'tight' -orig_img = Image.open(Path('assets') / 'astronaut.jpg') -# if you change the seed, make sure that the randomly-applied transforms -# properly show that the image can be both transformed and *not* transformed! -torch.manual_seed(0) - - -def plot(imgs, with_orig=True, row_title=None, **imshow_kwargs): - if not isinstance(imgs[0], list): - # Make a 2d grid even if there's just 1 row - imgs = [imgs] - - num_rows = len(imgs) - num_cols = len(imgs[0]) + with_orig - fig, axs = plt.subplots(nrows=num_rows, ncols=num_cols, squeeze=False) - for row_idx, row in enumerate(imgs): - row = [orig_img] + row if with_orig else row - for col_idx, img in enumerate(row): - ax = axs[row_idx, col_idx] - ax.imshow(np.asarray(img), **imshow_kwargs) - ax.set(xticklabels=[], yticklabels=[], xticks=[], yticks=[]) - - if with_orig: - axs[0, 0].set(title='Original image') - axs[0, 0].title.set_size(8) - if row_title is not None: - for row_idx in range(num_rows): - axs[row_idx, 0].set(ylabel=row_title[row_idx]) - - plt.tight_layout() +import torchvision.ops.boxes #################################### @@ -69,5 +38,31 @@ def plot(imgs, with_orig=True, row_title=None, **imshow_kwargs): # ~~~~~~~~~~~~~~~~~~~~~~~ # For example, the masks to bounding_boxes operation can be used to transform masks into bounding boxes that can be # used in methods like Faster RCNN and YOLO. -padded_imgs = [T.Pad(padding=padding)(orig_img) for padding in (3, 10, 30, 50)] -plot(padded_imgs) + +image, labels = skimage.draw.random_shapes((512, 256), 4, min_size=32, multichannel=False) + +labeled_image = skimage.measure.label(image, background=255) + +labeled_image = skimage.img_as_ubyte(labeled_image) + +masks = numpy.zeros((len(labels), *labeled_image.shape), numpy.ubyte) + +for index in numpy.unique(labeled_image): + masks[index - 1] = numpy.where(labeled_image == index, index, 0) + +bounding_boxes = torchvision.ops.boxes.masks_to_boxes(torch.tensor(masks)) + +figure = matplotlib.pyplot.figure() + +a = figure.add_subplot(121) +b = figure.add_subplot(122) + +a.imshow(labeled_image) +b.imshow(labeled_image) + +for bounding_box in bounding_boxes.tolist(): + x0, y0, x1, y1 = bounding_box + + rectangle = matplotlib.patches.Rectangle((x0, y0), x1 - x0, y1 - y0, linewidth=1, edgecolor='r', facecolor='none') + + b.add_patch(rectangle) diff --git a/torchvision/ops/boxes.py b/torchvision/ops/boxes.py index cd122513609..fe6535de4a9 100644 --- a/torchvision/ops/boxes.py +++ b/torchvision/ops/boxes.py @@ -314,20 +314,18 @@ def masks_to_boxes(masks: torch.Tensor) -> torch.Tensor: Tensor[N, 4]: bounding boxes """ if masks.numel() == 0: - return torch.zeros((0, 4), device=masks.device) + return torch.zeros((0, 4)) - h, w = masks.shape[-2:] + n = masks.shape[0] - y = torch.arange(0, h, dtype=torch.float) - x = torch.arange(0, w, dtype=torch.float) - y, x = torch.meshgrid(y, x) + bounding_boxes = torch.zeros((n, 4), dtype=torch.int) - x_mask = masks * x.unsqueeze(0) - x_max = x_mask.flatten(1).max(-1)[0] - x_min = x_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + for index, mask in enumerate(masks): + y, x = torch.where(masks[index] != 0) - y_mask = masks * y.unsqueeze(0) - y_max = y_mask.flatten(1).max(-1)[0] - y_min = y_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + bounding_boxes[index, 0] = torch.min(x) + bounding_boxes[index, 1] = torch.min(y) + bounding_boxes[index, 2] = torch.max(x) + bounding_boxes[index, 3] = torch.max(y) - return torch.stack([x_min, y_min, x_max, y_max], 1) + return bounding_boxes From db8fb7b9d37ca05c6c12f9bee445093932665c43 Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Fri, 17 Sep 2021 10:27:53 -0400 Subject: [PATCH 17/31] use torch --- gallery/plot_repurposing_annotations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gallery/plot_repurposing_annotations.py b/gallery/plot_repurposing_annotations.py index bd3350b320a..397acf74de6 100644 --- a/gallery/plot_repurposing_annotations.py +++ b/gallery/plot_repurposing_annotations.py @@ -45,10 +45,10 @@ labeled_image = skimage.img_as_ubyte(labeled_image) -masks = numpy.zeros((len(labels), *labeled_image.shape), numpy.ubyte) +masks = torch.zeros((len(labels), *labeled_image.shape), dtype=torch.int) for index in numpy.unique(labeled_image): - masks[index - 1] = numpy.where(labeled_image == index, index, 0) + masks[index - 1] = torch.where(labeled_image == index, index, 0) bounding_boxes = torchvision.ops.boxes.masks_to_boxes(torch.tensor(masks)) From f7a2c1ebb9b952bd82d311ef6e1c6b93f97b15f1 Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Fri, 17 Sep 2021 10:30:01 -0400 Subject: [PATCH 18/31] use torch --- gallery/plot_repurposing_annotations.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gallery/plot_repurposing_annotations.py b/gallery/plot_repurposing_annotations.py index 397acf74de6..7c78222137e 100644 --- a/gallery/plot_repurposing_annotations.py +++ b/gallery/plot_repurposing_annotations.py @@ -45,9 +45,11 @@ labeled_image = skimage.img_as_ubyte(labeled_image) +labeled_image = torch.tensor(labeled_image) + masks = torch.zeros((len(labels), *labeled_image.shape), dtype=torch.int) -for index in numpy.unique(labeled_image): +for index in torch.unique(labeled_image): masks[index - 1] = torch.where(labeled_image == index, index, 0) bounding_boxes = torchvision.ops.boxes.masks_to_boxes(torch.tensor(masks)) From c7dfcdf93113781f13e9e66384d9c2c2cda90335 Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Fri, 17 Sep 2021 10:31:00 -0400 Subject: [PATCH 19/31] use torch --- gallery/plot_repurposing_annotations.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gallery/plot_repurposing_annotations.py b/gallery/plot_repurposing_annotations.py index 7c78222137e..575c176ddcd 100644 --- a/gallery/plot_repurposing_annotations.py +++ b/gallery/plot_repurposing_annotations.py @@ -10,7 +10,6 @@ import matplotlib.patches import matplotlib.pyplot -import numpy import skimage.draw import skimage.measure import torch From 5e6198af7af9ad9376de0cab140c826eec9b43bc Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Fri, 17 Sep 2021 10:47:53 -0400 Subject: [PATCH 20/31] use torch --- gallery/plot_repurposing_annotations.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/gallery/plot_repurposing_annotations.py b/gallery/plot_repurposing_annotations.py index 575c176ddcd..9c0a32af2e7 100644 --- a/gallery/plot_repurposing_annotations.py +++ b/gallery/plot_repurposing_annotations.py @@ -38,15 +38,13 @@ # For example, the masks to bounding_boxes operation can be used to transform masks into bounding boxes that can be # used in methods like Faster RCNN and YOLO. -image, labels = skimage.draw.random_shapes((512, 256), 4, min_size=32, multichannel=False) +image, labels = skimage.draw.random_shapes((224, 224), 8, min_size=32, multichannel=False) labeled_image = skimage.measure.label(image, background=255) -labeled_image = skimage.img_as_ubyte(labeled_image) - labeled_image = torch.tensor(labeled_image) -masks = torch.zeros((len(labels), *labeled_image.shape), dtype=torch.int) +masks = torch.zeros((len(labels), *labeled_image.shape)) for index in torch.unique(labeled_image): masks[index - 1] = torch.where(labeled_image == index, index, 0) @@ -61,9 +59,9 @@ a.imshow(labeled_image) b.imshow(labeled_image) -for bounding_box in bounding_boxes.tolist(): +for bounding_box in bounding_boxes: x0, y0, x1, y1 = bounding_box - rectangle = matplotlib.patches.Rectangle((x0, y0), x1 - x0, y1 - y0, linewidth=1, edgecolor='r', facecolor='none') + rectangle = matplotlib.patches.Rectangle((x0, y0), x1 - x0, y1 - y0, linewidth=1, edgecolor="r", facecolor="none") b.add_patch(rectangle) From 7c78271def90a49731eaa28c475e22eb931b2b39 Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Fri, 17 Sep 2021 11:03:59 -0400 Subject: [PATCH 21/31] updated docs and test --- gallery/plot_repurposing_annotations.py | 6 +++++- test/test_masks_to_boxes.py | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/gallery/plot_repurposing_annotations.py b/gallery/plot_repurposing_annotations.py index 9c0a32af2e7..03acf1acdbd 100644 --- a/gallery/plot_repurposing_annotations.py +++ b/gallery/plot_repurposing_annotations.py @@ -15,6 +15,7 @@ import torch import torchvision.ops.boxes +matplotlib.pyplot.rcParams["savefig.bbox"] = "tight" #################################### # Masks @@ -49,7 +50,7 @@ for index in torch.unique(labeled_image): masks[index - 1] = torch.where(labeled_image == index, index, 0) -bounding_boxes = torchvision.ops.boxes.masks_to_boxes(torch.tensor(masks)) +bounding_boxes = torchvision.ops.boxes.masks_to_boxes(masks) figure = matplotlib.pyplot.figure() @@ -65,3 +66,6 @@ rectangle = matplotlib.patches.Rectangle((x0, y0), x1 - x0, y1 - y0, linewidth=1, edgecolor="r", facecolor="none") b.add_patch(rectangle) + +a.set(xticklabels=[], yticklabels=[], xticks=[], yticks=[]) +b.set(xticklabels=[], yticklabels=[], xticks=[], yticks=[]) diff --git a/test/test_masks_to_boxes.py b/test/test_masks_to_boxes.py index 8c3be468d06..023d334ed32 100644 --- a/test/test_masks_to_boxes.py +++ b/test/test_masks_to_boxes.py @@ -4,15 +4,14 @@ import numpy import torch -import torchvision.ops -import torchvision.ops.boxes +from torchvision.ops import masks_to_boxes ASSETS_DIRECTORY = os.path.join(os.path.dirname(os.path.abspath(__file__)), "assets") def test_masks_to_boxes(): with PIL.Image.open(os.path.join(ASSETS_DIRECTORY, "masks.tiff")) as image: - frames = numpy.zeros((image.n_frames, image.height, image.width), numpy.int) + frames = numpy.zeros((image.n_frames, image.height, image.width), int) for index in range(image.n_frames): image.seek(index) @@ -23,12 +22,13 @@ def test_masks_to_boxes(): expected = torch.tensor( [[127, 2, 165, 40], - [4, 100, 88, 184], - [168, 189, 294, 300], - [556, 272, 700, 416], - [800, 560, 990, 725], - [294, 828, 594, 1092], - [756, 1036, 1064, 1491]] + [2, 50, 44, 92], + [56, 63, 98, 100], + [139, 68, 175, 104], + [160, 112, 198, 145], + [49, 138, 99, 182], + [108, 148, 152, 213]], + dtype=torch.int32 ) - torch.testing.assert_close(torchvision.ops.boxes.masks_to_boxes(masks), expected) + torch.testing.assert_close(masks_to_boxes(masks), expected) From b9055c26ee823f315eeb1eaa2f65e74d0187bc3a Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Fri, 17 Sep 2021 11:42:28 -0400 Subject: [PATCH 22/31] cleanup --- docs/requirements.txt | 1 - gallery/plot_repurposing_annotations.py | 22 +++++++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index ce60904ced0..d2eb35aac8e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1,5 @@ matplotlib numpy -scikit-image>=0.18.3 sphinx-copybutton>=0.3.1 sphinx-gallery>=0.9.0 sphinx==3.5.4 diff --git a/gallery/plot_repurposing_annotations.py b/gallery/plot_repurposing_annotations.py index 03acf1acdbd..c634b04b329 100644 --- a/gallery/plot_repurposing_annotations.py +++ b/gallery/plot_repurposing_annotations.py @@ -7,14 +7,17 @@ object localization annotations for different tasks (e.g. transforming masks used by instance and panoptic segmentation methods into bounding boxes used by object detection methods). """ +import os.path +import PIL.Image import matplotlib.patches import matplotlib.pyplot -import skimage.draw -import skimage.measure +import numpy import torch import torchvision.ops.boxes +ASSETS_DIRECTORY = "../test/assets" + matplotlib.pyplot.rcParams["savefig.bbox"] = "tight" #################################### @@ -39,16 +42,15 @@ # For example, the masks to bounding_boxes operation can be used to transform masks into bounding boxes that can be # used in methods like Faster RCNN and YOLO. -image, labels = skimage.draw.random_shapes((224, 224), 8, min_size=32, multichannel=False) - -labeled_image = skimage.measure.label(image, background=255) +with PIL.Image.open(os.path.join(ASSETS_DIRECTORY, "masks.tiff")) as image: + frames = numpy.zeros((image.n_frames, image.height, image.width), int) -labeled_image = torch.tensor(labeled_image) + for index in range(image.n_frames): + image.seek(index) -masks = torch.zeros((len(labels), *labeled_image.shape)) + frames[index] = numpy.array(image) -for index in torch.unique(labeled_image): - masks[index - 1] = torch.where(labeled_image == index, index, 0) + masks = torch.tensor(frames) bounding_boxes = torchvision.ops.boxes.masks_to_boxes(masks) @@ -57,6 +59,8 @@ a = figure.add_subplot(121) b = figure.add_subplot(122) +labeled_image = torch.sum(masks, 0) + a.imshow(labeled_image) b.imshow(labeled_image) From 540c6a1b5b56822bbaf096c6b7e1fba6b4f95a8e Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Fri, 17 Sep 2021 17:42:30 -0400 Subject: [PATCH 23/31] updated import --- gallery/plot_repurposing_annotations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gallery/plot_repurposing_annotations.py b/gallery/plot_repurposing_annotations.py index c634b04b329..b214e8f47b1 100644 --- a/gallery/plot_repurposing_annotations.py +++ b/gallery/plot_repurposing_annotations.py @@ -14,7 +14,7 @@ import matplotlib.pyplot import numpy import torch -import torchvision.ops.boxes +from torchvision.ops.boxes import masks_to_boxes ASSETS_DIRECTORY = "../test/assets" @@ -52,7 +52,7 @@ masks = torch.tensor(frames) -bounding_boxes = torchvision.ops.boxes.masks_to_boxes(masks) +bounding_boxes = masks_to_boxes(masks) figure = matplotlib.pyplot.figure() From 4c78297d1311ac4e548f451a29dadea4da57b6d4 Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Mon, 20 Sep 2021 14:17:43 -0400 Subject: [PATCH 24/31] use torch --- gallery/plot_repurposing_annotations.py | 6 +++--- torchvision/ops/boxes.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gallery/plot_repurposing_annotations.py b/gallery/plot_repurposing_annotations.py index b214e8f47b1..6b1a1c0262c 100644 --- a/gallery/plot_repurposing_annotations.py +++ b/gallery/plot_repurposing_annotations.py @@ -43,14 +43,14 @@ # used in methods like Faster RCNN and YOLO. with PIL.Image.open(os.path.join(ASSETS_DIRECTORY, "masks.tiff")) as image: - frames = numpy.zeros((image.n_frames, image.height, image.width), int) + masks = torch.zeros((image.n_frames, image.height, image.width), dtype=torch.int) for index in range(image.n_frames): image.seek(index) - frames[index] = numpy.array(image) + frame = numpy.array(image) - masks = torch.tensor(frames) + masks[index] = torch.tensor(frame) bounding_boxes = masks_to_boxes(masks) diff --git a/torchvision/ops/boxes.py b/torchvision/ops/boxes.py index fe6535de4a9..4f51aec8f54 100644 --- a/torchvision/ops/boxes.py +++ b/torchvision/ops/boxes.py @@ -318,7 +318,7 @@ def masks_to_boxes(masks: torch.Tensor) -> torch.Tensor: n = masks.shape[0] - bounding_boxes = torch.zeros((n, 4), dtype=torch.int) + bounding_boxes = torch.zeros((n, 4), device=masks.device, dtype=torch.int) for index, mask in enumerate(masks): y, x = torch.where(masks[index] != 0) From 140e42975cbbb7101a552f273fcdf0161c4647ca Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Mon, 20 Sep 2021 14:56:30 -0400 Subject: [PATCH 25/31] Update gallery/plot_repurposing_annotations.py Co-authored-by: Aditya Oke <47158509+oke-aditya@users.noreply.github.com> --- gallery/plot_repurposing_annotations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallery/plot_repurposing_annotations.py b/gallery/plot_repurposing_annotations.py index 6b1a1c0262c..9e1886d9e73 100644 --- a/gallery/plot_repurposing_annotations.py +++ b/gallery/plot_repurposing_annotations.py @@ -38,7 +38,7 @@ # localization tasks. # # Masks to bounding boxes -# ~~~~~~~~~~~~~~~~~~~~~~~ +# ---------------------------------------- # For example, the masks to bounding_boxes operation can be used to transform masks into bounding boxes that can be # used in methods like Faster RCNN and YOLO. From 8f2cd4a9804bc919932717899b7a58869a68abef Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Mon, 20 Sep 2021 14:56:45 -0400 Subject: [PATCH 26/31] Update gallery/plot_repurposing_annotations.py Co-authored-by: Aditya Oke <47158509+oke-aditya@users.noreply.github.com> --- gallery/plot_repurposing_annotations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallery/plot_repurposing_annotations.py b/gallery/plot_repurposing_annotations.py index 9e1886d9e73..0c7f75c7955 100644 --- a/gallery/plot_repurposing_annotations.py +++ b/gallery/plot_repurposing_annotations.py @@ -22,7 +22,7 @@ #################################### # Masks -# -------------------------------------- +# ----- # In tasks like instance and panoptic segmentation, masks are commonly defined, and are defined by this package, # as a multi-dimensional array (e.g. a NumPy array or a PyTorch tensor) with the following shape: # From 7252723421567755cd1626f0e0f1d992c4fc7f93 Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Mon, 20 Sep 2021 14:56:54 -0400 Subject: [PATCH 27/31] Update gallery/plot_repurposing_annotations.py Co-authored-by: Aditya Oke <47158509+oke-aditya@users.noreply.github.com> --- gallery/plot_repurposing_annotations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallery/plot_repurposing_annotations.py b/gallery/plot_repurposing_annotations.py index 0c7f75c7955..92ec4020b10 100644 --- a/gallery/plot_repurposing_annotations.py +++ b/gallery/plot_repurposing_annotations.py @@ -14,7 +14,7 @@ import matplotlib.pyplot import numpy import torch -from torchvision.ops.boxes import masks_to_boxes +from torchvision.ops import masks_to_boxes ASSETS_DIRECTORY = "../test/assets" From 2c2d5dd27bb847c9ff0811988a2667c8479c5910 Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Tue, 21 Sep 2021 13:19:59 -0400 Subject: [PATCH 28/31] Autodoc --- docs/source/ops.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/source/ops.rst b/docs/source/ops.rst index ecef74dd8a6..5fd4b75e59d 100644 --- a/docs/source/ops.rst +++ b/docs/source/ops.rst @@ -9,19 +9,20 @@ torchvision.ops All operators have native support for TorchScript. -.. autofunction:: nms .. autofunction:: batched_nms -.. autofunction:: remove_small_boxes -.. autofunction:: clip_boxes_to_image -.. autofunction:: box_convert .. autofunction:: box_area +.. autofunction:: box_convert .. autofunction:: box_iou +.. autofunction:: clip_boxes_to_image +.. autofunction:: deform_conv2d .. autofunction:: generalized_box_iou -.. autofunction:: roi_align +.. autofunction:: masks_to_boxes +.. autofunction:: nms .. autofunction:: ps_roi_align -.. autofunction:: roi_pool .. autofunction:: ps_roi_pool -.. autofunction:: deform_conv2d +.. autofunction:: remove_small_boxes +.. autofunction:: roi_align +.. autofunction:: roi_pool .. autofunction:: sigmoid_focal_loss .. autofunction:: stochastic_depth From 3a919579749ef8fc4c6d1b2c6acf99919f97073f Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Tue, 21 Sep 2021 13:23:59 -0400 Subject: [PATCH 29/31] use torch instead of numpy in tests --- test/test_masks_to_boxes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_masks_to_boxes.py b/test/test_masks_to_boxes.py index 023d334ed32..7182ebcae9f 100644 --- a/test/test_masks_to_boxes.py +++ b/test/test_masks_to_boxes.py @@ -11,14 +11,14 @@ def test_masks_to_boxes(): with PIL.Image.open(os.path.join(ASSETS_DIRECTORY, "masks.tiff")) as image: - frames = numpy.zeros((image.n_frames, image.height, image.width), int) + masks = torch.zeros((image.n_frames, image.height, image.width), dtype=torch.int) for index in range(image.n_frames): image.seek(index) - frames[index] = numpy.array(image) + frame = numpy.array(image) - masks = torch.tensor(frames) + masks[index] = torch.tensor(frame) expected = torch.tensor( [[127, 2, 165, 40], From e24805c497a5e11904fa9f2380cfee9d823586bd Mon Sep 17 00:00:00 2001 From: Allen Goodman Date: Tue, 21 Sep 2021 13:25:29 -0400 Subject: [PATCH 30/31] fix build_docs failure --- gallery/plot_repurposing_annotations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gallery/plot_repurposing_annotations.py b/gallery/plot_repurposing_annotations.py index 92ec4020b10..2decefcc815 100644 --- a/gallery/plot_repurposing_annotations.py +++ b/gallery/plot_repurposing_annotations.py @@ -3,9 +3,9 @@ Repurposing annotations ======================= -The following example illustrates the operations available in :ref:`the torchvision.ops module ` for repurposing -object localization annotations for different tasks (e.g. transforming masks used by instance and panoptic -segmentation methods into bounding boxes used by object detection methods). +The following example illustrates the operations available in the torchvision.ops module for repurposing object +localization annotations for different tasks (e.g. transforming masks used by instance and panoptic segmentation +methods into bounding boxes used by object detection methods). """ import os.path From 6c89be7cc39752553c42a25adb35fef9d352038f Mon Sep 17 00:00:00 2001 From: Vasilis Vryniotis Date: Tue, 21 Sep 2021 19:04:26 +0100 Subject: [PATCH 31/31] Closing quotes. --- torchvision/ops/boxes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchvision/ops/boxes.py b/torchvision/ops/boxes.py index 4f51aec8f54..6dafcf1c190 100644 --- a/torchvision/ops/boxes.py +++ b/torchvision/ops/boxes.py @@ -304,7 +304,7 @@ def masks_to_boxes(masks: torch.Tensor) -> torch.Tensor: Compute the bounding boxes around the provided masks Returns a [N, 4] tensor. Both sets of boxes are expected to be in ``(x1, y1, x2, y2)`` format with - ``0 <= x1 < x2`` and ``0 <= y1 < y2. + ``0 <= x1 < x2`` and ``0 <= y1 < y2``. Args: masks (Tensor[N, H, W]): masks to transform where N is the number of