Implement smoothed Vision Transformer as defense against patch evasion attacks

.github/workflows/ci-lingvo.yml

@@ -50,7 +50,7 @@ jobs:
           sudo apt-get update
           sudo apt-get -y -q install ffmpeg libavcodec-extra
           python -m pip install --upgrade pip setuptools wheel
-          pip install -q -r <(sed '/^scipy/d;/^matplotlib/d;/^pandas/d;/^statsmodels/d;/^numba/d;/^jax/d;/^h5py/d;/^Pillow/d;/^pytest/d;/^pytest-mock/d;/^torch/d;/^torchaudio/d;/^torchvision/d;/^xgboost/d;/^requests/d;/^tensorflow/d;/^keras/d;/^kornia/d;/^librosa/d;/^tqdm/d' requirements_test.txt)
+          pip install -q -r <(sed '/^scipy/d;/^matplotlib/d;/^pandas/d;/^statsmodels/d;/^numba/d;/^jax/d;/^h5py/d;/^Pillow/d;/^pytest/d;/^pytest-mock/d;/^torch/d;/^torchaudio/d;/^torchvision/d;/^xgboost/d;/^requests/d;/^tensorflow/d;/^keras/d;/^kornia/d;/^librosa/d;/^tqdm/d;/^timm/d' requirements_test.txt)
           pip install scipy==1.5.4
           pip install matplotlib==3.3.4
           pip install pandas==1.1.5

art/estimators/certification/__init__.py

@@ -6,7 +6,6 @@
 from art.estimators.certification.randomized_smoothing.numpy import NumpyRandomizedSmoothing
 from art.estimators.certification.randomized_smoothing.tensorflow import TensorFlowV2RandomizedSmoothing
 from art.estimators.certification.randomized_smoothing.pytorch import PyTorchRandomizedSmoothing
-from art.estimators.certification.derandomized_smoothing.derandomized_smoothing import DeRandomizedSmoothingMixin
 from art.estimators.certification.derandomized_smoothing.pytorch import PyTorchDeRandomizedSmoothing
 from art.estimators.certification.derandomized_smoothing.tensorflow import TensorFlowV2DeRandomizedSmoothing
 

art/estimators/certification/derandomized_smoothing/__init__.py

@@ -1,6 +1,5 @@
 """
 DeRandomized smoothing estimators.
 """
-from art.estimators.certification.derandomized_smoothing.derandomized_smoothing import DeRandomizedSmoothingMixin
 from art.estimators.certification.derandomized_smoothing.pytorch import PyTorchDeRandomizedSmoothing
 from art.estimators.certification.derandomized_smoothing.tensorflow import TensorFlowV2DeRandomizedSmoothing

art/estimators/certification/derandomized_smoothing/ablators/__init__.py

@@ -0,0 +1,12 @@
+"""
+This module contains the ablators for the certified smoothing approaches.
+"""
+import importlib
+
+from art.estimators.certification.derandomized_smoothing.ablators.tensorflow import ColumnAblator, BlockAblator
+
+if importlib.util.find_spec("torch") is not None:
+    from art.estimators.certification.derandomized_smoothing.ablators.pytorch import (
+        ColumnAblatorPyTorch,
+        BlockAblatorPyTorch,
+    )

art/estimators/certification/derandomized_smoothing/ablators/ablate.py

@@ -0,0 +1,90 @@
+# MIT License
+#
+# Copyright (C) The Adversarial Robustness Toolbox (ART) Authors 2022
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+"""
+This module implements the abstract base class for the ablators.
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from abc import ABC, abstractmethod
+from typing import Optional, Tuple, Union, TYPE_CHECKING
+
+import numpy as np
+
+if TYPE_CHECKING:
+    # pylint: disable=C0412
+    import tensorflow as tf
+    import torch
+
+
+class BaseAblator(ABC):
+    """
+    Base class defining the methods used for the ablators.
+    """
+
+    @abstractmethod
+    def __call__(
+        self, x: np.ndarray, column_pos: Optional[Union[int, list]] = None, row_pos: Optional[Union[int, list]] = None
+    ) -> np.ndarray:
+        """
+        Ablate the image x at location specified by "column_pos" for the case of column ablation or at the location
+        specified by "column_pos" and "row_pos" in the case of block ablation.
+
+        :param x: input image.
+        :param column_pos: column position to specify where to retain the image
+        :param row_pos: row position to specify where to retain the image. Not used for ablation type "column".
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def certify(
+        self, pred_counts: np.ndarray, size_to_certify: int, label: Union[np.ndarray, "tf.Tensor"]
+    ) -> Union[Tuple["tf.Tensor", "tf.Tensor", "tf.Tensor"], Tuple["torch.Tensor", "torch.Tensor", "torch.Tensor"]]:
+        """
+        Checks if based on the predictions supplied the classifications over the ablated datapoints result in a
+        certified prediction against a patch attack of size size_to_certify.
+
+        :param pred_counts: The cumulative predictions of the classifier over the ablation locations.
+        :param size_to_certify: The size of the patch to check against.
+        :param label: ground truth labels
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def ablate(self, x: np.ndarray, column_pos: int, row_pos: int) -> Union[np.ndarray, "torch.Tensor"]:
+        """
+        Ablate the image x at location specified by "column_pos" for the case of column ablation or at the location
+        specified by "column_pos" and "row_pos" in the case of block ablation.
+
+        :param x: input image.
+        :param column_pos: column position to specify where to retain the image
+        :param row_pos: row position to specify where to retain the image. Not used for ablation type "column".
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def forward(
+        self, x: np.ndarray, column_pos: Optional[int] = None, row_pos: Optional[int] = None
+    ) -> Union[np.ndarray, "torch.Tensor"]:
+        """
+        Ablate batch of data at locations specified by column_pos and row_pos
+
+        :param x: input image.
+        :param column_pos: column position to specify where to retain the image
+        :param row_pos: row position to specify where to retain the image. Not used for ablation type "column".
+        """
+        raise NotImplementedError