Merge pull request #2247 from Trusted-AI/dev_1.15.1

Update to ART 1.15.1

Author: beat-buesser

art/attacks/evasion/auto_conjugate_gradient.py

@@ -463,7 +463,9 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
 
                 # self.eta = np.full((self.batch_size, 1, 1, 1), 2 * self.eps_step).astype(ART_NUMPY_DTYPE)
                 _batch_size = x_k.shape[0]
-                eta = np.full((_batch_size, 1, 1, 1), self.eps_step).astype(ART_NUMPY_DTYPE)
+                eta = np.full((_batch_size,) + (1,) * len(self.estimator.input_shape), self.eps_step).astype(
+                    ART_NUMPY_DTYPE
+                )
                 self.count_condition_1 = np.zeros(shape=(_batch_size,))
                 gradk_1 = np.zeros_like(x_k)
                 cgradk_1 = np.zeros_like(x_k)
@@ -650,4 +652,4 @@ def get_beta(gradk, gradk_1, cgradk_1):
     betak = -(_gradk * delta_gradk).sum(axis=1) / (
         (_cgradk_1 * delta_gradk).sum(axis=1) + np.finfo(ART_NUMPY_DTYPE).eps
     )
-    return betak.reshape((_batch_size, 1, 1, 1))
+    return betak.reshape((_batch_size,) + (1,) * (len(gradk.shape) - 1))

art/attacks/evasion/auto_projected_gradient_descent.py

@@ -458,7 +458,9 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
 
                 # modification for image-wise stepsize update
                 _batch_size = x_k.shape[0]
-                eta = np.full((_batch_size, 1, 1, 1), self.eps_step).astype(ART_NUMPY_DTYPE)
+                eta = np.full((_batch_size,) + (1,) * len(self.estimator.input_shape), self.eps_step).astype(
+                    ART_NUMPY_DTYPE
+                )
                 self.count_condition_1 = np.zeros(shape=(_batch_size,))
 
                 for k_iter in trange(self.max_iter, desc="AutoPGD - iteration", leave=False, disable=not self.verbose):

art/attacks/poisoning/perturbations/image_perturbations.py

@@ -21,7 +21,6 @@
 from typing import Optional, Tuple
 
 import numpy as np
-from PIL import Image
 
 
 def add_single_bd(x: np.ndarray, distance: int = 2, pixel_value: int = 1) -> np.ndarray:
@@ -112,6 +111,8 @@ def insert_image(
     :param blend: The blending factor
     :return: Backdoored image.
     """
+    from PIL import Image
+
     n_dim = len(x.shape)
     if n_dim == 4:
         return np.array(

art/defences/preprocessor/spatial_smoothing.py

@@ -30,7 +30,7 @@
 from typing import Optional, Tuple
 
 import numpy as np
-from scipy.ndimage.filters import median_filter
+from scipy.ndimage import median_filter
 
 from art.utils import CLIP_VALUES_TYPE
 from art.defences.preprocessor.preprocessor import Preprocessor

art/defences/trainer/adversarial_trainer_trades_pytorch.py

@@ -33,6 +33,7 @@
 from art.estimators.classification.pytorch import PyTorchClassifier
 from art.data_generators import DataGenerator
 from art.attacks.attack import EvasionAttack
+from art.utils import check_and_transform_label_format
 
 if TYPE_CHECKING:
     import torch
@@ -97,6 +98,15 @@ def fit(
         ind = np.arange(len(x))
 
         logger.info("Adversarial Training TRADES")
+        y = check_and_transform_label_format(y, nb_classes=self.classifier.nb_classes)
+
+        if validation_data is not None:
+            (x_test, y_test) = validation_data
+            y_test = check_and_transform_label_format(y_test, nb_classes=self.classifier.nb_classes)
+
+            x_preprocessed_test, y_preprocessed_test = self._classifier._apply_preprocessing(  # pylint: disable=W0212
+                x_test, y_test, fit=True
+            )
 
         for i_epoch in trange(nb_epochs, desc="Adversarial Training TRADES - Epochs"):
             # Shuffle the examples
@@ -107,7 +117,6 @@ def fit(
             train_n = 0.0
 
             for batch_id in range(nb_batches):
-
                 # Create batch data
                 x_batch = x[ind[batch_id * batch_size : min((batch_id + 1) * batch_size, x.shape[0])]].copy()
                 y_batch = y[ind[batch_id * batch_size : min((batch_id + 1) * batch_size, x.shape[0])]]
@@ -125,9 +134,9 @@ def fit(
 
             # compute accuracy
             if validation_data is not None:
-                (x_test, y_test) = validation_data
-                output = np.argmax(self.predict(x_test), axis=1)
-                nb_correct_pred = np.sum(output == np.argmax(y_test, axis=1))
+                output = np.argmax(self.predict(x_preprocessed_test), axis=1)
+                nb_correct_pred = np.sum(output == np.argmax(y_preprocessed_test, axis=1))
+
                 logger.info(
                     "epoch: %s time(s): %.1f loss: %.4f acc(tr): %.4f acc(val): %.4f",
                     i_epoch,
@@ -188,7 +197,6 @@ def fit_generator(
             train_n = 0.0
 
             for batch_id in range(nb_batches):  # pylint: disable=W0612
-
                 # Create batch data
                 x_batch, y_batch = generator.get_batch()
                 x_batch = x_batch.copy()
@@ -232,6 +240,8 @@ def _batch_process(self, x_batch: np.ndarray, y_batch: np.ndarray) -> Tuple[floa
         x_batch_pert = self._attack.generate(x_batch, y=y_batch)
 
         # Apply preprocessing
+        y_batch = check_and_transform_label_format(y_batch, nb_classes=self.classifier.nb_classes)
+
         x_preprocessed, y_preprocessed = self._classifier._apply_preprocessing(  # pylint: disable=W0212
             x_batch, y_batch, fit=True
         )

art/estimators/certification/__init__.py

@@ -2,12 +2,26 @@
 This module contains certified classifiers.
 """
 import importlib
-from art.estimators.certification import randomized_smoothing
-from art.estimators.certification import derandomized_smoothing
+from art.estimators.certification.randomized_smoothing.randomized_smoothing import RandomizedSmoothingMixin
+from art.estimators.certification.randomized_smoothing.numpy import NumpyRandomizedSmoothing
+from art.estimators.certification.randomized_smoothing.tensorflow import TensorFlowV2RandomizedSmoothing
+from art.estimators.certification.randomized_smoothing.pytorch import PyTorchRandomizedSmoothing
+from art.estimators.certification.derandomized_smoothing.derandomized_smoothing import DeRandomizedSmoothingMixin
+from art.estimators.certification.derandomized_smoothing.pytorch import PyTorchDeRandomizedSmoothing
+from art.estimators.certification.derandomized_smoothing.tensorflow import TensorFlowV2DeRandomizedSmoothing
 
 if importlib.util.find_spec("torch") is not None:
-    from art.estimators.certification import deep_z
-    from art.estimators.certification import interval
+    from art.estimators.certification.deep_z.deep_z import ZonoDenseLayer
+    from art.estimators.certification.deep_z.deep_z import ZonoBounds
+    from art.estimators.certification.deep_z.deep_z import ZonoConv
+    from art.estimators.certification.deep_z.deep_z import ZonoReLU
+    from art.estimators.certification.deep_z.pytorch import PytorchDeepZ
+    from art.estimators.certification.interval.interval import PyTorchIntervalDense
+    from art.estimators.certification.interval.interval import PyTorchIntervalConv2D
+    from art.estimators.certification.interval.interval import PyTorchIntervalReLU
+    from art.estimators.certification.interval.interval import PyTorchIntervalFlatten
+    from art.estimators.certification.interval.interval import PyTorchIntervalBounds
+    from art.estimators.certification.interval.pytorch import PyTorchIBPClassifier
 else:
     import warnings
 

art/estimators/object_detection/pytorch_object_detector.py

@@ -219,7 +219,10 @@ def _preprocess_and_convert_inputs(
 
             # Set gradients
             if not no_grad:
-                x_tensor.requires_grad = True
+                if x_tensor.is_leaf:
+                    x_tensor.requires_grad = True
+                else:
+                    x_tensor.retain_grad()
 
             # Apply framework-specific preprocessing
             x_preprocessed, y_preprocessed = self._apply_preprocessing(x=x_tensor, y=y_tensor, fit=fit, no_grad=no_grad)
@@ -267,6 +270,12 @@ def _get_losses(
         x_preprocessed = x_preprocessed.to(self.device)
         y_preprocessed = [{k: v.to(self.device) for k, v in y_i.items()} for y_i in y_preprocessed]
 
+        # Set gradients again after inputs are moved to another device
+        if x_preprocessed.is_leaf:
+            x_preprocessed.requires_grad = True
+        else:
+            x_preprocessed.retain_grad()
+
         loss_components = self._model(x_preprocessed, y_preprocessed)
 
         return loss_components, x_preprocessed

art/estimators/object_detection/pytorch_yolo.py

@@ -358,6 +358,12 @@ def _get_losses(
         x_preprocessed = x_preprocessed.to(self.device)
         y_preprocessed_yolo = y_preprocessed_yolo.to(self.device)
 
+        # Set gradients again after inputs are moved to another device
+        if x_preprocessed.is_leaf:
+            x_preprocessed.requires_grad = True
+        else:
+            x_preprocessed.retain_grad()
+
         # Calculate loss components
         loss_components = self._model(x_preprocessed, y_preprocessed_yolo)
 

art/visualization.py

@@ -25,7 +25,6 @@
 from typing import List, Optional, TYPE_CHECKING
 
 import numpy as np
-from PIL import Image
 
 from art import config
 
@@ -97,6 +96,8 @@ def save_image(image_array: np.ndarray, f_name: str) -> None:
     :param image_array: Image to be saved.
     :param f_name: File name containing extension e.g., my_img.jpg, my_img.png, my_images/my_img.png.
     """
+    from PIL import Image
+
     file_name = os.path.join(config.ART_DATA_PATH, f_name)
     folder = os.path.split(file_name)[0]
     if not os.path.exists(folder):

tests/defences/trainer/test_adversarial_trainer_trades_pytorch.py

@@ -34,7 +34,7 @@ def _get_adv_trainer():
         if framework in ["tensorflow", "tensorflow2v1"]:
             trainer = None
         if framework == "pytorch":
-            classifier, _ = image_dl_estimator()
+            classifier, _ = image_dl_estimator(from_logits=True)
             attack = ProjectedGradientDescent(
                 classifier,
                 norm=np.inf,
@@ -63,22 +63,38 @@ def fix_get_mnist_subset(get_mnist_dataset):
     yield x_train_mnist[:n_train], y_train_mnist[:n_train], x_test_mnist[:n_test], y_test_mnist[:n_test]
 
 
-@pytest.mark.skip_framework("tensorflow", "keras", "scikitlearn", "mxnet", "kerastf")
-def test_adversarial_trainer_trades_pytorch_fit_and_predict(get_adv_trainer, fix_get_mnist_subset):
+@pytest.mark.only_with_platform("pytorch")
+@pytest.mark.parametrize("label_format", ["one_hot", "numerical"])
+def test_adversarial_trainer_trades_pytorch_fit_and_predict(get_adv_trainer, fix_get_mnist_subset, label_format):
     (x_train_mnist, y_train_mnist, x_test_mnist, y_test_mnist) = fix_get_mnist_subset
     x_test_mnist_original = x_test_mnist.copy()
 
+    if label_format == "one_hot":
+        assert y_train_mnist.shape[-1] == 10
+        assert y_test_mnist.shape[-1] == 10
+    if label_format == "numerical":
+        y_test_mnist = np.argmax(y_test_mnist, axis=1)
+        y_train_mnist = np.argmax(y_train_mnist, axis=1)
+
     trainer = get_adv_trainer()
     if trainer is None:
         logging.warning("Couldn't perform  this test because no trainer is defined for this framework configuration")
         return
 
     predictions = np.argmax(trainer.predict(x_test_mnist), axis=1)
-    accuracy = np.sum(predictions == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0]
+
+    if label_format == "one_hot":
+        accuracy = np.sum(predictions == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0]
+    else:
+        accuracy = np.sum(predictions == y_test_mnist) / x_test_mnist.shape[0]
 
     trainer.fit(x_train_mnist, y_train_mnist, nb_epochs=20)
     predictions_new = np.argmax(trainer.predict(x_test_mnist), axis=1)
-    accuracy_new = np.sum(predictions_new == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0]
+
+    if label_format == "one_hot":
+        accuracy_new = np.sum(predictions_new == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0]
+    else:
+        accuracy_new = np.sum(predictions_new == y_test_mnist) / x_test_mnist.shape[0]
 
     np.testing.assert_array_almost_equal(
         float(np.mean(x_test_mnist_original - x_test_mnist)),
@@ -92,13 +108,20 @@ def test_adversarial_trainer_trades_pytorch_fit_and_predict(get_adv_trainer, fix
     trainer.fit(x_train_mnist, y_train_mnist, nb_epochs=20, validation_data=(x_train_mnist, y_train_mnist))
 
 
-@pytest.mark.skip_framework("tensorflow", "keras", "scikitlearn", "mxnet", "kerastf")
+@pytest.mark.only_with_platform("pytorch")
+@pytest.mark.parametrize("label_format", ["one_hot", "numerical"])
 def test_adversarial_trainer_trades_pytorch_fit_generator_and_predict(
-    get_adv_trainer, fix_get_mnist_subset, image_data_generator
+    get_adv_trainer, fix_get_mnist_subset, image_data_generator, label_format
 ):
     (x_train_mnist, y_train_mnist, x_test_mnist, y_test_mnist) = fix_get_mnist_subset
     x_test_mnist_original = x_test_mnist.copy()
 
+    if label_format == "one_hot":
+        assert y_train_mnist.shape[-1] == 10
+        assert y_test_mnist.shape[-1] == 10
+    if label_format == "numerical":
+        y_test_mnist = np.argmax(y_test_mnist, axis=1)
+
     generator = image_data_generator()
 
     trainer = get_adv_trainer()
@@ -107,11 +130,18 @@ def test_adversarial_trainer_trades_pytorch_fit_generator_and_predict(
         return
 
     predictions = np.argmax(trainer.predict(x_test_mnist), axis=1)
-    accuracy = np.sum(predictions == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0]
+    if label_format == "one_hot":
+        accuracy = np.sum(predictions == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0]
+    else:
+        accuracy = np.sum(predictions == y_test_mnist) / x_test_mnist.shape[0]
 
     trainer.fit_generator(generator=generator, nb_epochs=20)
     predictions_new = np.argmax(trainer.predict(x_test_mnist), axis=1)
-    accuracy_new = np.sum(predictions_new == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0]
+
+    if label_format == "one_hot":
+        accuracy_new = np.sum(predictions_new == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0]
+    else:
+        accuracy_new = np.sum(predictions_new == y_test_mnist) / x_test_mnist.shape[0]
 
     np.testing.assert_array_almost_equal(
         float(np.mean(x_test_mnist_original - x_test_mnist)),