From 6fb35a5dc6294cbc99f975e588daf164a3c3b397 Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Thu, 28 Sep 2023 15:36:27 -0700 Subject: [PATCH] Move Perturber to Composer.Perturber (#231) * Composer-Functions. * Get both key and order arguments. * Implement mask-additive in config. * Comment. * Fix test. * Fix tests. * Move adversary.perturber to adversary.composer.perturber * Add return value. * Fix adversarial example visualizer. * Fix config. * Fix merge. * Update tests. * Fix a visualizer test. --- mart/attack/adversary.py | 22 ++---- mart/attack/composer.py | 27 +++++-- mart/configs/attack/adversary.yaml | 1 - mart/configs/attack/composer/default.yaml | 3 + .../{ => composer}/perturber/default.yaml | 0 .../perturber/initializer/constant.yaml | 0 .../perturber/initializer/image.yaml | 0 .../perturber/initializer/uniform.yaml | 0 .../perturber/initializer/uniform_lp.yaml | 0 .../projector/linf_additive_range.yaml | 0 .../projector/lp_additive_range.yaml | 0 .../perturber/projector/mask_range.yaml | 0 .../perturber/projector/range.yaml | 0 mart/configs/attack/fgm.yaml | 13 ++-- mart/configs/attack/linf.yaml | 3 +- mart/configs/attack/mask.yaml | 3 +- .../object_detection_mask_adversary.yaml | 9 ++- ...bject_detection_mask_adversary_missed.yaml | 9 ++- mart/configs/attack/pgd.yaml | 15 ++-- tests/test_adversary.py | 75 +++++++------------ tests/test_composer.py | 21 ++++-- 21 files changed, 101 insertions(+), 100 deletions(-) rename mart/configs/attack/{ => composer}/perturber/default.yaml (100%) rename mart/configs/attack/{ => composer}/perturber/initializer/constant.yaml (100%) rename mart/configs/attack/{ => composer}/perturber/initializer/image.yaml (100%) rename mart/configs/attack/{ => composer}/perturber/initializer/uniform.yaml (100%) rename mart/configs/attack/{ => composer}/perturber/initializer/uniform_lp.yaml (100%) rename mart/configs/attack/{ => composer}/perturber/projector/linf_additive_range.yaml (100%) rename mart/configs/attack/{ => composer}/perturber/projector/lp_additive_range.yaml (100%) rename mart/configs/attack/{ => composer}/perturber/projector/mask_range.yaml (100%) rename mart/configs/attack/{ => composer}/perturber/projector/range.yaml (100%) diff --git a/mart/attack/adversary.py b/mart/attack/adversary.py index 7f4cbefb..62538258 100644 --- a/mart/attack/adversary.py +++ b/mart/attack/adversary.py @@ -16,7 +16,6 @@ from mart.utils import silent from ..optim import OptimizerFactory -from ..utils import MonkeyPatch if TYPE_CHECKING: from .composer import Composer @@ -24,7 +23,6 @@ from .gain import Gain from .gradient_modifier import GradientModifier from .objective import Objective - from .perturber import Perturber __all__ = ["Adversary"] @@ -35,7 +33,6 @@ class Adversary(pl.LightningModule): def __init__( self, *, - perturber: Perturber, composer: Composer, optimizer: OptimizerFactory | Callable[[Any], torch.optim.Optimizer], gain: Gain, @@ -48,7 +45,6 @@ def __init__( """_summary_ Args: - perturber (Perturber): A MART Perturber. composer (Composer): A MART Composer. optimizer (OptimizerFactory | Callable[[Any], torch.optim.Optimizer]): A MART OptimizerFactory or partial that returns an Optimizer when given params. gain (Gain): An adversarial gain function, which is a differentiable estimate of adversarial objective. @@ -65,10 +61,9 @@ def __init__( lambda state_dict, *args, **kwargs: state_dict.clear() ) - # Hide the perturber module in a list, so that perturbation is not exported as a parameter in the model checkpoint. + # Hide the composer module in a list, so that perturbation is not exported as a parameter in the model checkpoint. # and DDP won't try to get the uninitialized parameters of perturbation. - self._perturber = [perturber] - self.composer = composer + self._composer = [composer] self.optimizer = optimizer if not isinstance(self.optimizer, OptimizerFactory): self.optimizer = OptimizerFactory(self.optimizer) @@ -103,13 +98,13 @@ def __init__( assert self._attacker.limit_train_batches > 0 @property - def perturber(self) -> Perturber: - # Hide the perturber module in a list, so that perturbation is not exported as a parameter in the model checkpoint, + def composer(self) -> Composer: + # Hide the composer module in a list, so that perturbation is not exported as a parameter in the model checkpoint, # and DDP won't try to get the uninitialized parameters of perturbation. - return self._perturber[0] + return self._composer[0] def configure_optimizers(self): - return self.optimizer(self.perturber) + return self.optimizer(self.composer) def training_step(self, batch_and_model, batch_idx): input, target, model = batch_and_model @@ -157,7 +152,7 @@ def fit(self, input, target, *, model: Callable): batch_and_model = (input, target, model) # Configure and reset perturbation for current inputs - self.perturber.configure_perturbation(input) + self.composer.configure_perturbation(input) # Attack, aka fit a perturbation, for one epoch by cycling over the same input batch. # We use Trainer.limit_train_batches to control the number of attack iterations. @@ -166,8 +161,7 @@ def fit(self, input, target, *, model: Callable): def forward(self, input, target): """Compose adversarial examples and enforce the threat model.""" - perturbation = self.perturber(input=input, target=target) - input_adv = self.composer(perturbation, input=input, target=target) + input_adv = self.composer(input=input, target=target) if self.enforcer is not None: self.enforcer(input_adv, input=input, target=target) diff --git a/mart/attack/composer.py b/mart/attack/composer.py index 83ecd0c7..cd6300f7 100644 --- a/mart/attack/composer.py +++ b/mart/attack/composer.py @@ -8,10 +8,13 @@ import abc from collections import OrderedDict -from typing import Any, Iterable +from typing import TYPE_CHECKING, Any, Iterable import torch +if TYPE_CHECKING: + from .perturber import Perturber + class Function(torch.nn.Module): def __init__(self, *args, order=0, **kwargs) -> None: @@ -32,22 +35,36 @@ def forward( pass -class Composer: - def __init__(self, functions: dict[str, Function]) -> None: +class Composer(torch.nn.Module): + def __init__(self, perturber: Perturber, functions: dict[str, Function]) -> None: + """_summary_ + + Args: + perturber (Perturber): Manage perturbations. + functions (dict[str, Function]): A dictionary of functions for composing pertured input. + """ + super().__init__() + + self.perturber = perturber + # Sort functions by function.order and the name. self.functions_dict = OrderedDict( sorted(functions.items(), key=lambda name_fn: (name_fn[1].order, name_fn[0])) ) self.functions = list(self.functions_dict.values()) - def __call__( + def configure_perturbation(self, input: torch.Tensor | Iterable[torch.Tensor]): + return self.perturber.configure_perturbation(input) + + def forward( self, - perturbation: torch.Tensor | Iterable[torch.Tensor], *, input: torch.Tensor | Iterable[torch.Tensor], target: torch.Tensor | Iterable[torch.Tensor] | Iterable[dict[str, Any]], **kwargs, ) -> torch.Tensor | Iterable[torch.Tensor]: + perturbation = self.perturber(input=input, target=target) + if isinstance(perturbation, torch.Tensor) and isinstance(input, torch.Tensor): return self._compose(perturbation, input=input, target=target) diff --git a/mart/configs/attack/adversary.yaml b/mart/configs/attack/adversary.yaml index 5f65f99d..1cdb5f2c 100644 --- a/mart/configs/attack/adversary.yaml +++ b/mart/configs/attack/adversary.yaml @@ -4,7 +4,6 @@ defaults: _target_: mart.attack.Adversary _convert_: all -perturber: ??? optimizer: maximize: True gain: ??? diff --git a/mart/configs/attack/composer/default.yaml b/mart/configs/attack/composer/default.yaml index 91a25390..02bb8374 100644 --- a/mart/configs/attack/composer/default.yaml +++ b/mart/configs/attack/composer/default.yaml @@ -1,2 +1,5 @@ +defaults: + - perturber: default + _target_: mart.attack.Composer functions: ??? diff --git a/mart/configs/attack/perturber/default.yaml b/mart/configs/attack/composer/perturber/default.yaml similarity index 100% rename from mart/configs/attack/perturber/default.yaml rename to mart/configs/attack/composer/perturber/default.yaml diff --git a/mart/configs/attack/perturber/initializer/constant.yaml b/mart/configs/attack/composer/perturber/initializer/constant.yaml similarity index 100% rename from mart/configs/attack/perturber/initializer/constant.yaml rename to mart/configs/attack/composer/perturber/initializer/constant.yaml diff --git a/mart/configs/attack/perturber/initializer/image.yaml b/mart/configs/attack/composer/perturber/initializer/image.yaml similarity index 100% rename from mart/configs/attack/perturber/initializer/image.yaml rename to mart/configs/attack/composer/perturber/initializer/image.yaml diff --git a/mart/configs/attack/perturber/initializer/uniform.yaml b/mart/configs/attack/composer/perturber/initializer/uniform.yaml similarity index 100% rename from mart/configs/attack/perturber/initializer/uniform.yaml rename to mart/configs/attack/composer/perturber/initializer/uniform.yaml diff --git a/mart/configs/attack/perturber/initializer/uniform_lp.yaml b/mart/configs/attack/composer/perturber/initializer/uniform_lp.yaml similarity index 100% rename from mart/configs/attack/perturber/initializer/uniform_lp.yaml rename to mart/configs/attack/composer/perturber/initializer/uniform_lp.yaml diff --git a/mart/configs/attack/perturber/projector/linf_additive_range.yaml b/mart/configs/attack/composer/perturber/projector/linf_additive_range.yaml similarity index 100% rename from mart/configs/attack/perturber/projector/linf_additive_range.yaml rename to mart/configs/attack/composer/perturber/projector/linf_additive_range.yaml diff --git a/mart/configs/attack/perturber/projector/lp_additive_range.yaml b/mart/configs/attack/composer/perturber/projector/lp_additive_range.yaml similarity index 100% rename from mart/configs/attack/perturber/projector/lp_additive_range.yaml rename to mart/configs/attack/composer/perturber/projector/lp_additive_range.yaml diff --git a/mart/configs/attack/perturber/projector/mask_range.yaml b/mart/configs/attack/composer/perturber/projector/mask_range.yaml similarity index 100% rename from mart/configs/attack/perturber/projector/mask_range.yaml rename to mart/configs/attack/composer/perturber/projector/mask_range.yaml diff --git a/mart/configs/attack/perturber/projector/range.yaml b/mart/configs/attack/composer/perturber/projector/range.yaml similarity index 100% rename from mart/configs/attack/perturber/projector/range.yaml rename to mart/configs/attack/composer/perturber/projector/range.yaml diff --git a/mart/configs/attack/fgm.yaml b/mart/configs/attack/fgm.yaml index 637761db..73a8bbc5 100644 --- a/mart/configs/attack/fgm.yaml +++ b/mart/configs/attack/fgm.yaml @@ -1,5 +1,5 @@ defaults: - - perturber/initializer: constant + - composer/perturber/initializer: constant - /optimizer@optimizer: sgd max_iters: 1 @@ -8,11 +8,12 @@ eps: ??? optimizer: lr: ${..eps} -perturber: - initializer: - constant: 0 - projector: - eps: ${...eps} +composer: + perturber: + initializer: + constant: 0 + projector: + eps: ${....eps} # We can turn off progress bar for one-step attack. callbacks: diff --git a/mart/configs/attack/linf.yaml b/mart/configs/attack/linf.yaml index 64cd5e55..71c50c9e 100644 --- a/mart/configs/attack/linf.yaml +++ b/mart/configs/attack/linf.yaml @@ -1,6 +1,5 @@ defaults: - - perturber: default - - perturber/projector: linf_additive_range + - composer/perturber/projector: linf_additive_range - enforcer: default - enforcer/constraints: lp diff --git a/mart/configs/attack/mask.yaml b/mart/configs/attack/mask.yaml index 22afe959..08206522 100644 --- a/mart/configs/attack/mask.yaml +++ b/mart/configs/attack/mask.yaml @@ -1,5 +1,4 @@ defaults: - - perturber: default - - perturber/projector: mask_range + - composer/perturber/projector: mask_range - enforcer: default - enforcer/constraints: [mask, pixel_range] diff --git a/mart/configs/attack/object_detection_mask_adversary.yaml b/mart/configs/attack/object_detection_mask_adversary.yaml index bc88ba3d..b2b6b394 100644 --- a/mart/configs/attack/object_detection_mask_adversary.yaml +++ b/mart/configs/attack/object_detection_mask_adversary.yaml @@ -2,7 +2,7 @@ defaults: - adversary - gradient_ascent - mask - - perturber/initializer: constant + - composer/perturber/initializer: constant - composer/functions: overlay - gradient_modifier: sign - gain: rcnn_training_loss @@ -12,6 +12,7 @@ max_iters: ??? lr: ??? # Start with grey perturbation in the overlay mode. -perturber: - initializer: - constant: 127 +composer: + perturber: + initializer: + constant: 127 diff --git a/mart/configs/attack/object_detection_mask_adversary_missed.yaml b/mart/configs/attack/object_detection_mask_adversary_missed.yaml index 6be9ec8b..f8b5298f 100644 --- a/mart/configs/attack/object_detection_mask_adversary_missed.yaml +++ b/mart/configs/attack/object_detection_mask_adversary_missed.yaml @@ -2,7 +2,7 @@ defaults: - adversary - gradient_ascent - mask - - perturber/initializer: constant + - composer/perturber/initializer: constant - composer/functions: overlay - gradient_modifier: sign - gain: rcnn_class_background @@ -12,6 +12,7 @@ max_iters: ??? lr: ??? # Start with grey perturbation in the overlay mode. -perturber: - initializer: - constant: 127 +composer: + perturber: + initializer: + constant: 127 diff --git a/mart/configs/attack/pgd.yaml b/mart/configs/attack/pgd.yaml index 1ad2ac81..9af4985a 100644 --- a/mart/configs/attack/pgd.yaml +++ b/mart/configs/attack/pgd.yaml @@ -1,5 +1,5 @@ defaults: - - perturber/initializer: uniform + - composer/perturber/initializer: uniform - /optimizer@optimizer: sgd max_iters: ??? @@ -9,9 +9,10 @@ lr: ??? optimizer: lr: ${..lr} -perturber: - initializer: - min: ${negate:${...eps}} - max: ${...eps} - projector: - eps: ${...eps} +composer: + perturber: + initializer: + min: ${negate:${....eps}} + max: ${....eps} + projector: + eps: ${....eps} diff --git a/tests/test_adversary.py b/tests/test_adversary.py index b68a1052..15dc6e28 100644 --- a/tests/test_adversary.py +++ b/tests/test_adversary.py @@ -19,16 +19,14 @@ def test_with_model(input_data, target_data, perturbation): perturber = Mock(spec=Perturber, return_value=perturbation) - composer = mart.attack.composer.Composer( - functions={"additive": mart.attack.composer.Additive()} - ) + functions = {"additive": mart.attack.composer.Additive()} + composer = Composer(perturber=perturber, functions=functions) gain = Mock() enforcer = Mock() attacker = Mock(max_epochs=0, limit_train_batches=1, fit_loop=Mock(max_epochs=0)) model = Mock() adversary = Adversary( - perturber=perturber, composer=composer, optimizer=None, gain=gain, @@ -54,19 +52,16 @@ def test_with_model(input_data, target_data, perturbation): def test_hidden_params(): initializer = Mock() - composer = mart.attack.composer.Composer( - functions={"additive": mart.attack.composer.Additive()} - ) projector = Mock() - perturber = Perturber(initializer=initializer, projector=projector) + functions = {"additive": mart.attack.composer.Additive()} + composer = Composer(perturber=perturber, functions=functions) gain = Mock() enforcer = Mock() attacker = Mock(max_epochs=0, limit_train_batches=1, fit_loop=Mock(max_epochs=0)) adversary = Adversary( - perturber=perturber, composer=composer, optimizer=None, gain=gain, @@ -85,12 +80,10 @@ def test_hidden_params(): def test_hidden_params_after_forward(input_data, target_data, perturbation): initializer = Mock() - composer = mart.attack.composer.Composer( - functions={"additive": mart.attack.composer.Additive()} - ) projector = Mock() - perturber = Perturber(initializer=initializer, projector=projector) + functions = {"additive": mart.attack.composer.Additive()} + composer = Composer(perturber=perturber, functions=functions) gain = Mock() enforcer = Mock() @@ -98,7 +91,6 @@ def test_hidden_params_after_forward(input_data, target_data, perturbation): model = Mock() adversary = Adversary( - perturber=perturber, composer=composer, optimizer=None, gain=gain, @@ -121,19 +113,16 @@ def test_hidden_params_after_forward(input_data, target_data, perturbation): def test_loading_perturbation_from_state_dict(): initializer = Mock() - composer = mart.attack.composer.Composer( - functions={"additive": mart.attack.composer.Additive()} - ) projector = Mock() - perturber = Perturber(initializer=initializer, projector=projector) + functions = {"additive": mart.attack.composer.Additive()} + composer = Composer(perturber=perturber, functions=functions) gain = Mock() enforcer = Mock() attacker = Mock(max_epochs=0, limit_train_batches=1, fit_loop=Mock(max_epochs=0)) adversary = Adversary( - perturber=perturber, composer=composer, optimizer=None, gain=gain, @@ -147,21 +136,19 @@ def test_loading_perturbation_from_state_dict(): adversary.load_state_dict({"perturber.perturbation": torch.tensor([1.0, 2.0])}) # Adversary ignores load_state_dict() quietly, so perturbation is still None. - assert adversary.perturber.perturbation is None + assert adversary.composer.perturber.perturbation is None def test_perturbation(input_data, target_data, perturbation): perturber = Mock(spec=Perturber, return_value=perturbation) - composer = mart.attack.composer.Composer( - functions={"additive": mart.attack.composer.Additive()} - ) + functions = {"additive": mart.attack.composer.Additive()} + composer = Composer(perturber=perturber, functions=functions) gain = Mock() enforcer = Mock() attacker = Mock(max_epochs=0, limit_train_batches=1, fit_loop=Mock(max_epochs=0)) model = Mock() adversary = Adversary( - perturber=perturber, composer=composer, optimizer=None, gain=gain, @@ -185,9 +172,6 @@ def test_perturbation(input_data, target_data, perturbation): def test_forward_with_model(input_data, target_data): - composer = mart.attack.composer.Composer( - functions={"additive": mart.attack.composer.Additive()} - ) enforcer = Mock() optimizer = partial(SGD, lr=1.0, maximize=True) @@ -207,9 +191,10 @@ def initializer(x): initializer=initializer, projector=None, ) + functions = {"additive": mart.attack.composer.Additive()} + composer = Composer(perturber=perturber, functions=functions) adversary = Adversary( - perturber=perturber, composer=composer, optimizer=optimizer, gain=gain, @@ -231,14 +216,12 @@ def model(input, target): def test_configure_optimizers(): perturber = Mock() - composer = mart.attack.composer.Composer( - functions={"additive": mart.attack.composer.Additive()} - ) + functions = {"additive": mart.attack.composer.Additive()} + composer = Composer(perturber=perturber, functions=functions) optimizer = Mock(spec=mart.optim.OptimizerFactory) gain = Mock() adversary = Adversary( - perturber=perturber, composer=composer, optimizer=optimizer, gain=gain, @@ -252,16 +235,14 @@ def test_configure_optimizers(): def test_training_step(input_data, target_data, perturbation): perturber = Mock(spec=Perturber, return_value=perturbation) - composer = mart.attack.composer.Composer( - functions={"additive": mart.attack.composer.Additive()} - ) + functions = {"additive": mart.attack.composer.Additive()} + composer = Composer(perturber=perturber, functions=functions) optimizer = Mock(spec=mart.optim.OptimizerFactory) gain = Mock(return_value=torch.tensor(1337)) model = Mock(spec="__call__", return_value={}) # Set target_size manually because the test bypasses the convert() step that reads target_size. adversary = Adversary( - perturber=perturber, composer=composer, optimizer=optimizer, gain=gain, @@ -275,16 +256,14 @@ def test_training_step(input_data, target_data, perturbation): def test_training_step_with_many_gain(input_data, target_data, perturbation): perturber = Mock(spec=Perturber, return_value=perturbation) - composer = mart.attack.composer.Composer( - functions={"additive": mart.attack.composer.Additive()} - ) + functions = {"additive": mart.attack.composer.Additive()} + composer = Composer(perturber=perturber, functions=functions) optimizer = Mock(spec=mart.optim.OptimizerFactory) gain = Mock(return_value=torch.tensor([1234, 5678])) model = Mock(spec="__call__", return_value={}) # Set target_size manually because the test bypasses the convert() step that reads target_size. adversary = Adversary( - perturber=perturber, composer=composer, optimizer=optimizer, gain=gain, @@ -297,9 +276,8 @@ def test_training_step_with_many_gain(input_data, target_data, perturbation): def test_training_step_with_objective(input_data, target_data, perturbation): perturber = Mock(spec=Perturber, return_value=perturbation) - composer = mart.attack.composer.Composer( - functions={"additive": mart.attack.composer.Additive()} - ) + functions = {"additive": mart.attack.composer.Additive()} + composer = Composer(perturber=perturber, functions=functions) optimizer = Mock(spec=mart.optim.OptimizerFactory) gain = Mock(return_value=torch.tensor([1234, 5678])) # The model has no attack_step() or training_step(). @@ -308,7 +286,6 @@ def test_training_step_with_objective(input_data, target_data, perturbation): # Set target_size manually because the test bypasses the convert() step that reads target_size. adversary = Adversary( - perturber=perturber, composer=composer, optimizer=optimizer, objective=objective, @@ -324,17 +301,17 @@ def test_training_step_with_objective(input_data, target_data, perturbation): def test_configure_gradient_clipping(): perturber = Mock() - composer = mart.attack.composer.Composer( - functions={"additive": mart.attack.composer.Additive()} - ) + functions = {"additive": mart.attack.composer.Additive()} + composer = Composer(perturber=perturber, functions=functions) + optimizer = Mock( - spec=mart.optim.OptimizerFactory, param_groups=[{"params": Mock()}, {"params": Mock()}] + spec=mart.optim.OptimizerFactory, + param_groups=[{"params": Mock()}, {"params": Mock()}], ) gradient_modifier = Mock() gain = Mock() adversary = Adversary( - perturber=perturber, composer=composer, optimizer=optimizer, gradient_modifier=gradient_modifier, diff --git a/tests/test_composer.py b/tests/test_composer.py index 04851c5f..3da44a27 100644 --- a/tests/test_composer.py +++ b/tests/test_composer.py @@ -4,23 +4,29 @@ # SPDX-License-Identifier: BSD-3-Clause # +from unittest.mock import Mock + import torch from mart.attack.composer import Additive, Composer, Mask, Overlay def test_additive_composer_forward(input_data, target_data, perturbation): - composer = Composer(functions={"additive": Additive()}) + perturber = Mock(return_value=perturbation) + functions = {"additive": Additive()} + composer = Composer(perturber=perturber, functions=functions) - output = composer(perturbation, input=input_data, target=target_data) + output = composer(input=input_data, target=target_data) expected_output = input_data + perturbation torch.testing.assert_close(output, expected_output, equal_nan=True) def test_overlay_composer_forward(input_data, target_data, perturbation): - composer = Composer(functions={"overlay": Overlay()}) + perturber = Mock(return_value=perturbation) + functions = {"overlay": Overlay()} + composer = Composer(perturber=perturber, functions=functions) - output = composer(perturbation, input=input_data, target=target_data) + output = composer(input=input_data, target=target_data) mask = target_data["perturbable_mask"] mask = mask.to(input_data) expected_output = input_data * (1 - mask) + perturbation @@ -33,6 +39,9 @@ def test_mask_additive_composer_forward(): target = {"perturbable_mask": torch.eye(2)} expected_output = torch.eye(2) - composer = Composer(functions={"mask": Mask(order=0), "additive": Additive(order=1)}) - output = composer(perturbation, input=input, target=target) + perturber = Mock(return_value=perturbation) + functions = {"mask": Mask(order=0), "additive": Additive(order=1)} + composer = Composer(perturber=perturber, functions=functions) + + output = composer(input=input, target=target) torch.testing.assert_close(output, expected_output, equal_nan=True)