Script for creating dataset of 80x80 patches

.gitignore

@@ -129,3 +129,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+*.out
+results/

dataset.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+#SBATCH -c 4 
+#SBATCH --gres=gpu:1 
+#SBATCH -p gpu
+#SBATCH --time=00:30:00
+
+source /home/hanfeld/.front-env/bin/activate
+python src/attacks.py --file $1

dataset.yaml

@@ -0,0 +1,30 @@
+path: 'results/dataset'
+# +x is distance
+# +y is left
+# +z is top
+targets:
+  x : [ 1.0, 1.5, 0.5, 2.0, 1.0, 1.0]
+  y : [ 0.0, 0.0, 0.0, 0.0, 1.0, -1.0]
+  z : [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
+stlc: # small-time locally controllability
+  offsets:
+    # base
+    - target: [0,0,0]
+      position: [0,0,0]
+lr_pos: 0.001
+lr_patch: 0.01
+prob_weight: 1.0
+num_hl_iter: 50
+num_pos_restarts: 3
+num_pos_epochs: 1
+num_patch_epochs: 1
+batch_size: 32
+num_patches: 1
+scale_min: 0.2
+scale_max: 0.7
+patch: 
+  mode: 'random'  # face, white, random
+  size: [80, 80] # patch size in [height, width]
+  path: 'src/custom_patches/custom_patch_96x160.npy' # if mode == 'face' set a path, otherwise path is ignored
+mode: "joint" # fixed, joint, split, hybrid
+quantized: false

settings.yaml

@@ -1,11 +1,11 @@
-path: "eval/demo/multi/"
+path: "test/"
 # +x is distance
 # +y is left
 # +z is top
 targets:
-  x : [ 1.0, 1.5, 0.5, 1.0, 1.0] #, 1.0] #, 1.0] #, 1.0] #, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0]
-  y : [ 0.0, 0.0, 0.0, 1.0, -1.0] #, -1.0] #, 1.0] #, 0.0] #, 0.0, -1.0, -1.0, 1.0, 1.0, 0.0]
-  z : [ 0.0, 0.0, 0.0, 0.0, 0.0] #, 0.0] #, 0.0] #, -1.0] #, 1.0, -1.0, 1.0, -1.0, 1.0, 0.0]
+  x : [0.5, 1.0] #, 1.0] #, 1.0] #, 1.0] #, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0]
+  y : [0.0, 1.0] #, -1.0] #, 1.0] #, 0.0] #, 0.0, -1.0, -1.0, 1.0, 1.0, 0.0]
+  z : [0.0, 0.0] #, 0.0] #, 0.0] #, -1.0] #, 1.0, -1.0, 1.0, -1.0, 1.0, 0.0]
 stlc: # small-time locally controllability
   offsets:
     # base
@@ -34,18 +34,19 @@ stlc: # small-time locally controllability
     # - target: [0,-0.5,0]
     #   position: [0,-0.3,0]
 lr_pos: 0.001
-lr_patch: 0.001
+lr_patch: 0.01
 prob_weight: 1.0
-num_hl_iter: 100
+num_hl_iter: 10
 num_pos_restarts: 3
 num_pos_epochs: 1
 num_patch_epochs: 1
 batch_size: 32
-num_patches: 2
+num_patches: 1
 scale_min: 0.2
-scale_max: 0.4
+scale_max: 0.7
 patch: 
   mode: 'random'  # face, white, random
+  size: [80, 80] # patch size in [height, width]
   path: 'src/custom_patches/custom_patch_96x160.npy' # if mode == 'face' set a path, otherwise path is ignored
 mode: "joint" # fixed, joint, split, hybrid
-quantized: true
+quantized: false

src/attacks.py

@@ -16,18 +16,27 @@ def get_transformation(sf, tx, ty):
     eye = torch.eye(2, 2).unsqueeze(0).to(sf.device)
     scale = eye * sf
 
-    # print(scale.shape, translation_vector.shape)
+    # # print(scale.shape, translation_vector.shape)
 
     transformation_matrix = torch.cat([scale, translation_vector], dim=2)
+
+    # M = torch.eye(3,3).unsqueeze(0).to(sf.device)
+    # M[..., :2, :2] *= sf
+    # M[..., 0, 2] = tx
+    # M[..., 1, 2] = ty
     return transformation_matrix.float()
 
-def norm_transformation(sf, tx, ty, scale_min=0.3, scale_max=0.5):
-    tx_tanh = torch.tanh(tx) #* 0.8
-    ty_tanh = torch.tanh(ty) #* 0.8
+def norm_transformation(sf, tx, ty, scale_min=0.3, scale_max=0.5, tx_min=-10., tx_max=100., ty_min=-10., ty_max=80.):
+    # tx_tanh = torch.tanh(tx) #* 0.8
+    # ty_tanh = torch.tanh(ty) #* 0.8
+
+    # new patch placement implementation might need different tx, ty limits!:
+    tx_norm = (tx_max - tx_min) * (torch.tanh(tx) + 1) * 0.5 + tx_min
+    ty_norm = (ty_max - ty_min) * (torch.tanh(ty) + 1) * 0.5 + ty_min
 
     scaling_norm = (scale_max - scale_min) * (torch.tanh(sf) + 1) * 0.5 + scale_min # normalizes scaling factor to range [0.3, 0.5]
 
-    return scaling_norm, tx_tanh, ty_tanh
+    return scaling_norm, tx_norm, ty_norm
 
 # def get_rotation(yaw, pitch, roll):
 #     rotation_yaw = np.array([[np.cos(yaw), -np.sin(yaw), 0.0, 0.0],
@@ -127,7 +136,10 @@ def targeted_attack_joint(dataset, patch, model, positions, assignment, targets,
                         patch_batches = torch.cat([x.repeat(len(batch), 1, 1, 1) for x in patch_t[active_patches]]) # get batch_sized batches of each patch in patches, size should be batch_size*num_patches
                         batch_multi = batch.clone().repeat(len(patch_t[active_patches]), 1, 1, 1)
                         transformations_multi = noisy_transformations.view(len(patch_t[active_patches])*len(batch), 2, 3) # reshape transformation matrices
-                        #print(transformations_multi.shape)
+                        # print("before patch placement:")
+                        # print("transformations shape: ", transformations_multi.shape)
+                        # print("patch batch shape: ", patch_batches.shape)
+
 
                         mod_img = place_patch(batch_multi, patch_batches, transformations_multi) 
 
@@ -520,11 +532,13 @@ def calc_eval_loss(dataset, patch, transformation_matrix, model, target, quantiz
 
     # or start from a random patch
     if settings['patch']['mode'] == 'random':
-        patch_start = torch.rand(num_patches, 1, 96, 160).to(device)
+        size = settings['patch']['size']
+        patch_start = torch.rand(num_patches, 1, size[0], size[1]).to(device)
 
     # or start from a white patch
     if settings['patch']['mode'] == 'white':
-        patch_start = torch.ones(num_patches, 1, 96, 160).to(device)
+        size = settings['patch']['size']
+        patch_start = torch.ones(num_patches, 1, size[0], size[1]).to(device)
 
     optimization_pos_losses = []
     optimization_pos_vectors = []
@@ -539,6 +553,10 @@ def calc_eval_loss(dataset, patch, transformation_matrix, model, target, quantiz
     stats_p_all = []
 
     positions = torch.FloatTensor(len(targets), num_patches, 3, 1).uniform_(-1., 1.).to(device)
+    # sf = torch.FloatTensor(len(targets), num_patches, 1).uniform_(-1, 1.).to(device)
+    # tx = torch.FloatTensor(len(targets), num_patches, 1).uniform_(-1, 1.).to(device)
+    # ty = torch.FloatTensor(len(targets), num_patches, 1).uniform_(-1, 1.).to(device)
+    # positions = torch.stack([sf, tx, ty]).moveaxis(0, 2)
 
     optimization_pos_vectors.append(positions)
 
@@ -547,7 +565,7 @@ def calc_eval_loss(dataset, patch, transformation_matrix, model, target, quantiz
     optimization_patches.append(patch.clone())
 
     # assignment: we start by assigning all targets to all patches
-    A = np.ones((num_patches, len(targets)), dtype=np.bool8)
+    A = np.ones((num_patches, len(targets)), dtype=np.bool_)
 
     # # debug
     # A = np.zeros((num_patches, len(targets)), dtype=np.bool8)
@@ -658,85 +676,85 @@ def calc_eval_loss(dataset, patch, transformation_matrix, model, target, quantiz
 
     # save all results in numpy arrays for later use
     np.save(path / 'patches.npy', optimization_patches.cpu().numpy())
-    np.save(path / 'patch_losses.npy', optimization_patch_losses.cpu().numpy())
+    # np.save(path / 'patch_losses.npy', optimization_patch_losses.cpu().numpy())
     # np.save(path / 'positions.npy', optimization_pos_vectors.cpu().numpy())
     np.save(path / 'positions_norm.npy', np.array([all_sf.cpu().numpy(), all_tx.cpu().numpy(), all_ty.cpu().numpy()]))
-    np.save(path / 'position_losses.npy', optimization_pos_losses.cpu().numpy())
-    np.save(path / 'losses_train.npy', train_losses.cpu().numpy())
-    np.save(path / 'losses_test.npy', test_losses.cpu().numpy())
+    # np.save(path / 'position_losses.npy', optimization_pos_losses.cpu().numpy())
+    # np.save(path / 'losses_train.npy', train_losses.cpu().numpy())
+    # np.save(path / 'losses_test.npy', test_losses.cpu().numpy())
 
     np.save(path / 'stats.npy', stats_all)
     np.save(path / 'stats_p.npy', stats_p_all)
 
-    # final evaluation on test set
-    print("Evaluation...")
-
-    test_batch, test_gt = test_set.dataset[:]
-    test_batch = test_batch.to(device) / 255. # limit images to range [0-1]
-
-    boxplot_data = []
-    #target_mask = torch.tensor(target_mask).to(patch.device)
-    for target_idx, target in enumerate(targets):
-        pred_base = model(test_batch.float() * 255.)
-        pred_base = torch.stack(pred_base[:3]).squeeze(2).mT
-        target_batch = target.repeat(len(test_batch), 1)
-        target_batch = torch.where(torch.isnan(target_batch), pred_base, target_batch)
-        loss_base = torch.tensor([mse_loss(target_batch[i], pred_base[i]) for i in range(len(test_batch))])
-
-        loss_start_patch_best = None
-        loss_start_patch_best_value = np.inf
-        loss_opt_patch_best = None
-        loss_opt_patch_best_value = np.inf
-        for patch_idx in range(num_patches):
-            scale_norm, tx_norm, ty_norm = norm_transformation(*optimization_pos_vectors[-1][target_idx][patch_idx], scale_min, scale_max)
-            transformation_matrix = get_transformation(scale_norm, tx_norm, ty_norm).to(device)
-
-            mod_img = place_patch(test_batch, patch_start[patch_idx:patch_idx+1], transformation_matrix)
-            mod_img *= 255. # convert input images back to range [0-255.]
-            mod_img.clamp_(0., 255.)
-            pred_start_patch = model(mod_img.float())
-            pred_start_patch = torch.stack(pred_start_patch[:3]).squeeze(2).mT
-            target_batch = target.repeat(len(test_batch), 1)
-            target_batch = torch.where(torch.isnan(target_batch), pred_start_patch, target_batch)
-            loss_start_patch = torch.tensor([mse_loss(target_batch[i], pred_start_patch[i]) for i in range(len(test_batch))])
-            if torch.sum(loss_start_patch) < loss_start_patch_best_value:
-                loss_start_patch_best_value = torch.sum(loss_start_patch)
-                loss_start_patch_best = loss_start_patch
-
-            mod_img = place_patch(test_batch, patch[patch_idx:patch_idx+1], transformation_matrix)
-            mod_img *= 255. # convert input images back to range [0-255.]
-            mod_img.clamp_(0., 255.)
-            pred_opt_patch = model(mod_img.float())
-            pred_opt_patch = torch.stack(pred_opt_patch[:3]).squeeze(2).mT
-            target_batch = target.repeat(len(test_batch), 1)
-            target_batch = torch.where(torch.isnan(target_batch), pred_opt_patch, target_batch)
-            loss_opt_patch = torch.tensor([mse_loss(target_batch[i], pred_opt_patch[i]) for i in range(len(test_batch))])
-            if torch.sum(loss_opt_patch) < loss_opt_patch_best_value:
-                loss_opt_patch_best_value = torch.sum(loss_opt_patch)
-                loss_opt_patch_best = loss_opt_patch
-
-        boxplot_data.append(torch.stack([loss_base.detach().cpu(), loss_start_patch_best.detach().cpu(), loss_opt_patch_best.detach().cpu()]))
-
-
-    np.save(path / 'boxplot_data.npy', torch.stack(boxplot_data).cpu().numpy())
-
-    # STLC Analysis
-    for target_idx, target in enumerate(targets):
-        for patch_idx in range(num_patches):
-
-            scale_norm, tx_norm, ty_norm = norm_transformation(*optimization_pos_vectors[-1][target_idx][patch_idx], scale_min, scale_max)
-            for target_offset, position_offset in zip(stlc_target_offsets, stlc_position_offsets):
-                transformation_matrix = get_transformation(scale_norm + position_offset[0], tx_norm + position_offset[1], ty_norm + position_offset[2]).to(device)
-
-                mod_img = place_patch(test_batch, patch[patch_idx:patch_idx+1], transformation_matrix)
-                mod_img *= 255. # convert input images back to range [0-255.]
-                mod_img.clamp_(0., 255.)
-                pred_opt_patch = model(mod_img.float())
-                pred_opt_patch = torch.stack(pred_opt_patch[:3]).squeeze(2).mT
-                target_batch = (target + target_offset).repeat(len(test_batch), 1)
-                target_batch = torch.where(torch.isnan(target_batch), pred_opt_patch, target_batch)
-                loss_opt_patch = torch.tensor([mse_loss(target_batch[i], pred_opt_patch[i]) for i in range(len(test_batch))])
-                print("STLC", target_offset.detach().cpu().numpy(), torch.mean(loss_opt_patch).detach().cpu().numpy())
-
-    from plots import plot_results
-    plot_results(path)
+    # # final evaluation on test set
+    # print("Evaluation...")
+
+    # test_batch, test_gt = test_set.dataset[:]
+    # test_batch = test_batch.to(device) / 255. # limit images to range [0-1]
+
+    # boxplot_data = []
+    # #target_mask = torch.tensor(target_mask).to(patch.device)
+    # for target_idx, target in enumerate(targets):
+    #     pred_base = model(test_batch.float() * 255.)
+    #     pred_base = torch.stack(pred_base[:3]).squeeze(2).mT
+    #     target_batch = target.repeat(len(test_batch), 1)
+    #     target_batch = torch.where(torch.isnan(target_batch), pred_base, target_batch)
+    #     loss_base = torch.tensor([mse_loss(target_batch[i], pred_base[i]) for i in range(len(test_batch))])
+
+    #     loss_start_patch_best = None
+    #     loss_start_patch_best_value = np.inf
+    #     loss_opt_patch_best = None
+    #     loss_opt_patch_best_value = np.inf
+    #     for patch_idx in range(num_patches):
+    #         scale_norm, tx_norm, ty_norm = norm_transformation(*optimization_pos_vectors[-1][target_idx][patch_idx], scale_min, scale_max)
+    #         transformation_matrix = get_transformation(scale_norm, tx_norm, ty_norm).to(device)
+
+    #         mod_img = place_patch(test_batch, patch_start[patch_idx:patch_idx+1], transformation_matrix)
+    #         mod_img *= 255. # convert input images back to range [0-255.]
+    #         mod_img.clamp_(0., 255.)
+    #         pred_start_patch = model(mod_img.float())
+    #         pred_start_patch = torch.stack(pred_start_patch[:3]).squeeze(2).mT
+    #         target_batch = target.repeat(len(test_batch), 1)
+    #         target_batch = torch.where(torch.isnan(target_batch), pred_start_patch, target_batch)
+    #         loss_start_patch = torch.tensor([mse_loss(target_batch[i], pred_start_patch[i]) for i in range(len(test_batch))])
+    #         if torch.sum(loss_start_patch) < loss_start_patch_best_value:
+    #             loss_start_patch_best_value = torch.sum(loss_start_patch)
+    #             loss_start_patch_best = loss_start_patch
+
+    #         mod_img = place_patch(test_batch, patch[patch_idx:patch_idx+1], transformation_matrix)
+    #         mod_img *= 255. # convert input images back to range [0-255.]
+    #         mod_img.clamp_(0., 255.)
+    #         pred_opt_patch = model(mod_img.float())
+    #         pred_opt_patch = torch.stack(pred_opt_patch[:3]).squeeze(2).mT
+    #         target_batch = target.repeat(len(test_batch), 1)
+    #         target_batch = torch.where(torch.isnan(target_batch), pred_opt_patch, target_batch)
+    #         loss_opt_patch = torch.tensor([mse_loss(target_batch[i], pred_opt_patch[i]) for i in range(len(test_batch))])
+    #         if torch.sum(loss_opt_patch) < loss_opt_patch_best_value:
+    #             loss_opt_patch_best_value = torch.sum(loss_opt_patch)
+    #             loss_opt_patch_best = loss_opt_patch
+
+    #     boxplot_data.append(torch.stack([loss_base.detach().cpu(), loss_start_patch_best.detach().cpu(), loss_opt_patch_best.detach().cpu()]))
+
+
+    # np.save(path / 'boxplot_data.npy', torch.stack(boxplot_data).cpu().numpy())
+
+    # # STLC Analysis
+    # for target_idx, target in enumerate(targets):
+    #     for patch_idx in range(num_patches):
+
+    #         scale_norm, tx_norm, ty_norm = norm_transformation(*optimization_pos_vectors[-1][target_idx][patch_idx], scale_min, scale_max)
+    #         for target_offset, position_offset in zip(stlc_target_offsets, stlc_position_offsets):
+    #             transformation_matrix = get_transformation(scale_norm + position_offset[0], tx_norm + position_offset[1], ty_norm + position_offset[2]).to(device)
+
+    #             mod_img = place_patch(test_batch, patch[patch_idx:patch_idx+1], transformation_matrix)
+    #             mod_img *= 255. # convert input images back to range [0-255.]
+    #             mod_img.clamp_(0., 255.)
+    #             pred_opt_patch = model(mod_img.float())
+    #             pred_opt_patch = torch.stack(pred_opt_patch[:3]).squeeze(2).mT
+    #             target_batch = (target + target_offset).repeat(len(test_batch), 1)
+    #             target_batch = torch.where(torch.isnan(target_batch), pred_opt_patch, target_batch)
+    #             loss_opt_patch = torch.tensor([mse_loss(target_batch[i], pred_opt_patch[i]) for i in range(len(test_batch))])
+    #             print("STLC", target_offset.detach().cpu().numpy(), torch.mean(loss_opt_patch).detach().cpu().numpy())
+
+    # from plots import plot_results
+    # plot_results(path)