Merge pull request #46 from MichiganCOG/dev

Dev

.gitignore

@@ -13,3 +13,4 @@ runs/*
 models/HGC3D
 *.json
 pbs/*
+*.pt

README.md

@@ -1,4 +1,4 @@
-# Video Platform for Recognition and Detection in Pytorch
+# [Video Platform for Recognition and Detection in Pytorch](https://arxiv.org/abs/1910.02793)
 
 A platform for quick and easy development of deep learning networks for recognition and detection in videos. Includes popular models like C3D and SSD.
 
@@ -9,13 +9,39 @@ Check out our [wiki!](https://github.com/MichiganCOG/ViP/wiki)
 ### Recognition
 | Model Architecture | Dataset | ViP Accuracy (%) | 
 |:--------------------:|:------------------:|:---------------------:|
+| I3D | HMDB51 (Split 1) | 72.75 |
 | C3D | HMDB51 (Split 1) | 50.14 ± 0.777 |
 | C3D | UCF101 (Split 1) | 80.40 ± 0.399 |
 
 ### Object Detection
 | Model Architecture | Dataset | ViP Accuracy (%) | 
 |:--------------------:|:------------------:|:---------------------:|
 | SSD300 | VOC2007 | 76.58 |
+
+### Video Object Grounding
+| Model Architecture | Dataset | ViP Accuracy (%) | 
+|:--------------------:|:------------------:|:---------------------:|
+| DVSA (+fw, obj) | YC2-BB (Validation) | 30.09 |
+
+**fw**: framewise weighting, **obj**: object interaction
+
+
+## Citation
+
+Please cite ViP when releasing any work that used this platform: https://arxiv.org/abs/1910.02793
+
+```
+@article{ganesh2019vip,
+ title={ViP: Video Platform for PyTorch},
+ author={Ganesh, Madan Ravi and Hofesmann, Eric and Louis, Nathan and Corso, Jason},
+ journal={arXiv preprint arXiv:1910.02793},
+ year={2019}
+}
+
+```
+
+
+
 ## Table of Contents
 
 * [Datasets](#configured-datasets)
@@ -38,12 +64,16 @@ Check out our [wiki!](https://github.com/MichiganCOG/ViP/wiki)
 |[ImageNetVID](http://bvisionweb1.cs.unc.edu/ilsvrc2015/download-videos-3j16.php) | Video Object Detection |
 |[MSCOCO 2014](http://cocodataset.org/#download) | Object Detection, Keypoints|
 |[VOC2007](http://host.robots.ox.ac.uk/pascal/VOC/voc2007/) | Object Detection, Classification|
+|[YC2-BB](http://youcook2.eecs.umich.edu/download)| Video Object Grounding|
+|[DHF1K](https://github.com/wenguanwang/DHF1K) | Video Saliency Prediction|
 
 ## Models
 | Model | Task(s) |
 |:------------------------------------------------:|:--------------------:|
 |[C3D](https://github.com/jfzhang95/pytorch-video-recognition/blob/master/network/C3D_model.py) | Activity Recognition |
+|[I3D](https://github.com/piergiaj/pytorch-i3d) | Activity Recognition |
 |[SSD300](https://github.com/amdegroot/ssd.pytorch) | Object Detection |
+|[DVSA (+fw, obj)](https://github.com/MichiganCOG/Video-Grounding-from-Text)| Video Object Grounding|
 
 ## Requirements
 

config_default_example.yaml

@@ -1,25 +1,24 @@
 # Preprocessing
 clip_length: 16 # Number of frames within a clip 
 clip_offset: 0 # Frame offset between beginning of video and clip (1st clip only) 
-clip_stride: 0 # Frame offset between successive frames
+clip_stride: 1 # Frame offset between successive clips, must be >= 1 
 crop_shape: [112,112] # (Height, Width) of frame 
 crop_type: Random # Type of cropping operation (Random, Central and None) 
 final_shape: [112,112] # (Height, Width) of input to be given to CNN
 num_clips: -1 # Number clips to be generated from a video (<0: uniform sampling, 0: Divide entire video into clips, >0: Defines number of clips) 
 random_offset: 0 # Boolean switch to generate a clip length sized clip from a video 
 resize_shape: [128,171] # (Height, Width) to resize original data 
-sample_duration: 16 # Temporal size of video to be provided as input to the model 
-sample_size: 112 # Height of frame to be provided as input to the model
 subtract_mean: '' # Subtract mean (R,G,B) from all frames during preprocessing
 
 # Experiment Setup 
 acc_metric: Accuracy # Accuracy metric 
-batch_size: 3 # Numbers of videos in a mini-batch 
+batch_size: 15 # Numbers of videos in a mini-batch 
 dataset: HMDB51 # Name of dataset 
 debug: 0 # If True, do not plot, save, or create data files 
 epoch: 30 # Total number of epochs 
 exp: exp # Experiment name
 gamma: 0.1 # Multiplier with which to change learning rate
+grad_max_norm: 0 # Norm for gradient clipping
 json_path: /z/dat/HMDB51/ # Path to the json file for the given dataset
 labels: 51 # Number of total classes in the dataset
 load_type: train # Environment selection, to include only training/training and validation/testing dataset
@@ -37,3 +36,4 @@ rerun: 1 # Number of trials to repeat an experim
 save_dir: './results' # Path to results directory
 seed: 999 # Seed for reproducibility 
 weight_decay: 0.0005 # Weight decay
+resume: 0 # Flag to resume training or switch to alternate objective after loading

datasets/DHF1K.py

@@ -0,0 +1,113 @@
+import torch
+try:
+ from .abstract_datasets import DetectionDataset 
+except:
+ from abstract_datasets import DetectionDataset 
+import cv2
+import os
+import numpy as np
+import json
+try:
+ import datasets.preprocessing_transforms as pt
+except:
+ import preprocessing_transforms as pt
+
+class DHF1K(DetectionDataset):
+ def __init__(self, *args, **kwargs):
+ super(DHF1K, self).__init__(*args, **kwargs)
+
+ # Get model object in case preprocessing other than default is used
+ self.model_object = kwargs['model_obj']
+ self.load_type = kwargs['load_type']
+
+ print(self.load_type)
+ if self.load_type=='train':
+ self.transforms = kwargs['model_obj'].train_transforms
+
+ else:
+ self.transforms = kwargs['model_obj'].test_transforms
+
+
+
+
+ def __getitem__(self, idx):
+ vid_info = self.samples[idx]
+
+
+ base_path = vid_info['base_path']
+ vid_size = vid_info['frame_size']
+
+ input_data = []
+ map_data = []
+ bin_data = []
+
+ for frame_ind in range(len(vid_info['frames'])):
+ frame = vid_info['frames'][frame_ind]
+ frame_path = frame['img_path']
+ map_path = frame['map_path']
+ bin_path = frame['bin_path']
+
+ # Load frame, convert to RGB from BGR and normalize from 0 to 1
+ input_data.append(cv2.imread(os.path.join(base_path, frame_path))[...,::-1]/255.)
+
+ # Load frame, Normalize from 0 to 1
+ # All frame channels have repeated values
+ map_data.append(cv2.imread(map_path)/255.)
+ bin_data.append(cv2.imread(bin_path)/255.)
+
+
+
+ vid_data = self.transforms(input_data)
+
+ # Annotations must be resized in the loss/metric
+ map_data = torch.Tensor(map_data)
+ bin_data = torch.Tensor(bin_data)
+
+ # Permute the PIL dimensions (Frame, Height, Width, Chan) to pytorch (Chan, frame, height, width) 
+ vid_data = vid_data.permute(3, 0, 1, 2)
+ map_data = map_data.permute(3, 0, 1, 2)
+ bin_data = bin_data.permute(3, 0, 1, 2)
+ # All channels are repeated so remove the unnecessary channels
+ map_data = map_data[0].unsqueeze(0)
+ bin_data = bin_data[0].unsqueeze(0)
+
+
+ ret_dict = dict() 
+ ret_dict['data'] = vid_data 
+
+ annot_dict = dict()
+ annot_dict['map'] = map_data
+ annot_dict['bin'] = bin_data
+ annot_dict['input_shape'] = vid_data.size()
+ annot_dict['name'] = base_path
+ ret_dict['annots'] = annot_dict
+
+ return ret_dict
+
+
+if __name__=='__main__':
+
+ class tts():
+ def __call__(self, x):
+ return pt.ToTensorClip()(x)
+ class debug_model():
+ def __init__(self):
+ self.train_transforms = tts()
+
+
+ json_path = '/path/to/DHF1K' #### Change this when testing ####
+
+
+ dataset = DHF1K(model_obj=debug_model(), json_path=json_path, load_type='train', clip_length=16, clip_offset=0, clip_stride=1, num_clips=0, random_offset=0, resize_shape=0, crop_shape=0, crop_type='Center', final_shape=0, batch_size=1)
+ train_loader = torch.utils.data.DataLoader(dataset=dataset, batch_size=1, shuffle=False)
+
+
+ import matplotlib.pyplot as plt
+ for x in enumerate(train_loader):
+ dat = x[1]['data'][0,:,0].permute(1,2,0).numpy()
+ bin = x[1]['annots']['bin'][0,:,0].permute(1,2,0).numpy().repeat(3,axis=2)
+ map = x[1]['annots']['map'][0,:,0].permute(1,2,0).numpy().repeat(3, axis=2)
+ img = np.concatenate([dat,bin,map], axis=0)
+ plt.imshow(img)
+ plt.show()
+ import pdb; pdb.set_trace()

datasets/HMDB51.py

@@ -40,8 +40,9 @@ def __getitem__(self, idx):
  base_path = vid_info['base_path']
 
  input_data = []
- vid_data = np.zeros((self.clip_length, self.final_shape[0], self.final_shape[1], 3))-1
- labels = np.zeros((self.clip_length))-1
+ vid_length = len(vid_info['frames'])
+ vid_data = np.zeros((vid_length, self.final_shape[0], self.final_shape[1], 3))-1
+ labels = np.zeros((vid_length))-1
  input_data = []
 
  for frame_ind in range(len(vid_info['frames'])):

datasets/ImageNetVID.py

@@ -42,10 +42,12 @@ def __getitem__(self, idx):
  vid_size = vid_info['frame_size']
 
  input_data = []
- vid_data = np.zeros((self.clip_length, self.final_shape[0], self.final_shape[1], 3))-1
- bbox_data = np.zeros((self.clip_length, self.max_objects, 4))-1
- labels = np.zeros((self.clip_length, self.max_objects))-1
- occlusions = np.zeros((self.clip_length, self.max_objects))-1
+
+ vid_length = len(vid_info['frames'])
+ vid_data = np.zeros((vid_length, self.final_shape[0], self.final_shape[1], 3))-1
+ bbox_data = np.zeros((vid_length, self.max_objects, 4))-1
+ labels = np.zeros((vid_length, self.max_objects))-1
+ occlusions = np.zeros((vid_length, self.max_objects))-1
 
 
 

datasets/KTH.py

@@ -0,0 +1,79 @@
+import torch
+from .abstract_datasets import RecognitionDataset 
+from PIL import Image
+import cv2
+import os
+import numpy as np
+from torchvision import transforms
+
+class KTH(RecognitionDataset):
+ def __init__(self, *args, **kwargs):
+ """
+ Initialize KTH class 
+ Args:
+ load_type (String): Select training or testing set 
+ resize_shape (Int): [Int, Int] Array indicating desired height and width to resize input
+ crop_shape (Int): [Int, Int] Array indicating desired height and width to crop input
+ final_shape (Int): [Int, Int] Array indicating desired height and width of input to deep network
+ preprocess (String): Keyword to select different preprocessing types 
+
+ Return:
+ None
+ """
+ super(KTH, self).__init__(*args, **kwargs)
+
+ self.load_type = kwargs['load_type']
+ self.resize_shape = kwargs['resize_shape']
+ self.crop_shape = kwargs['crop_shape']
+ self.final_shape = kwargs['final_shape']
+ self.preprocess = kwargs['preprocess']
+
+ if self.load_type=='train':
+ self.transforms = kwargs['model_obj'].train_transforms
+
+ else:
+ self.transforms = kwargs['model_obj'].test_transforms
+
+
+ def __getitem__(self, idx):
+ vid_info = self.samples[idx]
+ base_path = vid_info['base_path']
+
+ input_data = []
+
+ vid_length = len(vid_info['frames'])
+ vid_data = np.zeros((vid_length, self.final_shape[0], self.final_shape[1], 3))-1
+ labels = np.zeros((vid_length))-1
+ input_data = []
+
+ for frame_ind in range(len(vid_info['frames'])):
+ frame_path = os.path.join(base_path, vid_info['frames'][frame_ind]['img_path'])
+
+ for frame_labels in vid_info['frames'][frame_ind]['actions']:
+ labels[frame_ind] = frame_labels['action_class']
+
+ # Load frame image data and preprocess image accordingly
+ input_data.append(cv2.imread(frame_path)[...,::-1]/1.)
+
+
+ # Preprocess data
+ vid_data = self.transforms(input_data)
+ labels = torch.from_numpy(labels).float()
+
+ # Permute the PIL dimensions (Frame, Height, Width, Chan) to pytorch (Chan, frame, height, width) 
+ vid_data = vid_data.permute(3, 0, 1, 2)
+
+ ret_dict = dict() 
+ ret_dict['data'] = vid_data 
+
+ annot_dict = dict()
+ annot_dict['labels'] = labels
+
+ ret_dict['annots'] = annot_dict
+
+ return ret_dict
+
+
+#dataset = HMDB51(json_path='/z/dat/HMDB51', dataset_type='train', clip_length=100, num_clips=0)
+#dat = dataset.__getitem__(0)
+#import pdb; pdb.set_trace()

datasets/MSCOCO.py

@@ -1,6 +1,6 @@
 import torch
 from .abstract_datasets import DetectionDataset 
-from PIL import Image
+import cv2
 import os
 import numpy as np
 import datasets.preprocessing_transforms as pt
@@ -34,10 +34,11 @@ def __getitem__(self, idx):
  vid_size = vid_info['frame_size']
 
  input_data = []
- vid_data = np.zeros((self.clip_length, self.final_shape[0], self.final_shape[1], 3))-1
- bbox_data = np.zeros((self.clip_length, self.max_objects, 4))-1
- labels = np.zeros((self.clip_length, self.max_objects))-1
- iscrowds = np.zeros((self.clip_length, self.max_objects))-1
+ vid_length = len(vid_info['frames'])
+ vid_data = np.zeros((vid_length, self.final_shape[0], self.final_shape[1], 3))-1
+ bbox_data = np.zeros((vid_length, self.max_objects, 4))-1
+ labels = np.zeros((vid_length, self.max_objects))-1
+ iscrowds = np.zeros((vid_length, self.max_objects))-1
 
 
 
@@ -62,7 +63,7 @@ def __getitem__(self, idx):
  iscrowds[frame_ind, trackid] = iscrowd
 
 
- input_data.append(Image.open(os.path.join(base_path, frame_path)))
+ input_data.append(cv2.imread(os.path.join(base_path, frame_path))[...,::-1])
 
  vid_data, bbox_data = self.transforms(input_data, bbox_data)