From 0109290e79f95ed9d7ffb91844a9bfb12b4191e7 Mon Sep 17 00:00:00 2001 From: knangia04 Date: Sun, 22 Aug 2021 19:10:02 -0500 Subject: [PATCH 1/2] PyTorch Versions of TC1 and P1B2 Benchmarks * baseline_pytorch.py is the main function of the benchmark. * Updated pytorch_utils.py to add additional functionality. * torch_deps folder consists of dependencies that are required for the benchmark to function correctly, including the model, dataset, classification network, training, validation, and test functions, and weight initialization. --- Pilot1/P1B2/p1b2_baseline_pytorch.py | 144 +++++++++ Pilot1/P1B2/p1b2_default_model_pytorch.txt | 28 ++ Pilot1/P1B2/pytorch_utils.py | 140 +++++++++ .../torch_deps/p1b2_classification_net.py | 95 ++++++ Pilot1/P1B2/torch_deps/p1b2_clf_func.py | 113 +++++++ .../P1B2/torch_deps/p1b2_dataset_pytorch.py | 132 ++++++++ Pilot1/P1B2/torch_deps/p1b2_pytorch_model.py | 297 ++++++++++++++++++ Pilot1/P1B2/torch_deps/random_seeding.py | 36 +++ Pilot1/P1B2/torch_deps/weight_init.py | 63 ++++ Pilot1/TC1/pytorch_utils.py | 139 ++++++++ Pilot1/TC1/tc1_baseline_pytorch.py | 138 ++++++++ Pilot1/TC1/tc1_default_model_pytorch.txt | 25 ++ Pilot1/TC1/torch_deps/classification_net.py | 134 ++++++++ Pilot1/TC1/torch_deps/random_seeding.py | 36 +++ Pilot1/TC1/torch_deps/tc1_clf_func.py | 114 +++++++ Pilot1/TC1/torch_deps/tc1_dataset_pytorch.py | 119 +++++++ Pilot1/TC1/torch_deps/tc1_pytorch_model.py | 264 ++++++++++++++++ Pilot1/TC1/torch_deps/weight_init.py | 63 ++++ 18 files changed, 2080 insertions(+) create mode 100644 Pilot1/P1B2/p1b2_baseline_pytorch.py create mode 100644 Pilot1/P1B2/p1b2_default_model_pytorch.txt create mode 100644 Pilot1/P1B2/pytorch_utils.py create mode 100644 Pilot1/P1B2/torch_deps/p1b2_classification_net.py create mode 100644 Pilot1/P1B2/torch_deps/p1b2_clf_func.py create mode 100644 Pilot1/P1B2/torch_deps/p1b2_dataset_pytorch.py create mode 100644 Pilot1/P1B2/torch_deps/p1b2_pytorch_model.py create mode 100644 Pilot1/P1B2/torch_deps/random_seeding.py create mode 100644 Pilot1/P1B2/torch_deps/weight_init.py create mode 100644 Pilot1/TC1/pytorch_utils.py create mode 100644 Pilot1/TC1/tc1_baseline_pytorch.py create mode 100644 Pilot1/TC1/tc1_default_model_pytorch.txt create mode 100644 Pilot1/TC1/torch_deps/classification_net.py create mode 100644 Pilot1/TC1/torch_deps/random_seeding.py create mode 100644 Pilot1/TC1/torch_deps/tc1_clf_func.py create mode 100644 Pilot1/TC1/torch_deps/tc1_dataset_pytorch.py create mode 100644 Pilot1/TC1/torch_deps/tc1_pytorch_model.py create mode 100644 Pilot1/TC1/torch_deps/weight_init.py diff --git a/Pilot1/P1B2/p1b2_baseline_pytorch.py b/Pilot1/P1B2/p1b2_baseline_pytorch.py new file mode 100644 index 00000000..3b0a215d --- /dev/null +++ b/Pilot1/P1B2/p1b2_baseline_pytorch.py @@ -0,0 +1,144 @@ +from __future__ import print_function + +import numpy as np +import os +import sys + +import torch + + +if True: + print("Restricting #of GPUs to 1") + os.environ["CUDA_VISIBLE_DEVICES"]="0" + #os.environ["CUDA_VISIBLE_DEVICES"]="0,1,2,3,4,5,6,7" + #os.environ["CUDA_VISIBLE_DEVICES"]="0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15" + +file_path = os.path.dirname(os.path.realpath(__file__)) +lib_path2 = os.path.abspath(os.path.join(file_path, '..', '..', 'common')) +sys.path.append(lib_path2) +os.chdir(file_path) + +import p1b2 as bmk +import candle +from torch_deps.p1b2_pytorch_model import P1B2Model +from torch_deps.random_seeding import seed_random_state + +np.set_printoptions(precision=4) + +def initialize_parameters(default_model = 'p1b2_default_model_pytorch.txt'): + + # Build benchmark object + p1b2Bmk = bmk.BenchmarkP1B2(bmk.file_path, default_model, 'pytorch', + prog='p1b2_baseline', desc='Train Classifier - Pilot 1 Benchmark 2') + + print("Created P1B2 benchmark") + + # Initialize parameters + gParameters = candle.finalize_parameters(p1b2Bmk) + #benchmark.logger.info('Params: {}'.format(gParameters)) + print("Parameters initialized") + + return gParameters + + +def run(params): + + args = candle.ArgumentStruct(**params) + args.no_cuda = args.no_cuda if hasattr(args,'no_cuda') else False + args.multi_gpu = args.multi_gpu if hasattr(args,'multi_gpu') else True + args.max_num_batches = args.max_num_batches if hasattr(args,'max_num_batches') else 1000 + args.dry_run = args.dry_run if hasattr(args,'dry_run') else False + args.log_interval = args.log_interval if hasattr(args,'log_interval') else 10 + + + seed = args.rng_seed + candle.set_seed(seed) + # Setting up random seed for reproducible and deterministic results + seed_random_state(args.rng_seed) + + args.keras_defaults = candle.keras_default_config() + + # Construct extension to save validation results + ext = bmk.extension_from_parameters(params, '.pytorch') + + candle.verify_path(params['save_path']) + prefix = '{}{}'.format(params['save_path'], ext) + logfile = params['logfile'] if params['logfile'] else prefix+'.log' + candle.set_up_logger(logfile, bmk.logger, params['verbose']) + bmk.logger.info('Params: {}'.format(params)) + + args.tensorboard_dir = "tb/{}".format(ext) + args.logger = bmk.logger + + #Autosave model + model_name = params['model_name'] + args_filename = "{}.model.args".format(params['save_path']) + args.model_autosave_filename = "{}.autosave.model.pth".format(params['save_path']) + # CSV logging + args.csv_filename = '{}{}_training.log'.format(params['save_path'], ext) + + # Computation device config (cuda or cpu) + use_cuda = not args.no_cuda and torch.cuda.is_available() + device = torch.device('cuda' if use_cuda else 'cpu') + + # save args to file + import pickle + args_file = open(args_filename, 'wb') + pickle.dump(args, args_file) + args_file.close() + + modelP1B2 = P1B2Model(args, use_cuda, device) + + #model.summary() + #print(modelP1B2.p1b2_net) # Model summary + bmk.logger.info('Model summary: {}'.format(modelP1B2.p1b2_net)) # Model summary + + modelP1B2.train() + modelP1B2.print_final_stats() + + #Save model + model_name = params['model_name'] + model_filename = "{}.model_state_dict.pth".format(params['save_path']) + if hasattr(modelP1B2.p1b2_net,'module'): + # Saving the DataParallel model + torch.save(modelP1B2.p1b2_net.module.state_dict(), model_filename) + else: + torch.save(modelP1B2.p1b2_net.state_dict(), model_filename) + + #reload args from file + args_file = open(args_filename, 'rb') + loaded_args = pickle.load(args_file) + args_file.close() + + # load weights into new model + loaded_modelP1B2 = P1B2Model(loaded_args) + loaded_modelP1B2.p1b2_net.load_state_dict(torch.load(model_filename, map_location=torch.device('cpu'))) + print("Loaded torch model from disk") + + # evaluate loaded model on test data + loaded_modelP1B2.p1b2_net.eval() + val_acc,val_loss = loaded_modelP1B2.validation(0) + + print("Model State Dict Validation loss: %5.2f" % (val_loss)) + print("Model State Dict Validation accuracy: %5.2f%%" %(val_acc)) + + print('Test data: ') + test_acc,test_loss = loaded_modelP1B2.test() + + print("Model State Dict Test loss: %5.2f" % (test_loss)) + print("Model State Dict Test accuracy: %5.2f%%" %(test_acc)) + + +def main(): + + gParameters = initialize_parameters() + run(gParameters) + +if __name__ == '__main__': + main() + try: + tmp = 1 + except AttributeError: # theano does not have this function + pass + + diff --git a/Pilot1/P1B2/p1b2_default_model_pytorch.txt b/Pilot1/P1B2/p1b2_default_model_pytorch.txt new file mode 100644 index 00000000..fad61376 --- /dev/null +++ b/Pilot1/P1B2/p1b2_default_model_pytorch.txt @@ -0,0 +1,28 @@ +[Global_Params] +data_url = 'http://ftp.mcs.anl.gov/pub/candle/public/benchmarks/P1B2/' +train_data = 'P1B2.train.csv' +test_data = 'P1B2.test.csv' +model_name='p1b2' +dense=[1024, 512, 256] +batch_size=60 +epochs=20 +activation='sigmoid' +out_activation='log_softmax' +loss='nll' +optimizer='rmsprop' +learning_rate=0.001 +scaling='minmax' +dropout=0. +classes=10 +feature_subsample=0 +reg_l2=0.00001 +val_split=0.1 +rng_seed=2017 +initialization='glorot_uniform' +save_path='save' +shuffle = True + +# Miscellaneous settings ################################## +# multi_gpu=True +# no_cuda=True +# rng_seed=0 \ No newline at end of file diff --git a/Pilot1/P1B2/pytorch_utils.py b/Pilot1/P1B2/pytorch_utils.py new file mode 100644 index 00000000..3e56e876 --- /dev/null +++ b/Pilot1/P1B2/pytorch_utils.py @@ -0,0 +1,140 @@ +from __future__ import absolute_import + +import torch +import torch.nn +import torch.nn.init +import torch.optim +import torch.nn.functional as F + +from default_utils import set_seed as set_seed_defaultUtils + +def set_parallelism_threads(): # for compatibility + pass + +def set_seed(seed): + """ Set the random number seed to the desired value + + Parameters + ---------- + seed : integer + Random number seed. + """ + + set_seed_defaultUtils(seed) + torch.manual_seed(seed) + + +def get_function(name): + mapping = {} + + # loss + mapping['mse'] = torch.nn.MSELoss() + mapping['binary_crossentropy'] = torch.nn.BCELoss() + mapping['categorical_crossentropy'] = torch.nn.CrossEntropyLoss() + mapping['smoothL1'] = torch.nn.SmoothL1Loss() + + mapped = mapping.get(name) + if not mapped: + raise Exception('No pytorch function found for "{}"'.format(name)) + + return mapped + + +def build_activation(type, dim=1): + + # activation + + if type=='relu': + return torch.nn.ReLU() + elif type=='sigmoid': + return torch.nn.Sigmoid() + elif type=='tanh': + return torch.nn.Tanh() + elif type=='softmax': + return torch.nn.Softmax(dim) + elif type=='log_softmax': + return torch.nn.LogSoftmax(dim) + +def build_optimizer(model, type, lr, kerasDefaults, trainable_only=True): + + if trainable_only: + params = filter(lambda p: p.requires_grad, model.parameters()) + else: + params = model.parameters() + + #schedule = optimizers.optimizer.Schedule() # constant lr (equivalent to default keras setting) + + if type == 'sgd': + return torch.optim.SGD(params, + lr=lr, + momentum=kerasDefaults['momentum_sgd'], + nesterov=kerasDefaults['nesterov_sgd']) + #schedule=schedule) + + elif type == 'rmsprop': + return torch.optim.RMSprop(model.parameters(), + lr=lr, + alpha=kerasDefaults['rho'], + eps=kerasDefaults['epsilon'], + weight_decay=kerasDefaults['weight_decay']) + #schedule=schedule) + + elif type == 'adagrad': + return torch.optim.Adagrad(model.parameters(), + lr=lr, + eps=kerasDefaults['epsilon']) + + elif type == 'adadelta': + return torch.optim.Adadelta(params, + eps=kerasDefaults['epsilon'], + rho=kerasDefaults['rho']) + + elif type == 'adam': + return torch.optim.Adam(params, + lr=lr, + betas=[kerasDefaults['beta_1'], kerasDefaults['beta_2']], + eps=kerasDefaults['epsilon']) + + + +def initialize(weights, type, kerasDefaults, seed=None, constant=0.): + + if type == 'constant': + return torch.nn.init.constant_(weights, + val=constant) + + elif type == 'uniform': + return torch.nn.init.uniform(weights, + a=kerasDefaults['minval_uniform'], + b=kerasDefaults['maxval_uniform']) + + elif type == 'normal': + return torch.nn.init.normal(weights, + mean=kerasDefaults['mean_normal'], + std=kerasDefaults['stddev_normal']) + + elif type == 'glorot_normal': # not quite Xavier + return torch.nn.init.xavier_normal(weights) + + elif type == 'glorot_uniform': + return torch.nn.init.xavier_uniform_(weights) + + elif type == 'he_normal': + return torch.nn.init.kaiming_uniform(weights) + + +def xent(y_true, y_pred): + return F.cross_entropy(y_pred, y_true) + + +def mse(y_true, y_pred): + return F.mse_loss(y_pred, y_true) + +def build_loss(type, y_pred, y_true): + + if type=='categorical_crossentropy': + return xent(y_true, y_pred) + elif type=='mse': + return mse(y_true, y_pred) + elif type=='nll': + return F.nll_loss(y_pred, y_true) \ No newline at end of file diff --git a/Pilot1/P1B2/torch_deps/p1b2_classification_net.py b/Pilot1/P1B2/torch_deps/p1b2_classification_net.py new file mode 100644 index 00000000..584862c2 --- /dev/null +++ b/Pilot1/P1B2/torch_deps/p1b2_classification_net.py @@ -0,0 +1,95 @@ +import torch +import torch.nn as nn +import numpy as np +from pytorch_utils import build_activation +from torch_deps.weight_init import basic_weight_init, basic_weight_init_he_normal_relu, basic_weight_init_he_uniform_relu, basic_weight_init_glorut_uniform + +class P1B2Net(nn.Module): + + def __init__(self, + + layers: list, + activation: str, + out_activation: str, + dropout: int, + classes: int, + input_dim: int, + ): + + super(P1B2Net, self).__init__() + + self.__p1b2_net = nn.Sequential() + + module_index = 0 + prev_dim = list(input_dim) + + # Define MLP architecture + + if layers is not None: + if type(layers) != list: + layers = list(layers) + for i, layer in enumerate(layers): + if i == 0: + #model.add(Dense(layer, input_shape=(x_train_len, 1))) + self.__p1b2_net.add_module('dense_%d' % module_index, + nn.Linear(prev_dim[0], layer, True)) + prev_dim[0] = layer + + else: + self.__p1b2_net.add_module('dense_%d' % module_index, + nn.Linear(prev_dim[0], layer, True)) + prev_dim[0] = layer + + self.__p1b2_net.add_module('activation_%d' % module_index, + build_activation(activation)) + if dropout: + #x = Dropout(gParameters['dropout'])(x) + self.__p1b2_net.add_module('dropout_%d' % module_index, + nn.Dropout(p=dropout)) + module_index += 1 + + #output = Dense(output_dim, activation=activation, + # kernel_initializer=initializer_weights, + # bias_initializer=initializer_bias)(x) + + #model.add(Dense(gParameters['classes'])) + self.__p1b2_net.add_module('dense_%d' % module_index, + nn.Linear(prev_dim[0], classes)) + prev_dim[0] = classes + module_index += 1 + + #model.add(Activation(gParameters['out_activation'])) + self.__p1b2_net.add_module('activation_%d' % module_index, + build_activation(out_activation, dim=1)) + else: + #output = Dense(output_dim, activation=activation, + # kernel_initializer=initializer_weights, + # bias_initializer=initializer_bias)(input_vector) + self.__p1b2_net.add_module('dense_%d' % module_index, + nn.Linear(prev_dim[0], classes)) + prev_dim[0] = classes + module_index += 1 + + #model.add(Activation(gParameters['out_activation'])) + self.__p1b2_net.add_module('activation_%d' % module_index, + build_activation(out_activation, dim=1)) + + + + #kernel_initializer=initializer_weights, + # bias_initializer=initializer_bias, + # kernel_regularizer=l2(gParameters['reg_l2']), + # activity_regularizer=l2(gParameters['reg_l2'] + + + # Weight Initialization ############################################### + if activation == 'relu': + self.__p1b2_net.apply(basic_weight_init_he_uniform_relu) + else: + self.__p1b2_net.apply(basic_weight_init_glorut_uniform) + + + + def forward(self, x): + return self.__p1b2_net(x) + diff --git a/Pilot1/P1B2/torch_deps/p1b2_clf_func.py b/Pilot1/P1B2/torch_deps/p1b2_clf_func.py new file mode 100644 index 00000000..40c7caeb --- /dev/null +++ b/Pilot1/P1B2/torch_deps/p1b2_clf_func.py @@ -0,0 +1,113 @@ +import os +import torch +import torch.utils.data +import torch.nn as nn +import torch.nn.functional as F +from pytorch_utils import build_loss + + +def train_p1b2_clf(device: torch.device, + + category_clf_net: nn.Module, + data_loader: torch.utils.data.DataLoader, + loss_type: str, + max_num_batches: int, + optimizer: torch.optim, + scheduler: torch.optim.lr_scheduler, + epoch: int, + log_interval: int, + dry_run: bool = False, ): + + category_clf_net.train() + pid = os.getpid() + + correct_category = 0 + train_loss = 0 + + for batch_idx, (rnaseq, cl_category) \ + in enumerate(data_loader): + + if batch_idx >= max_num_batches: + break + + rnaseq, cl_category = \ + rnaseq.to(device), cl_category.to(device) + + category_clf_net.zero_grad() + + out_category = category_clf_net(rnaseq) + + loss = build_loss(loss_type, out_category, cl_category) + train_loss += data_loader.batch_size * loss.item() # sum up batch loss + loss.backward() + + optimizer.step() + + if batch_idx % log_interval == 0: + print('{}\tTrain Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:5.5f}'.format( + pid, epoch+1, (batch_idx+1) * len(rnaseq), len(data_loader.dataset), + 100. * (batch_idx+1) / len(data_loader), loss.item())) + if dry_run: + break + + + pred_category = out_category.max(1, keepdim=True)[1] + + correct_category += pred_category.eq( + cl_category.view_as(pred_category)).sum().item() + + + # Get overall accuracy + train_loss /= len(data_loader.dataset) + category_acc = 100. * correct_category / len(data_loader.dataset) + + + print('\tP1B2 classification: ' + '\n\t\tTraining Loss: \t\t\t%5.5f ' + '\n\t\tTraining Accuracy: \t\t%5.2f%%' + % (train_loss, category_acc)) + + return category_acc, train_loss + + + +def valid_p1b2_clf( + device: torch.device, + category_clf_net: nn.Module, + data_loader: torch.utils.data.DataLoader, + loss_type: str, ): + + category_clf_net.eval() + + correct_category = 0 + + test_loss = 0 + + with torch.no_grad(): + for rnaseq, cl_category in data_loader: + + rnaseq, cl_category = \ + rnaseq.to(device), cl_category.to(device) + + out_category = category_clf_net(rnaseq) + + loss = build_loss(loss_type, out_category, cl_category) + test_loss += data_loader.batch_size * loss.item() # sum up batch loss + + pred_category = out_category.max(1, keepdim=True)[1] + + correct_category += pred_category.eq( + cl_category.view_as(pred_category)).sum().item() + + + # Get overall accuracy + test_loss /= len(data_loader.dataset) + category_acc = 100. * correct_category / len(data_loader.dataset) + + + print('\tP1B2 classification: ' + '\n\t\tValidation Loss: \t\t%5.5f ' + '\n\t\tValidation Accuracy: \t\t%5.2f%%' + % (test_loss, category_acc)) + + return category_acc, test_loss diff --git a/Pilot1/P1B2/torch_deps/p1b2_dataset_pytorch.py b/Pilot1/P1B2/torch_deps/p1b2_dataset_pytorch.py new file mode 100644 index 00000000..e1d4a2a0 --- /dev/null +++ b/Pilot1/P1B2/torch_deps/p1b2_dataset_pytorch.py @@ -0,0 +1,132 @@ +""" +Implements the dataset for P1B2 +""" +import numpy as np +import pandas as pd +import torch +import torch.utils.data as data +import p1b2 as bmk + +class P1B2Dataset(data.Dataset): + + input_dim = [] + __is_loaded = False + __X_train = [] + __Y_train = [] + __X_val = [] + __Y_val = [] + __X_test = [] + __Y_test = [] + + def __init__( + self, + data_root: str, + data_src: str, + training: int, + data_url: str, + train_data: str, + test_data: str, + classes: int, + feature_subsample: int, + shuffle: bool, + val_split: float, + rand_state: int = 0, + summary: bool = True, + + # Data type settings (for storage and data loading) + int_dtype: type = np.int8, + float_dtype: type = np.float16, + output_dtype: type = np.float32, + data_type: type = np.float16, + + # Pre-processing settings + scaling: str = 'minmax', + ): + + + # Initialization ###################################################### + self.__data_root = data_root + + # Class-wise variables + self.data_source = data_src + self.training = training + self.__rand_state = rand_state + self.__output_dtype = output_dtype + + # Load the data ################################################# + params = {} + params['data_url'] = data_url + params['train_data'] = train_data + params['test_data'] = test_data + params['feature_subsample'] = feature_subsample + params['classes'] = classes + params['shuffle'] = shuffle + params['scaling'] = scaling + params['val_split'] = val_split + params['data_type'] = data_type + + + if not P1B2Dataset.__is_loaded: + (X_train, Y_train), (X_val, Y_val), (X_test, Y_test) = bmk.load_data_one_hot(params, seed=rand_state) + P1B2Dataset.__is_loaded = True + + input_shape = X_train.shape[1:] + + P1B2Dataset.__X_train = X_train + P1B2Dataset.__Y_train = np.argmax(Y_train, axis=1) + P1B2Dataset.__X_val = X_val + P1B2Dataset.__Y_val = np.argmax(Y_val, axis=1) + P1B2Dataset.__X_test = X_test + P1B2Dataset.__Y_test = np.argmax(Y_test, axis=1) + + P1B2Dataset.input_dim = input_shape + + + if training == 1: + self.__data = P1B2Dataset.__X_train + self.__label = P1B2Dataset.__Y_train + data_str = 'Training' + elif training == 2: + self.__data = P1B2Dataset.__X_val + self.__label = P1B2Dataset.__Y_val + data_str = 'Validation' + else: + self.__data = P1B2Dataset.__X_test + self.__label = P1B2Dataset.__Y_test + data_str = 'Test' + + self.__out_dtype = data_type + self.__len = len(self.__data) + + # Public attributes ################################################### + self.input_dim = self.__data.shape + self.num_cells = self.input_dim[0] + self.rnaseq_dim = self.input_dim[1] + + # Dataset summary ##################################################### + if summary: + print('=' * 80) + print(data_str + + ' P1B2 Dataset Summary:') + print('\t%i Cell Lines (feature dim: %4i).' + % (self.num_cells, self.rnaseq_dim)) + print('=' * 80) + + + def __len__(self): + return self.__len + + def __getitem__(self, index): + + item_data = self.__data[index] + item_label = self.__label[index] + + item_data = np.asarray(item_data, dtype=self.__output_dtype) + + # Note that PyTorch requires np.int64 for classification labels + item_label = np.int64(item_label) + + item_data = torch.as_tensor(item_data) + item_label = torch.as_tensor(item_label) + + return item_data, item_label diff --git a/Pilot1/P1B2/torch_deps/p1b2_pytorch_model.py b/Pilot1/P1B2/torch_deps/p1b2_pytorch_model.py new file mode 100644 index 00000000..6bcc4f44 --- /dev/null +++ b/Pilot1/P1B2/torch_deps/p1b2_pytorch_model.py @@ -0,0 +1,297 @@ +import time + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from numpy.core.multiarray import ndarray +from torch.optim.lr_scheduler import ReduceLROnPlateau, LambdaLR + +#from utils.optimizer import get_optimizer +from torch_deps.random_seeding import seed_random_state +from torch_deps.p1b2_dataset_pytorch import P1B2Dataset +from torch_deps.p1b2_classification_net import P1B2Net +from pytorch_utils import build_optimizer +from torch_deps.p1b2_clf_func import train_p1b2_clf, valid_p1b2_clf +from torch.utils.tensorboard import SummaryWriter + +# Number of workers for dataloader. Too many workers might lead to process +# hanging for PyTorch version 4.1. Set this number between 0 and 4. +NUM_WORKER = 4 +DATA_ROOT = '../../Data/Pilot1/' + + +class P1B2Model(object): + + def __init__(self, args, use_cuda=False, device=torch.device('cpu')): + + self.args = args + self.use_cuda = use_cuda + self.device = device + self.val_cl_clf_acc = [] + self.config_data_loaders() + self.build_data_loaders() + self.build_nn() + self.config_optimization() + + + def config_data_loaders(self): + + args = self.args + + + # Data loaders for training/validation #################################### + self.dataloader_kwargs = { + 'timeout': 0, + 'shuffle': 'False', + # 'num_workers': multiprocessing.cpu_count() if use_cuda else 0, + 'num_workers': NUM_WORKER if self.use_cuda else 0, + 'pin_memory': True if self.use_cuda else False, } + + # Drug response dataloaders for training/validation + self.p1b2_dataset_kwargs = { + 'data_root': DATA_ROOT, + 'rand_state': args.rng_seed, + 'summary': True, + + 'data_url': args.data_url, + 'train_data': args.train_data, + 'test_data': args.test_data, + 'feature_subsample': args.feature_subsample, + 'classes': args.classes, + 'data_type': args.data_type, + 'shuffle': args.shuffle, + 'val_split': args.val_split, + + 'int_dtype': np.int8, + 'float_dtype': np.float16, + 'output_dtype': np.float32, + + 'feature_subsample': args.feature_subsample, } + + + def build_data_loaders(self): + + args = self.args + + self.p1b2_trn_loader = torch.utils.data.DataLoader( + P1B2Dataset(data_src=args.train_data, + training=1, + **(self.p1b2_dataset_kwargs)), + batch_size=args.batch_size, + **(self.dataloader_kwargs)) + + self.args.input_dim = P1B2Dataset.input_dim + + # Data loader for validation + self.p1b2_val_loader = torch.utils.data.DataLoader( + P1B2Dataset(data_src=args.test_data,training=2, + **(self.p1b2_dataset_kwargs)), + batch_size=args.batch_size, + **(self.dataloader_kwargs)) + + # Data loader for test + self.p1b2_test_loader = torch.utils.data.DataLoader( + P1B2Dataset(data_src=args.test_data,training=0, + **(self.p1b2_dataset_kwargs)), + batch_size=args.batch_size, + **(self.dataloader_kwargs)) + + + def build_nn(self): + + args = self.args + device = self.device + #args.input_dim = [1, 28204] + + # Sequence classifier + self.p1b2_net_kwargs = { + + 'layers': args.dense, + 'activation': args.activation, + 'out_activation': args.out_activation, + 'dropout': args.dropout, + #'classes': args.classes, + 'input_dim': args.input_dim, } + + self.p1b2_net = P1B2Net( + classes=args.classes, + **(self.p1b2_net_kwargs)).to(device) + + # Multi-GPU settings + if self.use_cuda and args.multi_gpu: + self.p1b2_net = nn.DataParallel(self.p1b2_net) + + + def config_optimization(self): + + args = self.args + weight_decay = args.reg_l2 + + type = args.optimizer + lr = args.learning_rate + #kerasDefaults = {} + #kerasDefaults['momentum_sgd'] = 0 + #kerasDefaults['nesterov_sgd'] = False + kerasDefaults = args.keras_defaults + kerasDefaults['weight_decay'] = weight_decay + + self.p1b2_optimizer = build_optimizer(self.p1b2_net, type, lr, kerasDefaults, trainable_only=False) + + #reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=10, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0) + self.p1b2_scheduler = ReduceLROnPlateau(self.p1b2_optimizer, mode='min', factor=0.1, patience=10, verbose=True, eps=0.0001, cooldown=0, min_lr=0) + + + def train(self): + + args = self.args + device = self.device + + # Writer will output to ./runs/ directory by default + tensorboard_writer = SummaryWriter(args.tensorboard_dir) + + # Training/validation loops ############################################### + self.val_cl_clf_acc = [] + + # CSV Logger ############################################################## + import csv + # open the file in the write mode + csv_logger_f = open(args.csv_filename, 'w', encoding='UTF8', newline='') + # create the csv writer + csv_logger = csv.writer(csv_logger_f) + csv_header = ['epoch', 'lr', 'train_accuracy', 'train_loss', 'val_accuracy', 'val_loss'] + # write the header + csv_logger.writerow(csv_header) + args.logger.info('{}'.format(csv_header)) + ############################################################################ + + self.start_time = time.time() + + + for epoch in range(args.epochs): + + print('=' * 80 + '\nTraining Epoch %3i:' % (epoch + 1)) + epoch_start_time = time.time() + + curr_lr = self.p1b2_optimizer.param_groups[0]['lr'] + + # Training Disease Type classifier + train_acc,train_loss = train_p1b2_clf(device=device, + category_clf_net=self.p1b2_net, + data_loader=self.p1b2_trn_loader, + loss_type=args.loss, + max_num_batches=args.max_num_batches, + optimizer=self.p1b2_optimizer, + scheduler=self.p1b2_scheduler, + epoch=epoch, + log_interval=args.log_interval, + dry_run=args.dry_run) + + # Min validation loss until epock - 1 + val_cl_clf_acc = np.array(self.val_cl_clf_acc).reshape(-1, 2) + min_val_loss = np.amin(val_cl_clf_acc, axis=0)[1] if (epoch > 0) else 1e8 + + val_acc,val_loss = self.validation(epoch) + + # Save the model if val_loss has improved + if val_loss < min_val_loss: + print('Train Epoch %3d: val_loss improved from %5.5f to %5.5f, saving model to %s' + % (epoch+1, min_val_loss, val_loss, args.model_autosave_filename)) + + if hasattr(self.p1b2_net,'module'): + # Saving the DataParallel model + torch.save(self.p1b2_net.module.state_dict(), args.model_autosave_filename) + else: + torch.save(self.p1b2_net.state_dict(), args.model_autosave_filename) + else: + print('Train Epoch %3d: val_loss did not improve from %5.5f' + % (epoch+1, min_val_loss)) + + # CSV logger + csv_data = [epoch, curr_lr, train_acc, train_loss, val_acc, val_loss] + csv_logger.writerow(csv_data) + args.logger.info('{}'.format(csv_data)) + + # Tensorboard logger + tensorboard_writer.add_scalar("Loss/train", train_loss, epoch) + tensorboard_writer.add_scalar("Loss/val", val_loss, epoch) + tensorboard_writer.add_scalar("Accuracy/train", train_acc/100, epoch) + tensorboard_writer.add_scalar("Accuracy/val", val_acc/100, epoch) + tensorboard_writer.add_scalar("Train/lr", curr_lr, epoch) + + # Run the scheduler + self.p1b2_scheduler.step(val_loss) + + print('Epoch Running Time: %.1f Seconds.' + % (time.time() - epoch_start_time)) + + # close the csv logger file + csv_logger_f.close() + tensorboard_writer.flush() + tensorboard_writer.close() + + def validation(self, epoch): + + args = self.args + device = self.device + + + # Validating Disease Type Classifier + cl_category_acc, category_loss = \ + valid_p1b2_clf(device=device, + category_clf_net=self.p1b2_net, + data_loader=self.p1b2_val_loader, + loss_type=args.loss, ) + + self.val_cl_clf_acc.append([cl_category_acc, category_loss]) + + return cl_category_acc,category_loss + + def test(self): + + args = self.args + device = self.device + + + # Test Disease Type Classifier + cl_category_acc, category_loss = \ + valid_p1b2_clf(device=device, + category_clf_net=self.p1b2_net, + data_loader=self.p1b2_test_loader, + loss_type=args.loss, ) + + #self.val_cl_clf_acc.append([cl_category_acc, category_loss]) + + return cl_category_acc,category_loss + + def print_final_stats(self): + + args = self.args + + val_cl_clf_acc = np.array(self.val_cl_clf_acc).reshape(-1, 2) + + print('Test data: ') + test_acc,test_loss = self.test() + + print('Program Running Time: %.1f Seconds.' % (time.time() - self.start_time)) + + # Print overall validation results + print('=' * 80) + print('Overall Validation Results:\n') + + print('\tBest Results from P1B2 Models (Epochs):') + # Print best accuracy for Disease Type classifiers + clf_targets = ['Disease Type Classifier from Somatic SNPs', + ] + best_acc = np.amax(val_cl_clf_acc, axis=0) + best_acc_epochs = np.argmax(val_cl_clf_acc, axis=0) + + for index, clf_target in enumerate(clf_targets): + print('\t\t%-24s Best Accuracy: %.3f%% (Epoch = %3d)' + % (clf_target, best_acc[index], + best_acc_epochs[index] + 1 )) + + print('=' * 80) + print('Test Results: Accuracy: %.3f%% Loss=%5.5f\n' + %(test_acc, test_loss)) + print('=' * 80) \ No newline at end of file diff --git a/Pilot1/P1B2/torch_deps/random_seeding.py b/Pilot1/P1B2/torch_deps/random_seeding.py new file mode 100644 index 00000000..c2f3b931 --- /dev/null +++ b/Pilot1/P1B2/torch_deps/random_seeding.py @@ -0,0 +1,36 @@ +""" + File Name: UnoPytorch/random_seeding.py + Author: Xiaotian Duan (xduan7) + Email: xduan7@uchicago.edu + Date: 8/22/18 + Python Version: 3.6.6 + File Description: + +""" +import random +import numpy as np +import torch + + +def seed_random_state(rand_state: int=0): + """seed_random_state(0) + + This function sets up with random seed in multiple libraries possibly used + during PyTorch training and validation. + + Args: + rand_state (int): random seed + + Returns: + None + """ + + random.seed(rand_state) + + np.random.seed(rand_state) + + torch.manual_seed(rand_state) + torch.cuda.manual_seed_all(rand_state) + + # This must be set to True for deterministic results in PyTorch 4.1 + torch.backends.cudnn.deterministic = True diff --git a/Pilot1/P1B2/torch_deps/weight_init.py b/Pilot1/P1B2/torch_deps/weight_init.py new file mode 100644 index 00000000..d67747b7 --- /dev/null +++ b/Pilot1/P1B2/torch_deps/weight_init.py @@ -0,0 +1,63 @@ +import torch.nn as nn + +# TODO: more complex weight initialization function +# * choice of initialization strategy +# * choice of bias terms +# * choice of different initialization for different types (Linear, Conv, etc.) + + +def basic_weight_init(module: nn.Module): + """weight_init(model) or model.apply(weight_init) + + This function initializes the weights of a module using xavier_normal. + + Args: + module (nn.Module): PyTorch module to be initialized. + Returns: + None + """ + if type(module) in [nn.Linear, ]: + nn.init.xavier_normal_(module.weight) + +def basic_weight_init_glorut_uniform(module: nn.Module): + """weight_init(model) or model.apply(weight_init) + + This function initializes the weights of a module using xavier_uniform. + + Args: + module (nn.Module): PyTorch module to be initialized. + Returns: + None + """ + if type(module) in [nn.Linear, ]: + nn.init.xavier_uniform_(module.weight) + +def basic_weight_init_he_normal_relu(module: nn.Module): + """weight_init(model) or model.apply(weight_init) + + This function initializes the weights of a module using kaiming_normal. + + Args: + module (nn.Module): PyTorch module to be initialized. + Returns: + None + """ + if type(module) in [nn.Linear, ]: + nn.init.kaiming_normal_(module.weight, mode='fan_in', nonlinearity='relu') + nn.init.zeros_(module.bias) + +def basic_weight_init_he_uniform_relu(module: nn.Module): + """weight_init(model) or model.apply(weight_init) + + This function initializes the weights of a module using kaiming_uniform. + + Args: + module (nn.Module): PyTorch module to be initialized. + Returns: + None + """ + if type(module) in [nn.Linear, ]: + nn.init.kaiming_uniform_(module.weight, mode='fan_in', nonlinearity='relu') + nn.init.zeros_(module.bias) + + diff --git a/Pilot1/TC1/pytorch_utils.py b/Pilot1/TC1/pytorch_utils.py new file mode 100644 index 00000000..cce885a7 --- /dev/null +++ b/Pilot1/TC1/pytorch_utils.py @@ -0,0 +1,139 @@ +from __future__ import absolute_import + +import torch +import torch.nn +import torch.nn.init +import torch.optim +import torch.nn.functional as F + +from default_utils import set_seed as set_seed_defaultUtils + +def set_parallelism_threads(): # for compatibility + pass + +def set_seed(seed): + """ Set the random number seed to the desired value + + Parameters + ---------- + seed : integer + Random number seed. + """ + + set_seed_defaultUtils(seed) + torch.manual_seed(seed) + + +def get_function(name): + mapping = {} + + # loss + mapping['mse'] = torch.nn.MSELoss() + mapping['binary_crossentropy'] = torch.nn.BCELoss() + mapping['categorical_crossentropy'] = torch.nn.CrossEntropyLoss() + mapping['smoothL1'] = torch.nn.SmoothL1Loss() + + mapped = mapping.get(name) + if not mapped: + raise Exception('No pytorch function found for "{}"'.format(name)) + + return mapped + + +def build_activation(type, dim=1): + + # activation + + if type=='relu': + return torch.nn.ReLU() + elif type=='sigmoid': + return torch.nn.Sigmoid() + elif type=='tanh': + return torch.nn.Tanh() + elif type=='softmax': + return torch.nn.Softmax(dim) + elif type=='log_softmax': + return torch.nn.LogSoftmax(dim) + +def build_optimizer(model, type, lr, kerasDefaults, trainable_only=True): + + if trainable_only: + params = filter(lambda p: p.requires_grad, model.parameters()) + else: + params = model.parameters() + + #schedule = optimizers.optimizer.Schedule() # constant lr (equivalent to default keras setting) + + if type == 'sgd': + return torch.optim.SGD(params, + lr=lr, + momentum=kerasDefaults['momentum_sgd'], + nesterov=kerasDefaults['nesterov_sgd']) + #schedule=schedule) + + elif type == 'rmsprop': + return torch.optim.RMSprop(model.parameters(), + lr=lr, + alpha=kerasDefaults['rho'], + eps=kerasDefaults['epsilon']) + #schedule=schedule) + + elif type == 'adagrad': + return torch.optim.Adagrad(model.parameters(), + lr=lr, + eps=kerasDefaults['epsilon']) + + elif type == 'adadelta': + return torch.optim.Adadelta(params, + eps=kerasDefaults['epsilon'], + rho=kerasDefaults['rho']) + + elif type == 'adam': + return torch.optim.Adam(params, + lr=lr, + betas=[kerasDefaults['beta_1'], kerasDefaults['beta_2']], + eps=kerasDefaults['epsilon']) + + + +def initialize(weights, type, kerasDefaults, seed=None, constant=0.): + + if type == 'constant': + return torch.nn.init.constant_(weights, + val=constant) + + elif type == 'uniform': + return torch.nn.init.uniform(weights, + a=kerasDefaults['minval_uniform'], + b=kerasDefaults['maxval_uniform']) + + elif type == 'normal': + return torch.nn.init.normal(weights, + mean=kerasDefaults['mean_normal'], + std=kerasDefaults['stddev_normal']) + + elif type == 'glorot_normal': # not quite Xavier + return torch.nn.init.xavier_normal(weights) + + elif type == 'glorot_uniform': + return torch.nn.init.xavier_uniform_(weights) + + elif type == 'he_normal': + return torch.nn.init.kaiming_uniform(weights) + + +def xent(y_true, y_pred): + return F.cross_entropy(y_pred, y_true) + + +def mse(y_true, y_pred): + return F.mse_loss(y_pred, y_true) + +def build_loss(type, y_pred, y_true): + + if type=='categorical_crossentropy': + return xent(y_true, y_pred) + elif type=='mse': + return mse(y_true, y_pred) + elif type=='nll': + return F.nll_loss(y_pred, y_true) \ No newline at end of file diff --git a/Pilot1/TC1/tc1_baseline_pytorch.py b/Pilot1/TC1/tc1_baseline_pytorch.py new file mode 100644 index 00000000..cea7dcff --- /dev/null +++ b/Pilot1/TC1/tc1_baseline_pytorch.py @@ -0,0 +1,138 @@ +from __future__ import print_function + +import numpy as np +import os +import sys + +import datetime + +import torch + + +if True: + print("Restricting #of GPUs to 8") + os.environ["CUDA_VISIBLE_DEVICES"]="0,1,2,3,4,5,6,7" + #os.environ["CUDA_VISIBLE_DEVICES"]="0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15" + +file_path = os.path.dirname(os.path.realpath(__file__)) +lib_path2 = os.path.abspath(os.path.join(file_path, '..', '..', 'common')) +sys.path.append(lib_path2) +os.chdir(file_path) + +import tc1 as bmk +import candle +from torch_deps.tc1_pytorch_model import TC1Model +from torch_deps.random_seeding import seed_random_state + +np.set_printoptions(precision=4) + +def initialize_parameters(default_model = 'tc1_default_model_pytorch.txt'): + + # Build benchmark object + tc1Bmk = bmk.BenchmarkTC1(file_path, default_model, 'pytorch', + prog='tc1_baseline', desc='Multi-task (DNN) for data extraction from clinical reports - Pilot 3 Benchmark 1') + + print("Created Tc1 benchmark") + + # Initialize parameters + gParameters = candle.finalize_parameters(tc1Bmk) + #benchmark.logger.info('Params: {}'.format(gParameters)) + print("Parameters initialized") + + return gParameters + + +def run(params): + + args = candle.ArgumentStruct(**params) + args.no_cuda = args.no_cuda if hasattr(args,'no_cuda') else False + args.multi_gpu = args.multi_gpu if hasattr(args,'multi_gpu') else True + args.max_num_batches = args.max_num_batches if hasattr(args,'max_num_batches') else 1000 + args.dry_run = args.dry_run if hasattr(args,'dry_run') else False + args.log_interval = args.log_interval if hasattr(args,'log_interval') else 8 + + + seed = args.rng_seed + candle.set_seed(seed) + # Setting up random seed for reproducible and deterministic results + seed_random_state(args.rng_seed) + + # Construct extension to save validation results + now = datetime.datetime.now() + ext = '%02d%02d_%02d%02d_pytorch' \ + % (now.month, now.day, now.hour, now.minute) + + candle.verify_path(params['save_path']) + prefix = '{}{}'.format(params['save_path'], ext) + logfile = params['logfile'] if params['logfile'] else prefix+'.log' + candle.set_up_logger(logfile, bmk.logger, params['verbose']) + bmk.logger.info('Params: {}'.format(params)) + + args.tensorboard_dir = "tb/{}".format(ext) + args.logger = bmk.logger + + #Autosave model + model_name = params['model_name'] + args_filename = "{}.model.args".format(params['save_path']) + args.model_autosave_filename = "{}.autosave.model.pth".format(params['save_path']) + # CSV logging + args.csv_filename = '{}{}_training.log'.format(params['save_path'], ext) + + # Computation device config (cuda or cpu) + use_cuda = not args.no_cuda and torch.cuda.is_available() + device = torch.device('cuda' if use_cuda else 'cpu') + + # save args to file + import pickle + args_file = open(args_filename, 'wb') + pickle.dump(args, args_file) + args_file.close() + + modelTc1 = TC1Model(args, use_cuda, device) + + #model.summary() + #print(modelTc1.tc1_net) # Model summary + bmk.logger.info('Model summary: {}'.format(modelTc1.tc1_net)) # Model summary + + modelTc1.train() + modelTc1.print_final_stats() + + #Save model + model_name = params['model_name'] + model_filename = "{}.model_state_dict.pth".format(params['save_path']) + if hasattr(modelTc1.tc1_net,'module'): + # Saving the DataParallel model + torch.save(modelTc1.tc1_net.module.state_dict(), model_filename) + else: + torch.save(modelTc1.tc1_net.state_dict(), model_filename) + + #reload args from file + args_file = open(args_filename, 'rb') + loaded_args = pickle.load(args_file) + args_file.close() + + # load weights into new model + loaded_modelTc1 = TC1Model(loaded_args) + loaded_modelTc1.tc1_net.load_state_dict(torch.load(model_filename, map_location=torch.device('cpu'))) + print("Loaded torch model from disk") + + # evaluate loaded model on test data + loaded_modelTc1.tc1_net.eval() + val_acc,val_loss = loaded_modelTc1.validation(0) + + print("Model State Dict Test loss: %5.2f" % (val_loss)) + print("Model State Dict Test accuracy: %5.2f%%" %(val_acc)) + + +def main(): + + gParameters = initialize_parameters() + run(gParameters) + +if __name__ == '__main__': + main() + try: + tmp = 1 + except AttributeError: # theano does not have this function + pass + diff --git a/Pilot1/TC1/tc1_default_model_pytorch.txt b/Pilot1/TC1/tc1_default_model_pytorch.txt new file mode 100644 index 00000000..10e2a4cc --- /dev/null +++ b/Pilot1/TC1/tc1_default_model_pytorch.txt @@ -0,0 +1,25 @@ +[Global_Params] +data_url = 'http://ftp.mcs.anl.gov/pub/candle/public/benchmarks/Pilot1/type-class/' +train_data = 'type_18_300_train.csv' +test_data = 'type_18_300_test.csv' +model_name = 'tc1' +conv=[128, 20, 1, 128, 10, 1] +dense=[200,20] +activation='relu' +out_activation='log_softmax' +loss='nll' +optimizer='sgd' +metrics='accuracy' +epochs=75 +batch_size=128 +dropout=0.1 +classes=36 +feature_subsample=0 +pool=[1, 10] +output_dir='.' + +# Miscellaneous settings ################################## +# multi_gpu=True +# no_cuda=True +# rng_seed=0 +save_path='Output/TC1' \ No newline at end of file diff --git a/Pilot1/TC1/torch_deps/classification_net.py b/Pilot1/TC1/torch_deps/classification_net.py new file mode 100644 index 00000000..3c7a1794 --- /dev/null +++ b/Pilot1/TC1/torch_deps/classification_net.py @@ -0,0 +1,134 @@ +import torch +import torch.nn as nn +import numpy as np +from pytorch_utils import build_activation +from torch_deps.weight_init import basic_weight_init, basic_weight_init_he_normal_relu, basic_weight_init_he_uniform_relu, basic_weight_init_glorut_uniform + +class Tc1Net(nn.Module): + + def __init__(self, + + conv: list, + dense: list, + activation: str, + out_activation: str, + dropout: int, + classes: int, + pool: list, + locally_connected: bool, + input_dim: int, + + ): + + super(Tc1Net, self).__init__() + + self.__tc1_net = nn.Sequential() + + module_index = 0 + prev_dim = list(input_dim) + + dense_first = True + + layer_list = list(range(0, len(conv), 3)) + for l, i in enumerate(layer_list): + filters = conv[i] + filter_len = conv[i+1] + stride = conv[i+2] + print(i/3, filters, filter_len, stride) + if pool: + pool_list=pool + if type(pool_list) != list: + pool_list=list(pool_list) + + if filters <= 0 or filter_len <= 0 or stride <= 0: + break + dense_first = False + + if locally_connected: + test = 1 + # model.add(LocallyConnected1D(filters, filter_len, strides=stride, padding='valid', input_shape=(x_train_len, 1))) + else: + #input layer + if i == 0: + self.__tc1_net.add_module('conv_%d' % module_index, + nn.Conv1d(in_channels=1, out_channels=filters, kernel_size=filter_len, stride=stride, padding=0)) + prev_dim[0] = filters + prev_dim[1] = (prev_dim[1] - (filter_len - 1) -1)//stride + 1 #need to add stride + #model.add(Conv1D(filters=filters, kernel_size=filter_len, strides=stride, padding='valid', input_shape=(x_train_len, 1))) + else: + #model.add(Conv1D(filters=filters, kernel_size=filter_len, strides=stride, padding='valid')) + self.__tc1_net.add_module('conv_%d' % module_index, + nn.Conv1d(in_channels=prev_dim[0], out_channels=filters, kernel_size=filter_len, stride=stride, padding=0)) + prev_dim[0] = filters + prev_dim[1] = (prev_dim[1] - (filter_len - 1) -1)//stride + 1 #need to add stride + + #model.add(Activation(gParameters['activation'])) + self.__tc1_net.add_module('activation_%d' % module_index, + build_activation(activation)) + if pool: + #model.add(MaxPooling1D(pool_size=pool_list[i//3]))pool_list[i//3] + pool_size = pool_list[i//3] + self.__tc1_net.add_module('activation_%d' % module_index, + nn.MaxPool1d(kernel_size=pool_size, stride=pool_size, padding=0)) + prev_dim[1] = (prev_dim[1] - (pool_size - 1) -1)//pool_size + 1 #need to add stride + + module_index += 1 + if not dense_first: + #model.add(Flatten()) + self.__tc1_net.add_module('flatten_%d' % module_index, + nn.Flatten()) + prev_dim[0] = prev_dim[0]*prev_dim[1] + prev_dim[1] = 1 + module_index += 1 + + for i, layer in enumerate(dense): + if layer: + if i == 0 and dense_first: + #model.add(Dense(layer, input_shape=(x_train_len, 1))) + self.__tc1_net.add_module('dense_%d' % module_index, + nn.Linear(prev_dim[0], layer)) + prev_dim[0] = layer + else: + #model.add(Dense(layer)) + self.__tc1_net.add_module('dense_%d' % module_index, + nn.Linear(prev_dim[0], layer)) + prev_dim[0] = layer + #model.add(Activation(gParameters['activation'])) + self.__tc1_net.add_module('activation_%d' % module_index, + build_activation(activation)) + if dropout: + #model.add(Dropout(gParameters['dropout'])) + self.__tc1_net.add_module('dropout_%d' % module_index, + nn.Dropout(p=dropout)) + module_index += 1 + + # Weight Initialization ############################################### + if activation == 'relu': + self.__tc1_net.apply(basic_weight_init_he_uniform_relu) + else: + self.__tc1_net.apply(basic_weight_init) + + + if dense_first: + #model.add(Flatten()) + self.__tc1_net.add_module('flatten_%d' % module_index, + nn.Flatten()) + prev_dim[0] = prev_dim[0]*prev_dim[1] + prev_dim[1] = 1 + module_index += 1 + + #model.add(Dense(gParameters['classes'])) + self.__tc1_net.add_module('dense_%d' % module_index, + nn.Linear(prev_dim[0], classes)) + prev_dim[0] = classes + module_index += 1 + + #model.add(Activation(gParameters['out_activation'])) + self.__tc1_net.add_module('activation_%d' % module_index, + build_activation(out_activation, dim=1)) + + + + def forward(self, x): + return self.__tc1_net(x) + diff --git a/Pilot1/TC1/torch_deps/random_seeding.py b/Pilot1/TC1/torch_deps/random_seeding.py new file mode 100644 index 00000000..c2f3b931 --- /dev/null +++ b/Pilot1/TC1/torch_deps/random_seeding.py @@ -0,0 +1,36 @@ +""" + File Name: UnoPytorch/random_seeding.py + Author: Xiaotian Duan (xduan7) + Email: xduan7@uchicago.edu + Date: 8/22/18 + Python Version: 3.6.6 + File Description: + +""" +import random +import numpy as np +import torch + + +def seed_random_state(rand_state: int=0): + """seed_random_state(0) + + This function sets up with random seed in multiple libraries possibly used + during PyTorch training and validation. + + Args: + rand_state (int): random seed + + Returns: + None + """ + + random.seed(rand_state) + + np.random.seed(rand_state) + + torch.manual_seed(rand_state) + torch.cuda.manual_seed_all(rand_state) + + # This must be set to True for deterministic results in PyTorch 4.1 + torch.backends.cudnn.deterministic = True diff --git a/Pilot1/TC1/torch_deps/tc1_clf_func.py b/Pilot1/TC1/torch_deps/tc1_clf_func.py new file mode 100644 index 00000000..ed05b7bb --- /dev/null +++ b/Pilot1/TC1/torch_deps/tc1_clf_func.py @@ -0,0 +1,114 @@ +import os +import torch +import torch.utils.data +import torch.nn as nn +import torch.nn.functional as F +from pytorch_utils import build_loss + + +def train_tc1_clf(device: torch.device, + + category_clf_net: nn.Module, + data_loader: torch.utils.data.DataLoader, + loss_type: str, + max_num_batches: int, + optimizer: torch.optim, + scheduler: torch.optim.lr_scheduler, + epoch: int, + log_interval: int, + dry_run: bool = False, ): + + category_clf_net.train() + pid = os.getpid() + + correct_category = 0 + train_loss = 0 + + for batch_idx, (rnaseq, cl_category) \ + in enumerate(data_loader): + + if batch_idx >= max_num_batches: + break + + rnaseq, cl_category = \ + rnaseq.to(device), cl_category.to(device) + + category_clf_net.zero_grad() + + out_category = category_clf_net(rnaseq) + + loss = build_loss(loss_type, out_category, cl_category) + train_loss += data_loader.batch_size * loss.item() # sum up batch loss + loss.backward() + + optimizer.step() + + if batch_idx % log_interval == 0: + print('{}\tTrain Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:5.5f}'.format( + pid, epoch+1, (batch_idx+1) * len(rnaseq), len(data_loader.dataset), + 100. * (batch_idx+1) / len(data_loader), loss.item())) + if dry_run: + break + + + pred_category = out_category.max(1, keepdim=True)[1] + + correct_category += pred_category.eq( + cl_category.view_as(pred_category)).sum().item() + + + # Get overall accuracy + train_loss /= len(data_loader.dataset) + category_acc = 100. * correct_category / len(data_loader.dataset) + + #scheduler.step(loss) + + print('\tRNA-seq classification: ' + '\n\t\tTraining Loss: \t\t\t%5.5f ' + '\n\t\tTraining Accuracy: \t\t%5.2f%%' + % (train_loss, category_acc)) + + return category_acc, train_loss + + + +def valid_tc1_clf( + device: torch.device, + category_clf_net: nn.Module, + data_loader: torch.utils.data.DataLoader, + loss_type: str, ): + + category_clf_net.eval() + + correct_category = 0 + + test_loss = 0 + + with torch.no_grad(): + for rnaseq, cl_category in data_loader: + + rnaseq, cl_category = \ + rnaseq.to(device), cl_category.to(device) + + out_category = category_clf_net(rnaseq) + + loss = build_loss(loss_type, out_category, cl_category) + test_loss += data_loader.batch_size * loss.item() # sum up batch loss + + pred_category = out_category.max(1, keepdim=True)[1] + + correct_category += pred_category.eq( + cl_category.view_as(pred_category)).sum().item() + + + # Get overall accuracy + test_loss /= len(data_loader.dataset) + category_acc = 100. * correct_category / len(data_loader.dataset) + + + print('\tRNA-seq classification: ' + '\n\t\tValidation Loss: \t\t%5.5f ' + '\n\t\tValidation Accuracy: \t\t%5.2f%%' + % (test_loss, category_acc)) + + return category_acc, test_loss diff --git a/Pilot1/TC1/torch_deps/tc1_dataset_pytorch.py b/Pilot1/TC1/torch_deps/tc1_dataset_pytorch.py new file mode 100644 index 00000000..e6dc5944 --- /dev/null +++ b/Pilot1/TC1/torch_deps/tc1_dataset_pytorch.py @@ -0,0 +1,119 @@ +""" +Implements the dataset for TC1 +""" +import numpy as np +import pandas as pd +import torch +import torch.utils.data as data +import tc1 as bmk + +class TC1Dataset(data.Dataset): + + input_dim = [] + __is_loaded = False + __X_train = [] + __Y_train = [] + __X_test = [] + __Y_test = [] + + def __init__( + self, + data_root: str, + data_src: str, + training: bool, + data_url: str, + train_data: str, + test_data: str, + classes: int, + feature_subsample: int, + rand_state: int = 0, + summary: bool = True, + + # Data type settings (for storage and data loading) + int_dtype: type = np.int8, + float_dtype: type = np.float16, + output_dtype: type = np.float32, + data_type: type = np.float16, + + # Pre-processing settings + scaling: str = 'maxabs', + ): + + # Initialization ###################################################### + self.__data_root = data_root + + # Class-wise variables + self.data_source = data_src + self.training = training + self.__rand_state = rand_state + self.__output_dtype = output_dtype + + # Load the data ################################################# + params = {} + params['data_url'] = data_url + params['train_data'] = train_data + params['test_data'] = test_data + params['feature_subsample'] = feature_subsample + params['classes'] = classes + params['data_type'] = data_type + + if not TC1Dataset.__is_loaded: + X_train, Y_train, X_test, Y_test = bmk.load_data(params) + TC1Dataset.__is_loaded = True + + # channel first + X_train = X_train.reshape(X_train.shape[0], 1, X_train.shape[1]) + X_test = X_test.reshape(X_test.shape[0], 1, X_test.shape[1]) + + input_shape = X_train.shape[1:] + + TC1Dataset.__X_train = X_train + TC1Dataset.__Y_train = np.argmax(Y_train, axis=1) + TC1Dataset.__X_test = X_test + TC1Dataset.__Y_test = np.argmax(Y_test, axis=1) + + TC1Dataset.input_dim = input_shape + + if training: + self.__data = TC1Dataset.__X_train + self.__label = TC1Dataset.__Y_train + + else: + self.__data = TC1Dataset.__X_test + self.__label = TC1Dataset.__Y_test + + self.__out_dtype = data_type + self.__len = len(self.__data) + + # Public attributes ################################################### + self.input_dim = self.__data.shape + self.num_cells = self.input_dim[0] + self.rnaseq_dim = self.input_dim[2] + + # Dataset summary ##################################################### + if summary: + print('=' * 80) + print(('Training' if self.training else 'Validation') + + ' TC1 Dataset Summary:') + print('\t%i Cell Lines (feature dim: %4i).' + % (self.num_cells, self.rnaseq_dim)) + print('=' * 80) + + + def __len__(self): + return self.__len + + def __getitem__(self, index): + + item_data = self.__data[index] + item_label = self.__label[index] + + item_data = np.asarray(item_data, dtype=self.__output_dtype) + + # Note that PyTorch requires np.int64 for classification labels + item_label = np.int64(item_label) + + item_data = torch.as_tensor(item_data) + item_label = torch.as_tensor(item_label) + + return item_data, item_label diff --git a/Pilot1/TC1/torch_deps/tc1_pytorch_model.py b/Pilot1/TC1/torch_deps/tc1_pytorch_model.py new file mode 100644 index 00000000..8bd8b340 --- /dev/null +++ b/Pilot1/TC1/torch_deps/tc1_pytorch_model.py @@ -0,0 +1,264 @@ +import time + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from numpy.core.multiarray import ndarray +from torch.optim.lr_scheduler import ReduceLROnPlateau + +#from utils.optimizer import get_optimizer +from torch_deps.random_seeding import seed_random_state +from torch_deps.tc1_dataset_pytorch import TC1Dataset +from torch_deps.classification_net import Tc1Net +from pytorch_utils import build_optimizer +from torch_deps.tc1_clf_func import train_tc1_clf, valid_tc1_clf +from torch.utils.tensorboard import SummaryWriter + +# Number of workers for dataloader. Too many workers might lead to process +# hanging for PyTorch version 4.1. Set this number between 0 and 4. +NUM_WORKER = 4 +DATA_ROOT = '../../Data/Pilot1/' + + +class TC1Model(object): + + def __init__(self, args, use_cuda=False, device=torch.device('cpu')): + + self.args = args + self.use_cuda = use_cuda + self.device = device + self.val_cl_clf_acc = [] + self.config_data_loaders() + self.build_data_loaders() + self.build_nn() + self.config_optimization() + + + def config_data_loaders(self): + + args = self.args + + + # Data loaders for training/validation #################################### + self.dataloader_kwargs = { + 'timeout': 0, + 'shuffle': 'False', + # 'num_workers': multiprocessing.cpu_count() if use_cuda else 0, + 'num_workers': NUM_WORKER if self.use_cuda else 0, + 'pin_memory': True if self.use_cuda else False, } + + # Drug response dataloaders for training/validation + self.tc1_dataset_kwargs = { + 'data_root': DATA_ROOT, + 'rand_state': args.rng_seed, + 'summary': True, + + 'data_url': args.data_url, + 'train_data': args.train_data, + 'test_data': args.test_data, + 'feature_subsample': args.feature_subsample, + 'classes': args.classes, + 'data_type': args.data_type, + + 'int_dtype': np.int8, + 'float_dtype': np.float16, + 'output_dtype': np.float32, + + 'feature_subsample': args.feature_subsample, } + + + def build_data_loaders(self): + + args = self.args + + self.tc1_trn_loader = torch.utils.data.DataLoader( + TC1Dataset(data_src=args.train_data, + training=True, + **(self.tc1_dataset_kwargs)), + batch_size=args.batch_size, + **(self.dataloader_kwargs)) + + self.args.input_dim = TC1Dataset.input_dim + + # Data loader for validation + self.tc1_val_loader = torch.utils.data.DataLoader( + TC1Dataset(data_src=args.test_data,training=False, + **(self.tc1_dataset_kwargs)), + batch_size=args.batch_size, + **(self.dataloader_kwargs)) + + + def build_nn(self): + + args = self.args + device = self.device + #args.input_dim = [1, 60483] + + # Sequence classifier + self.tc1_net_kwargs = { + + 'conv': args.conv, + 'dense': args.dense, + 'activation': args.activation, + 'out_activation': args.out_activation, + 'dropout': args.dropout, + #'classes': args.classes, + 'pool': args.pool if hasattr(args,'pool') else False, + 'locally_connected': args.locally_connected if hasattr(args,'locally_connected') else False, + 'input_dim': args.input_dim, } + + self.tc1_net = Tc1Net( + classes=args.classes, + **(self.tc1_net_kwargs)).to(device) + + # Multi-GPU settings + if self.use_cuda and args.multi_gpu: + self.tc1_net = nn.DataParallel(self.tc1_net) + + + def config_optimization(self): + + args = self.args + + type = args.optimizer + lr = 0.01 + kerasDefaults = {} + kerasDefaults['momentum_sgd'] = 0 + kerasDefaults['nesterov_sgd'] = False + self.tc1_optimizer = build_optimizer(self.tc1_net, type, lr, kerasDefaults, trainable_only=False) + + #reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=10, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0) + self.tc1_scheduler = ReduceLROnPlateau(self.tc1_optimizer, mode='min', factor=0.1, patience=10, verbose=True, eps=0.0001, cooldown=0, min_lr=0) + + + def train(self): + + args = self.args + device = self.device + + # Writer will output to ./runs/ directory by default + tensorboard_writer = SummaryWriter(args.tensorboard_dir) + + # Training/validation loops ############################################### + self.val_cl_clf_acc = [] + + # CSV Logger ############################################################## + import csv + # open the file in the write mode + csv_logger_f = open(args.csv_filename, 'w', encoding='UTF8', newline='') + # create the csv writer + csv_logger = csv.writer(csv_logger_f) + csv_header = ['epoch', 'lr', 'train_accuracy', 'train_loss', 'val_accuracy', 'val_loss'] + # write the header + csv_logger.writerow(csv_header) + args.logger.info('{}'.format(csv_header)) + ############################################################################ + + self.start_time = time.time() + + + for epoch in range(args.epochs): + + print('=' * 80 + '\nTraining Epoch %3i:' % (epoch + 1)) + epoch_start_time = time.time() + + curr_lr = self.tc1_optimizer.param_groups[0]['lr'] + + # Training RNA-seq gene expression classifier + train_acc,train_loss = train_tc1_clf(device=device, + category_clf_net=self.tc1_net, + data_loader=self.tc1_trn_loader, + loss_type=args.loss, + max_num_batches=args.max_num_batches, + optimizer=self.tc1_optimizer, + scheduler=self.tc1_scheduler, + epoch=epoch, + log_interval=args.log_interval, + dry_run=args.dry_run) + + # Min validation loss until epock - 1 + val_cl_clf_acc = np.array(self.val_cl_clf_acc).reshape(-1, 2) + min_val_loss = np.amin(val_cl_clf_acc, axis=0)[1] if (epoch > 0) else 1e8 + + val_acc,val_loss = self.validation(epoch) + + # Save the model if val_loss has improved + if val_loss < min_val_loss: + print('Train Epoch %3d: val_loss improved from %5.5f to %5.5f, saving model to %s' + % (epoch+1, min_val_loss, val_loss, args.model_autosave_filename)) + + if hasattr(self.tc1_net,'module'): + # Saving the DataParallel model + torch.save(self.tc1_net.module.state_dict(), args.model_autosave_filename) + else: + torch.save(self.tc1_net.state_dict(), args.model_autosave_filename) + else: + print('Train Epoch %3d: val_loss did not improve from %5.5f' + % (epoch+1, min_val_loss)) + + # CSV logger + csv_data = [epoch, curr_lr, train_acc, train_loss, val_acc, val_loss] + csv_logger.writerow(csv_data) + args.logger.info('{}'.format(csv_data)) + + # Tensorboard logger + tensorboard_writer.add_scalar("Loss/train", train_loss, epoch) + tensorboard_writer.add_scalar("Loss/val", val_loss, epoch) + tensorboard_writer.add_scalar("Accuracy/train", train_acc/100, epoch) + tensorboard_writer.add_scalar("Accuracy/val", val_acc/100, epoch) + tensorboard_writer.add_scalar("Train/lr", curr_lr, epoch) + + # Run the scheduler + #self.tc1_scheduler.step(torch.as_tensor(val_loss)) + self.tc1_scheduler.step(val_loss) + + print('Epoch Running Time: %.1f Seconds.' + % (time.time() - epoch_start_time)) + + # close the csv logger file + csv_logger_f.close() + tensorboard_writer.flush() + tensorboard_writer.close() + + def validation(self, epoch): + + args = self.args + device = self.device + + + # Validating RNA-seq gene expression classifier + cl_category_acc, category_loss = \ + valid_tc1_clf(device=device, + category_clf_net=self.tc1_net, + data_loader=self.tc1_val_loader, + loss_type=args.loss, ) + + self.val_cl_clf_acc.append([cl_category_acc, category_loss]) + + return cl_category_acc,category_loss + + + def print_final_stats(self): + + args = self.args + + val_cl_clf_acc = np.array(self.val_cl_clf_acc).reshape(-1, 2) + + print('Program Running Time: %.1f Seconds.' % (time.time() - self.start_time)) + + # Print overall validation results + print('=' * 80) + print('Overall Validation Results:\n') + + print('\tBest Results from TC1 Models (Epochs):') + # Print best accuracy for RNA-seq gene expression classifiers + clf_targets = ['RNA-seq gene expression Categories', + ] + best_acc = np.amax(val_cl_clf_acc, axis=0) + best_acc_epochs = np.argmax(val_cl_clf_acc, axis=0) + + for index, clf_target in enumerate(clf_targets): + print('\t\t%-24s Best Accuracy: %.3f%% (Epoch = %3d)' + % (clf_target, best_acc[index], + best_acc_epochs[index] + 1 )) diff --git a/Pilot1/TC1/torch_deps/weight_init.py b/Pilot1/TC1/torch_deps/weight_init.py new file mode 100644 index 00000000..d67747b7 --- /dev/null +++ b/Pilot1/TC1/torch_deps/weight_init.py @@ -0,0 +1,63 @@ +import torch.nn as nn + +# TODO: more complex weight initialization function +# * choice of initialization strategy +# * choice of bias terms +# * choice of different initialization for different types (Linear, Conv, etc.) + + +def basic_weight_init(module: nn.Module): + """weight_init(model) or model.apply(weight_init) + + This function initializes the weights of a module using xavier_normal. + + Args: + module (nn.Module): PyTorch module to be initialized. + Returns: + None + """ + if type(module) in [nn.Linear, ]: + nn.init.xavier_normal_(module.weight) + +def basic_weight_init_glorut_uniform(module: nn.Module): + """weight_init(model) or model.apply(weight_init) + + This function initializes the weights of a module using xavier_uniform. + + Args: + module (nn.Module): PyTorch module to be initialized. + Returns: + None + """ + if type(module) in [nn.Linear, ]: + nn.init.xavier_uniform_(module.weight) + +def basic_weight_init_he_normal_relu(module: nn.Module): + """weight_init(model) or model.apply(weight_init) + + This function initializes the weights of a module using kaiming_normal. + + Args: + module (nn.Module): PyTorch module to be initialized. + Returns: + None + """ + if type(module) in [nn.Linear, ]: + nn.init.kaiming_normal_(module.weight, mode='fan_in', nonlinearity='relu') + nn.init.zeros_(module.bias) + +def basic_weight_init_he_uniform_relu(module: nn.Module): + """weight_init(model) or model.apply(weight_init) + + This function initializes the weights of a module using kaiming_uniform. + + Args: + module (nn.Module): PyTorch module to be initialized. + Returns: + None + """ + if type(module) in [nn.Linear, ]: + nn.init.kaiming_uniform_(module.weight, mode='fan_in', nonlinearity='relu') + nn.init.zeros_(module.bias) + + From 2654b2e20376c894aef0f499f59bd2c2fd9bb436 Mon Sep 17 00:00:00 2001 From: knangia04 Date: Sat, 18 Sep 2021 00:00:48 -0500 Subject: [PATCH 2/2] Updated to resolve comments on pull request 1. Updated to use the original default_model txt file. 2. Updated common/pytorch_utils.py based on the the local pytorch_utils.py and deleted the local versions. --- Pilot1/P1B2/p1b2_baseline_pytorch.py | 14 +-- Pilot1/P1B2/p1b2_default_model_pytorch.txt | 28 ----- Pilot1/P1B2/pytorch_utils.py | 140 --------------------- Pilot1/TC1/pytorch_utils.py | 139 -------------------- Pilot1/TC1/tc1_baseline_pytorch.py | 11 +- Pilot1/TC1/tc1_default_model_pytorch.txt | 25 ---- common/pytorch_utils.py | 30 +++-- 7 files changed, 34 insertions(+), 353 deletions(-) delete mode 100644 Pilot1/P1B2/p1b2_default_model_pytorch.txt delete mode 100644 Pilot1/P1B2/pytorch_utils.py delete mode 100644 Pilot1/TC1/pytorch_utils.py delete mode 100644 Pilot1/TC1/tc1_default_model_pytorch.txt diff --git a/Pilot1/P1B2/p1b2_baseline_pytorch.py b/Pilot1/P1B2/p1b2_baseline_pytorch.py index 3b0a215d..54a9e042 100644 --- a/Pilot1/P1B2/p1b2_baseline_pytorch.py +++ b/Pilot1/P1B2/p1b2_baseline_pytorch.py @@ -6,13 +6,6 @@ import torch - -if True: - print("Restricting #of GPUs to 1") - os.environ["CUDA_VISIBLE_DEVICES"]="0" - #os.environ["CUDA_VISIBLE_DEVICES"]="0,1,2,3,4,5,6,7" - #os.environ["CUDA_VISIBLE_DEVICES"]="0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15" - file_path = os.path.dirname(os.path.realpath(__file__)) lib_path2 = os.path.abspath(os.path.join(file_path, '..', '..', 'common')) sys.path.append(lib_path2) @@ -25,7 +18,7 @@ np.set_printoptions(precision=4) -def initialize_parameters(default_model = 'p1b2_default_model_pytorch.txt'): +def initialize_parameters(default_model = 'p1b2_default_model.txt'): # Build benchmark object p1b2Bmk = bmk.BenchmarkP1B2(bmk.file_path, default_model, 'pytorch', @@ -50,7 +43,12 @@ def run(params): args.dry_run = args.dry_run if hasattr(args,'dry_run') else False args.log_interval = args.log_interval if hasattr(args,'log_interval') else 10 + args.classes = args.classes if hasattr(args,'classes') else 10 + if args.loss=='categorical_crossentropy': + args.out_activation='log_softmax' + args.loss='nll' + seed = args.rng_seed candle.set_seed(seed) # Setting up random seed for reproducible and deterministic results diff --git a/Pilot1/P1B2/p1b2_default_model_pytorch.txt b/Pilot1/P1B2/p1b2_default_model_pytorch.txt deleted file mode 100644 index fad61376..00000000 --- a/Pilot1/P1B2/p1b2_default_model_pytorch.txt +++ /dev/null @@ -1,28 +0,0 @@ -[Global_Params] -data_url = 'http://ftp.mcs.anl.gov/pub/candle/public/benchmarks/P1B2/' -train_data = 'P1B2.train.csv' -test_data = 'P1B2.test.csv' -model_name='p1b2' -dense=[1024, 512, 256] -batch_size=60 -epochs=20 -activation='sigmoid' -out_activation='log_softmax' -loss='nll' -optimizer='rmsprop' -learning_rate=0.001 -scaling='minmax' -dropout=0. -classes=10 -feature_subsample=0 -reg_l2=0.00001 -val_split=0.1 -rng_seed=2017 -initialization='glorot_uniform' -save_path='save' -shuffle = True - -# Miscellaneous settings ################################## -# multi_gpu=True -# no_cuda=True -# rng_seed=0 \ No newline at end of file diff --git a/Pilot1/P1B2/pytorch_utils.py b/Pilot1/P1B2/pytorch_utils.py deleted file mode 100644 index 3e56e876..00000000 --- a/Pilot1/P1B2/pytorch_utils.py +++ /dev/null @@ -1,140 +0,0 @@ -from __future__ import absolute_import - -import torch -import torch.nn -import torch.nn.init -import torch.optim -import torch.nn.functional as F - -from default_utils import set_seed as set_seed_defaultUtils - -def set_parallelism_threads(): # for compatibility - pass - -def set_seed(seed): - """ Set the random number seed to the desired value - - Parameters - ---------- - seed : integer - Random number seed. - """ - - set_seed_defaultUtils(seed) - torch.manual_seed(seed) - - -def get_function(name): - mapping = {} - - # loss - mapping['mse'] = torch.nn.MSELoss() - mapping['binary_crossentropy'] = torch.nn.BCELoss() - mapping['categorical_crossentropy'] = torch.nn.CrossEntropyLoss() - mapping['smoothL1'] = torch.nn.SmoothL1Loss() - - mapped = mapping.get(name) - if not mapped: - raise Exception('No pytorch function found for "{}"'.format(name)) - - return mapped - - -def build_activation(type, dim=1): - - # activation - - if type=='relu': - return torch.nn.ReLU() - elif type=='sigmoid': - return torch.nn.Sigmoid() - elif type=='tanh': - return torch.nn.Tanh() - elif type=='softmax': - return torch.nn.Softmax(dim) - elif type=='log_softmax': - return torch.nn.LogSoftmax(dim) - -def build_optimizer(model, type, lr, kerasDefaults, trainable_only=True): - - if trainable_only: - params = filter(lambda p: p.requires_grad, model.parameters()) - else: - params = model.parameters() - - #schedule = optimizers.optimizer.Schedule() # constant lr (equivalent to default keras setting) - - if type == 'sgd': - return torch.optim.SGD(params, - lr=lr, - momentum=kerasDefaults['momentum_sgd'], - nesterov=kerasDefaults['nesterov_sgd']) - #schedule=schedule) - - elif type == 'rmsprop': - return torch.optim.RMSprop(model.parameters(), - lr=lr, - alpha=kerasDefaults['rho'], - eps=kerasDefaults['epsilon'], - weight_decay=kerasDefaults['weight_decay']) - #schedule=schedule) - - elif type == 'adagrad': - return torch.optim.Adagrad(model.parameters(), - lr=lr, - eps=kerasDefaults['epsilon']) - - elif type == 'adadelta': - return torch.optim.Adadelta(params, - eps=kerasDefaults['epsilon'], - rho=kerasDefaults['rho']) - - elif type == 'adam': - return torch.optim.Adam(params, - lr=lr, - betas=[kerasDefaults['beta_1'], kerasDefaults['beta_2']], - eps=kerasDefaults['epsilon']) - - - -def initialize(weights, type, kerasDefaults, seed=None, constant=0.): - - if type == 'constant': - return torch.nn.init.constant_(weights, - val=constant) - - elif type == 'uniform': - return torch.nn.init.uniform(weights, - a=kerasDefaults['minval_uniform'], - b=kerasDefaults['maxval_uniform']) - - elif type == 'normal': - return torch.nn.init.normal(weights, - mean=kerasDefaults['mean_normal'], - std=kerasDefaults['stddev_normal']) - - elif type == 'glorot_normal': # not quite Xavier - return torch.nn.init.xavier_normal(weights) - - elif type == 'glorot_uniform': - return torch.nn.init.xavier_uniform_(weights) - - elif type == 'he_normal': - return torch.nn.init.kaiming_uniform(weights) - - -def xent(y_true, y_pred): - return F.cross_entropy(y_pred, y_true) - - -def mse(y_true, y_pred): - return F.mse_loss(y_pred, y_true) - -def build_loss(type, y_pred, y_true): - - if type=='categorical_crossentropy': - return xent(y_true, y_pred) - elif type=='mse': - return mse(y_true, y_pred) - elif type=='nll': - return F.nll_loss(y_pred, y_true) \ No newline at end of file diff --git a/Pilot1/TC1/pytorch_utils.py b/Pilot1/TC1/pytorch_utils.py deleted file mode 100644 index cce885a7..00000000 --- a/Pilot1/TC1/pytorch_utils.py +++ /dev/null @@ -1,139 +0,0 @@ -from __future__ import absolute_import - -import torch -import torch.nn -import torch.nn.init -import torch.optim -import torch.nn.functional as F - -from default_utils import set_seed as set_seed_defaultUtils - -def set_parallelism_threads(): # for compatibility - pass - -def set_seed(seed): - """ Set the random number seed to the desired value - - Parameters - ---------- - seed : integer - Random number seed. - """ - - set_seed_defaultUtils(seed) - torch.manual_seed(seed) - - -def get_function(name): - mapping = {} - - # loss - mapping['mse'] = torch.nn.MSELoss() - mapping['binary_crossentropy'] = torch.nn.BCELoss() - mapping['categorical_crossentropy'] = torch.nn.CrossEntropyLoss() - mapping['smoothL1'] = torch.nn.SmoothL1Loss() - - mapped = mapping.get(name) - if not mapped: - raise Exception('No pytorch function found for "{}"'.format(name)) - - return mapped - - -def build_activation(type, dim=1): - - # activation - - if type=='relu': - return torch.nn.ReLU() - elif type=='sigmoid': - return torch.nn.Sigmoid() - elif type=='tanh': - return torch.nn.Tanh() - elif type=='softmax': - return torch.nn.Softmax(dim) - elif type=='log_softmax': - return torch.nn.LogSoftmax(dim) - -def build_optimizer(model, type, lr, kerasDefaults, trainable_only=True): - - if trainable_only: - params = filter(lambda p: p.requires_grad, model.parameters()) - else: - params = model.parameters() - - #schedule = optimizers.optimizer.Schedule() # constant lr (equivalent to default keras setting) - - if type == 'sgd': - return torch.optim.SGD(params, - lr=lr, - momentum=kerasDefaults['momentum_sgd'], - nesterov=kerasDefaults['nesterov_sgd']) - #schedule=schedule) - - elif type == 'rmsprop': - return torch.optim.RMSprop(model.parameters(), - lr=lr, - alpha=kerasDefaults['rho'], - eps=kerasDefaults['epsilon']) - #schedule=schedule) - - elif type == 'adagrad': - return torch.optim.Adagrad(model.parameters(), - lr=lr, - eps=kerasDefaults['epsilon']) - - elif type == 'adadelta': - return torch.optim.Adadelta(params, - eps=kerasDefaults['epsilon'], - rho=kerasDefaults['rho']) - - elif type == 'adam': - return torch.optim.Adam(params, - lr=lr, - betas=[kerasDefaults['beta_1'], kerasDefaults['beta_2']], - eps=kerasDefaults['epsilon']) - - - -def initialize(weights, type, kerasDefaults, seed=None, constant=0.): - - if type == 'constant': - return torch.nn.init.constant_(weights, - val=constant) - - elif type == 'uniform': - return torch.nn.init.uniform(weights, - a=kerasDefaults['minval_uniform'], - b=kerasDefaults['maxval_uniform']) - - elif type == 'normal': - return torch.nn.init.normal(weights, - mean=kerasDefaults['mean_normal'], - std=kerasDefaults['stddev_normal']) - - elif type == 'glorot_normal': # not quite Xavier - return torch.nn.init.xavier_normal(weights) - - elif type == 'glorot_uniform': - return torch.nn.init.xavier_uniform_(weights) - - elif type == 'he_normal': - return torch.nn.init.kaiming_uniform(weights) - - -def xent(y_true, y_pred): - return F.cross_entropy(y_pred, y_true) - - -def mse(y_true, y_pred): - return F.mse_loss(y_pred, y_true) - -def build_loss(type, y_pred, y_true): - - if type=='categorical_crossentropy': - return xent(y_true, y_pred) - elif type=='mse': - return mse(y_true, y_pred) - elif type=='nll': - return F.nll_loss(y_pred, y_true) \ No newline at end of file diff --git a/Pilot1/TC1/tc1_baseline_pytorch.py b/Pilot1/TC1/tc1_baseline_pytorch.py index cea7dcff..c8dae1f8 100644 --- a/Pilot1/TC1/tc1_baseline_pytorch.py +++ b/Pilot1/TC1/tc1_baseline_pytorch.py @@ -9,11 +9,6 @@ import torch -if True: - print("Restricting #of GPUs to 8") - os.environ["CUDA_VISIBLE_DEVICES"]="0,1,2,3,4,5,6,7" - #os.environ["CUDA_VISIBLE_DEVICES"]="0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15" - file_path = os.path.dirname(os.path.realpath(__file__)) lib_path2 = os.path.abspath(os.path.join(file_path, '..', '..', 'common')) sys.path.append(lib_path2) @@ -26,7 +21,7 @@ np.set_printoptions(precision=4) -def initialize_parameters(default_model = 'tc1_default_model_pytorch.txt'): +def initialize_parameters(default_model = 'tc1_default_model.txt'): # Build benchmark object tc1Bmk = bmk.BenchmarkTC1(file_path, default_model, 'pytorch', @@ -50,7 +45,11 @@ def run(params): args.max_num_batches = args.max_num_batches if hasattr(args,'max_num_batches') else 1000 args.dry_run = args.dry_run if hasattr(args,'dry_run') else False args.log_interval = args.log_interval if hasattr(args,'log_interval') else 8 + args.save_path = args.save_path if hasattr(args,'save_path') else 'Output/TC1' + if args.loss=='categorical_crossentropy': + args.out_activation='log_softmax' + args.loss='nll' seed = args.rng_seed candle.set_seed(seed) diff --git a/Pilot1/TC1/tc1_default_model_pytorch.txt b/Pilot1/TC1/tc1_default_model_pytorch.txt deleted file mode 100644 index 10e2a4cc..00000000 --- a/Pilot1/TC1/tc1_default_model_pytorch.txt +++ /dev/null @@ -1,25 +0,0 @@ -[Global_Params] -data_url = 'http://ftp.mcs.anl.gov/pub/candle/public/benchmarks/Pilot1/type-class/' -train_data = 'type_18_300_train.csv' -test_data = 'type_18_300_test.csv' -model_name = 'tc1' -conv=[128, 20, 1, 128, 10, 1] -dense=[200,20] -activation='relu' -out_activation='log_softmax' -loss='nll' -optimizer='sgd' -metrics='accuracy' -epochs=75 -batch_size=128 -dropout=0.1 -classes=36 -feature_subsample=0 -pool=[1, 10] -output_dir='.' - -# Miscellaneous settings ################################## -# multi_gpu=True -# no_cuda=True -# rng_seed=0 -save_path='Output/TC1' \ No newline at end of file diff --git a/common/pytorch_utils.py b/common/pytorch_utils.py index e756ef4b..c99b7b70 100644 --- a/common/pytorch_utils.py +++ b/common/pytorch_utils.py @@ -42,7 +42,7 @@ def get_function(name): return mapped -def build_activation(type): +def build_activation(type, dim=1): # activation if type == 'relu': @@ -51,7 +51,10 @@ def build_activation(type): return torch.nn.Sigmoid() elif type == 'tanh': return torch.nn.Tanh() - + elif type == 'softmax': + return torch.nn.Softmax(dim) + elif type == 'log_softmax': + return torch.nn.LogSoftmax(dim) def build_optimizer(model, type, lr, kerasDefaults, trainable_only=True): if trainable_only: @@ -62,16 +65,21 @@ def build_optimizer(model, type, lr, kerasDefaults, trainable_only=True): # schedule = optimizers.optimizer.Schedule() # constant lr (equivalent to default keras setting) if type == 'sgd': - return torch.optim.GradientDescentMomentum(params, - lr=lr, - momentum_coef=kerasDefaults['momentum_sgd'], - nesterov=kerasDefaults['nesterov_sgd']) + #return torch.optim.GradientDescentMomentum(params, + # lr=lr, + # momentum_coef=kerasDefaults['momentum_sgd'], + # nesterov=kerasDefaults['nesterov_sgd']) + return torch.optim.SGD(params, + lr=lr, + momentum=kerasDefaults['momentum_sgd'], + nesterov=kerasDefaults['nesterov_sgd']) elif type == 'rmsprop': return torch.optim.RMSprop(model.parameters(), lr=lr, alpha=kerasDefaults['rho'], - eps=kerasDefaults['epsilon']) + eps=kerasDefaults['epsilon'], + weight_decay=kerasDefaults['weight_decay']) elif type == 'adagrad': return torch.optim.Adagrad(model.parameters(), @@ -121,3 +129,11 @@ def xent(y_true, y_pred): def mse(y_true, y_pred): return F.mse_loss(y_pred, y_true) + +def build_loss(type, y_pred, y_true): + if type=='categorical_crossentropy': + return xent(y_true, y_pred) + elif type=='mse': + return mse(y_true, y_pred) + elif type=='nll': + return F.nll_loss(y_pred, y_true) \ No newline at end of file