diff --git a/CHANGE_LOG.md b/CHANGE_LOG.md index 544dbac47..ad22f311c 100644 --- a/CHANGE_LOG.md +++ b/CHANGE_LOG.md @@ -1,5 +1,7 @@ # Change Log +* Fixes hard-coded repository path for Aim experiment tracking. + ## 23/07/15/2023 Update * Added DeFixmatch algorithm. diff --git a/requirements.txt b/requirements.txt index 9b5701c77..f656c0935 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ fonttools==4.33.3 google-auth==2.9.0 google-auth-oauthlib==0.4.6 grpcio==1.53.0 -huggingface-hub==0.8.1 +huggingface-hub==0.14.1 idna==3.3 imageio==2.19.3 importlib-metadata==4.12.0 diff --git a/scripts/config_generator_classic_cv.py b/scripts/config_generator_classic_cv.py index 90ace8141..ee023389b 100644 --- a/scripts/config_generator_classic_cv.py +++ b/scripts/config_generator_classic_cv.py @@ -8,266 +8,285 @@ def create_configuration(cfg, cfg_file): - cfg['save_name'] = "{alg}_{dataset}_{num_lb}_{seed}".format( - alg=cfg['algorithm'], - dataset=cfg['dataset'], - num_lb=cfg['num_labels'], - seed=cfg['seed'], + cfg["save_name"] = "{alg}_{dataset}_{num_lb}_{seed}".format( + alg=cfg["algorithm"], + dataset=cfg["dataset"], + num_lb=cfg["num_labels"], + seed=cfg["seed"], ) # resume - cfg['resume'] = True - cfg['load_path'] = '{}/{}/latest_model.pth'.format(cfg['save_dir'], cfg['save_name']) + cfg["resume"] = True + cfg["load_path"] = "{}/{}/latest_model.pth".format( + cfg["save_dir"], cfg["save_name"] + ) - alg_file = cfg_file + cfg['algorithm'] + '/' + alg_file = cfg_file + cfg["algorithm"] + "/" if not os.path.exists(alg_file): os.mkdir(alg_file) - print(alg_file + cfg['save_name'] + '.yaml') - with open(alg_file + cfg['save_name'] + '.yaml', 'w', encoding='utf-8') as w: + print(alg_file + cfg["save_name"] + ".yaml") + with open(alg_file + cfg["save_name"] + ".yaml", "w", encoding="utf-8") as w: lines = [] for k, v in cfg.items(): - line = str(k) + ': ' + str(v) + line = str(k) + ": " + str(v) lines.append(line) for line in lines: w.writelines(line) - w.write('\n') - + w.write("\n") - -def create_classific_config(alg, seed, - dataset, net, num_classes, num_labels, img_size, - port, weight_decay): +def create_classic_config( + alg, seed, dataset, net, num_classes, num_labels, img_size, port, weight_decay +): cfg = {} - cfg['algorithm'] = alg + cfg["algorithm"] = alg # save config - cfg['save_dir'] = './saved_models/classic_cv' - cfg['save_name'] = None - cfg['resume'] = False - cfg['load_path'] = None - cfg['overwrite'] = True - cfg['use_tensorboard'] = True - cfg['use_wandb'] = True + cfg["save_dir"] = "./saved_models/classic_cv" + cfg["save_name"] = None + cfg["resume"] = False + cfg["load_path"] = None + cfg["overwrite"] = True + cfg["use_tensorboard"] = True + cfg["use_wandb"] = True + cfg["use_aim"] = False # algorithm config - cfg['epoch'] = 1024 - cfg['num_train_iter'] = 2 ** 20 - cfg['num_eval_iter'] = 5120 - cfg['num_log_iter'] = 256 - cfg['num_labels'] = num_labels - cfg['batch_size'] = 64 - cfg['eval_batch_size'] = 256 - if alg == 'fixmatch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - cfg['uratio'] = 7 - elif alg == 'adamatch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - cfg['ema_p'] = 0.999 - cfg['uratio'] = 7 - elif alg == 'flexmatch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['thresh_warmup'] = True - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - cfg['uratio'] = 7 - elif alg == 'uda': - cfg['tsa_schedule'] = 'none' - cfg['T'] = 0.4 - cfg['p_cutoff'] = 0.8 - cfg['ulb_loss_ratio'] = 1.0 - cfg['uratio'] = 7 - elif alg == 'pseudolabel': - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - cfg['uratio'] = 1 - cfg['unsup_warm_up'] = 0.4 - elif alg == 'mixmatch': - cfg['uratio'] = 1 - cfg['mixup_alpha'] = 0.5 - cfg['T'] = 0.5 - if dataset == 'cifar10': - cfg['ulb_loss_ratio'] = 100 - elif dataset == 'cifar100': - cfg['ulb_loss_ratio'] = 150 + cfg["epoch"] = 1024 + cfg["num_train_iter"] = 2**20 + cfg["num_eval_iter"] = 5120 + cfg["num_log_iter"] = 256 + cfg["num_labels"] = num_labels + cfg["batch_size"] = 64 + cfg["eval_batch_size"] = 256 + if alg == "fixmatch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + cfg["uratio"] = 7 + elif alg == "adamatch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + cfg["ema_p"] = 0.999 + cfg["uratio"] = 7 + elif alg == "flexmatch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["thresh_warmup"] = True + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + cfg["uratio"] = 7 + elif alg == "uda": + cfg["tsa_schedule"] = "none" + cfg["T"] = 0.4 + cfg["p_cutoff"] = 0.8 + cfg["ulb_loss_ratio"] = 1.0 + cfg["uratio"] = 7 + elif alg == "pseudolabel": + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + cfg["uratio"] = 1 + cfg["unsup_warm_up"] = 0.4 + elif alg == "mixmatch": + cfg["uratio"] = 1 + cfg["mixup_alpha"] = 0.5 + cfg["T"] = 0.5 + if dataset == "cifar10": + cfg["ulb_loss_ratio"] = 100 + elif dataset == "cifar100": + cfg["ulb_loss_ratio"] = 150 else: - cfg['ulb_loss_ratio'] = 100 - cfg['unsup_warm_up'] = 0.4 # 16000 / 1024 / 1024 - elif alg == 'remixmatch': - cfg['mixup_alpha'] = 0.75 - cfg['T'] = 0.5 - cfg['kl_loss_ratio'] = 0.5 - cfg['ulb_loss_ratio'] = 1.5 - cfg['rot_loss_ratio'] = 0.5 - cfg['unsup_warm_up'] = 1 / 64 - cfg['uratio'] = 1 - elif alg == 'crmatch': - cfg['hard_label'] = True - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - cfg['uratio'] = 7 - elif alg == 'comatch': - cfg['hard_label'] = False - cfg['p_cutoff'] = 0.95 - cfg['contrast_p_cutoff'] = 0.8 - cfg['contrast_loss_ratio'] = 1.0 - cfg['ulb_loss_ratio'] = 1.0 - cfg['proj_size'] = 64 - cfg['queue_batch'] = 5 - cfg['smoothing_alpha'] = 0.9 - cfg['uratio'] = 7 - cfg['T'] = 0.2 - cfg['da_len'] = 32 - - if dataset == 'stl10': - cfg['contrast_loss_ratio'] = 5.0 - - if dataset == 'imagenet': - cfg['p_cutoff'] = 0.6 - cfg['contrast_p_cutoff'] = 0.3 - cfg['contrast_loss_ratio'] = 10.0 - cfg['ulb_loss_ratio'] = 10.0 - cfg['smoothing_alpha'] = 0.9 - cfg['T'] = 0.1 - cfg['proj_size'] = 128 - - elif alg == 'simmatch': - cfg['p_cutoff'] = 0.95 - cfg['in_loss_ratio'] = 1.0 - cfg['ulb_loss_ratio'] = 1.0 - cfg['proj_size'] = 128 - cfg['K'] = 256 - cfg['da_len'] = 32 - cfg['smoothing_alpha'] = 0.9 - cfg['uratio'] = 7 - if dataset in ['cifar10', 'svhn', 'cifar100', 'stl10']: - cfg['T'] = 0.1 + cfg["ulb_loss_ratio"] = 100 + cfg["unsup_warm_up"] = 0.4 # 16000 / 1024 / 1024 + elif alg == "remixmatch": + cfg["mixup_alpha"] = 0.75 + cfg["T"] = 0.5 + cfg["kl_loss_ratio"] = 0.5 + cfg["ulb_loss_ratio"] = 1.5 + cfg["rot_loss_ratio"] = 0.5 + cfg["unsup_warm_up"] = 1 / 64 + cfg["uratio"] = 1 + elif alg == "crmatch": + cfg["hard_label"] = True + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + cfg["uratio"] = 7 + elif alg == "comatch": + cfg["hard_label"] = False + cfg["p_cutoff"] = 0.95 + cfg["contrast_p_cutoff"] = 0.8 + cfg["contrast_loss_ratio"] = 1.0 + cfg["ulb_loss_ratio"] = 1.0 + cfg["proj_size"] = 64 + cfg["queue_batch"] = 5 + cfg["smoothing_alpha"] = 0.9 + cfg["uratio"] = 7 + cfg["T"] = 0.2 + cfg["da_len"] = 32 + + if dataset == "stl10": + cfg["contrast_loss_ratio"] = 5.0 + + if dataset == "imagenet": + cfg["p_cutoff"] = 0.6 + cfg["contrast_p_cutoff"] = 0.3 + cfg["contrast_loss_ratio"] = 10.0 + cfg["ulb_loss_ratio"] = 10.0 + cfg["smoothing_alpha"] = 0.9 + cfg["T"] = 0.1 + cfg["proj_size"] = 128 + + elif alg == "simmatch": + cfg["p_cutoff"] = 0.95 + cfg["in_loss_ratio"] = 1.0 + cfg["ulb_loss_ratio"] = 1.0 + cfg["proj_size"] = 128 + cfg["K"] = 256 + cfg["da_len"] = 32 + cfg["smoothing_alpha"] = 0.9 + cfg["uratio"] = 7 + if dataset in ["cifar10", "svhn", "cifar100", "stl10"]: + cfg["T"] = 0.1 else: - cfg['T'] = 0.2 - elif alg == 'meanteacher': - cfg['uratio'] = 1 - cfg['ulb_loss_ratio'] = 50 - cfg['unsup_warm_up'] = 0.4 - elif alg == 'pimodel': - cfg['ulb_loss_ratio'] = 10 - cfg['uratio'] = 1 - cfg['unsup_warm_up'] = 0.4 - elif alg == 'dash': - cfg['gamma'] = 1.27 - cfg['C'] = 1.0001 - cfg['rho_min'] = 0.05 - cfg['num_wu_iter'] = 2048 - cfg['T'] = 0.5 - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - cfg['uratio'] = 7 - elif alg == 'mpl': - cfg['tsa_schedule'] = 'none' - cfg['T'] = 0.7 - cfg['p_cutoff'] = 0.6 - cfg['ulb_loss_ratio'] = 8.0 - cfg['uratio'] = 7 - cfg['teacher_lr'] = 0.03 - cfg['label_smoothing'] = 0.1 - cfg['num_uda_warmup_iter'] = 5000 - cfg['num_stu_wait_iter'] = 3000 - - elif alg == 'freematch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['ema_p'] = 0.999 - cfg['ent_loss_ratio'] = 0.001 - cfg['uratio'] = 7 - cfg['use_quantile'] = False - if dataset == 'svhn': - cfg['clip_thresh'] = True - elif alg == 'softmatch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['dist_align'] = True - cfg['dist_uniform'] = True - cfg['per_class'] = False - cfg['ema_p'] = 0.999 - cfg['ulb_loss_ratio'] = 1.0 - cfg['n_sigma'] = 2 - cfg['uratio'] = 7 - if dataset == 'imagenet': - cfg['ulb_loss_ratio'] = 1.0 - elif alg == 'defixmatch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 0.5 - cfg['uratio'] = 7 + cfg["T"] = 0.2 + elif alg == "meanteacher": + cfg["uratio"] = 1 + cfg["ulb_loss_ratio"] = 50 + cfg["unsup_warm_up"] = 0.4 + elif alg == "pimodel": + cfg["ulb_loss_ratio"] = 10 + cfg["uratio"] = 1 + cfg["unsup_warm_up"] = 0.4 + elif alg == "dash": + cfg["gamma"] = 1.27 + cfg["C"] = 1.0001 + cfg["rho_min"] = 0.05 + cfg["num_wu_iter"] = 2048 + cfg["T"] = 0.5 + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + cfg["uratio"] = 7 + elif alg == "mpl": + cfg["tsa_schedule"] = "none" + cfg["T"] = 0.7 + cfg["p_cutoff"] = 0.6 + cfg["ulb_loss_ratio"] = 8.0 + cfg["uratio"] = 7 + cfg["teacher_lr"] = 0.03 + cfg["label_smoothing"] = 0.1 + cfg["num_uda_warmup_iter"] = 5000 + cfg["num_stu_wait_iter"] = 3000 + + elif alg == "freematch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["ema_p"] = 0.999 + cfg["ent_loss_ratio"] = 0.001 + cfg["uratio"] = 7 + cfg["use_quantile"] = False + if dataset == "svhn": + cfg["clip_thresh"] = True + elif alg == "softmatch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["dist_align"] = True + cfg["dist_uniform"] = True + cfg["per_class"] = False + cfg["ema_p"] = 0.999 + cfg["ulb_loss_ratio"] = 1.0 + cfg["n_sigma"] = 2 + cfg["uratio"] = 7 + if dataset == "imagenet": + cfg["ulb_loss_ratio"] = 1.0 + elif alg == "defixmatch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 0.5 + cfg["uratio"] = 7 # cfg['img'] - cfg['ema_m'] = 0.999 - cfg['crop_ratio'] = 0.875 - cfg['img_size'] = img_size + cfg["ema_m"] = 0.999 + cfg["crop_ratio"] = 0.875 + cfg["img_size"] = img_size # optim config - cfg['optim'] = 'SGD' - cfg['lr'] = 0.03 - cfg['momentum'] = 0.9 - cfg['weight_decay'] = weight_decay - cfg['layer_decay'] = 1.0 - cfg['amp'] = False - cfg['clip'] = 0.0 - cfg['use_cat'] = True + cfg["optim"] = "SGD" + cfg["lr"] = 0.03 + cfg["momentum"] = 0.9 + cfg["weight_decay"] = weight_decay + cfg["layer_decay"] = 1.0 + cfg["amp"] = False + cfg["clip"] = 0.0 + cfg["use_cat"] = True # net config - cfg['net'] = net - cfg['net_from_name'] = False + cfg["net"] = net + cfg["net_from_name"] = False # data config - cfg['data_dir'] = './data' - cfg['dataset'] = dataset - cfg['train_sampler'] = 'RandomSampler' - cfg['num_classes'] = num_classes - cfg['num_workers'] = 1 + cfg["data_dir"] = "./data" + cfg["dataset"] = dataset + cfg["train_sampler"] = "RandomSampler" + cfg["num_classes"] = num_classes + cfg["num_workers"] = 1 # basic config - cfg['seed'] = seed + cfg["seed"] = seed # distributed config - cfg['world_size'] = 1 - cfg['rank'] = 0 - cfg['multiprocessing_distributed'] = True - cfg['dist_url'] = 'tcp://127.0.0.1:' + str(port) - cfg['dist_backend'] = 'nccl' - cfg['gpu'] = None + cfg["world_size"] = 1 + cfg["rank"] = 0 + cfg["multiprocessing_distributed"] = True + cfg["dist_url"] = "tcp://127.0.0.1:" + str(port) + cfg["dist_backend"] = "nccl" + cfg["gpu"] = None # other config - cfg['overwrite'] = True - cfg['amp'] = False - cfg['use_wandb'] = False + cfg["overwrite"] = True + cfg["amp"] = False + cfg["use_wandb"] = False + cfg["use_aim"] = False return cfg - # prepare the configuration for baseline model, use_penalty == False -def exp_classific_cv(label_amount): - config_file = r'./config/classic_cv/' - save_path = r'./saved_models/classic_cv' +def exp_classic_cv(label_amount): + config_file = r"./config/classic_cv/" + save_path = r"./saved_models/classic_cv" if not os.path.exists(config_file): os.makedirs(config_file) if not os.path.exists(save_path): os.makedirs(save_path) - - algs = ['flexmatch', 'fixmatch', 'uda', 'pseudolabel', 'fullysupervised', 'supervised', 'remixmatch', 'mixmatch', 'meanteacher', - 'pimodel', 'vat', 'dash', 'crmatch', 'comatch', 'simmatch', 'adamatch', 'freematch', 'softmatch', 'defixmatch'] - datasets = ['cifar100', 'svhn', 'stl10', 'cifar10'] + algs = [ + "flexmatch", + "fixmatch", + "uda", + "pseudolabel", + "fullysupervised", + "supervised", + "remixmatch", + "mixmatch", + "meanteacher", + "pimodel", + "vat", + "dash", + "crmatch", + "comatch", + "simmatch", + "adamatch", + "freematch", + "softmatch", + "defixmatch", + ] + datasets = ["cifar100", "svhn", "stl10", "cifar10"] seeds = [0] @@ -278,15 +297,15 @@ def exp_classific_cv(label_amount): for dataset in datasets: for seed in seeds: # change the configuration of each dataset - if dataset == 'cifar10': + if dataset == "cifar10": # net = 'WideResNet' num_classes = 10 num_labels = label_amount[0] weight_decay = 5e-4 - net = 'wrn_28_2' + net = "wrn_28_2" img_size = 32 - elif dataset == 'cifar100': + elif dataset == "cifar100": # net = 'WideResNet' num_classes = 100 num_labels = label_amount[1] @@ -294,55 +313,64 @@ def exp_classific_cv(label_amount): # depth = 28 # widen_factor = 8 # net = 'wrn_28_8' - net = 'wrn_28_2' - img_size = 32 + net = "wrn_28_2" + img_size = 32 - elif dataset == 'svhn': + elif dataset == "svhn": # net = 'WideResNet' num_classes = 10 num_labels = label_amount[2] weight_decay = 5e-4 # depth = 28 # widen_factor = 2 - net = 'wrn_28_2' + net = "wrn_28_2" img_size = 32 - elif dataset == 'stl10': + elif dataset == "stl10": # net = 'WideResNetVar' num_classes = 10 num_labels = label_amount[3] weight_decay = 5e-4 - net = 'wrn_var_37_2' + net = "wrn_var_37_2" img_size = 96 - elif dataset == 'imagenet': - if alg not in ['fixmatch', 'flexmatch']: + elif dataset == "imagenet": + if alg not in ["fixmatch", "flexmatch"]: continue - net = 'resnet50' + net = "resnet50" num_classes = 1000 num_labels = 100000 # 128000 weight_decay = 3e-4 port = dist_port[count] # prepare the configuration file - cfg = create_classific_config(alg, seed, - dataset, net, num_classes, num_labels, img_size, - port, weight_decay) + cfg = create_classic_config( + alg, + seed, + dataset, + net, + num_classes, + num_labels, + img_size, + port, + weight_decay, + ) count += 1 create_configuration(cfg, config_file) - -if __name__ == '__main__': +if __name__ == "__main__": # if not os.path.exists('./saved_models/classic_cv/'): # os.mkdir('./saved_models/classic_cv/') - if not os.path.exists('./config/classic_cv/'): - os.mkdir('./config/classic_cv/') + if not os.path.exists("./config/classic_cv/"): + os.mkdir("./config/classic_cv/") - # classific cv - label_amount = {'s': [40, 400, 40, 40], - 'm': [250, 2500, 250, 250], - 'l': [4000, 10000, 1000, 1000]} + # classic cv + label_amount = { + "s": [40, 400, 40, 40], + "m": [250, 2500, 250, 250], + "l": [4000, 10000, 1000, 1000], + } for i in label_amount: - exp_classific_cv(label_amount=label_amount[i]) + exp_classic_cv(label_amount=label_amount[i]) diff --git a/scripts/config_generator_classic_cv_imb.py b/scripts/config_generator_classic_cv_imb.py index fada32811..ebe214dfd 100644 --- a/scripts/config_generator_classic_cv_imb.py +++ b/scripts/config_generator_classic_cv_imb.py @@ -5,210 +5,275 @@ Create the .yaml for each experiment """ import os -from config_generator_classic_cv import create_classific_config -from config_generator_usb_cv import create_usb_cv_config - +from config_generator_classic_cv import create_classic_config +from config_generator_usb_cv import create_usb_cv_config def create_configuration(cfg, cfg_file): - if 'imb_algorithm' in cfg: - cfg['save_name'] = "{alg}_{imb_alg}_{dataset}_lb{num_lb}_{imb_lb}_ulb{num_ulb}_{imb_ulb}_{seed}".format( - imb_alg=cfg['imb_algorithm'], - alg=cfg['algorithm'], - dataset=cfg['dataset'], - num_lb=cfg['num_labels'], - imb_lb=cfg['lb_imb_ratio'], - num_ulb=cfg['ulb_num_labels'], - imb_ulb=cfg['ulb_imb_ratio'], - seed=cfg['seed'], + if "imb_algorithm" in cfg: + cfg[ + "save_name" + ] = "{alg}_{imb_alg}_{dataset}_lb{num_lb}_{imb_lb}_ulb{num_ulb}_{imb_ulb}_{seed}".format( # noqa: E501 + imb_alg=cfg["imb_algorithm"], + alg=cfg["algorithm"], + dataset=cfg["dataset"], + num_lb=cfg["num_labels"], + imb_lb=cfg["lb_imb_ratio"], + num_ulb=cfg["ulb_num_labels"], + imb_ulb=cfg["ulb_imb_ratio"], + seed=cfg["seed"], ) else: - cfg['save_name'] = "{alg}_{dataset}_lb{num_lb}_{imb_lb}_ulb{num_ulb}_{imb_ulb}_{seed}".format( - alg=cfg['algorithm'], - dataset=cfg['dataset'], - num_lb=cfg['num_labels'], - imb_lb=cfg['lb_imb_ratio'], - num_ulb=cfg['ulb_num_labels'], - - imb_ulb=cfg['ulb_imb_ratio'], - seed=cfg['seed'], + cfg[ + "save_name" + ] = "{alg}_{dataset}_lb{num_lb}_{imb_lb}_ulb{num_ulb}_{imb_ulb}_{seed}".format( + alg=cfg["algorithm"], + dataset=cfg["dataset"], + num_lb=cfg["num_labels"], + imb_lb=cfg["lb_imb_ratio"], + num_ulb=cfg["ulb_num_labels"], + imb_ulb=cfg["ulb_imb_ratio"], + seed=cfg["seed"], ) # resume - cfg['resume'] = True - cfg['load_path'] = '{}/{}/latest_model.pth'.format(cfg['save_dir'], cfg['save_name']) + cfg["resume"] = True + cfg["load_path"] = "{}/{}/latest_model.pth".format( + cfg["save_dir"], cfg["save_name"] + ) - if 'imb_algorithm' in cfg: - alg_file = cfg_file + cfg['algorithm'] + '_' + cfg['imb_algorithm'] + '/' + if "imb_algorithm" in cfg: + alg_file = cfg_file + cfg["algorithm"] + "_" + cfg["imb_algorithm"] + "/" else: - alg_file = cfg_file + cfg['algorithm'] + '/' - + alg_file = cfg_file + cfg["algorithm"] + "/" + if not os.path.exists(alg_file): os.mkdir(alg_file) - print(alg_file + cfg['save_name'] + '.yaml') - with open(alg_file + cfg['save_name'] + '.yaml', 'w', encoding='utf-8') as w: + print(alg_file + cfg["save_name"] + ".yaml") + with open(alg_file + cfg["save_name"] + ".yaml", "w", encoding="utf-8") as w: lines = [] for k, v in cfg.items(): - line = str(k) + ': ' + str(v) + line = str(k) + ": " + str(v) lines.append(line) for line in lines: w.writelines(line) - w.write('\n') - - -def create_classic_cv_imb_config(alg, seed, dataset, net, num_classes, num_labels, img_size, port, lr, weight_decay, - imb_alg, lb_imb_ratio, ulb_imb_ratio, ulb_num_labels, - amp=False, layer_decay=1.0, pretrain_name=None): - + w.write("\n") + + +def create_classic_cv_imb_config( + alg, + seed, + dataset, + net, + num_classes, + num_labels, + img_size, + port, + lr, + weight_decay, + imb_alg, + lb_imb_ratio, + ulb_imb_ratio, + ulb_num_labels, + amp=False, + layer_decay=1.0, + pretrain_name=None, +): # get core algorithm related configs - if dataset in ['cifar10', 'cifar100', 'stl10']: - cfg = create_classific_config(alg, seed, dataset, net, num_classes, num_labels, img_size, port, weight_decay) - cfg['net'] = net - cfg['optim'] = 'SGD' - cfg['lr'] = lr - cfg['weight_decay'] = weight_decay - cfg['num_workers'] = 1 - cfg['epoch'] = 256 - cfg['num_train_iter'] = cfg['epoch'] * 1024 - cfg['num_eval_iter'] = 1024 - cfg['lr'] = lr - elif dataset in ['semi_aves', 'semi_aves_out']: - cfg = create_usb_cv_config(alg, seed, dataset, net, num_classes, num_labels, img_size, 0.875, port, lr, weight_decay, layer_decay, pretrain_name, warmup=0, amp=False) - - cfg['optim'] = 'AdamW' - cfg['lr'] = lr - cfg['weight_decay'] = weight_decay - cfg['num_workers'] = 4 - cfg['epoch'] = 128 - cfg['num_train_iter'] = cfg['epoch'] * 1024 - cfg['num_eval_iter'] = 1024 - elif dataset in ['imagenet', 'imagenet127']: - cfg = create_classific_config(alg, seed, dataset, net, num_classes, num_labels, img_size, port, weight_decay) - cfg['net'] = net - cfg['optim'] = 'SGD' - cfg['lr'] = lr - cfg['weight_decay'] = weight_decay - cfg['num_workers'] = 4 - cfg['epoch'] = 200 - cfg['num_train_iter'] = cfg['epoch'] * 2500 - cfg['num_eval_iter'] = 2500 - cfg['batch_size'] = 256 - cfg['img_size'] = img_size - cfg['crop_ratio']= 0.875 - - cfg['num_log_iter'] = 256 - - cfg['include_lb_to_ulb'] = False - cfg['use_cat'] = True - cfg['net'] = net - cfg['lb_imb_ratio'] = lb_imb_ratio - cfg['ulb_imb_ratio'] = ulb_imb_ratio - cfg['ulb_num_labels'] = ulb_num_labels + if dataset in ["cifar10", "cifar100", "stl10"]: + cfg = create_classic_config( + alg, + seed, + dataset, + net, + num_classes, + num_labels, + img_size, + port, + weight_decay, + ) + cfg["net"] = net + cfg["optim"] = "SGD" + cfg["lr"] = lr + cfg["weight_decay"] = weight_decay + cfg["num_workers"] = 1 + cfg["epoch"] = 256 + cfg["num_train_iter"] = cfg["epoch"] * 1024 + cfg["num_eval_iter"] = 1024 + cfg["lr"] = lr + elif dataset in ["semi_aves", "semi_aves_out"]: + cfg = create_usb_cv_config( + alg, + seed, + dataset, + net, + num_classes, + num_labels, + img_size, + 0.875, + port, + lr, + weight_decay, + layer_decay, + pretrain_name, + warmup=0, + amp=False, + ) + + cfg["optim"] = "AdamW" + cfg["lr"] = lr + cfg["weight_decay"] = weight_decay + cfg["num_workers"] = 4 + cfg["epoch"] = 128 + cfg["num_train_iter"] = cfg["epoch"] * 1024 + cfg["num_eval_iter"] = 1024 + elif dataset in ["imagenet", "imagenet127"]: + cfg = create_classic_config( + alg, + seed, + dataset, + net, + num_classes, + num_labels, + img_size, + port, + weight_decay, + ) + cfg["net"] = net + cfg["optim"] = "SGD" + cfg["lr"] = lr + cfg["weight_decay"] = weight_decay + cfg["num_workers"] = 4 + cfg["epoch"] = 200 + cfg["num_train_iter"] = cfg["epoch"] * 2500 + cfg["num_eval_iter"] = 2500 + cfg["batch_size"] = 256 + cfg["img_size"] = img_size + cfg["crop_ratio"] = 0.875 + + cfg["num_log_iter"] = 256 + + cfg["include_lb_to_ulb"] = False + cfg["use_cat"] = True + cfg["net"] = net + cfg["lb_imb_ratio"] = lb_imb_ratio + cfg["ulb_imb_ratio"] = ulb_imb_ratio + cfg["ulb_num_labels"] = ulb_num_labels # rewrite configs for imbalanced setting if imb_alg is not None: - cfg['imb_algorithm'] = imb_alg - - cfg['save_dir'] = './saved_models/classic_cv_imb' - cfg['data_dir'] = './data' - - if alg == 'fixmatch': - cfg['uratio'] = 2 - elif alg == 'remixmatch': - cfg['uratio'] = 1 - cfg['dist_align_target'] = 'uniform' - - - if imb_alg == 'crest+': - cfg['epoch'] = 128 - cfg['num_train_iter'] = 2 ** 16 - cfg['crest_num_gens'] = 6 - cfg['crest_pro_dist_align'] = True - cfg['crest_alpha'] = 3 - cfg['crest_dist_align_t'] = 0.5 - if dataset in ['imagenet', 'imagenet127']: - cfg['epoch'] = 100 - cfg['num_train_iter'] = cfg['epoch'] * 2500 - cfg['crest_num_gens'] = 3 - cfg['crest_alpha'] = 0.7 - cfg['crest_dist_align_t'] = 0.5 - elif imb_alg == 'crest': - cfg['epoch'] = 128 - cfg['num_train_iter'] = 2 ** 16 - cfg['crest_num_gens'] = 6 - cfg['crest_pro_dist_align'] = False - cfg['crest_alpha'] = 3 - if dataset in ['imagenet', 'imagenet127']: - cfg['epoch'] = 100 - cfg['num_train_iter'] =cfg['epoch'] * 2500 - cfg['crest_num_gens'] = 3 - cfg['crest_alpha'] = 0.7 - cfg['crest_dist_align_t'] = 0.5 - elif imb_alg == 'darp': + cfg["imb_algorithm"] = imb_alg + + cfg["save_dir"] = "./saved_models/classic_cv_imb" + cfg["data_dir"] = "./data" + + if alg == "fixmatch": + cfg["uratio"] = 2 + elif alg == "remixmatch": + cfg["uratio"] = 1 + cfg["dist_align_target"] = "uniform" + + if imb_alg == "crest+": + cfg["epoch"] = 128 + cfg["num_train_iter"] = 2**16 + cfg["crest_num_gens"] = 6 + cfg["crest_pro_dist_align"] = True + cfg["crest_alpha"] = 3 + cfg["crest_dist_align_t"] = 0.5 + if dataset in ["imagenet", "imagenet127"]: + cfg["epoch"] = 100 + cfg["num_train_iter"] = cfg["epoch"] * 2500 + cfg["crest_num_gens"] = 3 + cfg["crest_alpha"] = 0.7 + cfg["crest_dist_align_t"] = 0.5 + elif imb_alg == "crest": + cfg["epoch"] = 128 + cfg["num_train_iter"] = 2**16 + cfg["crest_num_gens"] = 6 + cfg["crest_pro_dist_align"] = False + cfg["crest_alpha"] = 3 + if dataset in ["imagenet", "imagenet127"]: + cfg["epoch"] = 100 + cfg["num_train_iter"] = cfg["epoch"] * 2500 + cfg["crest_num_gens"] = 3 + cfg["crest_alpha"] = 0.7 + cfg["crest_dist_align_t"] = 0.5 + elif imb_alg == "darp": # cfg['epoch'] = 512 # cfg['num_train_iter'] = 2 ** 18 - cfg['darp_warmup_epochs'] = 200 - if dataset in ['imagenet', 'imagenet127']: - cfg['darp_warmup_epochs'] = 150 - cfg['darp_alpha'] = 2.0 - cfg['darp_num_refine_iter'] = 10 - cfg['darp_iter_T'] = 10 - elif imb_alg == 'abc': + cfg["darp_warmup_epochs"] = 200 + if dataset in ["imagenet", "imagenet127"]: + cfg["darp_warmup_epochs"] = 150 + cfg["darp_alpha"] = 2.0 + cfg["darp_num_refine_iter"] = 10 + cfg["darp_iter_T"] = 10 + elif imb_alg == "abc": # cfg['epoch'] = 512 # cfg['num_train_iter'] = 2 ** 18 - cfg['abc_p_cutoff'] = 0.95 - cfg['abc_loss_ratio'] = 1.0 - elif imb_alg == 'daso': - cfg['daso_queue_len'] = 256 - cfg['daso_T_proto'] = 0.05 - cfg['daso_interp_alpha'] = 0.5 - cfg['daso_with_dist_aware'] = True - cfg['daso_assign_loss_ratio'] = 1.0 - cfg['daso_num_pl_dist_iter'] = 100 - cfg['daso_num_pretrain_iter'] = 5120 - if dataset == 'cifar10': - cfg['daso_T_dist'] = 1.5 + cfg["abc_p_cutoff"] = 0.95 + cfg["abc_loss_ratio"] = 1.0 + elif imb_alg == "daso": + cfg["daso_queue_len"] = 256 + cfg["daso_T_proto"] = 0.05 + cfg["daso_interp_alpha"] = 0.5 + cfg["daso_with_dist_aware"] = True + cfg["daso_assign_loss_ratio"] = 1.0 + cfg["daso_num_pl_dist_iter"] = 100 + cfg["daso_num_pretrain_iter"] = 5120 + if dataset == "cifar10": + cfg["daso_T_dist"] = 1.5 else: - cfg['daso_T_dist'] = 0.3 - elif imb_alg == 'cossl': - cfg['cossl_max_lam'] = 0.6 - cfg['cossl_tfe_augment'] = 'strong' - cfg['cossl_tfe_u_ratio'] = 1 - cfg['cossl_warm_epoch'] = 200 - if dataset in ['imagenet127']: - cfg['cossl_warm_epoch'] = 150 - cfg['cossl_tfe_warm_epoch'] = 10 - cfg['cossl_tfe_warm_lr'] = 0.02 - cfg['cossl_tfe_warm_ema_decay'] = 0.999 - cfg['cossl_tfe_warm_wd'] = 5e-4 - cfg['cossl_tfe_warm_bs'] = 64 - elif imb_alg == 'tras': - cfg['tras_A'] = 2 - cfg['tras_B'] = 2 - cfg['tras_tro'] = 1.0 - cfg['tras_warmup_epochs'] = 1 + cfg["daso_T_dist"] = 0.3 + elif imb_alg == "cossl": + cfg["cossl_max_lam"] = 0.6 + cfg["cossl_tfe_augment"] = "strong" + cfg["cossl_tfe_u_ratio"] = 1 + cfg["cossl_warm_epoch"] = 200 + if dataset in ["imagenet127"]: + cfg["cossl_warm_epoch"] = 150 + cfg["cossl_tfe_warm_epoch"] = 10 + cfg["cossl_tfe_warm_lr"] = 0.02 + cfg["cossl_tfe_warm_ema_decay"] = 0.999 + cfg["cossl_tfe_warm_wd"] = 5e-4 + cfg["cossl_tfe_warm_bs"] = 64 + elif imb_alg == "tras": + cfg["tras_A"] = 2 + cfg["tras_B"] = 2 + cfg["tras_tro"] = 1.0 + cfg["tras_warmup_epochs"] = 1 else: pass - cfg['use_wandb'] = False - # cfg['use_aim'] = True + cfg["use_wandb"] = False + cfg["use_aim"] = False return cfg - def exp_classic_cv_imb(settings): - config_file = r'./config/classic_cv_imb/' - save_path = r'./saved_models/classic_cv_imb' + config_file = r"./config/classic_cv_imb/" + save_path = r"./saved_models/classic_cv_imb" if not os.path.exists(config_file): os.mkdir(config_file) if not os.path.exists(save_path): os.mkdir(save_path) - algs = ['supervised', 'fixmatch', 'remixmatch'] - imb_algs = [None, 'crest', 'crest+', 'darp', 'abc', 'daso', 'saw', 'adsh', 'cossl', 'debiaspl', 'simis'] + algs = ["supervised", "fixmatch", "remixmatch"] + imb_algs = [ + None, + "crest", + "crest+", + "darp", + "abc", + "daso", + "saw", + "adsh", + "cossl", + "debiaspl", + "simis", + ] datasets = list(settings.keys()) seeds = [0] @@ -219,123 +284,210 @@ def exp_classic_cv_imb(settings): for alg in algs: for imb_alg in imb_algs: - - if alg == 'remixmatch' and imb_alg in ['adsh', 'tars']: + if alg == "remixmatch" and imb_alg in ["adsh", "tars"]: continue for dataset in datasets: for seed in seeds: - if dataset == 'cifar10': + if dataset == "cifar10": num_classes = 10 img_size = 32 - net = 'wrn_28_2' + net = "wrn_28_2" lr = 0.03 weight_decay = 5e-4 - elif dataset == 'cifar100': - + elif dataset == "cifar100": num_classes = 100 img_size = 32 - net = 'wrn_28_2' + net = "wrn_28_2" lr = 0.03 weight_decay = 5e-4 - elif dataset == 'stl10': + elif dataset == "stl10": num_classes = 10 img_size = 32 - net = 'wrn_28_2' + net = "wrn_28_2" lr = 0.03 weight_decay = 5e-4 - elif dataset == 'semi_aves' or dataset == 'semi_aves_out': + elif dataset == "semi_aves" or dataset == "semi_aves_out": num_classes = 200 img_size = 224 - net = 'vit_small_patch16_224' - pretrain_name = 'https://github.com/microsoft/Semi-supervised-learning/releases/download/v.0.0.0/vit_small_patch16_224_mlp_im_1k_224.pth' + net = "vit_small_patch16_224" + pretrain_name = "https://github.com/microsoft/Semi-supervised-learning/releases/download/v.0.0.0/vit_small_patch16_224_mlp_im_1k_224.pth" # noqa: E501 lr = 1e-3 layer_decay = 0.65 - elif dataset == 'imagenet127': - + elif dataset == "imagenet127": num_classes = 127 img_size = 112 lr = 0.1 weight_decay = 1e-4 - net = 'resnet50' - - if dataset in ['cifar10', 'cifar100']: - + net = "resnet50" + + if dataset in ["cifar10", "cifar100"]: setting_dicts = settings[dataset] for setting_dict in setting_dicts: - lb_imb_ratio = setting_dict['lb_imb_ratio'] - ulb_imb_ratio = setting_dict['ulb_imb_ratio'] - lb_num_labels = setting_dict['lb_num_labels'] - ulb_num_labels = setting_dict['ulb_num_labels'] - - if alg == 'supervised': + lb_imb_ratio = setting_dict["lb_imb_ratio"] + ulb_imb_ratio = setting_dict["ulb_imb_ratio"] + lb_num_labels = setting_dict["lb_num_labels"] + ulb_num_labels = setting_dict["ulb_num_labels"] + + if alg == "supervised": port = dist_port[count] # prepare the configuration file - cfg = create_classic_cv_imb_config(alg, seed, dataset, net, num_classes, lb_num_labels, img_size, port, lr, weight_decay, - None, lb_imb_ratio, ulb_imb_ratio, ulb_num_labels, amp=False) + cfg = create_classic_cv_imb_config( + alg, + seed, + dataset, + net, + num_classes, + lb_num_labels, + img_size, + port, + lr, + weight_decay, + None, + lb_imb_ratio, + ulb_imb_ratio, + ulb_num_labels, + amp=False, + ) count += 1 create_configuration(cfg, config_file) else: port = dist_port[count] # prepare the configuration file - cfg = create_classic_cv_imb_config(alg, seed, dataset, net, num_classes, lb_num_labels, img_size, port, lr, weight_decay, - imb_alg, lb_imb_ratio, ulb_imb_ratio, ulb_num_labels, amp=False) + cfg = create_classic_cv_imb_config( + alg, + seed, + dataset, + net, + num_classes, + lb_num_labels, + img_size, + port, + lr, + weight_decay, + imb_alg, + lb_imb_ratio, + ulb_imb_ratio, + ulb_num_labels, + amp=False, + ) count += 1 create_configuration(cfg, config_file) - - elif dataset == 'stl10': - + + elif dataset == "stl10": setting_dicts = settings[dataset] for setting_dict in setting_dicts: - lb_imb_ratio = setting_dict['lb_imb_ratio'] + lb_imb_ratio = setting_dict["lb_imb_ratio"] ulb_imb_ratio = 1.0 - lb_num_labels = setting_dict['lb_num_labels'] + lb_num_labels = setting_dict["lb_num_labels"] ulb_num_labels = None - - if alg == 'supervised': + + if alg == "supervised": port = dist_port[count] # prepare the configuration file - cfg = create_classic_cv_imb_config(alg, seed, dataset, net, num_classes, lb_num_labels, img_size, port, lr, weight_decay, - None, lb_imb_ratio, ulb_imb_ratio, ulb_num_labels, amp=False) + cfg = create_classic_cv_imb_config( + alg, + seed, + dataset, + net, + num_classes, + lb_num_labels, + img_size, + port, + lr, + weight_decay, + None, + lb_imb_ratio, + ulb_imb_ratio, + ulb_num_labels, + amp=False, + ) count += 1 create_configuration(cfg, config_file) else: port = dist_port[count] # prepare the configuration file - cfg = create_classic_cv_imb_config(alg, seed, dataset, net, num_classes, lb_num_labels, img_size, port, lr, weight_decay, - imb_alg, lb_imb_ratio, ulb_imb_ratio, ulb_num_labels, amp=False) + cfg = create_classic_cv_imb_config( + alg, + seed, + dataset, + net, + num_classes, + lb_num_labels, + img_size, + port, + lr, + weight_decay, + imb_alg, + lb_imb_ratio, + ulb_imb_ratio, + ulb_num_labels, + amp=False, + ) count += 1 create_configuration(cfg, config_file) - elif dataset in ['semi_aves', 'semi_aves_out']: - + elif dataset in ["semi_aves", "semi_aves_out"]: lb_num_labels = 3957 lb_imb_ratio = 1.0 ulb_num_labels = None ulb_imb_ratio = 1.0 - if alg == 'supervised': + if alg == "supervised": port = dist_port[count] # prepare the configuration file - cfg = create_classic_cv_imb_config(alg, seed, dataset, net, num_classes, lb_num_labels, img_size, port, lr, weight_decay, - None, lb_imb_ratio, ulb_imb_ratio, ulb_num_labels, amp=False, layer_decay=layer_decay, pretrain_name=pretrain_name) + cfg = create_classic_cv_imb_config( + alg, + seed, + dataset, + net, + num_classes, + lb_num_labels, + img_size, + port, + lr, + weight_decay, + None, + lb_imb_ratio, + ulb_imb_ratio, + ulb_num_labels, + amp=False, + layer_decay=layer_decay, + pretrain_name=pretrain_name, + ) count += 1 create_configuration(cfg, config_file) else: port = dist_port[count] # prepare the configuration file - cfg = create_classic_cv_imb_config(alg, seed, dataset, net, num_classes, lb_num_labels, img_size, port, lr, weight_decay, - imb_alg, lb_imb_ratio, ulb_imb_ratio, ulb_num_labels, amp=False, layer_decay=layer_decay, pretrain_name=pretrain_name) + cfg = create_classic_cv_imb_config( + alg, + seed, + dataset, + net, + num_classes, + lb_num_labels, + img_size, + port, + lr, + weight_decay, + imb_alg, + lb_imb_ratio, + ulb_imb_ratio, + ulb_num_labels, + amp=False, + layer_decay=layer_decay, + pretrain_name=pretrain_name, + ) count += 1 create_configuration(cfg, config_file) - elif dataset in ['imagenet127']: - + elif dataset in ["imagenet127"]: lb_num_labels = 128101 lb_imb_ratio = 286 ulb_num_labels = None @@ -343,56 +495,122 @@ def exp_classic_cv_imb(settings): img_size = 112 seed = 0 - if alg == 'supervised': + if alg == "supervised": port = dist_port[count] # prepare the configuration file - cfg = create_classic_cv_imb_config(alg, seed, dataset, net, num_classes, lb_num_labels, img_size, port, lr, weight_decay, - None, lb_imb_ratio, ulb_imb_ratio, ulb_num_labels, amp=False) + cfg = create_classic_cv_imb_config( + alg, + seed, + dataset, + net, + num_classes, + lb_num_labels, + img_size, + port, + lr, + weight_decay, + None, + lb_imb_ratio, + ulb_imb_ratio, + ulb_num_labels, + amp=False, + ) count += 1 create_configuration(cfg, config_file) else: port = dist_port[count] # prepare the configuration file - cfg = create_classic_cv_imb_config(alg, seed, dataset, net, num_classes, lb_num_labels, img_size, port, lr, weight_decay, - imb_alg, lb_imb_ratio, ulb_imb_ratio, ulb_num_labels, amp=False) + cfg = create_classic_cv_imb_config( + alg, + seed, + dataset, + net, + num_classes, + lb_num_labels, + img_size, + port, + lr, + weight_decay, + imb_alg, + lb_imb_ratio, + ulb_imb_ratio, + ulb_num_labels, + amp=False, + ) count += 1 create_configuration(cfg, config_file) - -if __name__ == '__main__': - if not os.path.exists('./saved_models/classic_cv_imb/'): - os.makedirs('./saved_models/classic_cv_imb/', exist_ok=True) - if not os.path.exists('./config/classic_cv_imb/'): - os.makedirs('./config/classic_cv_imb/', exist_ok=True) +if __name__ == "__main__": + if not os.path.exists("./saved_models/classic_cv_imb/"): + os.makedirs("./saved_models/classic_cv_imb/", exist_ok=True) + if not os.path.exists("./config/classic_cv_imb/"): + os.makedirs("./config/classic_cv_imb/", exist_ok=True) settings = { - 'cifar10': [ - {'lb_num_labels': 1500, 'ulb_num_labels': 3000, 'lb_imb_ratio': 100, 'ulb_imb_ratio': 100}, - {'lb_num_labels': 500, 'ulb_num_labels': 4000, 'lb_imb_ratio': 100, 'ulb_imb_ratio': 100}, - - {'lb_num_labels': 1500, 'ulb_num_labels': 3000, 'lb_imb_ratio': 150, 'ulb_imb_ratio': 150}, - {'lb_num_labels': 500, 'ulb_num_labels': 4000, 'lb_imb_ratio': 150, 'ulb_imb_ratio': 150}, - - - {'lb_num_labels': 1500, 'ulb_num_labels': 3000, 'lb_imb_ratio': 100, 'ulb_imb_ratio': -100}, - {'lb_num_labels': 500, 'ulb_num_labels': 4000, 'lb_imb_ratio': 100, 'ulb_imb_ratio': -100}, + "cifar10": [ + { + "lb_num_labels": 1500, + "ulb_num_labels": 3000, + "lb_imb_ratio": 100, + "ulb_imb_ratio": 100, + }, + { + "lb_num_labels": 500, + "ulb_num_labels": 4000, + "lb_imb_ratio": 100, + "ulb_imb_ratio": 100, + }, + { + "lb_num_labels": 1500, + "ulb_num_labels": 3000, + "lb_imb_ratio": 150, + "ulb_imb_ratio": 150, + }, + { + "lb_num_labels": 500, + "ulb_num_labels": 4000, + "lb_imb_ratio": 150, + "ulb_imb_ratio": 150, + }, + { + "lb_num_labels": 1500, + "ulb_num_labels": 3000, + "lb_imb_ratio": 100, + "ulb_imb_ratio": -100, + }, + { + "lb_num_labels": 500, + "ulb_num_labels": 4000, + "lb_imb_ratio": 100, + "ulb_imb_ratio": -100, + }, ], - - 'cifar100': [ - {'lb_num_labels': 150, 'ulb_num_labels': 300, 'lb_imb_ratio': 10, 'ulb_imb_ratio': 10}, - - {'lb_num_labels': 150, 'ulb_num_labels': 300, 'lb_imb_ratio': 15, 'ulb_imb_ratio': 15}, - - {'lb_num_labels': 150, 'ulb_num_labels': 300, 'lb_imb_ratio': 10, 'ulb_imb_ratio': -10}, + "cifar100": [ + { + "lb_num_labels": 150, + "ulb_num_labels": 300, + "lb_imb_ratio": 10, + "ulb_imb_ratio": 10, + }, + { + "lb_num_labels": 150, + "ulb_num_labels": 300, + "lb_imb_ratio": 15, + "ulb_imb_ratio": 15, + }, + { + "lb_num_labels": 150, + "ulb_num_labels": 300, + "lb_imb_ratio": 10, + "ulb_imb_ratio": -10, + }, ], - - 'stl10': [ - {'lb_num_labels': 150, 'lb_imb_ratio': 10}, - {'lb_num_labels': 150, 'lb_imb_ratio': 20}, + "stl10": [ + {"lb_num_labels": 150, "lb_imb_ratio": 10}, + {"lb_num_labels": 150, "lb_imb_ratio": 20}, ], - - 'imagenet127': [], + "imagenet127": [], } exp_classic_cv_imb(settings) diff --git a/scripts/config_generator_usb_audio.py b/scripts/config_generator_usb_audio.py index 5bab23bda..1be337b2b 100644 --- a/scripts/config_generator_usb_audio.py +++ b/scripts/config_generator_usb_audio.py @@ -8,237 +8,266 @@ def create_configuration(cfg, cfg_file): - cfg['save_name'] = "{alg}_{dataset}_{num_lb}_{seed}".format( - alg=cfg['algorithm'], - dataset=cfg['dataset'], - num_lb=cfg['num_labels'], - seed=cfg['seed'], + cfg["save_name"] = "{alg}_{dataset}_{num_lb}_{seed}".format( + alg=cfg["algorithm"], + dataset=cfg["dataset"], + num_lb=cfg["num_labels"], + seed=cfg["seed"], ) # resume - cfg['resume'] = True - cfg['load_path'] = '{}/{}/latest_model.pth'.format(cfg['save_dir'], cfg['save_name']) + cfg["resume"] = True + cfg["load_path"] = "{}/{}/latest_model.pth".format( + cfg["save_dir"], cfg["save_name"] + ) - alg_file = cfg_file + cfg['algorithm'] + '/' + alg_file = cfg_file + cfg["algorithm"] + "/" if not os.path.exists(alg_file): os.mkdir(alg_file) - print(alg_file + cfg['save_name'] + '.yaml') - with open(alg_file + cfg['save_name'] + '.yaml', 'w', encoding='utf-8') as w: + print(alg_file + cfg["save_name"] + ".yaml") + with open(alg_file + cfg["save_name"] + ".yaml", "w", encoding="utf-8") as w: lines = [] for k, v in cfg.items(): - line = str(k) + ': ' + str(v) + line = str(k) + ": " + str(v) lines.append(line) for line in lines: w.writelines(line) - w.write('\n') - - - -def create_usb_audio_config(alg, seed, - dataset, net, num_classes, num_labels, - port, lr, weight_decay, layer_decay, max_length_seconds, sample_rate): + w.write("\n") + + +def create_usb_audio_config( + alg, + seed, + dataset, + net, + num_classes, + num_labels, + port, + lr, + weight_decay, + layer_decay, + max_length_seconds, + sample_rate, +): cfg = {} - cfg['algorithm'] = alg + cfg["algorithm"] = alg # save config - cfg['save_dir'] = './saved_models/usb_audio' - cfg['save_name'] = None - cfg['resume'] = True - cfg['load_path'] = None - cfg['overwrite'] = True - cfg['use_tensorboard'] = True - - cfg['epoch'] = 100 - cfg['num_train_iter'] = 1024 * 100 - cfg['num_eval_iter'] = 2048 - cfg['num_log_iter'] = 256 - cfg['num_warmup_iter'] = int(1024 * 5) - - cfg['num_labels'] = num_labels - cfg['batch_size'] = 8 - cfg['eval_batch_size'] = 16 - cfg['ema_m'] = 0.0 - - if alg == 'fixmatch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - elif alg == 'adamatch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - cfg['ema_p'] = 0.999 - elif alg == 'flexmatch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['thresh_warmup'] = True - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - elif alg == 'uda': - cfg['tsa_schedule'] = 'none' - cfg['T'] = 0.4 - cfg['p_cutoff'] = 0.8 - cfg['ulb_loss_ratio'] = 1.0 - elif alg == 'pseudolabel': - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - cfg['unsup_warm_up'] = 0.4 - elif alg == 'mixmatch': - cfg['mixup_alpha'] = 0.5 - cfg['T'] = 0.5 - cfg['ulb_loss_ratio'] = 10 - cfg['unsup_warm_up'] = 0.4 - cfg['mixup_manifold'] = True - elif alg == 'remixmatch': - cfg['mixup_alpha'] = 0.75 - cfg['T'] = 0.5 - cfg['kl_loss_ratio'] = 0.5 - cfg['ulb_loss_ratio'] = 1.5 - cfg['rot_loss_ratio'] = 0.0 - cfg['unsup_warm_up'] = 1 / 64 - cfg['mixup_manifold'] = True - elif alg == 'crmatch': - cfg['hard_label'] = True - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - cfg['rot_loss_ratio'] = 0.0 - elif alg == 'comatch': - cfg['hard_label'] = False - cfg['p_cutoff'] = 0.95 - cfg['contrast_p_cutoff'] = 0.8 - cfg['contrast_loss_ratio'] = 1.0 - cfg['ulb_loss_ratio'] = 1.0 - cfg['proj_size'] = 64 - cfg['queue_batch'] = 128 - cfg['smoothing_alpha'] = 0.9 - cfg['T'] = 0.2 - cfg['da_len'] = 32 - elif alg == 'simmatch': - cfg['p_cutoff'] = 0.95 - cfg['in_loss_ratio'] = 1.0 - cfg['ulb_loss_ratio'] = 1.0 - cfg['proj_size'] = 128 - cfg['K'] = 256 - cfg['da_len'] = 32 - cfg['smoothing_alpha'] = 0.9 - cfg['T'] = 0.2 - cfg['ema_m'] = 0.999 - elif alg == 'meanteacher': - cfg['ulb_loss_ratio'] = 50 - cfg['unsup_warm_up'] = 0.4 - cfg['ema_m'] = 0.999 - elif alg == 'pimodel': - cfg['ulb_loss_ratio'] = 10 - cfg['unsup_warm_up'] = 0.4 - elif alg == 'dash': - cfg['gamma'] = 1.27 - cfg['C'] = 1.0001 - cfg['rho_min'] = 0.05 - cfg['num_wu_iter'] = 2048 - cfg['T'] = 0.5 - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - elif alg == 'mpl': - cfg['tsa_schedule'] = 'none' - cfg['T'] = 0.7 - cfg['p_cutoff'] = 0.6 - cfg['ulb_loss_ratio'] = 8.0 - cfg['teacher_lr'] = 0.03 - cfg['label_smoothing'] = 0.1 - cfg['num_uda_warmup_iter'] = 5000 - cfg['num_stu_wait_iter'] = 3000 - cfg['ema_m'] = 0.999 - elif alg == 'freematch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['ema_p'] = 0.999 - cfg['ent_loss_ratio'] = 0.001 - if dataset == 'imagenet': - cfg['ulb_loss_ratio'] = 1.0 - elif alg == 'softmatch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['dist_align'] = True - cfg['dist_uniform'] = True - cfg['per_class'] = False - cfg['ema_p'] = 0.999 - cfg['ulb_loss_ratio'] = 1.0 - cfg['n_sigma'] = 2 - if dataset == 'imagenet': - cfg['ulb_loss_ratio'] = 1.0 - elif alg == 'defixmatch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 0.5 - - - cfg['uratio'] = 1 - cfg['use_cat'] = False + cfg["save_dir"] = "./saved_models/usb_audio" + cfg["save_name"] = None + cfg["resume"] = True + cfg["load_path"] = None + cfg["overwrite"] = True + cfg["use_tensorboard"] = True + + cfg["epoch"] = 100 + cfg["num_train_iter"] = 1024 * 100 + cfg["num_eval_iter"] = 2048 + cfg["num_log_iter"] = 256 + cfg["num_warmup_iter"] = int(1024 * 5) + + cfg["num_labels"] = num_labels + cfg["batch_size"] = 8 + cfg["eval_batch_size"] = 16 + cfg["ema_m"] = 0.0 + + if alg == "fixmatch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + elif alg == "adamatch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + cfg["ema_p"] = 0.999 + elif alg == "flexmatch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["thresh_warmup"] = True + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + elif alg == "uda": + cfg["tsa_schedule"] = "none" + cfg["T"] = 0.4 + cfg["p_cutoff"] = 0.8 + cfg["ulb_loss_ratio"] = 1.0 + elif alg == "pseudolabel": + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + cfg["unsup_warm_up"] = 0.4 + elif alg == "mixmatch": + cfg["mixup_alpha"] = 0.5 + cfg["T"] = 0.5 + cfg["ulb_loss_ratio"] = 10 + cfg["unsup_warm_up"] = 0.4 + cfg["mixup_manifold"] = True + elif alg == "remixmatch": + cfg["mixup_alpha"] = 0.75 + cfg["T"] = 0.5 + cfg["kl_loss_ratio"] = 0.5 + cfg["ulb_loss_ratio"] = 1.5 + cfg["rot_loss_ratio"] = 0.0 + cfg["unsup_warm_up"] = 1 / 64 + cfg["mixup_manifold"] = True + elif alg == "crmatch": + cfg["hard_label"] = True + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + cfg["rot_loss_ratio"] = 0.0 + elif alg == "comatch": + cfg["hard_label"] = False + cfg["p_cutoff"] = 0.95 + cfg["contrast_p_cutoff"] = 0.8 + cfg["contrast_loss_ratio"] = 1.0 + cfg["ulb_loss_ratio"] = 1.0 + cfg["proj_size"] = 64 + cfg["queue_batch"] = 128 + cfg["smoothing_alpha"] = 0.9 + cfg["T"] = 0.2 + cfg["da_len"] = 32 + elif alg == "simmatch": + cfg["p_cutoff"] = 0.95 + cfg["in_loss_ratio"] = 1.0 + cfg["ulb_loss_ratio"] = 1.0 + cfg["proj_size"] = 128 + cfg["K"] = 256 + cfg["da_len"] = 32 + cfg["smoothing_alpha"] = 0.9 + cfg["T"] = 0.2 + cfg["ema_m"] = 0.999 + elif alg == "meanteacher": + cfg["ulb_loss_ratio"] = 50 + cfg["unsup_warm_up"] = 0.4 + cfg["ema_m"] = 0.999 + elif alg == "pimodel": + cfg["ulb_loss_ratio"] = 10 + cfg["unsup_warm_up"] = 0.4 + elif alg == "dash": + cfg["gamma"] = 1.27 + cfg["C"] = 1.0001 + cfg["rho_min"] = 0.05 + cfg["num_wu_iter"] = 2048 + cfg["T"] = 0.5 + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + elif alg == "mpl": + cfg["tsa_schedule"] = "none" + cfg["T"] = 0.7 + cfg["p_cutoff"] = 0.6 + cfg["ulb_loss_ratio"] = 8.0 + cfg["teacher_lr"] = 0.03 + cfg["label_smoothing"] = 0.1 + cfg["num_uda_warmup_iter"] = 5000 + cfg["num_stu_wait_iter"] = 3000 + cfg["ema_m"] = 0.999 + elif alg == "freematch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["ema_p"] = 0.999 + cfg["ent_loss_ratio"] = 0.001 + if dataset == "imagenet": + cfg["ulb_loss_ratio"] = 1.0 + elif alg == "softmatch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["dist_align"] = True + cfg["dist_uniform"] = True + cfg["per_class"] = False + cfg["ema_p"] = 0.999 + cfg["ulb_loss_ratio"] = 1.0 + cfg["n_sigma"] = 2 + if dataset == "imagenet": + cfg["ulb_loss_ratio"] = 1.0 + elif alg == "defixmatch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 0.5 + + cfg["uratio"] = 1 + cfg["use_cat"] = False # optim config - cfg['optim'] = 'AdamW' - cfg['lr'] = lr - cfg['momentum'] = 0.9 - cfg['weight_decay'] = weight_decay - cfg['layer_decay'] = layer_decay - cfg['amp'] = False - cfg['clip'] = 0.0 + cfg["optim"] = "AdamW" + cfg["lr"] = lr + cfg["momentum"] = 0.9 + cfg["weight_decay"] = weight_decay + cfg["layer_decay"] = layer_decay + cfg["amp"] = False + cfg["clip"] = 0.0 # net config - cfg['net'] = net - cfg['net_from_name'] = False + cfg["net"] = net + cfg["net_from_name"] = False # data config - cfg['data_dir'] = './data' - cfg['dataset'] = dataset - cfg['train_sampler'] = 'RandomSampler' - cfg['num_classes'] = num_classes - cfg['num_workers'] = 4 - cfg['max_length_seconds'] = max_length_seconds - cfg['sample_rate'] = sample_rate + cfg["data_dir"] = "./data" + cfg["dataset"] = dataset + cfg["train_sampler"] = "RandomSampler" + cfg["num_classes"] = num_classes + cfg["num_workers"] = 4 + cfg["max_length_seconds"] = max_length_seconds + cfg["sample_rate"] = sample_rate # basic config - cfg['seed'] = seed + cfg["seed"] = seed # distributed config - cfg['world_size'] = 1 - cfg['rank'] = 0 - cfg['multiprocessing_distributed'] = False - cfg['dist_url'] = 'tcp://127.0.0.1:' + str(port) - cfg['dist_backend'] = 'nccl' - cfg['gpu'] = None + cfg["world_size"] = 1 + cfg["rank"] = 0 + cfg["multiprocessing_distributed"] = False + cfg["dist_url"] = "tcp://127.0.0.1:" + str(port) + cfg["dist_backend"] = "nccl" + cfg["gpu"] = None # other config - cfg['overwrite'] = True - cfg['use_wandb'] = False + cfg["overwrite"] = True + cfg["use_wandb"] = False + cfg["use_aim"] = False return cfg - def exp_usb_speech(label_amount): - config_file = r'./config/usb_audio/' - save_path = r'./saved_models/usb_audio' + config_file = r"./config/usb_audio/" + save_path = r"./saved_models/usb_audio" if not os.path.exists(config_file): os.mkdir(config_file) if not os.path.exists(save_path): os.mkdir(save_path) - - algs = ['flexmatch', 'fixmatch', 'uda', 'pseudolabel', 'fullysupervised', 'supervised', 'remixmatch', 'mixmatch', 'meanteacher', - 'pimodel', 'vat', 'dash', 'crmatch', 'comatch', 'simmatch', 'adamatch', 'freematch', 'softmatch', 'defixmatch'] - datasets = ['esc50', 'fsdnoisy', 'urbansound8k', 'gtzan', 'superbks'] + algs = [ + "flexmatch", + "fixmatch", + "uda", + "pseudolabel", + "fullysupervised", + "supervised", + "remixmatch", + "mixmatch", + "meanteacher", + "pimodel", + "vat", + "dash", + "crmatch", + "comatch", + "simmatch", + "adamatch", + "freematch", + "softmatch", + "defixmatch", + ] + datasets = ["esc50", "fsdnoisy", "urbansound8k", "gtzan", "superbks"] # seeds = [0, 1, 2] # 1, 22, 333 seeds = [0] dist_port = range(10001, 31120, 1) - count = 0 + count = 0 weight_decay = 2e-5 sampling_rate = 16000 @@ -246,65 +275,76 @@ def exp_usb_speech(label_amount): for dataset in datasets: for seed in seeds: # change the configuration of each dataset - if dataset == 'esc50': + if dataset == "esc50": num_classes = 50 num_labels = label_amount[0] * num_classes max_length_seconds = 5.0 - net = 'hubert_base' + net = "hubert_base" lr = 1e-4 layer_decay = 0.85 - elif dataset == 'fsdnoisy': + elif dataset == "fsdnoisy": num_classes = 20 num_labels = 1773 max_length_seconds = 5.0 - net = 'hubert_base' + net = "hubert_base" lr = 5e-4 layer_decay = 0.75 - elif dataset == 'urbansound8k': + elif dataset == "urbansound8k": num_classes = 10 num_labels = label_amount[2] * num_classes max_length_seconds = 4.0 - net = 'hubert_base' + net = "hubert_base" lr = 5e-5 layer_decay = 0.75 - elif dataset == 'gtzan': + elif dataset == "gtzan": num_classes = 10 num_labels = label_amount[3] * num_classes max_length_seconds = 3.0 - net = 'wave2vecv2_base' + net = "wave2vecv2_base" lr = 2e-5 layer_decay = 1.0 - elif dataset == 'superbks': + elif dataset == "superbks": num_classes = 10 num_labels = label_amount[4] * num_classes max_length_seconds = 1.0 - net = 'wave2vecv2_base' + net = "wave2vecv2_base" lr = 5e-5 layer_decay = 0.75 - elif dataset == 'superbsi': + elif dataset == "superbsi": num_classes = 1251 num_labels = label_amount[5] * num_classes max_length_seconds = 3.0 port = dist_port[count] # prepare the configuration file - cfg = create_usb_audio_config(alg, seed, - dataset, net, num_classes, num_labels, - port, lr, weight_decay, layer_decay, max_length_seconds, sampling_rate) + cfg = create_usb_audio_config( + alg, + seed, + dataset, + net, + num_classes, + num_labels, + port, + lr, + weight_decay, + layer_decay, + max_length_seconds, + sampling_rate, + ) count += 1 create_configuration(cfg, config_file) -if __name__ == '__main__': - if not os.path.exists('./saved_models/usb_audio/'): - os.makedirs('./saved_models/usb_audio/', exist_ok=True) - if not os.path.exists('./config/usb_audio/'): - os.makedirs('./config/usb_audio/', exist_ok=True) +if __name__ == "__main__": + if not os.path.exists("./saved_models/usb_audio/"): + os.makedirs("./saved_models/usb_audio/", exist_ok=True) + if not os.path.exists("./config/usb_audio/"): + os.makedirs("./config/usb_audio/", exist_ok=True) # usb speech label_amount = { - 's': [5, 10, 10, 10, 5, 20], - 'm': [10, 40, 40, 40, 10, 40], + "s": [5, 10, 10, 10, 5, 20], + "m": [10, 40, 40, 40, 10, 40], } for i in label_amount: diff --git a/scripts/config_generator_usb_cv.py b/scripts/config_generator_usb_cv.py index 785c0ca41..ba422a98f 100644 --- a/scripts/config_generator_usb_cv.py +++ b/scripts/config_generator_usb_cv.py @@ -6,288 +6,318 @@ """ import os + def create_configuration(cfg, cfg_file): - cfg['save_name'] = "{alg}_{dataset}_{num_lb}_{seed}".format( - alg=cfg['algorithm'], - dataset=cfg['dataset'], - num_lb=cfg['num_labels'], - seed=cfg['seed'], + cfg["save_name"] = "{alg}_{dataset}_{num_lb}_{seed}".format( + alg=cfg["algorithm"], + dataset=cfg["dataset"], + num_lb=cfg["num_labels"], + seed=cfg["seed"], ) # resume - cfg['resume'] = True - cfg['load_path'] = '{}/{}/latest_model.pth'.format(cfg['save_dir'], cfg['save_name']) + cfg["resume"] = True + cfg["load_path"] = "{}/{}/latest_model.pth".format( + cfg["save_dir"], cfg["save_name"] + ) - alg_file = cfg_file + cfg['algorithm'] + '/' + alg_file = cfg_file + cfg["algorithm"] + "/" if not os.path.exists(alg_file): os.mkdir(alg_file) - print(alg_file + cfg['save_name'] + '.yaml') - with open(alg_file + cfg['save_name'] + '.yaml', 'w', encoding='utf-8') as w: + print(alg_file + cfg["save_name"] + ".yaml") + with open(alg_file + cfg["save_name"] + ".yaml", "w", encoding="utf-8") as w: lines = [] for k, v in cfg.items(): - line = str(k) + ': ' + str(v) + line = str(k) + ": " + str(v) lines.append(line) for line in lines: w.writelines(line) - w.write('\n') - - - - -def create_usb_cv_config(alg, seed, - dataset, net, num_classes, num_labels, img_size, crop_ratio, - port, lr, weight_decay, layer_decay, pretrain_path, warmup=5, amp=False): + w.write("\n") + + +def create_usb_cv_config( + alg, + seed, + dataset, + net, + num_classes, + num_labels, + img_size, + crop_ratio, + port, + lr, + weight_decay, + layer_decay, + pretrain_path, + warmup=5, + amp=False, +): cfg = {} - cfg['algorithm'] = alg + cfg["algorithm"] = alg # save config - cfg['save_dir'] = './saved_models/usb_cv/' - cfg['save_name'] = None - cfg['resume'] = False - cfg['load_path'] = None - cfg['overwrite'] = True - cfg['use_tensorboard'] = True - cfg['use_wandb'] = False - - if dataset == 'imagenet': - cfg['epoch'] = 500 - cfg['num_train_iter'] = 1024 * 500 - cfg['num_log_iter'] = 256 - cfg['num_eval_iter'] = 5120 - cfg['batch_size'] = 256 - cfg['eval_batch_size'] = 512 + cfg["save_dir"] = "./saved_models/usb_cv/" + cfg["save_name"] = None + cfg["resume"] = False + cfg["load_path"] = None + cfg["overwrite"] = True + cfg["use_tensorboard"] = True + cfg["use_wandb"] = False + cfg["use_aim"] = False + + if dataset == "imagenet": + cfg["epoch"] = 500 + cfg["num_train_iter"] = 1024 * 500 + cfg["num_log_iter"] = 256 + cfg["num_eval_iter"] = 5120 + cfg["batch_size"] = 256 + cfg["eval_batch_size"] = 512 else: - cfg['epoch'] = 200 - cfg['num_train_iter'] = 1024 * 200 - cfg['num_log_iter'] = 256 - cfg['num_eval_iter'] = 2048 - cfg['batch_size'] = 8 - cfg['eval_batch_size'] = 16 - - cfg['num_warmup_iter'] = int(1024 * warmup) - cfg['num_labels'] = num_labels - - cfg['uratio'] = 1 - cfg['ema_m'] = 0.0 - - if alg == 'fixmatch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - if dataset == 'imagenet': - cfg['ulb_loss_ratio'] = 10.0 - cfg['p_cutoff'] = 0.7 - elif alg == 'adamatch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - cfg['ema_p'] = 0.999 - elif alg == 'flexmatch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['thresh_warmup'] = True - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - if dataset == 'imagenet': - cfg['ulb_loss_ratio'] = 10.0 - cfg['p_cutoff'] = 0.7 - elif alg == 'uda': - cfg['tsa_schedule'] = 'none' - cfg['T'] = 0.4 - cfg['p_cutoff'] = 0.8 - cfg['ulb_loss_ratio'] = 1.0 - if dataset == 'imagenet': - cfg['ulb_loss_ratio'] = 10.0 - elif alg == 'pseudolabel': - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - cfg['unsup_warm_up'] = 0.4 - elif alg == 'mixmatch': - cfg['mixup_alpha'] = 0.5 - cfg['T'] = 0.5 - cfg['ulb_loss_ratio'] = 10 - cfg['unsup_warm_up'] = 0.4 # 16000 / 1024 / 1024 - elif alg == 'remixmatch': - cfg['mixup_alpha'] = 0.75 - cfg['T'] = 0.5 - cfg['kl_loss_ratio'] = 0.5 - cfg['ulb_loss_ratio'] = 1.5 - cfg['rot_loss_ratio'] = 0.5 - cfg['unsup_warm_up'] = 1 / 64 - elif alg == 'crmatch': - cfg['hard_label'] = True - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - - elif alg == 'comatch': - cfg['hard_label'] = False - cfg['p_cutoff'] = 0.95 - cfg['contrast_p_cutoff'] = 0.8 - cfg['contrast_loss_ratio'] = 1.0 - cfg['ulb_loss_ratio'] = 1.0 - cfg['proj_size'] = 64 - cfg['queue_batch'] = 32 - cfg['smoothing_alpha'] = 0.9 - cfg['T'] = 0.2 - cfg['da_len'] = 32 - cfg['ema_m'] = 0.999 - if dataset == 'stl10': - cfg['contrast_loss_ratio'] = 5.0 - - if dataset == 'imagenet': - cfg['p_cutoff'] = 0.6 - cfg['contrast_p_cutoff'] = 0.3 - cfg['contrast_loss_ratio'] = 10.0 - cfg['ulb_loss_ratio'] = 10.0 - cfg['smoothing_alpha'] = 0.9 - cfg['T'] = 0.1 - cfg['proj_size'] = 128 - cfg['queue_batch'] = 128 - - elif alg == 'simmatch': - cfg['p_cutoff'] = 0.95 - cfg['in_loss_ratio'] = 1.0 - cfg['ulb_loss_ratio'] = 1.0 - cfg['proj_size'] = 128 - cfg['K'] = 256 - cfg['da_len'] = 32 - cfg['smoothing_alpha'] = 0.9 - cfg['ema_m'] = 0.999 - - if dataset in ['cifar10', 'svhn', 'cifar100', 'stl10']: - cfg['T'] = 0.1 + cfg["epoch"] = 200 + cfg["num_train_iter"] = 1024 * 200 + cfg["num_log_iter"] = 256 + cfg["num_eval_iter"] = 2048 + cfg["batch_size"] = 8 + cfg["eval_batch_size"] = 16 + + cfg["num_warmup_iter"] = int(1024 * warmup) + cfg["num_labels"] = num_labels + + cfg["uratio"] = 1 + cfg["ema_m"] = 0.0 + + if alg == "fixmatch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + if dataset == "imagenet": + cfg["ulb_loss_ratio"] = 10.0 + cfg["p_cutoff"] = 0.7 + elif alg == "adamatch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + cfg["ema_p"] = 0.999 + elif alg == "flexmatch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["thresh_warmup"] = True + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + if dataset == "imagenet": + cfg["ulb_loss_ratio"] = 10.0 + cfg["p_cutoff"] = 0.7 + elif alg == "uda": + cfg["tsa_schedule"] = "none" + cfg["T"] = 0.4 + cfg["p_cutoff"] = 0.8 + cfg["ulb_loss_ratio"] = 1.0 + if dataset == "imagenet": + cfg["ulb_loss_ratio"] = 10.0 + elif alg == "pseudolabel": + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + cfg["unsup_warm_up"] = 0.4 + elif alg == "mixmatch": + cfg["mixup_alpha"] = 0.5 + cfg["T"] = 0.5 + cfg["ulb_loss_ratio"] = 10 + cfg["unsup_warm_up"] = 0.4 # 16000 / 1024 / 1024 + elif alg == "remixmatch": + cfg["mixup_alpha"] = 0.75 + cfg["T"] = 0.5 + cfg["kl_loss_ratio"] = 0.5 + cfg["ulb_loss_ratio"] = 1.5 + cfg["rot_loss_ratio"] = 0.5 + cfg["unsup_warm_up"] = 1 / 64 + elif alg == "crmatch": + cfg["hard_label"] = True + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + + elif alg == "comatch": + cfg["hard_label"] = False + cfg["p_cutoff"] = 0.95 + cfg["contrast_p_cutoff"] = 0.8 + cfg["contrast_loss_ratio"] = 1.0 + cfg["ulb_loss_ratio"] = 1.0 + cfg["proj_size"] = 64 + cfg["queue_batch"] = 32 + cfg["smoothing_alpha"] = 0.9 + cfg["T"] = 0.2 + cfg["da_len"] = 32 + cfg["ema_m"] = 0.999 + if dataset == "stl10": + cfg["contrast_loss_ratio"] = 5.0 + + if dataset == "imagenet": + cfg["p_cutoff"] = 0.6 + cfg["contrast_p_cutoff"] = 0.3 + cfg["contrast_loss_ratio"] = 10.0 + cfg["ulb_loss_ratio"] = 10.0 + cfg["smoothing_alpha"] = 0.9 + cfg["T"] = 0.1 + cfg["proj_size"] = 128 + cfg["queue_batch"] = 128 + + elif alg == "simmatch": + cfg["p_cutoff"] = 0.95 + cfg["in_loss_ratio"] = 1.0 + cfg["ulb_loss_ratio"] = 1.0 + cfg["proj_size"] = 128 + cfg["K"] = 256 + cfg["da_len"] = 32 + cfg["smoothing_alpha"] = 0.9 + cfg["ema_m"] = 0.999 + + if dataset in ["cifar10", "svhn", "cifar100", "stl10"]: + cfg["T"] = 0.1 else: - cfg['T'] = 0.2 - - if dataset == 'imagenet': - cfg['in_loss_ratio'] = 5.0 - cfg['ulb_loss_ratio'] = 10.0 - cfg['T'] = 0.1 - cfg['p_cutoff'] = 0.7 - cfg['da_len'] = 256 - cfg['ema_m'] = 0.999 - - - elif alg == 'meanteacher': - - cfg['ulb_loss_ratio'] = 50 - cfg['unsup_warm_up'] = 0.4 - cfg['ema_m'] = 0.999 - - elif alg == 'pimodel': - cfg['ulb_loss_ratio'] = 10 - - cfg['unsup_warm_up'] = 0.4 - elif alg == 'dash': - cfg['gamma'] = 1.27 - cfg['C'] = 1.0001 - cfg['rho_min'] = 0.05 - cfg['num_wu_iter'] = 2048 - cfg['T'] = 0.5 - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - - elif alg == 'mpl': - cfg['tsa_schedule'] = 'none' - cfg['T'] = 0.7 - cfg['p_cutoff'] = 0.6 - cfg['ulb_loss_ratio'] = 8.0 - - cfg['teacher_lr'] = 0.03 - cfg['label_smoothing'] = 0.1 - cfg['num_uda_warmup_iter'] = 5000 - cfg['num_stu_wait_iter'] = 3000 - - - elif alg == 'freematch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['ema_p'] = 0.999 - cfg['ent_loss_ratio'] = 0.001 - if dataset == 'imagenet': - cfg['ulb_loss_ratio'] = 1.0 - elif alg == 'softmatch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['dist_align'] = True - cfg['dist_uniform'] = True - cfg['per_class'] = False - cfg['ema_p'] = 0.999 - cfg['ulb_loss_ratio'] = 1.0 - cfg['n_sigma'] = 2 - if dataset == 'imagenet': - cfg['ulb_loss_ratio'] = 1.0 - elif alg == 'defixmatch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 0.5 - - cfg['img_size'] = img_size - cfg['crop_ratio'] = crop_ratio + cfg["T"] = 0.2 + + if dataset == "imagenet": + cfg["in_loss_ratio"] = 5.0 + cfg["ulb_loss_ratio"] = 10.0 + cfg["T"] = 0.1 + cfg["p_cutoff"] = 0.7 + cfg["da_len"] = 256 + cfg["ema_m"] = 0.999 + + elif alg == "meanteacher": + cfg["ulb_loss_ratio"] = 50 + cfg["unsup_warm_up"] = 0.4 + cfg["ema_m"] = 0.999 + + elif alg == "pimodel": + cfg["ulb_loss_ratio"] = 10 + + cfg["unsup_warm_up"] = 0.4 + elif alg == "dash": + cfg["gamma"] = 1.27 + cfg["C"] = 1.0001 + cfg["rho_min"] = 0.05 + cfg["num_wu_iter"] = 2048 + cfg["T"] = 0.5 + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + + elif alg == "mpl": + cfg["tsa_schedule"] = "none" + cfg["T"] = 0.7 + cfg["p_cutoff"] = 0.6 + cfg["ulb_loss_ratio"] = 8.0 + + cfg["teacher_lr"] = 0.03 + cfg["label_smoothing"] = 0.1 + cfg["num_uda_warmup_iter"] = 5000 + cfg["num_stu_wait_iter"] = 3000 + + elif alg == "freematch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["ema_p"] = 0.999 + cfg["ent_loss_ratio"] = 0.001 + if dataset == "imagenet": + cfg["ulb_loss_ratio"] = 1.0 + elif alg == "softmatch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["dist_align"] = True + cfg["dist_uniform"] = True + cfg["per_class"] = False + cfg["ema_p"] = 0.999 + cfg["ulb_loss_ratio"] = 1.0 + cfg["n_sigma"] = 2 + if dataset == "imagenet": + cfg["ulb_loss_ratio"] = 1.0 + elif alg == "defixmatch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 0.5 + + cfg["img_size"] = img_size + cfg["crop_ratio"] = crop_ratio # optim config - cfg['optim'] = 'AdamW' - cfg['lr'] = lr - cfg['layer_decay'] = layer_decay - cfg['momentum'] = 0.9 - cfg['weight_decay'] = weight_decay - cfg['amp'] = amp - cfg['clip'] = 0.0 - cfg['use_cat'] = True + cfg["optim"] = "AdamW" + cfg["lr"] = lr + cfg["layer_decay"] = layer_decay + cfg["momentum"] = 0.9 + cfg["weight_decay"] = weight_decay + cfg["amp"] = amp + cfg["clip"] = 0.0 + cfg["use_cat"] = True # net config - cfg['net'] = net - cfg['net_from_name'] = False + cfg["net"] = net + cfg["net_from_name"] = False # data config - cfg['data_dir'] = './data' - cfg['dataset'] = dataset - cfg['train_sampler'] = 'RandomSampler' - cfg['num_classes'] = num_classes - cfg['num_workers'] = 4 + cfg["data_dir"] = "./data" + cfg["dataset"] = dataset + cfg["train_sampler"] = "RandomSampler" + cfg["num_classes"] = num_classes + cfg["num_workers"] = 4 # basic config - cfg['seed'] = seed + cfg["seed"] = seed # distributed config - cfg['world_size'] = 1 - cfg['rank'] = 0 - cfg['multiprocessing_distributed'] = True - cfg['dist_url'] = 'tcp://127.0.0.1:' + str(port) - cfg['dist_backend'] = 'nccl' - cfg['gpu'] = None + cfg["world_size"] = 1 + cfg["rank"] = 0 + cfg["multiprocessing_distributed"] = True + cfg["dist_url"] = "tcp://127.0.0.1:" + str(port) + cfg["dist_backend"] = "nccl" + cfg["gpu"] = None - if alg == 'crmatch' and dataset == 'stl10': - cfg['multiprocessing_distributed'] = False - cfg['gpu'] = 0 + if alg == "crmatch" and dataset == "stl10": + cfg["multiprocessing_distributed"] = False + cfg["gpu"] = 0 # other config - cfg['overwrite'] = True - cfg['use_pretrain'] = True - cfg['pretrain_path'] = pretrain_path + cfg["overwrite"] = True + cfg["use_pretrain"] = True + cfg["pretrain_path"] = pretrain_path return cfg - # prepare the configuration for baseline model, use_penalty == False def exp_usb_cv(label_amount): - config_file = r'./config/usb_cv/' - save_path = r'./saved_models/usb_cv' + config_file = r"./config/usb_cv/" + save_path = r"./saved_models/usb_cv" if not os.path.exists(config_file): os.mkdir(config_file) if not os.path.exists(save_path): os.mkdir(save_path) - - algs = ['flexmatch', 'fixmatch', 'uda', 'pseudolabel', 'fullysupervised', 'supervised', 'remixmatch', 'mixmatch', 'meanteacher', - 'pimodel', 'vat', 'dash', 'crmatch', 'comatch', 'simmatch', 'adamatch', 'freematch', 'softmatch', 'defixmatch'] - datasets = ['cifar100', 'eurosat', 'semi_aves', 'tissuemnist', 'stl10'] + algs = [ + "flexmatch", + "fixmatch", + "uda", + "pseudolabel", + "fullysupervised", + "supervised", + "remixmatch", + "mixmatch", + "meanteacher", + "pimodel", + "vat", + "dash", + "crmatch", + "comatch", + "simmatch", + "adamatch", + "freematch", + "softmatch", + "defixmatch", + ] + datasets = ["cifar100", "eurosat", "semi_aves", "tissuemnist", "stl10"] # algs = ['fixmatch', 'flexmatch', 'comatch', 'simmatch'] # datasets = ['imagenet'] # seeds = [0, 1, 2] # 1, 22, 333 @@ -295,8 +325,8 @@ def exp_usb_cv(label_amount): dist_port = range(10001, 11120, 1) count = 0 - - pretrain_path = 'https://github.com/microsoft/Semi-supervised-learning/releases/download/v.0.0.0' + + pretrain_path = "https://github.com/microsoft/Semi-supervised-learning/releases/download/v.0.0.0" # noqa: E501 weight_decay = 5e-4 # lr = 5e-5 warmup = 0 @@ -306,19 +336,19 @@ def exp_usb_cv(label_amount): for dataset in datasets: for seed in seeds: # change the configuration of each dataset - if dataset == 'cifar10': + if dataset == "cifar10": num_classes = 10 num_labels = label_amount[0] * num_classes img_size = 32 crop_ratio = 0.875 - net = 'vit_tiny_patch2_32' - pretrain_name = 'vit_tiny_patch2_32_mlp_im_1k_32.pth' + net = "vit_tiny_patch2_32" + pretrain_name = "vit_tiny_patch2_32_mlp_im_1k_32.pth" lr = 5e-4 - layer_decay = 0.5 + layer_decay = 0.5 - elif dataset == 'cifar100': + elif dataset == "cifar100": num_classes = 100 num_labels = label_amount[1] * num_classes @@ -326,37 +356,36 @@ def exp_usb_cv(label_amount): # widen_factor = 8 img_size = 32 crop_ratio = 0.875 - net = 'vit_small_patch2_32' - pretrain_name = 'vit_small_patch2_32_mlp_im_1k_32.pth' + net = "vit_small_patch2_32" + pretrain_name = "vit_small_patch2_32_mlp_im_1k_32.pth" lr = 5e-4 - layer_decay = 0.5 - + layer_decay = 0.5 - elif dataset == 'svhn': + elif dataset == "svhn": img_size = 32 crop_ratio = 0.875 num_classes = 10 num_labels = label_amount[2] * num_classes - net = 'vit_tiny_patch2_32' - pretrain_name = 'vit_tiny_patch2_32_mlp_im_1k_32.pth' + net = "vit_tiny_patch2_32" + pretrain_name = "vit_tiny_patch2_32_mlp_im_1k_32.pth" - elif dataset == 'stl10': + elif dataset == "stl10": num_classes = 10 num_labels = label_amount[3] * num_classes img_size = 96 crop_ratio = 0.875 warmup = 5 - net = 'vit_base_patch16_96' - pretrain_name = 'mae_pretrain_vit_base.pth' + net = "vit_base_patch16_96" + pretrain_name = "mae_pretrain_vit_base.pth" lr = 1e-4 - layer_decay = 0.65 - - elif dataset == 'semi_aves': + layer_decay = 0.65 + + elif dataset == "semi_aves": num_classes = 200 # num_labels = label_amount[4] * num_classes num_labels = 3959 @@ -364,45 +393,44 @@ def exp_usb_cv(label_amount): img_size = 224 crop_ratio = 0.875 - net = 'vit_small_patch16_224' - pretrain_name = 'vit_small_patch16_224_mlp_im_1k_224.pth' + net = "vit_small_patch16_224" + pretrain_name = "vit_small_patch16_224_mlp_im_1k_224.pth" lr = 1e-3 layer_decay = 0.65 - + # NOTE: resize to 32 x 32 - elif dataset == 'eurosat': + elif dataset == "eurosat": num_classes = 10 num_labels = label_amount[5] * num_classes img_size = 32 crop_ratio = 0.875 - net = 'vit_small_patch2_32' - pretrain_name = 'vit_small_patch2_32_mlp_im_1k_32.pth' + net = "vit_small_patch2_32" + pretrain_name = "vit_small_patch2_32_mlp_im_1k_32.pth" lr = 5e-5 layer_decay = 1.0 - - elif dataset == 'tissuemnist': + elif dataset == "tissuemnist": num_classes = 8 num_labels = label_amount[6] * num_classes img_size = 32 crop_ratio = 0.95 - net = 'vit_tiny_patch2_32' - pretrain_name = 'vit_tiny_patch2_32_mlp_im_1k_32.pth' + net = "vit_tiny_patch2_32" + pretrain_name = "vit_tiny_patch2_32_mlp_im_1k_32.pth" lr = 5e-5 layer_decay = 0.95 - elif dataset == 'imagenet': - net = 'vit_base_path16_224' - pretrain_name = 'mae_pretrain_vit_base.pth' + elif dataset == "imagenet": + net = "vit_base_path16_224" + pretrain_name = "mae_pretrain_vit_base.pth" num_classes = 1000 num_labels = 100000 # 128000 - lr = 1e-3 + lr = 1e-3 weight_decay = 0.01 warmup = 5 amp = True @@ -411,21 +439,34 @@ def exp_usb_cv(label_amount): port = dist_port[count] # prepare the configuration file - cfg = create_usb_cv_config(alg, seed, - dataset, net, num_classes, num_labels, img_size, crop_ratio, - port, lr, weight_decay, layer_decay, pretrain_path=os.path.join(pretrain_path, pretrain_name), - warmup=warmup, amp=amp) + cfg = create_usb_cv_config( + alg, + seed, + dataset, + net, + num_classes, + num_labels, + img_size, + crop_ratio, + port, + lr, + weight_decay, + layer_decay, + pretrain_path=os.path.join(pretrain_path, pretrain_name), + warmup=warmup, + amp=amp, + ) count += 1 create_configuration(cfg, config_file) -if __name__ == '__main__': - if not os.path.exists('./saved_models/usb_cv/'): - os.makedirs('./saved_models/usb_cv/', exist_ok=True) - if not os.path.exists('./config/usb_cv/'): - os.makedirs('./config/usb_cv/', exist_ok=True) - label_amount = {'s': [2, 2, 2, 4, 2, 2, 10], - 'm': [4, 4, 4, 10, 2, 4, 50]} +if __name__ == "__main__": + if not os.path.exists("./saved_models/usb_cv/"): + os.makedirs("./saved_models/usb_cv/", exist_ok=True) + if not os.path.exists("./config/usb_cv/"): + os.makedirs("./config/usb_cv/", exist_ok=True) + + label_amount = {"s": [2, 2, 2, 4, 2, 2, 10], "m": [4, 4, 4, 10, 2, 4, 50]} for i in label_amount: exp_usb_cv(label_amount=label_amount[i]) diff --git a/scripts/config_generator_usb_nlp.py b/scripts/config_generator_usb_nlp.py index 6e6455437..3ee97a105 100644 --- a/scripts/config_generator_usb_nlp.py +++ b/scripts/config_generator_usb_nlp.py @@ -8,236 +8,274 @@ def create_configuration(cfg, cfg_file): - cfg['save_name'] = "{alg}_{dataset}_{num_lb}_{seed}".format( - alg=cfg['algorithm'], - dataset=cfg['dataset'], - num_lb=cfg['num_labels'], - seed=cfg['seed'], + cfg["save_name"] = "{alg}_{dataset}_{num_lb}_{seed}".format( + alg=cfg["algorithm"], + dataset=cfg["dataset"], + num_lb=cfg["num_labels"], + seed=cfg["seed"], ) - + # resume - cfg['resume'] = True - cfg['load_path'] = '{}/{}/latest_model.pth'.format(cfg['save_dir'], cfg['save_name']) + cfg["resume"] = True + cfg["load_path"] = "{}/{}/latest_model.pth".format( + cfg["save_dir"], cfg["save_name"] + ) - alg_file = cfg_file + cfg['algorithm'] + '/' + alg_file = cfg_file + cfg["algorithm"] + "/" if not os.path.exists(alg_file): os.mkdir(alg_file) - print(alg_file + cfg['save_name'] + '.yaml') - with open(alg_file + cfg['save_name'] + '.yaml', 'w', encoding='utf-8') as w: + print(alg_file + cfg["save_name"] + ".yaml") + with open(alg_file + cfg["save_name"] + ".yaml", "w", encoding="utf-8") as w: lines = [] for k, v in cfg.items(): - line = str(k) + ': ' + str(v) + line = str(k) + ": " + str(v) lines.append(line) for line in lines: w.writelines(line) - w.write('\n') - - - -def create_usb_nlp_config(alg, seed, - dataset, net, num_classes, num_labels, - port, lr, weight_decay, layer_decay, max_length, warmup_epoch=5, amp=False): + w.write("\n") + + +def create_usb_nlp_config( + alg, + seed, + dataset, + net, + num_classes, + num_labels, + port, + lr, + weight_decay, + layer_decay, + max_length, + warmup_epoch=5, + amp=False, +): cfg = {} - cfg['algorithm'] = alg + cfg["algorithm"] = alg # save config - cfg['save_dir'] = './saved_models/usb_nlp' - cfg['save_name'] = None - cfg['resume'] = True - cfg['load_path'] = None - cfg['overwrite'] = True - cfg['use_tensorboard'] = True - cfg['use_wandb'] = False + cfg["save_dir"] = "./saved_models/usb_nlp" + cfg["save_name"] = None + cfg["resume"] = True + cfg["load_path"] = None + cfg["overwrite"] = True + cfg["use_tensorboard"] = True + cfg["use_wandb"] = False + cfg["use_aim"] = False # algorithm config - cfg['epoch'] = 100 - cfg['num_train_iter'] = 1024 * 100 - cfg['num_warmup_iter'] = int(1024 * warmup_epoch) - cfg['num_log_iter'] = 256 - cfg['num_eval_iter'] = 2048 - cfg['num_labels'] = num_labels - cfg['batch_size'] = 8 - cfg['eval_batch_size'] = 8 - cfg['ema_m'] = 0.0 - - if alg == 'fixmatch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - elif alg == 'adamatch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - cfg['ema_p'] = 0.999 - elif alg == 'flexmatch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['thresh_warmup'] = True - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - elif alg == 'uda': - cfg['tsa_schedule'] = 'exp' - cfg['T'] = 0.4 - cfg['p_cutoff'] = 0.8 - cfg['ulb_loss_ratio'] = 1.0 - elif alg == 'pseudolabel': - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - cfg['unsup_warm_up'] = 0.4 - elif alg == 'mixmatch': - cfg['mixup_alpha'] = 0.5 - cfg['T'] = 0.5 - cfg['ulb_loss_ratio'] = 100 - cfg['unsup_warm_up'] = 0.4 - cfg['mixup_manifold'] = True - elif alg == 'remixmatch': - cfg['mixup_alpha'] = 0.75 - cfg['T'] = 0.5 - cfg['kl_loss_ratio'] = 0.5 - cfg['ulb_loss_ratio'] = 1.5 - cfg['rot_loss_ratio'] = 0.0 - cfg['unsup_warm_up'] = 1 / 64 - cfg['mixup_manifold'] = True - elif alg == 'crmatch': - cfg['hard_label'] = True - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - cfg['rot_loss_ratio'] = 0.0 - elif alg == 'comatch': - cfg['hard_label'] = False - cfg['p_cutoff'] = 0.95 - cfg['contrast_p_cutoff'] = 0.8 - cfg['contrast_loss_ratio'] = 1.0 - cfg['ulb_loss_ratio'] = 1.0 - cfg['proj_size'] = 64 - cfg['queue_batch'] = 128 - cfg['smoothing_alpha'] = 0.9 - cfg['T'] = 0.2 - cfg['da_len'] = 32 - elif alg == 'simmatch': - cfg['p_cutoff'] = 0.95 - cfg['in_loss_ratio'] = 1.0 - cfg['ulb_loss_ratio'] = 1.0 - cfg['proj_size'] = 128 - cfg['K'] = 256 - cfg['da_len'] = 32 - cfg['smoothing_alpha'] = 0.9 - cfg['T'] = 0.1 - cfg['ema_m'] = 0.0 - elif alg == 'meanteacher': - cfg['ulb_loss_ratio'] = 50 - cfg['unsup_warm_up'] = 0.4 - elif alg == 'pimodel': - cfg['ulb_loss_ratio'] = 10 - cfg['unsup_warm_up'] = 0.4 - cfg['ema_m'] = 0.999 - elif alg == 'dash': - cfg['gamma'] = 1.27 - cfg['C'] = 1.0001 - cfg['rho_min'] = 0.05 - cfg['num_wu_iter'] = 2048 - cfg['T'] = 0.5 - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 1.0 - elif alg == 'mpl': - cfg['tsa_schedule'] = 'exp' - cfg['T'] = 0.7 - cfg['p_cutoff'] = 0.6 - cfg['ulb_loss_ratio'] = 8.0 - cfg['teacher_lr'] = 0.03 - cfg['label_smoothing'] = 0.1 - cfg['num_uda_warmup_iter'] = 5000 - cfg['num_stu_wait_iter'] = 3000 - elif alg == 'vat': - cfg['vat_embed'] = True - elif alg == 'freematch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['ema_p'] = 0.999 - cfg['ent_loss_ratio'] = 0.001 - if dataset == 'imagenet': - cfg['ulb_loss_ratio'] = 1.0 - elif alg == 'softmatch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['dist_align'] = True - cfg['dist_uniform'] = True - cfg['per_class'] = False - cfg['ema_p'] = 0.999 - cfg['ulb_loss_ratio'] = 1.0 - cfg['n_sigma'] = 2 - if dataset == 'imagenet': - cfg['ulb_loss_ratio'] = 1.0 - elif alg == 'defixmatch': - cfg['hard_label'] = True - cfg['T'] = 0.5 - cfg['p_cutoff'] = 0.95 - cfg['ulb_loss_ratio'] = 0.5 - - cfg['uratio'] = 1 - cfg['use_cat'] = False + cfg["epoch"] = 100 + cfg["num_train_iter"] = 1024 * 100 + cfg["num_warmup_iter"] = int(1024 * warmup_epoch) + cfg["num_log_iter"] = 256 + cfg["num_eval_iter"] = 2048 + cfg["num_labels"] = num_labels + cfg["batch_size"] = 8 + cfg["eval_batch_size"] = 8 + cfg["ema_m"] = 0.0 + + if alg == "fixmatch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + elif alg == "adamatch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + cfg["ema_p"] = 0.999 + elif alg == "flexmatch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["thresh_warmup"] = True + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + elif alg == "uda": + cfg["tsa_schedule"] = "exp" + cfg["T"] = 0.4 + cfg["p_cutoff"] = 0.8 + cfg["ulb_loss_ratio"] = 1.0 + elif alg == "pseudolabel": + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + cfg["unsup_warm_up"] = 0.4 + elif alg == "mixmatch": + cfg["mixup_alpha"] = 0.5 + cfg["T"] = 0.5 + cfg["ulb_loss_ratio"] = 100 + cfg["unsup_warm_up"] = 0.4 + cfg["mixup_manifold"] = True + elif alg == "remixmatch": + cfg["mixup_alpha"] = 0.75 + cfg["T"] = 0.5 + cfg["kl_loss_ratio"] = 0.5 + cfg["ulb_loss_ratio"] = 1.5 + cfg["rot_loss_ratio"] = 0.0 + cfg["unsup_warm_up"] = 1 / 64 + cfg["mixup_manifold"] = True + elif alg == "crmatch": + cfg["hard_label"] = True + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + cfg["rot_loss_ratio"] = 0.0 + elif alg == "comatch": + cfg["hard_label"] = False + cfg["p_cutoff"] = 0.95 + cfg["contrast_p_cutoff"] = 0.8 + cfg["contrast_loss_ratio"] = 1.0 + cfg["ulb_loss_ratio"] = 1.0 + cfg["proj_size"] = 64 + cfg["queue_batch"] = 128 + cfg["smoothing_alpha"] = 0.9 + cfg["T"] = 0.2 + cfg["da_len"] = 32 + elif alg == "simmatch": + cfg["p_cutoff"] = 0.95 + cfg["in_loss_ratio"] = 1.0 + cfg["ulb_loss_ratio"] = 1.0 + cfg["proj_size"] = 128 + cfg["K"] = 256 + cfg["da_len"] = 32 + cfg["smoothing_alpha"] = 0.9 + cfg["T"] = 0.1 + cfg["ema_m"] = 0.0 + elif alg == "meanteacher": + cfg["ulb_loss_ratio"] = 50 + cfg["unsup_warm_up"] = 0.4 + elif alg == "pimodel": + cfg["ulb_loss_ratio"] = 10 + cfg["unsup_warm_up"] = 0.4 + cfg["ema_m"] = 0.999 + elif alg == "dash": + cfg["gamma"] = 1.27 + cfg["C"] = 1.0001 + cfg["rho_min"] = 0.05 + cfg["num_wu_iter"] = 2048 + cfg["T"] = 0.5 + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 1.0 + elif alg == "mpl": + cfg["tsa_schedule"] = "exp" + cfg["T"] = 0.7 + cfg["p_cutoff"] = 0.6 + cfg["ulb_loss_ratio"] = 8.0 + cfg["teacher_lr"] = 0.03 + cfg["label_smoothing"] = 0.1 + cfg["num_uda_warmup_iter"] = 5000 + cfg["num_stu_wait_iter"] = 3000 + elif alg == "vat": + cfg["vat_embed"] = True + elif alg == "freematch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["ema_p"] = 0.999 + cfg["ent_loss_ratio"] = 0.001 + if dataset == "imagenet": + cfg["ulb_loss_ratio"] = 1.0 + elif alg == "softmatch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["dist_align"] = True + cfg["dist_uniform"] = True + cfg["per_class"] = False + cfg["ema_p"] = 0.999 + cfg["ulb_loss_ratio"] = 1.0 + cfg["n_sigma"] = 2 + if dataset == "imagenet": + cfg["ulb_loss_ratio"] = 1.0 + elif alg == "defixmatch": + cfg["hard_label"] = True + cfg["T"] = 0.5 + cfg["p_cutoff"] = 0.95 + cfg["ulb_loss_ratio"] = 0.5 + + cfg["uratio"] = 1 + cfg["use_cat"] = False # optim config - cfg['optim'] = 'AdamW' - cfg['lr'] = lr - cfg['momentum'] = 0.9 - cfg['weight_decay'] = weight_decay - cfg['layer_decay'] = layer_decay - cfg['amp'] = amp - cfg['clip'] = 0.0 + cfg["optim"] = "AdamW" + cfg["lr"] = lr + cfg["momentum"] = 0.9 + cfg["weight_decay"] = weight_decay + cfg["layer_decay"] = layer_decay + cfg["amp"] = amp + cfg["clip"] = 0.0 # net config - cfg['net'] = net - cfg['net_from_name'] = False + cfg["net"] = net + cfg["net_from_name"] = False # data config - cfg['data_dir'] = './data' - cfg['dataset'] = dataset - cfg['train_sampler'] = 'RandomSampler' - cfg['num_classes'] = num_classes - cfg['num_workers'] = 4 - cfg['max_length'] = max_length + cfg["data_dir"] = "./data" + cfg["dataset"] = dataset + cfg["train_sampler"] = "RandomSampler" + cfg["num_classes"] = num_classes + cfg["num_workers"] = 4 + cfg["max_length"] = max_length # basic config - cfg['seed'] = seed + cfg["seed"] = seed # distributed config - cfg['world_size'] = 1 - cfg['rank'] = 0 - cfg['multiprocessing_distributed'] = False - cfg['dist_url'] = 'tcp://127.0.0.1:' + str(port) - cfg['dist_backend'] = 'nccl' - cfg['gpu'] = None + cfg["world_size"] = 1 + cfg["rank"] = 0 + cfg["multiprocessing_distributed"] = False + cfg["dist_url"] = "tcp://127.0.0.1:" + str(port) + cfg["dist_backend"] = "nccl" + cfg["gpu"] = None # other config - cfg['overwrite'] = True + cfg["overwrite"] = True return cfg - def exp_usb_nlp(label_amount): - config_file = r'./config/usb_nlp/' - save_path = r'./saved_models/usb_nlp' + config_file = r"./config/usb_nlp/" + save_path = r"./saved_models/usb_nlp" if not os.path.exists(config_file): os.mkdir(config_file) if not os.path.exists(save_path): os.mkdir(save_path) - - algs = ['flexmatch', 'fixmatch', 'uda', 'pseudolabel', 'fullysupervised', 'supervised', 'remixmatch', 'mixmatch', 'meanteacher', - 'pimodel', 'vat', 'dash', 'comatch', 'crmatch', 'simmatch', 'adamatch', 'softmatch', 'freematch', 'defixmatch'] - datasets = ['aclImdb', 'ag_news', 'amazon_review', 'dbpedia', 'yahoo_answers', 'yelp_review'] + algs = [ + "flexmatch", + "fixmatch", + "uda", + "pseudolabel", + "fullysupervised", + "supervised", + "remixmatch", + "mixmatch", + "meanteacher", + "pimodel", + "vat", + "dash", + "comatch", + "crmatch", + "simmatch", + "adamatch", + "softmatch", + "freematch", + "defixmatch", + ] + datasets = [ + "aclImdb", + "ag_news", + "amazon_review", + "dbpedia", + "yahoo_answers", + "yelp_review", + ] seeds = [0] dist_port = range(10001, 31120, 1) - count = 0 - net = 'bert_base_uncased' + count = 0 + net = "bert_base_uncased" weight_decay = 5e-4 max_length = 512 @@ -245,38 +283,38 @@ def exp_usb_nlp(label_amount): for dataset in datasets: for seed in seeds: # change the configuration of each dataset - if dataset == 'aclImdb': + if dataset == "aclImdb": num_classes = 2 num_labels = label_amount[0] * num_classes lr = 5e-5 layer_decay = 0.75 - elif dataset == 'ag_news': + elif dataset == "ag_news": num_classes = 4 num_labels = label_amount[1] * num_classes lr = 5e-5 layer_decay = 0.65 - elif dataset == 'amazon_review': + elif dataset == "amazon_review": num_classes = 5 num_labels = label_amount[2] * num_classes lr = 1e-5 layer_decay = 0.75 - elif dataset == 'dbpedia': + elif dataset == "dbpedia": num_classes = 14 num_labels = label_amount[3] * num_classes - elif dataset == 'yahoo_answers': + elif dataset == "yahoo_answers": num_classes = 10 num_labels = label_amount[4] * num_classes lr = 1e-4 layer_decay = 0.65 - elif dataset == 'yelp_review': + elif dataset == "yelp_review": num_classes = 5 num_labels = label_amount[5] * num_classes @@ -285,25 +323,33 @@ def exp_usb_nlp(label_amount): port = dist_port[count] # prepare the configuration file - cfg = create_usb_nlp_config(alg, seed, - dataset, net, num_classes, num_labels, - port, lr, weight_decay, layer_decay, max_length) + cfg = create_usb_nlp_config( + alg, + seed, + dataset, + net, + num_classes, + num_labels, + port, + lr, + weight_decay, + layer_decay, + max_length, + ) count += 1 create_configuration(cfg, config_file) - -if __name__ == '__main__': - if not os.path.exists('./saved_models/usb_nlp/'): - os.makedirs('./saved_models/usb_nlp/', exist_ok=True) - if not os.path.exists('./config/usb_nlp/'): - os.makedirs('./config/usb_nlp/', exist_ok=True) - +if __name__ == "__main__": + if not os.path.exists("./saved_models/usb_nlp/"): + os.makedirs("./saved_models/usb_nlp/", exist_ok=True) + if not os.path.exists("./config/usb_nlp/"): + os.makedirs("./config/usb_nlp/", exist_ok=True) # usb nlp label_amount = { - 's': [10, 10, 50, 5, 50, 50], - 'm': [50, 50, 200, 20, 200, 200], + "s": [10, 10, 50, 5, 50, 50], + "m": [50, 50, 200, 20, 200, 200], } for i in label_amount: diff --git a/semilearn/core/hooks/aim.py b/semilearn/core/hooks/aim.py index 826c35d55..7693f98c6 100644 --- a/semilearn/core/hooks/aim.py +++ b/semilearn/core/hooks/aim.py @@ -1,8 +1,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. - import os + import aim + from .hook import Hook @@ -13,26 +14,36 @@ class AimHook(Hook): def __init__(self): super().__init__() - self.log_key_list = ['train/sup_loss', 'train/unsup_loss', 'train/total_loss', 'train/util_ratio', - 'train/run_time', 'train/prefetch_time', 'lr', - 'eval/top-1-acc', 'eval/precision', 'eval/recall', 'eval/F1'] + self.log_key_list = [ + "train/sup_loss", + "train/unsup_loss", + "train/total_loss", + "train/util_ratio", + "train/run_time", + "train/prefetch_time", + "lr", + "eval/top-1-acc", + "eval/precision", + "eval/recall", + "eval/F1", + ] def before_run(self, algorithm): # initialize aim run name = algorithm.save_name - project = algorithm.save_dir.split('/')[-1] - self.run = aim.Run(experiment=name, repo='/mnt/default/projects/USB_formal_run/221124/aim_data') + project = algorithm.save_dir.split("/")[-1] + repo = os.path.join(algorithm.args.save_dir, "aim", algorithm.args.save_name) + self.run = aim.Run(experiment=name, repo=repo) # set configuration - self.run['hparams'] = algorithm.args.__dict__ - + self.run["hparams"] = algorithm.args.__dict__ # set tag - benchmark = f'benchmark: {project}' - dataset = f'dataset: {algorithm.args.dataset}' - data_setting = f'setting: {algorithm.args.dataset}_lb{algorithm.args.num_labels}_{algorithm.args.lb_imb_ratio}_ulb{algorithm.args.ulb_num_labels}_{algorithm.args.ulb_imb_ratio}' - alg = f'alg: {algorithm.args.algorithm}' - imb_alg = f'imb_alg: {algorithm.args.imb_algorithm}' + benchmark = f"benchmark: {project}" + dataset = f"dataset: {algorithm.args.dataset}" + data_setting = f"setting: {algorithm.args.dataset}_lb{algorithm.args.num_labels}_{algorithm.args.lb_imb_ratio}_ulb{algorithm.args.ulb_num_labels}_{algorithm.args.ulb_imb_ratio}" # noqa: E501 + alg = f"alg: {algorithm.args.algorithm}" + imb_alg = f"imb_alg: {algorithm.args.imb_algorithm}" self.run.add_tag(benchmark) self.run.add_tag(dataset) self.run.add_tag(data_setting) @@ -44,6 +55,8 @@ def after_train_step(self, algorithm): for key, item in algorithm.log_dict.items(): if key in self.log_key_list: self.run.track(item, name=key, step=algorithm.it) - + if self.every_n_iters(algorithm, algorithm.num_eval_iter): - self.run.track(algorithm.best_eval_acc, name='eval/best-acc', step=algorithm.it) + self.run.track( + algorithm.best_eval_acc, name="eval/best-acc", step=algorithm.it + ) diff --git a/train.py b/train.py index af2693625..3ba78ba31 100644 --- a/train.py +++ b/train.py @@ -2,197 +2,303 @@ # Licensed under the MIT License. import argparse -import os import logging +import os import random import warnings import numpy as np import torch -import torch.nn as nn -import torch.nn.parallel import torch.backends.cudnn as cudnn import torch.distributed as dist import torch.multiprocessing as mp - +import torch.nn.parallel from semilearn.algorithms import get_algorithm, name2alg +from semilearn.core.utils import ( + TBLog, + count_parameters, + get_logger, + get_net_builder, + get_port, + over_write_args_from_file, + send_model_cuda, +) from semilearn.imb_algorithms import get_imb_algorithm, name2imbalg -from semilearn.core.utils import get_net_builder, get_logger, get_port, send_model_cuda, count_parameters, over_write_args_from_file, TBLog def get_config(): from semilearn.algorithms.utils import str2bool - parser = argparse.ArgumentParser(description='Semi-Supervised Learning (USB)') + parser = argparse.ArgumentParser(description="Semi-Supervised Learning (USB)") - ''' + """ Saving & loading of the model. - ''' - parser.add_argument('--save_dir', type=str, default='./saved_models') - parser.add_argument('-sn', '--save_name', type=str, default='fixmatch') - parser.add_argument('--resume', action='store_true') - parser.add_argument('--load_path', type=str) - parser.add_argument('-o', '--overwrite', action='store_true', default=True) - parser.add_argument('--use_tensorboard', action='store_true', help='Use tensorboard to plot and save curves') - parser.add_argument('--use_wandb', action='store_true', help='Use wandb to plot and save curves') - parser.add_argument('--use_aim', action='store_true', help='Use aim to plot and save curves') - - ''' + """ + parser.add_argument("--save_dir", type=str, default="./saved_models") + parser.add_argument("-sn", "--save_name", type=str, default="fixmatch") + parser.add_argument("--resume", action="store_true") + parser.add_argument("--load_path", type=str) + parser.add_argument("-o", "--overwrite", action="store_true", default=True) + parser.add_argument( + "--use_tensorboard", + action="store_true", + help="Use tensorboard to plot and save curves", + ) + parser.add_argument( + "--use_wandb", action="store_true", help="Use wandb to plot and save curves" + ) + parser.add_argument( + "--use_aim", action="store_true", help="Use aim to plot and save curves" + ) + + """ Training Configuration of FixMatch - ''' - parser.add_argument('--epoch', type=int, default=1) - parser.add_argument('--num_train_iter', type=int, default=20, - help='total number of training iterations') - parser.add_argument('--num_warmup_iter', type=int, default=0, - help='cosine linear warmup iterations') - parser.add_argument('--num_eval_iter', type=int, default=10, - help='evaluation frequency') - parser.add_argument('--num_log_iter', type=int, default=5, - help='logging frequency') - parser.add_argument('-nl', '--num_labels', type=int, default=400) - parser.add_argument('-bsz', '--batch_size', type=int, default=8) - parser.add_argument('--uratio', type=int, default=1, - help='the ratio of unlabeled data to labeled data in each mini-batch') - parser.add_argument('--eval_batch_size', type=int, default=16, - help='batch size of evaluation data loader (it does not affect the accuracy)') - parser.add_argument('--ema_m', type=float, default=0.999, help='ema momentum for eval_model') - parser.add_argument('--ulb_loss_ratio', type=float, default=1.0) - - ''' + """ + parser.add_argument("--epoch", type=int, default=1) + parser.add_argument( + "--num_train_iter", + type=int, + default=20, + help="total number of training iterations", + ) + parser.add_argument( + "--num_warmup_iter", type=int, default=0, help="cosine linear warmup iterations" + ) + parser.add_argument( + "--num_eval_iter", type=int, default=10, help="evaluation frequency" + ) + parser.add_argument("--num_log_iter", type=int, default=5, help="logging frequency") + parser.add_argument("-nl", "--num_labels", type=int, default=400) + parser.add_argument("-bsz", "--batch_size", type=int, default=8) + parser.add_argument( + "--uratio", + type=int, + default=1, + help="the ratio of unlabeled data to labeled data in each mini-batch", + ) + parser.add_argument( + "--eval_batch_size", + type=int, + default=16, + help="batch size of evaluation data loader (it does not affect the accuracy)", + ) + parser.add_argument( + "--ema_m", type=float, default=0.999, help="ema momentum for eval_model" + ) + parser.add_argument("--ulb_loss_ratio", type=float, default=1.0) + + """ Optimizer configurations - ''' - parser.add_argument('--optim', type=str, default='SGD') - parser.add_argument('--lr', type=float, default=3e-2) - parser.add_argument('--momentum', type=float, default=0.9) - parser.add_argument('--weight_decay', type=float, default=5e-4) - parser.add_argument('--layer_decay', type=float, default=1.0, help='layer-wise learning rate decay, default to 1.0 which means no layer decay') - - ''' + """ + parser.add_argument("--optim", type=str, default="SGD") + parser.add_argument("--lr", type=float, default=3e-2) + parser.add_argument("--momentum", type=float, default=0.9) + parser.add_argument("--weight_decay", type=float, default=5e-4) + parser.add_argument( + "--layer_decay", + type=float, + default=1.0, + help="layer-wise learning rate decay, default to 1.0 which means no layer " + "decay", + ) + + """ Backbone Net Configurations - ''' - parser.add_argument('--net', type=str, default='wrn_28_2') - parser.add_argument('--net_from_name', type=str2bool, default=False) - parser.add_argument('--use_pretrain', default=False, type=str2bool) - parser.add_argument('--pretrain_path', default='', type=str) + """ + parser.add_argument("--net", type=str, default="wrn_28_2") + parser.add_argument("--net_from_name", type=str2bool, default=False) + parser.add_argument("--use_pretrain", default=False, type=str2bool) + parser.add_argument("--pretrain_path", default="", type=str) - ''' + """ Algorithms Configurations - ''' + """ ## core algorithm setting - parser.add_argument('-alg', '--algorithm', type=str, default='fixmatch', help='ssl algorithm') - parser.add_argument('--use_cat', type=str2bool, default=True, help='use cat operation in algorithms') - parser.add_argument('--amp', type=str2bool, default=False, help='use mixed precision training or not') - parser.add_argument('--clip_grad', type=float, default=0) + parser.add_argument( + "-alg", "--algorithm", type=str, default="fixmatch", help="ssl algorithm" + ) + parser.add_argument( + "--use_cat", type=str2bool, default=True, help="use cat operation in algorithms" + ) + parser.add_argument( + "--amp", + type=str2bool, + default=False, + help="use mixed precision training or not", + ) + parser.add_argument("--clip_grad", type=float, default=0) ## imbalance algorithm setting - parser.add_argument('-imb_alg', '--imb_algorithm', type=str, default=None, help='imbalance ssl algorithm') - - ''' + parser.add_argument( + "-imb_alg", + "--imb_algorithm", + type=str, + default=None, + help="imbalance ssl algorithm", + ) + + """ Data Configurations - ''' + """ ## standard setting configurations - parser.add_argument('--data_dir', type=str, default='./data') - parser.add_argument('-ds', '--dataset', type=str, default='cifar10') - parser.add_argument('-nc', '--num_classes', type=int, default=10) - parser.add_argument('--train_sampler', type=str, default='RandomSampler') - parser.add_argument('--num_workers', type=int, default=1) - parser.add_argument('--include_lb_to_ulb', type=str2bool, default='True', help='flag of including labeled data into unlabeled data, default to True') + parser.add_argument("--data_dir", type=str, default="./data") + parser.add_argument("-ds", "--dataset", type=str, default="cifar10") + parser.add_argument("-nc", "--num_classes", type=int, default=10) + parser.add_argument("--train_sampler", type=str, default="RandomSampler") + parser.add_argument("--num_workers", type=int, default=1) + parser.add_argument( + "--include_lb_to_ulb", + type=str2bool, + default="True", + help="flag of including labeled data into unlabeled data, default to True", + ) ## imbalanced setting arguments - parser.add_argument('--lb_imb_ratio', type=int, default=1, help="imbalance ratio of labeled data, default to 1") - parser.add_argument('--ulb_imb_ratio', type=int, default=1, help="imbalance ratio of unlabeled data, default to 1") - parser.add_argument('--ulb_num_labels', type=int, default=None, help="number of labels for unlabeled data, used for determining the maximum number of labels in imbalanced setting") + parser.add_argument( + "--lb_imb_ratio", + type=int, + default=1, + help="imbalance ratio of labeled data, default to 1", + ) + parser.add_argument( + "--ulb_imb_ratio", + type=int, + default=1, + help="imbalance ratio of unlabeled data, default to 1", + ) + parser.add_argument( + "--ulb_num_labels", + type=int, + default=None, + help="number of labels for unlabeled data, used for determining the maximum " + "number of labels in imbalanced setting", + ) ## cv dataset arguments - parser.add_argument('--img_size', type=int, default=32) - parser.add_argument('--crop_ratio', type=float, default=0.875) + parser.add_argument("--img_size", type=int, default=32) + parser.add_argument("--crop_ratio", type=float, default=0.875) - ## nlp dataset arguments - parser.add_argument('--max_length', type=int, default=512) + ## nlp dataset arguments + parser.add_argument("--max_length", type=int, default=512) ## speech dataset algorithms - parser.add_argument('--max_length_seconds', type=float, default=4.0) - parser.add_argument('--sample_rate', type=int, default=16000) + parser.add_argument("--max_length_seconds", type=float, default=4.0) + parser.add_argument("--sample_rate", type=int, default=16000) - ''' + """ multi-GPUs & Distributed Training - ''' - - ## args for distributed training (from https://github.com/pytorch/examples/blob/master/imagenet/main.py) - parser.add_argument('--world-size', default=1, type=int, - help='number of nodes for distributed training') - parser.add_argument('--rank', default=0, type=int, - help='**node rank** for distributed training') - parser.add_argument('-du', '--dist-url', default='tcp://127.0.0.1:11111', type=str, - help='url used to set up distributed training') - parser.add_argument('--dist-backend', default='nccl', type=str, - help='distributed backend') - parser.add_argument('--seed', default=1, type=int, - help='seed for initializing training. ') - parser.add_argument('--gpu', default=None, type=int, - help='GPU id to use.') - parser.add_argument('--multiprocessing-distributed', type=str2bool, default=False, - help='Use multi-processing distributed training to launch ' - 'N processes per node, which has N GPUs. This is the ' - 'fastest way to use PyTorch for either single node or ' - 'multi node data parallel training') + """ + + ## args for distributed training (from https://github.com/pytorch/examples/blob/master/imagenet/main.py) # noqa: E501 + parser.add_argument( + "--world-size", + default=1, + type=int, + help="number of nodes for distributed training", + ) + parser.add_argument( + "--rank", default=0, type=int, help="**node rank** for distributed training" + ) + parser.add_argument( + "-du", + "--dist-url", + default="tcp://127.0.0.1:11111", + type=str, + help="url used to set up distributed training", + ) + parser.add_argument( + "--dist-backend", default="nccl", type=str, help="distributed backend" + ) + parser.add_argument( + "--seed", default=1, type=int, help="seed for initializing training. " + ) + parser.add_argument("--gpu", default=None, type=int, help="GPU id to use.") + parser.add_argument( + "--multiprocessing-distributed", + type=str2bool, + default=False, + help="Use multi-processing distributed training to launch " + "N processes per node, which has N GPUs. This is the " + "fastest way to use PyTorch for either single node or " + "multi node data parallel training", + ) # config file - parser.add_argument('--c', type=str, default='') + parser.add_argument("--c", type=str, default="") # add algorithm specific parameters args = parser.parse_args() over_write_args_from_file(args, args.c) for argument in name2alg[args.algorithm].get_argument(): - parser.add_argument(argument.name, type=argument.type, default=argument.default, help=argument.help) + parser.add_argument( + argument.name, + type=argument.type, + default=argument.default, + help=argument.help, + ) # add imbalanced algorithm specific parameters args = parser.parse_args() over_write_args_from_file(args, args.c) if args.imb_algorithm is not None: for argument in name2imbalg[args.imb_algorithm].get_argument(): - parser.add_argument(argument.name, type=argument.type, default=argument.default, help=argument.help) + parser.add_argument( + argument.name, + type=argument.type, + default=argument.default, + help=argument.help, + ) args = parser.parse_args() over_write_args_from_file(args, args.c) return args - def main(args): - ''' + """ For (Distributed)DataParallelism, main(args) spawn each process (main_worker) to each GPU. - ''' + """ - assert args.num_train_iter % args.epoch == 0, \ - f"# total training iter. {args.num_train_iter} is not divisible by # epochs {args.epoch}" + assert ( + args.num_train_iter % args.epoch == 0 + ), f"# total training iter. {args.num_train_iter} is not divisible by # epochs {args.epoch}" # noqa: E501 save_path = os.path.join(args.save_dir, args.save_name) - if os.path.exists(save_path) and args.overwrite and args.resume == False: + if os.path.exists(save_path) and args.overwrite and args.resume is False: import shutil + shutil.rmtree(save_path) if os.path.exists(save_path) and not args.overwrite: - raise Exception('already existing model: {}'.format(save_path)) + raise Exception("already existing model: {}".format(save_path)) if args.resume: if args.load_path is None: - raise Exception('Resume of training requires --load_path in the args') - if os.path.abspath(save_path) == os.path.abspath(args.load_path) and not args.overwrite: - raise Exception('Saving & Loading paths are same. \ - If you want over-write, give --overwrite in the argument.') + raise Exception("Resume of training requires --load_path in the args") + if ( + os.path.abspath(save_path) == os.path.abspath(args.load_path) + and not args.overwrite + ): + raise Exception( + "Saving & Loading paths are same. \ + If you want over-write, give --overwrite in the argument." + ) if args.seed is not None: - warnings.warn('You have chosen to seed training. ' - 'This will turn on the CUDNN deterministic setting, ' - 'which can slow down your training considerably! ' - 'You may see unexpected behavior when restarting ' - 'from checkpoints.') - - if args.gpu == 'None': + warnings.warn( + "You have chosen to seed training. " + "This will turn on the CUDNN deterministic setting, " + "which can slow down your training considerably! " + "You may see unexpected behavior when restarting " + "from checkpoints." + ) + + if args.gpu == "None": args.gpu = None if args.gpu is not None: - warnings.warn('You have chosen a specific GPU. This will completely ' - 'disable data parallelism.') + warnings.warn( + "You have chosen a specific GPU. This will completely " + "disable data parallelism." + ) if args.dist_url == "env://" and args.world_size == -1: args.world_size = int(os.environ["WORLD_SIZE"]) @@ -212,14 +318,15 @@ def main(args): def main_worker(gpu, ngpus_per_node, args): - ''' + """ main_worker is conducted on each GPU. - ''' + """ global best_acc1 args.gpu = gpu - # random seed has to be set for the synchronization of labeled data sampling in each process. + # random seed has to be set for the synchronization of labeled data sampling in each + # process. assert args.seed is not None random.seed(args.seed) torch.manual_seed(args.seed) @@ -236,15 +343,19 @@ def main_worker(gpu, ngpus_per_node, args): args.rank = args.rank * ngpus_per_node + gpu # compute global rank # set distributed group: - dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, - world_size=args.world_size, rank=args.rank) + dist.init_process_group( + backend=args.dist_backend, + init_method=args.dist_url, + world_size=args.world_size, + rank=args.rank, + ) # SET save_path and logger save_path = os.path.join(args.save_dir, args.save_name) logger_level = "WARNING" tb_log = None if args.rank % ngpus_per_node == 0: - tb_log = TBLog(save_path, 'tensorboard', use_tensorboard=args.use_tensorboard) + tb_log = TBLog(save_path, "tensorboard", use_tensorboard=args.use_tensorboard) logger_level = "INFO" logger = get_logger(args.save_name, save_path, logger_level) @@ -256,7 +367,7 @@ def main_worker(gpu, ngpus_per_node, args): model = get_imb_algorithm(args, _net_builder, tb_log, logger) else: model = get_algorithm(args, _net_builder, tb_log, logger) - logger.info(f'Number of Trainable Params: {count_parameters(model.model)}') + logger.info(f"Number of Trainable Params: {count_parameters(model.model)}") # SET Devices for (Distributed) DataParallel model.model = send_model_cuda(args, model.model) @@ -268,12 +379,12 @@ def main_worker(gpu, ngpus_per_node, args): try: model.load_model(args.load_path) except: - logger.info("Fail to resume load path {}".format(args.load_path)) + logger.info("Fail to resume load path {}".format(args.load_path)) args.resume = False else: logger.info("Resume load path {} does not exist".format(args.load_path)) - if hasattr(model, 'warmup'): + if hasattr(model, "warmup"): logger.info(("Warmup stage")) model.warmup() @@ -285,7 +396,7 @@ def main_worker(gpu, ngpus_per_node, args): for key, item in model.results_dict.items(): logger.info(f"Model result - {key} : {item}") - if hasattr(model, 'finetune'): + if hasattr(model, "finetune"): logger.info("Finetune stage") model.finetune()