resnet50-training-fgvc8.ipynb

{"cells":[{"cell_type":"code","execution_count":15,"metadata":{},"outputs":[],"source":["import comet_ml\n","from comet_ml import Experiment"]},{"cell_type":"code","execution_count":16,"metadata":{},"outputs":[],"source":["import torch_optimizer as opt"]},{"metadata":{"trusted":true},"cell_type":"code","source":["import seaborn as sns\n","import pandas as pd\n","import time\n","import cv2\n","import os\n","import albumentations\n","import timm\n","from pytorch_lightning.loggers import CometLogger\n","import pickle\n","from sklearn.model_selection import StratifiedKFold\n","from pytorch_lightning.callbacks import ModelCheckpoint,EarlyStopping\n","from torchvision.models import resnet50,resnet18\n","from sklearn.preprocessing import LabelEncoder,MultiLabelBinarizer\n","from albumentations.pytorch import ToTensorV2\n","import json\n","import torch.nn.functional as F\n","import numpy as np\n","import random\n","import pytorch_lightning as pl\n","import torch.optim as optim\n","import torch.nn as nn\n","\n","from torch.utils.data import Dataset,DataLoader\n","import torch\n","from sklearn.utils.class_weight import compute_class_weight"],"execution_count":17,"outputs":[]},{"cell_type":"code","execution_count":18,"metadata":{},"outputs":[],"source":["class cfg:\n","    img_size = 512\n","    model_name = \"resnet50\"\n","    max_epochs = 100\n","    batch_size = 32\n","    classes = np.array(['complex','frog_eye_leaf_spot','powdery_mildew','rust','scab'])\n","    train_root_dir = \"preprocessed\"\n","    val_root_dir = \"train_images\"\n","    patience = [5,2]\n","    factor= .1\n","    folds=5\n","    min_lr=1e-8\n"]},{"metadata":{"trusted":true},"cell_type":"code","source":["def seeding(seed=2021):\n","    np.random.seed(seed)\n","    os.environ['PYTHONHASHSEED'] = str(seed)\n","    torch.random.manual_seed(seed)\n","    random.seed(seed)\n","seeding()"],"execution_count":19,"outputs":[]},{"cell_type":"code","execution_count":20,"metadata":{},"outputs":[],"source":["@torch.jit.script\n","def mish(input):\n","    return input * F.tanh(F.softplus(input))"]},{"cell_type":"code","execution_count":29,"metadata":{},"outputs":[],"source":["class Mish(nn.Module):\n","    def __init__(self):\n","        super().__init__()\n","    \n","    def forward(self,input):\n","        return mish(input)"]},{"cell_type":"code","execution_count":30,"metadata":{},"outputs":[],"source":["def train_augs():\n","    return albumentations.Compose([\n","       albumentations.RandomResizedCrop(cfg.img_size, cfg.img_size, scale=(0.9, 1), p=1), \n","       albumentations.HorizontalFlip(p=0.5),\n","       albumentations.VerticalFlip(p=0.5),\n","       albumentations.ShiftScaleRotate(p=0.5),\n","       albumentations.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=10, val_shift_limit=10, p=0.7),\n","       albumentations.RandomBrightnessContrast(brightness_limit=(-0.2,0.2), contrast_limit=(-0.2, 0.2), p=0.7),\n","       albumentations.CLAHE(clip_limit=(1,4), p=0.5),\n","       albumentations.OneOf([\n","           albumentations.OpticalDistortion(distort_limit=1.0),\n","           albumentations.GridDistortion(num_steps=5, distort_limit=1.),\n","           albumentations.ElasticTransform(alpha=3),\n","       ], p=0.2),\n","       albumentations.OneOf([\n","           albumentations.GaussNoise(var_limit=[10, 50]),\n","           albumentations.GaussianBlur(),\n","           albumentations.MotionBlur(),\n","           albumentations.MedianBlur(),\n","       ], p=0.2),\n","      albumentations.Resize(cfg.img_size, cfg.img_size),\n","      albumentations.OneOf([\n","          albumentations.JpegCompression(),\n","          albumentations.Downscale(scale_min=0.1, scale_max=0.15),\n","      ], p=0.2),\n","      albumentations.IAAPiecewiseAffine(p=0.2),\n","      albumentations.IAASharpen(p=0.2),\n","      albumentations.Cutout(max_h_size=int(cfg.img_size * 0.1), max_w_size=int(cfg.img_size * 0.1), num_holes=5, p=0.5),\n","      albumentations.Normalize(p=1.0),\n","      ToTensorV2()\n","    ])\n","    "]},{"metadata":{"trusted":true},"cell_type":"code","source":["\n","def val_augs():\n","    return albumentations.Compose([\n","        albumentations.Resize(cfg.img_size,cfg.img_size),\n","        albumentations.Normalize(p=1.0),\n","        ToTensorV2()\n","    ])\n","\n","\n","\n","class FGVC(Dataset):\n","    def __init__(self,df,mode='train',transforms=None):\n","        self.transforms = transforms\n","        self.dataf = df\n","        self.mode = mode\n","    def __len__(self):\n","        return len(self.dataf)\n","    def __getitem__(self,idx):\n","        img_name = self.dataf.loc[idx,'image']\n","        if self.mode == 'train':\n","            img = cv2.imread(os.path.join(cfg.train_root_dir,img_name))\n","        elif self.mode =='val':\n","            img = cv2.imread(os.path.join(cfg.val_root_dir,img_name))\n","        #print(img_name)\n","        label = self.dataf.iloc[idx][cfg.classes]\n","        if self.transforms is not None:\n","            img = self.transforms(image=img)['image']\n","        return img,torch.tensor(label)\n"],"execution_count":31,"outputs":[]},{"cell_type":"code","execution_count":32,"metadata":{},"outputs":[],"source":["mis = Mish()"]},{"metadata":{"trusted":true},"cell_type":"code","source":["class FGVCNet(pl.LightningModule):\n","    def __init__(self):\n","        super(FGVCNet,self).__init__()\n","\n","        if \"resnet\" in cfg.model_name:\n","            self.model = eval(cfg.model_name)(pretrained=False)\n","            for params in self.model.parameters():\n","                params.require_grad = True\n","            self.model = nn.Sequential(*list(self.model.children())[:-2])\n","            self.model.add_module(\"average_pool\",nn.AdaptiveAvgPool2d(output_size=(1,1)))\n","            self.model.add_module(\"Flatten\",nn.Flatten())\n","            self.model.add_module(\"FC1\",nn.Linear(2048,1024))\n","            self.model.add_module(\"ACT1\",mis)\n","            self.model.add_module(\"FC2\",nn.Linear(1024,1024))\n","            self.model.add_module(\"ACT2\",mis)\n","            self.model.add_module(\"CLASSIFIER\",nn.Linear(1024,5))\n","\n","        if \"efficient\" in cfg.model_name:\n","            self.model = timm.create_model(cfg.model_name,pretrained=True)\n","            num_ftrs = self.model.classifier.in_features\n","            self.model.classifier = nn.Linear(num_ftrs,5)\n","        \n","\n","    def forward(self,x):\n","        return self.model(x)\n","    \n","    def configure_optimizers(self):\n","        op = opt.Ranger(self.parameters(),lr=0.01)\n","        \n","        scheduler = {\n","            'scheduler':optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer=op,T_0=10),\n","            'monitor':'val_f1',\n","            'interval':'epoch',\n","            'frequency':1,\n","            'strict':True\n","        }\n","\n","        self.op = op\n","        self.scheduler = scheduler\n","        \n","        return [op],[scheduler]\n","\n","    def training_step(self,batch,batch_idx):\n","        X,y = batch\n","        y_hat = self.model(X)\n","        loss_tr = F.binary_cross_entropy_with_logits(y_hat,y.float())\n","        f1_tr = pl.metrics.functional.f1(y_hat.sigmoid(),y,5) \n","        self.log(\"TrainLoss\",loss_tr,prog_bar=True,on_step=False,on_epoch=True,logger=True)\n","        self.log(\"TrainF1\",f1_tr,prog_bar=True,on_epoch=True,on_step=False,logger=True)\n","        return loss_tr\n","    \n","    def validation_step(self,batch,batch_idx):\n","        X,y = batch\n","        y_hat = self.model(X)\n","        loss_val = F.binary_cross_entropy_with_logits(y_hat,y.float())\n","        f1_val = pl.metrics.functional.f1(y_hat.sigmoid(),y,5)\n","        self.log(\"val_loss\",loss_val,prog_bar=True,on_step=False,on_epoch=True,logger=True)\n","        self.log(\"val_f1\",f1_val,prog_bar=True,on_epoch=True,on_step=False,logger=True)\n","        return loss_val"],"execution_count":33,"outputs":[]},{"cell_type":"code","execution_count":34,"metadata":{},"outputs":[],"source":["train_data = pd.read_csv(\"train2.csv\")"]},{"metadata":{"tags":[],"trusted":true},"cell_type":"code","source":["def main():\n","    \n","    i=0\n","\n","    comet = CometLogger(api_key=\"TFI1zlhr3wXWanvKsvRrDaGc4\",workspace=\"thirurjst\",save_dir='comet',project_name=\"fgvc8\",experiment_name=f\"{cfg.model_name}_{cfg.img_size}_Model Fold - {i+1}/5\"+\"\\t\"+\"Time:\"+str(time.time()))\n","\n","    estp = EarlyStopping(monitor=\"val_f1\",mode=\"max\",patience=cfg.patience[0])\n","\n","    if os.path.isdir(f\"{cfg.model_name}_{cfg.img_size}_Model_Fold-{i+1}_2\"):\n","        print(\"Already Exists\")\n","    else:\n","        os.makedirs(f\"{cfg.model_name}_{cfg.img_size}_Model_Fold-{i+1}_2\")\n","    \n","\n","    mod_ckpt = ModelCheckpoint(\n","    monitor='val_loss',\n","    mode='min',\n","    dirpath=f\"{cfg.model_name}_{cfg.img_size}_Model_Fold-{i+1}_2\",\n","    filename='Resnet18_512_Checkpoint-ValLoss:{val_loss:.4f}-F1:{val_f1:.4f}',\n","    save_top_k=1,\n",")\n","    trainer = pl.Trainer(gpus=1,precision=16,max_epochs=cfg.max_epochs,progress_bar_refresh_rate=30,deterministic=True,benchmark=True,callbacks=[mod_ckpt,estp],logger=comet,accumulate_grad_batches=1)\n","    \n","    \n","    print(f\"Initializing model Fold - {i+1}/5\")\n","    model = FGVCNet()\n","    print(\"*** Model Initialization Completed ***\")\n","    train_recs = train_data[train_data.folds != i].reset_index(drop=True)\n","    val_recs = train_data[train_data.folds == i].reset_index(drop=True)\n","\n","    train_dataset = FGVC(df=train_recs,mode='train')\n","    val_dataset = FGVC(df=val_recs,mode='val',transforms=val_augs())\n","\n","    train_loader = DataLoader(train_dataset,batch_size=32,num_workers=4)\n","    val_loader = DataLoader(val_dataset,batch_size=32,num_workers=4)\n","\n","    trainer.fit(model,train_loader,val_loader)\n","\n","    print(\"Uploading Model to Comet\")\n","    comet.experiment.log_model(f\"Model{i+1}\",f\"./{cfg.model_name}_{cfg.img_size}_Model_Fold-{i+1}_2\")\n","    print(\"Upload OK...\")\n","    comet.experiment.end()\n"],"execution_count":35,"outputs":[]},{"cell_type":"code","execution_count":36,"metadata":{"tags":[]},"outputs":[{"output_type":"stream","name":"stderr","text":["CometLogger will be initialized in online mode\n","GPU available: True, used: True\n","TPU available: None, using: 0 TPU cores\n","LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n","Using native 16bit precision.\n","Already Exists\n","Initializing model Fold - 1/5\n","COMET INFO: ---------------------------\n","COMET INFO: Comet.ml Experiment Summary\n","COMET INFO: ---------------------------\n","COMET INFO:   Data:\n","COMET INFO:     display_summary_level : 1\n","COMET INFO:     url                   : https://www.comet.ml/thirurjst/fgvc8/334844889c914f2aa3cb4bbac781b74e\n","COMET INFO:   Others:\n","COMET INFO:     Name : resnet50_512_Model Fold - 1/5\tTime:1619071810.946502\n","COMET INFO:   Uploads:\n","COMET INFO:     environment details      : 1\n","COMET INFO:     filename                 : 1\n","COMET INFO:     git metadata             : 1\n","COMET INFO:     git-patch (uncompressed) : 1 (137 KB)\n","COMET INFO:     installed packages       : 1\n","COMET INFO:     os packages              : 1\n","COMET INFO: ---------------------------\n","*** Model Initialization Completed ***\n","COMET INFO: Experiment is live on comet.ml https://www.comet.ml/thirurjst/fgvc8/bcd6c91f31664ca3a242cd6bfe49e8d9\n","\n","\n","  | Name  | Type       | Params\n","-------------------------------------\n","0 | model | Sequential | 26.7 M\n","-------------------------------------\n","26.7 M    Trainable params\n","0         Non-trainable params\n","26.7 M    Total params\n","Validation sanity check: 0it [00:00, ?it/s]"]},{"output_type":"error","ename":"TypeError","evalue":"__init__() takes 1 positional argument but 2 were given","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)","\u001b[0;32m<ipython-input-36-263240bbee7e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;32m<ipython-input-35-3a0a2cdee515>\u001b[0m in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m     35\u001b[0m     \u001b[0mval_loader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDataLoader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_dataset\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mnum_workers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 37\u001b[0;31m     \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtrain_loader\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mval_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     38\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Uploading Model to Comet\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m~/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, model, train_dataloader, val_dataloaders, datamodule)\u001b[0m\n\u001b[1;32m    468\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'on_fit_start'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    469\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 470\u001b[0;31m         \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccelerator_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    471\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccelerator_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mteardown\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    472\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m~/.local/lib/python3.8/site-packages/pytorch_lightning/accelerators/gpu_accelerator.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     64\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m         \u001b[0;31m# train or test\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m         \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_or_test\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     67\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m~/.local/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py\u001b[0m in \u001b[0;36mtrain_or_test\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     63\u001b[0m             \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_test\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m             \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     66\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m~/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    490\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    491\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 492\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_sanity_check\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    493\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    494\u001b[0m         \u001b[0;31m# set stage for logging\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m~/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mrun_sanity_check\u001b[0;34m(self, ref_model)\u001b[0m\n\u001b[1;32m    688\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    689\u001b[0m             \u001b[0;31m# run eval step\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 690\u001b[0;31m             \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meval_results\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_evaluation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_mode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_batches\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_sanity_val_batches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    691\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    692\u001b[0m             \u001b[0;31m# allow no returns from eval\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m~/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mrun_evaluation\u001b[0;34m(self, test_mode, max_batches)\u001b[0m\n\u001b[1;32m    604\u001b[0m                 \u001b[0;31m# lightning module methods\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    605\u001b[0m                 \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"evaluation_step_and_end\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 606\u001b[0;31m                     \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluation_loop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluation_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_mode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdataloader_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    607\u001b[0m                     \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluation_loop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluation_step_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    608\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m~/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/evaluation_loop.py\u001b[0m in \u001b[0;36mevaluation_step\u001b[0;34m(self, test_mode, batch, batch_idx, dataloader_idx)\u001b[0m\n\u001b[1;32m    176\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    177\u001b[0m             \u001b[0mmodel_ref\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_current_fx_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"validation_step\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 178\u001b[0;31m             \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccelerator_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalidation_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    179\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    180\u001b[0m         \u001b[0;31m# capture any logged information\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m~/.local/lib/python3.8/site-packages/pytorch_lightning/accelerators/gpu_accelerator.py\u001b[0m in \u001b[0;36mvalidation_step\u001b[0;34m(self, args)\u001b[0m\n\u001b[1;32m     82\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     83\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mvalidation_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 84\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalidation_step\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     85\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mtest_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m~/.local/lib/python3.8/site-packages/pytorch_lightning/accelerators/gpu_accelerator.py\u001b[0m in \u001b[0;36m_step\u001b[0;34m(self, model_step, args)\u001b[0m\n\u001b[1;32m     72\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mamp_backend\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mAMPType\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNATIVE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mamp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautocast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 74\u001b[0;31m                 \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     75\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     76\u001b[0m             \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m<ipython-input-33-f8dd50e2f8b4>\u001b[0m in \u001b[0;36mvalidation_step\u001b[0;34m(self, batch, batch_idx)\u001b[0m\n\u001b[1;32m     52\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mvalidation_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mbatch_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m         \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 54\u001b[0;31m         \u001b[0my_hat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     55\u001b[0m         \u001b[0mloss_val\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbinary_cross_entropy_with_logits\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_hat\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m         \u001b[0mf1_val\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunctional\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_hat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msigmoid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    725\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 727\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    728\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    729\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    115\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 117\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    118\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    725\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 727\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    728\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    729\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m<ipython-input-29-6d1c6f9c5633>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mmish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;31mTypeError\u001b[0m: __init__() takes 1 positional argument but 2 were given"]}],"source":["main()"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]}],"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3.8.5 64-bit","metadata":{"interpreter":{"hash":"df0893f56f349688326838aaeea0de204df53a132722cbd565e54b24a8fec5f6"}}},"language_info":{"name":"python","version":"3.8.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":4}