show.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bd3a9167-f891-4dd2-b923-74b8b2d4061e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import random\n",
    "from torch import nn\n",
    "from torch.nn import MSELoss\n",
    "from torch.utils.data import DataLoader\n",
    "import get_data as get_data\n",
    "from transformers.optimization import AdamW, get_linear_schedule_with_warmup\n",
    "import model\n",
    "from metrics import multiclass_acc\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import pickle\n",
    "from model import save_model\n",
    "from transformers import  BertTokenizer\n",
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "\n",
    "def to_pickle(obj, path):\n",
    "    with open(path, 'wb') as f:\n",
    "        pickle.dump(obj, f)\n",
    "def load_pickle(path):\n",
    "    with open(path, 'rb') as f:\n",
    "        return pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a971ecd7-073d-4cab-b047-31a2b2d69959",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "=======OK!======\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HeJH\\Desktop\\搞快点\\实验部分\\code\\get_data.py:261: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at  C:\\cb\\pytorch_1000000000000\\work\\torch\\csrc\\utils\\tensor_new.cpp:210.)\n",
      "  all_visual = torch.tensor([f.visual for f in aoli], dtype=torch.float)\n"
     ]
    }
   ],
   "source": [
    "test = load_pickle('data/mosi_pro_test.pkl')\n",
    "DEVICE=torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "test_dataset= get_data.get_Dataset(test)\n",
    "test_dataloader = DataLoader(test_dataset, batch_size=48, shuffle=True)\n",
    "#test = get_data.convert_3_features(test,max_tlen=50,max_vlen=70,max_alen=80)\n",
    "#test_dataloader = DataLoader(test, batch_size=48, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c90364e4-5e51-46b2-9812-91881b50021d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.dense.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "word_size = 50\n",
    "w_len = 50\n",
    "v_len = 70\n",
    "a_len = 80\n",
    "\n",
    "hidden_size = 768\n",
    "max_position = 150\n",
    "modal_size = 3\n",
    "layer_norm_eps = 1e-05\n",
    "hidden_dropout_prob = 0.1 #0.125\n",
    "word_embedding_dim = 768\n",
    "visual_dim = 20 #20 35\n",
    "audio_dim = 5   #5  74\n",
    "attention_dropout_prob = 0.1 #0.125\n",
    "num_head = 8\n",
    "output_attention = 0\n",
    "num_layer = 3\n",
    "output_hidden_state = 0\n",
    "intermediate_size = 768*2\n",
    "num_head_modal = 8\n",
    "num_layer_modal = 0\n",
    "\n",
    "\n",
    "config = {'word_size': word_size , 'hidden_size': hidden_size , 'max_position': max_position ,\n",
    "          'word_embedding_dim':word_embedding_dim, 'audio_dim':audio_dim,'visual_dim':visual_dim,\n",
    "          'modal_size': modal_size , 'layer_norm_eps': layer_norm_eps , 'hidden_dropout_prob': hidden_dropout_prob ,\n",
    "          'attention_dropout_prob':attention_dropout_prob , 'num_head':num_head , 'output_attention':output_attention,\n",
    "          'num_layer':num_layer , 'output_hidden_state':output_hidden_state, 'intermediate_size':intermediate_size,\n",
    "          'num_head_modal':num_head_modal,'num_layer_modal':num_layer_modal,'w_len':w_len,'v_len':v_len,'a_len':a_len}\n",
    "\n",
    "TF_model = model.My_model(config).to(DEVICE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f4715a9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "params = list(TF_model.named_parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "197d3b71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([70, 768])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params[3][1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7092049d-8aba-4e03-8844-5aa236c24730",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load the best weight\n",
    "model = TF_model\n",
    "#model.load_state_dict(torch.load('./weights/xf_model.pt')['state_dict'],map_location='cuda:0')\n",
    "loaded_state = torch.load('./weights/version_2/excl_3_mosi_model.pt', map_location='cuda:0')\n",
    "model.load_state_dict(loaded_state['state_dict'],False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "440fb1a5-5de3-483f-b544-8030ec057463",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.3095, device='cuda:0'),\n",
       " tensor(0.3430, device='cuda:0'),\n",
       " tensor(0.2562, device='cuda:0'))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.state_dict()['transformer.Layer.0.attention.w_a'],model.state_dict()['transformer.Layer.1.attention.w_a'],model.state_dict()['transformer.Layer.2.attention.w_a']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a81b6f14",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.3017, device='cuda:0'),\n",
       " tensor(0.2949, device='cuda:0'),\n",
       " tensor(0.2774, device='cuda:0'))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.state_dict()['transformer.Layer.0.attention.w_b'],model.state_dict()['transformer.Layer.1.attention.w_b'],model.state_dict()['transformer.Layer.2.attention.w_b']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "51a10aa3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.3008, device='cuda:0'),\n",
       " tensor(0.2970, device='cuda:0'),\n",
       " tensor(0.2859, device='cuda:0'))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.state_dict()['transformer.Layer.0.attention.w_c'],model.state_dict()['transformer.Layer.1.attention.w_c'],model.state_dict()['transformer.Layer.2.attention.w_c']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "67e1263e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_epoch(model: nn.Module, test_dataloader: DataLoader):\n",
    "    model.eval()\n",
    "    preds = []\n",
    "    labels = []\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for batch in tqdm(test_dataloader):\n",
    "            batch = tuple(t.to(DEVICE) for t in batch)\n",
    "\n",
    "            input_ids, input_mask, my_mask, segment_ids, visual_embedding, audio_embedding, label_ids = batch\n",
    "            input_ids = torch.squeeze(input_ids, 1)\n",
    "            #tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(input_ids))\n",
    "            input_mask = torch.squeeze(input_mask, 1)\n",
    "            my_mask = torch.squeeze(my_mask, 1)\n",
    "            segment_ids = torch.squeeze(segment_ids, 1)\n",
    "            visual_embedding = torch.squeeze(visual_embedding, 1)\n",
    "            audio_embedding = torch.squeeze(audio_embedding, 1)\n",
    "            outputs = model(\n",
    "                input_ids,\n",
    "                input_mask,\n",
    "                my_mask,\n",
    "                segment_ids,\n",
    "                visual_embedding,\n",
    "                audio_embedding\n",
    "            )\n",
    "\n",
    "            logits = outputs\n",
    "\n",
    "            logits = logits.detach().cpu().numpy()\n",
    "            label_ids = label_ids.detach().cpu().numpy()\n",
    "\n",
    "            logits = np.squeeze(logits).tolist()\n",
    "            label_ids = np.squeeze(label_ids).tolist()\n",
    "\n",
    "            preds.extend(logits)\n",
    "            labels.extend(label_ids)\n",
    "\n",
    "        preds = np.array(preds)\n",
    "        labels = np.array(labels)\n",
    "\n",
    "    return preds, labels\n",
    "\n",
    "\n",
    "def test_score_model(model: nn.Module, test_dataloader: DataLoader, use_zero=False):\n",
    "\n",
    "    preds, y_test = test_epoch(model, test_dataloader)\n",
    "    non_zeros = np.array(\n",
    "        [i for i, e in enumerate(y_test) if e != 0 or use_zero])\n",
    "\n",
    "    preds = preds[non_zeros]\n",
    "    y_test = y_test[non_zeros]\n",
    "\n",
    "    test_preds_a7 = np.clip(preds, a_min=-3., a_max=3.)\n",
    "    test_truth_a7 = np.clip(y_test, a_min=-3., a_max=3.)\n",
    "    mult_a7 = multiclass_acc(test_preds_a7, test_truth_a7)\n",
    "\n",
    "    mae = np.mean(np.absolute(preds - y_test))\n",
    "    corr = np.corrcoef(preds, y_test)[0][1]\n",
    "\n",
    "    preds = preds >= 0\n",
    "    y_test = y_test >= 0\n",
    "\n",
    "    f_score = f1_score(y_test, preds, average=\"weighted\")\n",
    "    acc = accuracy_score(y_test, preds)\n",
    "    return acc, mae, corr, f_score,mult_a7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4189e3cb-7041-4245-838f-78c8ff8c01e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_epoch(model: nn.Module, test_dataloader: DataLoader):\n",
    "    model.eval()\n",
    "    preds = []\n",
    "    labels = []\n",
    "    tzs=[]\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for batch in tqdm(test_dataloader):\n",
    "            batch = tuple(t.to(DEVICE) for t in batch)\n",
    "\n",
    "            input_ids, input_mask, my_mask, segment_ids, visual_embedding, audio_embedding, label_ids = batch\n",
    "            input_ids = torch.squeeze(input_ids, 1)\n",
    "            input_mask = torch.squeeze(input_mask, 1)\n",
    "            my_mask = torch.squeeze(my_mask, 1)\n",
    "            segment_ids = torch.squeeze(segment_ids, 1)\n",
    "            visual_embedding = torch.squeeze(visual_embedding, 1)\n",
    "            audio_embedding = torch.squeeze(audio_embedding, 1)\n",
    "            outputs,features = model(\n",
    "                input_ids,\n",
    "                input_mask,\n",
    "                my_mask,\n",
    "                segment_ids,\n",
    "                visual_embedding,\n",
    "                audio_embedding\n",
    "            )\n",
    "\n",
    "            logits = outputs\n",
    "            tz=features.cpu().numpy()\n",
    "            \n",
    "\n",
    "            logits = logits.detach().cpu().numpy()\n",
    "            label_ids = label_ids.detach().cpu().numpy()\n",
    "\n",
    "            logits = np.squeeze(logits).tolist()\n",
    "            label_ids = np.squeeze(label_ids).tolist()\n",
    "\n",
    "            preds.extend(logits)\n",
    "            labels.extend(label_ids)\n",
    "            tzs.extend(tz)\n",
    "\n",
    "        preds = np.array(preds)\n",
    "        labels = np.array(labels)\n",
    "        tzs=np.array(tzs)\n",
    "\n",
    "    return preds, labels,tzs\n",
    "\n",
    "def test_score_model(model: nn.Module, test_dataloader: DataLoader, use_zero=False):\n",
    "\n",
    "    preds, y_test,tz = test_epoch(model, test_dataloader)\n",
    "    non_zeros = np.array(\n",
    "        [i for i, e in enumerate(y_test) if e != 0 or use_zero])\n",
    "\n",
    "    preds = preds[non_zeros]\n",
    "    y_test = y_test[non_zeros]\n",
    "    tz=tz[non_zeros,:]\n",
    "\n",
    "    test_preds_a7 = np.clip(preds, a_min=-3., a_max=3.)\n",
    "    test_truth_a7 = np.clip(y_test, a_min=-3., a_max=3.)\n",
    "    mult_a7 = multiclass_acc(test_preds_a7, test_truth_a7)\n",
    "\n",
    "    mae = np.mean(np.absolute(preds - y_test))\n",
    "    corr = np.corrcoef(preds, y_test)[0][1]\n",
    "\n",
    "    preds = preds >= 0\n",
    "    y_test = y_test >= 0\n",
    "\n",
    "    f_score = f1_score(y_test, preds, average=\"weighted\")\n",
    "    acc = accuracy_score(y_test, preds)\n",
    "    return acc, mae, corr, f_score,mult_a7,tz,y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8993ab6e-3180-4f8d-b7f5-da3a468c60a3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████| 15/15 [00:01<00:00,  7.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ACC: 0.8658536585365854\n",
      "MAE: 0.7189127605826532\n",
      "CORR: 0.7988340286966654\n",
      "F_SCORE: 0.864567138619036\n",
      "ACC7: 0.4634146341463415\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "acc, mae, corr, f_score,mult_a7= test_score_model(TF_model, test_dataloader, False)\n",
    "print(\"ACC:\", acc)\n",
    "print(\"MAE:\", mae)\n",
    "print(\"CORR:\", corr)\n",
    "print(\"F_SCORE:\", f_score)\n",
    "print(\"ACC7:\",mult_a7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db368d8c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PyCharm (sentiment_project)",
   "language": "python",
   "name": "pycharm-8caeba1a"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}