R-Bench-Script.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "db149db0",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "140cd4c8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-10-11 03:28:07,214 - modelscope - WARNING - Authentication has expired, please re-login with modelscope login --token \"YOUR_SDK_TOKEN\" if you need to access private models or datasets.\n"
     ]
    }
   ],
   "source": [
    "from modelscope.msdatasets import MsDataset\n",
    "\n",
    "ms_dataset = MsDataset.load(\n",
    "            'R-Bench', namespace='lcysyzxdxc',\n",
    "            subset_name='default', split='test')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34e37089",
   "metadata": {},
   "source": [
    "# Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "63f2035f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "MiniCPMV(\n",
       "  (llm): MiniCPMForCausalLM(\n",
       "    (model): MiniCPMModel(\n",
       "      (embed_tokens): Embedding(122753, 2304)\n",
       "      (layers): ModuleList(\n",
       "        (0-39): 40 x MiniCPMDecoderLayer(\n",
       "          (self_attn): MiniCPMAttention(\n",
       "            (q_proj): Linear(in_features=2304, out_features=2304, bias=False)\n",
       "            (k_proj): Linear(in_features=2304, out_features=2304, bias=False)\n",
       "            (v_proj): Linear(in_features=2304, out_features=2304, bias=False)\n",
       "            (o_proj): Linear(in_features=2304, out_features=2304, bias=False)\n",
       "            (rotary_emb): MiniCPMRotaryEmbedding()\n",
       "          )\n",
       "          (mlp): MiniCPMMLP(\n",
       "            (gate_proj): Linear(in_features=2304, out_features=5760, bias=False)\n",
       "            (up_proj): Linear(in_features=2304, out_features=5760, bias=False)\n",
       "            (down_proj): Linear(in_features=5760, out_features=2304, bias=False)\n",
       "            (act_fn): SiLU()\n",
       "          )\n",
       "          (input_layernorm): MiniCPMRMSNorm()\n",
       "          (post_attention_layernorm): MiniCPMRMSNorm()\n",
       "        )\n",
       "      )\n",
       "      (norm): MiniCPMRMSNorm()\n",
       "    )\n",
       "    (lm_head): Linear(in_features=2304, out_features=122753, bias=False)\n",
       "  )\n",
       "  (vpm): VisionTransformer(\n",
       "    (patch_embed): PatchEmbed(\n",
       "      (proj): Conv2d(3, 1152, kernel_size=(14, 14), stride=(14, 14))\n",
       "      (norm): Identity()\n",
       "    )\n",
       "    (pos_drop): Dropout(p=0.0, inplace=False)\n",
       "    (patch_drop): Identity()\n",
       "    (norm_pre): Identity()\n",
       "    (blocks): Sequential(\n",
       "      (0): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (1): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (2): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (3): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (4): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (5): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (6): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (7): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (8): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (9): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (10): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (11): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (12): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (13): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (14): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (15): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (16): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (17): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (18): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (19): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (20): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (21): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (22): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (23): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (24): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "      (25): Block(\n",
       "        (norm1): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (qkv): Linear(in_features=1152, out_features=3456, bias=True)\n",
       "          (q_norm): Identity()\n",
       "          (k_norm): Identity()\n",
       "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "          (proj): Linear(in_features=1152, out_features=1152, bias=True)\n",
       "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls1): Identity()\n",
       "        (drop_path1): Identity()\n",
       "        (norm2): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "        (mlp): Mlp(\n",
       "          (fc1): Linear(in_features=1152, out_features=4304, bias=True)\n",
       "          (act): GELU(approximate='none')\n",
       "          (drop1): Dropout(p=0.0, inplace=False)\n",
       "          (norm): Identity()\n",
       "          (fc2): Linear(in_features=4304, out_features=1152, bias=True)\n",
       "          (drop2): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (ls2): Identity()\n",
       "        (drop_path2): Identity()\n",
       "      )\n",
       "    )\n",
       "    (norm): LayerNorm((1152,), eps=1e-06, elementwise_affine=True)\n",
       "    (attn_pool): Identity()\n",
       "    (fc_norm): Identity()\n",
       "    (head_drop): Dropout(p=0.0, inplace=False)\n",
       "    (head): Identity()\n",
       "  )\n",
       "  (resampler): Resampler(\n",
       "    (kv_proj): Linear(in_features=1152, out_features=2304, bias=False)\n",
       "    (attn): MultiheadAttention(\n",
       "      (out_proj): Linear(in_features=2304, out_features=2304, bias=True)\n",
       "    )\n",
       "    (ln_q): LayerNorm((2304,), eps=1e-06, elementwise_affine=True)\n",
       "    (ln_kv): LayerNorm((2304,), eps=1e-06, elementwise_affine=True)\n",
       "    (ln_post): LayerNorm((2304,), eps=1e-06, elementwise_affine=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "from PIL import Image\n",
    "from transformers import AutoModel, AutoTokenizer\n",
    "\n",
    "name='MiniCPM'\n",
    "path='openbmb/MiniCPM-V-2'\n",
    "\n",
    "model = AutoModel.from_pretrained('openbmb/MiniCPM-V-2', trust_remote_code=True, torch_dtype=torch.bfloat16)\n",
    "# For Nvidia GPUs support BF16 (like A100, H100, RTX3090)\n",
    "model = model.to(device='cuda', dtype=torch.bfloat16)\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained('openbmb/MiniCPM-V-2', trust_remote_code=True)\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d7fce3b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def inference(question,choice,task,byte):\n",
    "    \n",
    "    if task=='MCQ':\n",
    "        prompt = question + \"\\n\" + choice +\"\\nAnswer with the option's letter from the given choices directly.\"\n",
    "    elif task=='VQA':\n",
    "        prompt = question+\". Please answer no more than 10 words\"\n",
    "    elif task=='CAP':\n",
    "        prompt =  \"Please describe this image in general. Directly provide the description, do not include prefix like 'This image depicts'\"\n",
    "    else:\n",
    "        raise ValueError('No task named'+task)\n",
    "        return\n",
    "                         \n",
    "    image_file = io.BytesIO(byte)\n",
    "    image = Image.open(image_file).convert('RGB')    \n",
    "\n",
    "    answer, context, _ = model.chat(\n",
    "        image=image,\n",
    "        msgs=[{'role': 'user', 'content': prompt}],\n",
    "        context=None,\n",
    "        tokenizer=tokenizer,\n",
    "        sampling=True,\n",
    "        temperature=0.7\n",
    "    )\n",
    "                         \n",
    "    return answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51068b6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame(columns=['name','answer','type','distortion','strength'])\n",
    "\n",
    "for num in range(len(val)):\n",
    "\n",
    "    answer=inference(question=ms_dataset[num]['question'],choice=ms_dataset[num]['choice'],task=ms_dataset[num]['type'],image=ms_dataset[num]['ref_image']['bytes'])\n",
    "    \n",
    "    results = results.append({'name': val['name'][num], 'answer': answer, 'type': val['type'][num], 'distortion': val['distortion'][num], 'strength': val['strength'][num]}, ignore_index=True)\n",
    "    if num%100==99:\n",
    "        print(str(num+1) +'reference item finished!')\n",
    "\n",
    "results.to_csv('./exp/'+model_name+'_ref.csv')\n",
    "\n",
    "results = pd.DataFrame(columns=['name','answer','type','distortion','strength'])\n",
    "\n",
    "for num in range(len(val)):\n",
    "\n",
    "    answer=inference(question=ms_dataset[num]['question'],choice=ms_dataset[num]['choice'],task=ms_dataset[num]['type'],image=ms_dataset[num]['dis_image']['bytes'])\n",
    "    \n",
    "    results = results.append({'name': val['name'][num], 'answer': answer, 'type': val['type'][num], 'distortion': val['distortion'][num], 'strength': val['strength'][num]}, ignore_index=True)\n",
    "    if num%100==99:\n",
    "        print(str(num+1) +' distorted item finished!')\n",
    "\n",
    "results.to_csv('./exp/'+model_name+'_dis.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "118628de",
   "metadata": {},
   "source": [
    "# Evaluate"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "370e408c",
   "metadata": {},
   "source": [
    "Use GPT-3.5 to assist evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f20139ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install openai\n",
    "import json\n",
    "\n",
    "from openai import OpenAI\n",
    "client = OpenAI(api_key = \"Your api key\")\n",
    "import time\n",
    "\n",
    "def get_completion(msg):\n",
    "    completion = client.chat.completions.create(\n",
    "        model=\"gpt-3.5-turbo\",\n",
    "        messages=[\n",
    "            {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
    "            {\"role\": \"user\", \"content\": msg}\n",
    "        ]\n",
    "    )\n",
    "    return completion.choices[0].message.content\n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c7529c9",
   "metadata": {},
   "source": [
    "you can also use your own text-modality LLM (not recommended)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69b1d6e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "\n",
    "# model_path = \"F:/chunyi/LLM/Yi-1.5-9B-Chat\"\n",
    "\n",
    "# tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)\n",
    "\n",
    "# # Since transformers 4.35.0, the GPT-Q/AWQ model can be loaded using AutoModelForCausalLM.\n",
    "# model = AutoModelForCausalLM.from_pretrained(\n",
    "#     model_path,\n",
    "#     device_map=\"cuda\",\n",
    "#     torch_dtype='auto'\n",
    "# ).eval()\n",
    "\n",
    "# def get_completion(msg):\n",
    "#     messages=[\n",
    "#         {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
    "#         {\"role\": \"user\", \"content\": msg},\n",
    "#     ]\n",
    "\n",
    "#     input_ids = tokenizer.apply_chat_template(conversation=messages, tokenize=True, return_tensors='pt')\n",
    "#     output_ids = model.generate(input_ids.to('cuda'), eos_token_id=tokenizer.eos_token_id, max_new_tokens=50)\n",
    "#     response = tokenizer.decode(output_ids[0][input_ids.shape[1]:], skip_special_tokens=True)\n",
    "#     return response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f3fe136",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "name='Your model name'\n",
    "\n",
    "ans_file=pd.read_csv(\"Ref_Release.csv\")\n",
    "dis_file=pd.read_csv('exp/'+name+'_dis.csv')\n",
    "ref_file=pd.read_csv('exp/'+name+'_ref.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92e2603d",
   "metadata": {},
   "outputs": [],
   "source": [
    "ans_file['result']=0\n",
    "for num in range(len(ans_file)):\n",
    "    question=ans_file['question'][num]\n",
    "    answers=ans_file['choice'][num]\n",
    "    correct_ans=ans_file['answer'][num]\n",
    "    answer=dis_file['answer'][num]\n",
    "\n",
    "    if ans_file['type'][num]=='MCQ':\n",
    "        for i in range(5):\n",
    "            msg = f'''You will now be provided with a question [{question}] and a set of options [{answers}] with option [{correct_ans}] being the correct answer.\n",
    "            Additionally, there will be an answer [{answer}] provided by a respondent. Please determine whether the respondent's answer is correct considering the context of the question.\n",
    "            Even if the word choice is not completely the same, you can decide based on the given options and see whether the one in the answer is close enough to the given correct answer\n",
    "            The result is 1 if the answer is correct and else the result is 0. Please only provide the result in the following format: Score:'''\n",
    "\n",
    "            try:\n",
    "                result=get_completion(msg)\n",
    "                if result[-1] in ['0','1']:\n",
    "                    tmp=int(result[-1])\n",
    "\n",
    "                else:\n",
    "                    tmp=0\n",
    "            except:\n",
    "                tmp=0\n",
    "                print('Error '+str(num))\n",
    "            ans_file.loc[num, \"result\"]=ans_file.loc[num, \"result\"]+tmp\n",
    "        ans_file.loc[num, \"result\"]=int(ans_file.loc[num, \"result\"]/5+0.5)\n",
    "\n",
    "\n",
    "    elif ans_file['type'][num]=='VQA':\n",
    "        for i in range(5):\n",
    "            msg= f'''Given the question [{question}], evaluate whether the response [{answer}] completely matches the correct answer [{correct_ans}]. \n",
    "            First, check the response and please rate score 0 if the response is not a valid answer.\n",
    "            Please rate score 2 if the response completely or almost completely matches the correct answer on completeness, accuracy, and relevance. \n",
    "            Please rate score 1 if the response partly matches the correct answer on completeness, accuracy, and relevance.\n",
    "            Please rate score 0 if the response doesn't match the correct answer on completeness, accuracy, and relevance at all.\n",
    "            Please only provide the result in the following format: Score:'''\n",
    "\n",
    "            try:\n",
    "                result=get_completion(msg)\n",
    "                if result[-1] in ['0','1','2']:\n",
    "                    tmp=int(result[-1])\n",
    "                else:\n",
    "                    tmp=0\n",
    "            except:\n",
    "                tmp=0\n",
    "                print('Error '+str(num))     \n",
    "\n",
    "            ans_file.loc[num, \"result\"]=ans_file.loc[num, \"result\"]+tmp\n",
    "        ans_file.loc[num, \"result\"]=ans_file.loc[num, \"result\"]/10\n",
    "\n",
    "    elif ans_file['type'][num]=='CAP':\n",
    "        corrects=eval(correct_ans)\n",
    "        for correct in corrects:\n",
    "            msg= f'''Evaluate whether the sentence [{answer}] completely matches the correct answer [{correct}]. \n",
    "            First, check the response and please rate score 0 if the response is not a valid answer.\n",
    "            Please rate score 2 if the response completely or almost completely matches the correct answer on completeness, accuracy, and relevance. \n",
    "            Please rate score 1 if the response partly matches the correct answer on completeness, accuracy, and relevance.\n",
    "            Please rate score 0 if the response doesn't match the correct answer on completeness, accuracy, and relevance at all.\n",
    "            Please only provide the result in the following format: Score:'''\n",
    "\n",
    "            try:\n",
    "                result=get_completion(msg)\n",
    "                if result[-1] in ['0','1','2']:\n",
    "                    tmp=int(result[-1])\n",
    "                else:\n",
    "                    tmp=0\n",
    "            except:\n",
    "                tmp=0\n",
    "                print('Error '+str(num))   \n",
    "\n",
    "            ans_file.loc[num, \"result\"]=ans_file.loc[num, \"result\"]+tmp\n",
    "        ans_file.loc[num, \"result\"]=ans_file.loc[num, \"result\"]/(2*len(corrects))\n",
    "    else:\n",
    "        print('Error:'+str(num))\n",
    "\n",
    "    if num%20==1:\n",
    "        print(num)\n",
    "        time.sleep(5)\n",
    "\n",
    "ans_file.to_csv('./result/Result_dis_'+name+'.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b485e8fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "ans_file['result']=0\n",
    "for num in range(len(ans_file)):\n",
    "    question=ans_file['question'][num]\n",
    "    answers=ans_file['choice'][num]\n",
    "    correct_ans=ans_file['answer'][num]\n",
    "    answer=ref_file['answer'][num]\n",
    "\n",
    "    if ans_file['type'][num]=='MCQ':\n",
    "        for i in range(5):\n",
    "            msg = f'''You will now be provided with a question [{question}] and a set of options [{answers}] with option [{correct_ans}] being the correct answer.\n",
    "            Additionally, there will be an answer [{answer}] provided by a respondent. Please determine whether the respondent's answer is correct considering the context of the question.\n",
    "            Even if the word choice is not completely the same, you can decide based on the given options and see whether the one in the answer is close enough to the given correct answer\n",
    "            The result is 1 if the answer is correct and else the result is 0. Please only provide the result in the following format: Score:'''\n",
    "\n",
    "            try:\n",
    "                result=get_completion(msg)\n",
    "                if result[-1] in ['0','1']:\n",
    "                    tmp=int(result[-1])\n",
    "\n",
    "                else:\n",
    "                    tmp=0\n",
    "            except:\n",
    "                tmp=0\n",
    "                print('Error '+str(num))\n",
    "            ans_file.loc[num, \"result\"]=ans_file.loc[num, \"result\"]+tmp\n",
    "        ans_file.loc[num, \"result\"]=int(ans_file.loc[num, \"result\"]/5+0.5)\n",
    "\n",
    "\n",
    "    elif ans_file['type'][num]=='VQA':\n",
    "        for i in range(5):\n",
    "            msg= f'''Given the question [{question}], evaluate whether the response [{answer}] completely matches the correct answer [{correct_ans}]. \n",
    "            First, check the response and please rate score 0 if the response is not a valid answer.\n",
    "            Please rate score 2 if the response completely or almost completely matches the correct answer on completeness, accuracy, and relevance. \n",
    "            Please rate score 1 if the response partly matches the correct answer on completeness, accuracy, and relevance.\n",
    "            Please rate score 0 if the response doesn't match the correct answer on completeness, accuracy, and relevance at all.\n",
    "            Please only provide the result in the following format: Score:'''\n",
    "\n",
    "            try:\n",
    "                result=get_completion(msg)\n",
    "                if result[-1] in ['0','1','2']:\n",
    "                    tmp=int(result[-1])\n",
    "                else:\n",
    "                    tmp=0\n",
    "            except:\n",
    "                tmp=0\n",
    "                print('Error '+str(num))     \n",
    "\n",
    "            ans_file.loc[num, \"result\"]=ans_file.loc[num, \"result\"]+tmp\n",
    "        ans_file.loc[num, \"result\"]=ans_file.loc[num, \"result\"]/10\n",
    "\n",
    "    elif ans_file['type'][num]=='CAP':\n",
    "        corrects=eval(correct_ans)\n",
    "        for correct in corrects:\n",
    "            msg= f'''Evaluate whether the sentence [{answer}] completely matches the correct answer [{correct}]. \n",
    "            First, check the response and please rate score 0 if the response is not a valid answer.\n",
    "            Please rate score 2 if the response completely or almost completely matches the correct answer on completeness, accuracy, and relevance. \n",
    "            Please rate score 1 if the response partly matches the correct answer on completeness, accuracy, and relevance.\n",
    "            Please rate score 0 if the response doesn't match the correct answer on completeness, accuracy, and relevance at all.\n",
    "            Please only provide the result in the following format: Score:'''\n",
    "\n",
    "            try:\n",
    "                result=get_completion(msg)\n",
    "                if result[-1] in ['0','1','2']:\n",
    "                    tmp=int(result[-1])\n",
    "                else:\n",
    "                    tmp=0\n",
    "            except:\n",
    "                tmp=0\n",
    "                print('Error '+str(num))   \n",
    "\n",
    "            ans_file.loc[num, \"result\"]=ans_file.loc[num, \"result\"]+tmp\n",
    "        ans_file.loc[num, \"result\"]=ans_file.loc[num, \"result\"]/(2*len(corrects))\n",
    "    else:\n",
    "        print('Error:'+str(num))\n",
    "\n",
    "    if num%20==1:\n",
    "        print(num)\n",
    "        time.sleep(5)\n",
    "\n",
    "ans_file.to_csv('./result/Result_ref_'+name+'.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3340e37",
   "metadata": {},
   "outputs": [],
   "source": [
    "ans_file['result']=0\n",
    "for num in range(len(ans_file)):\n",
    "    question=ans_file['question'][num]\n",
    "    answers=ans_file['choice'][num]\n",
    "    correct_ans=ref_file['answer'][num]\n",
    "    answer=dis_file['answer'][num]\n",
    "\n",
    "    if ans_file['type'][num]=='MCQ':\n",
    "        for i in range(5):\n",
    "            msg = f'''You will now be provided with a question [{question}] and a set of options [{answers}] with option [{correct_ans}] being the correct answer.\n",
    "            Additionally, there will be an answer [{answer}] provided by a respondent. Please determine whether the respondent's answer is correct considering the context of the question.\n",
    "            Even if the word choice is not completely the same, you can decide based on the given options and see whether the one in the answer is close enough to the given correct answer\n",
    "            The result is 1 if the answer is correct and else the result is 0. Please only provide the result in the following format: Score:'''\n",
    "\n",
    "            try:\n",
    "                result=get_completion(msg)\n",
    "                if result[-1] in ['0','1']:\n",
    "                    tmp=int(result[-1])\n",
    "\n",
    "                else:\n",
    "                    tmp=0\n",
    "            except:\n",
    "                tmp=0\n",
    "                print('Error '+str(num))\n",
    "            ans_file.loc[num, \"result\"]=ans_file.loc[num, \"result\"]+tmp\n",
    "        ans_file.loc[num, \"result\"]=int(ans_file.loc[num, \"result\"]/5+0.5)\n",
    "\n",
    "\n",
    "    elif ans_file['type'][num]=='VQA':\n",
    "        for i in range(5):\n",
    "            msg= f'''Given the question [{question}], evaluate whether the response [{answer}] completely matches the correct answer [{correct_ans}]. \n",
    "            First, check the response and please rate score 0 if the response is not a valid answer.\n",
    "            Please rate score 2 if the response completely or almost completely matches the correct answer on completeness, accuracy, and relevance. \n",
    "            Please rate score 1 if the response partly matches the correct answer on completeness, accuracy, and relevance.\n",
    "            Please rate score 0 if the response doesn't match the correct answer on completeness, accuracy, and relevance at all.\n",
    "            Please only provide the result in the following format: Score:'''\n",
    "\n",
    "            try:\n",
    "                result=get_completion(msg)\n",
    "                if result[-1] in ['0','1','2']:\n",
    "                    tmp=int(result[-1])\n",
    "                else:\n",
    "                    tmp=0\n",
    "            except:\n",
    "                tmp=0\n",
    "                print('Error '+str(num))     \n",
    "\n",
    "            ans_file.loc[num, \"result\"]=ans_file.loc[num, \"result\"]+tmp\n",
    "        ans_file.loc[num, \"result\"]=ans_file.loc[num, \"result\"]/10\n",
    "\n",
    "    elif ans_file['type'][num]=='CAP':\n",
    "        for i in range(5):\n",
    "            msg= f'''Evaluate whether the sentence [{answer}] completely matches the correct answer [{correct_ans}]. \n",
    "            First, check the response and please rate score 0 if the response is not a valid answer.\n",
    "            Please rate score 2 if the response completely or almost completely matches the correct answer on completeness, accuracy, and relevance. \n",
    "            Please rate score 1 if the response partly matches the correct answer on completeness, accuracy, and relevance.\n",
    "            Please rate score 0 if the response doesn't match the correct answer on completeness, accuracy, and relevance at all.\n",
    "            Please only provide the result in the following format: Score:'''\n",
    "\n",
    "            try:\n",
    "                result=get_completion(msg)\n",
    "                if result[-1] in ['0','1','2']:\n",
    "                    tmp=int(result[-1])\n",
    "                else:\n",
    "                    tmp=0\n",
    "            except:\n",
    "                tmp=0\n",
    "                print('Error '+str(num))   \n",
    "\n",
    "            ans_file.loc[num, \"result\"]=ans_file.loc[num, \"result\"]+tmp\n",
    "        ans_file.loc[num, \"result\"]=ans_file.loc[num, \"result\"]/10\n",
    "    else:\n",
    "        print('Error:'+str(num))\n",
    "\n",
    "    if num%20==1:\n",
    "        print(num)\n",
    "        time.sleep(5)\n",
    "\n",
    "ans_file.to_csv('./result/Result_ref_'+name+'.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "580f72bd",
   "metadata": {},
   "source": [
    "# Show final result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "f28db425",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "name=\"llava15\"\n",
    "dis=pd.read_csv(\"result/Result_dis_\"+name+\".csv\")\n",
    "com=pd.read_csv(\"result/Result_com_\"+name+\".csv\")\n",
    "ref=pd.read_csv(\"result/Result_ref_\"+name+\".csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "e6d47605",
   "metadata": {},
   "outputs": [],
   "source": [
    "Type=[]\n",
    "Type.append(dis[(dis['type'] =='MCQ') ].index.tolist())\n",
    "Type.append(dis[(dis['type'] =='VQA') ].index.tolist())\n",
    "Type.append(dis[(dis['type'] =='CAP') ].index.tolist())\n",
    "Strength=[]\n",
    "Strength.append(dis[(dis['strength'] == 1) ].index.tolist())\n",
    "Strength.append(dis[(dis['strength'] == 2) ].index.tolist())\n",
    "Strength.append(dis[(dis['strength'] == 3) ].index.tolist())\n",
    "Step=[]\n",
    "Step.append([index for index in range(len(dis)) if dis['distortion'][index] in [1,2,3,4]])\n",
    "Step.append([index for index in range(len(dis)) if dis['distortion'][index] in [5,6,7,8]])\n",
    "Step.append([index for index in range(len(dis)) if dis['distortion'][index] in [9,10,11,12,13]])\n",
    "Step.append([index for index in range(len(dis)) if dis['distortion'][index] in [14,15,16,17,18]])\n",
    "Step.append([index for index in range(len(dis)) if dis['distortion'][index] in [19,20,21,22]])\n",
    "Step.append([index for index in range(len(dis)) if dis['distortion'][index] in [23,24,25,26,27]])\n",
    "Step.append([index for index in range(len(dis)) if dis['distortion'][index] in [28,29,30,31,32,33]])\n",
    "Group=[]\n",
    "Group.append([index for index in range(len(dis)) if dis['distortion'][index] in [1,5,28]])\n",
    "Group.append([index for index in range(len(dis)) if dis['distortion'][index] in [25,26,27]])\n",
    "Group.append([index for index in range(len(dis)) if dis['distortion'][index] in [14,23,24,29,30,32,33]])\n",
    "Group.append([index for index in range(len(dis)) if dis['distortion'][index] in [6,13,18,19,20,21,22]])\n",
    "Group.append([index for index in range(len(dis)) if dis['distortion'][index] in [9,10,11,12,31]])\n",
    "Group.append([index for index in range(len(dis)) if dis['distortion'][index] in [15,16,17]])\n",
    "Group.append([index for index in range(len(dis)) if dis['distortion'][index] in [2,3,4,7,8]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "318909ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "TaS=[]\n",
    "TaS.append(dis[(dis['type'] =='MCQ') & (dis['strength'] == 1)].index.tolist())\n",
    "TaS.append(dis[(dis['type'] =='MCQ') & (dis['strength'] == 2)].index.tolist())\n",
    "TaS.append(dis[(dis['type'] =='MCQ') & (dis['strength'] == 3)].index.tolist())\n",
    "TaS.append(dis[(dis['type'] =='VQA') & (dis['strength'] == 1)].index.tolist())\n",
    "TaS.append(dis[(dis['type'] =='VQA') & (dis['strength'] == 2)].index.tolist())\n",
    "TaS.append(dis[(dis['type'] =='VQA') & (dis['strength'] == 3)].index.tolist())\n",
    "TaS.append(dis[(dis['type'] =='CAP') & (dis['strength'] == 1)].index.tolist())\n",
    "TaS.append(dis[(dis['type'] =='CAP') & (dis['strength'] == 2)].index.tolist())\n",
    "TaS.append(dis[(dis['type'] =='CAP') & (dis['strength'] == 3)].index.tolist())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2223cc4",
   "metadata": {},
   "source": [
    "format:<br>\n",
    "Task:&emsp;&emsp;MCQ&emsp;VQA&emsp;CAP<br>\n",
    "Strength:&emsp;high&emsp;mid&emsp;low<br>\n",
    "Step:&emsp;&emsp;Enviro&emsp;Camera&emsp;Analog&emsp;Source&emsp;Channel&emsp;Receive&emsp;Enhance<br>\n",
    "Group:&emsp;&emsp;Blur&emsp;Lumi&emsp;Chromi&emsp;Spatial&emsp;Noise&emsp;Comp&emsp;Wild<br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "4bf7b327",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "llava15\n",
      "Absolute Robustness\n",
      "0.4364\t0.5980\t0.3455\t\n",
      "0.4679\t0.4398\t0.4721\t\n",
      "0.4024\t0.4417\t0.4560\t0.4964\t0.5067\t0.4862\t0.4304\t\n",
      "0.4459\t0.4570\t0.4654\t0.4781\t0.4600\t0.5193\t0.4014\t\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "print(name)\n",
    "print('Absolute Robustness')\n",
    "for tmp in [Type,Strength,Step,Group]:\n",
    "    for num in range(len(tmp)):\n",
    "        print('%.4f' % np.mean(dis['result'][tmp[num]]),end='\\t')\n",
    "    print('')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "5e1d71ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "llava15\n",
      "Relative Robustness\n",
      "0.3758\t0.4920\t0.1896\t\n",
      "0.3774\t0.3273\t0.3526\t\n",
      "0.3309\t0.3542\t0.3298\t0.3677\t0.4094\t0.3619\t0.3258\t\n",
      "0.3658\t0.3543\t0.3453\t0.3772\t0.3431\t0.3830\t0.3096\t\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "print(name)\n",
    "print('Relative Robustness')\n",
    "for tmp in [Type,Strength,Step,Group]:\n",
    "    for num in range(len(tmp)):\n",
    "        print('%.4f' % np.mean(com['result'][tmp[num]]*ref['result'][tmp[num]]),end='\\t')\n",
    "    print('')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}