notebooks/01-DownloadCaspData.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download CASP data\n",
    "\n",
    "This notebook downloads all raw protein data from the CASP website.\n",
    "\n",
    "Each CASP edition is downloaded into `./CASP*` which is a symlink to `../data/CASP*`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib agg\n",
    "import io\n",
    "import os\n",
    "import re\n",
    "import time\n",
    "import json\n",
    "import pickle\n",
    "import hashlib\n",
    "import tarfile\n",
    "import requests\n",
    "import tempfile\n",
    "import warnings\n",
    "import functools\n",
    "import contextlib\n",
    "import subprocess\n",
    "from pathlib import Path\n",
    "\n",
    "import bs4\n",
    "import docker\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.animation\n",
    "import natsort as ns\n",
    "import tqdm.notebook as tqdm\n",
    "\n",
    "import Bio.PDB\n",
    "import Bio.SeqIO\n",
    "import Bio.Align.AlignInfo\n",
    "import Bio.AlignIO\n",
    "\n",
    "from loguru import logger\n",
    "from joblib import Parallel, delayed\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "from IPython.display import display, Markdown, HTML, Video\n",
    "\n",
    "from graphqa.data.aminoacids import *\n",
    "from graphqa.data.decoys import ca_coord_and_orientation\n",
    "\n",
    "@functools.lru_cache(maxsize=128)\n",
    "def requests_get(url):\n",
    "    return requests.get(url)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Documentation of CASP download area for CASP 11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CASP11\n",
      "   SUMMARY_TABLES\t    - text version of the result tables (per target and archived)\n",
      "   extra_experiments\t    - input data (selected contacts) for Tp, Tc, Ts and Tx contact-assisted categories) \n",
      "\t\t\t\tand starting models for the TR refinement category\n",
      "   predictions              - all predictions (DR, FN, QA, RR, 3D)\n",
      "   predictions_trimmed_to_domains - all tertiary structure (TS) predictions trimmed according to \n",
      "\t\t\t       the official domain definitions\n",
      "   results_LGA_sda          - results (complete data and summaries) of sequence-dependent LGA analysis for all models\n",
      "   results_LGA_sia          - results (complete data and summaries) of sequence-independent LGA analysis for all models\n",
      "   server_predictions\t    - 3D-structure predictions from servers\n",
      "   targets                  - target sequences; preprocessed target structures\n",
      "   templates\t\t    - results of LGA sequence-independent comparison of the best 25 templates \n"
     ]
    }
   ],
   "source": [
    "response = requests_get('https://predictioncenter.org/download_area/README')\n",
    "readme = response.text.splitlines()\n",
    "i = readme.index('CASP11')\n",
    "print(*readme[i: i+12], sep='\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "incorrectly_encoded_metadata": "toc-hr-collapsed=true toc-nb-collapsed=true"
   },
   "source": [
    "## Primary structure\n",
    "\n",
    "Download all sequences from the CASP website and save them as `CASP*/sequences.fasta`.\n",
    "\n",
    "Metadata about all sequences is saved in [`sequences.csv`](./sequences.csv)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Targets</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Edition</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>90</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Targets\n",
       "Edition         \n",
       "9            129\n",
       "10           115\n",
       "11           105\n",
       "12            90\n",
       "13            90"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regex_name = re.compile(r'(T|H)\\d\\d\\d\\d')\n",
    "regex_subunit = re.compile(r'\\bsubunit\\s+(\\d+)')\n",
    "\n",
    "def is_qa_target(target):\n",
    "    return bool(regex_name.fullmatch(target.name))\n",
    "\n",
    "def rename(target):\n",
    "    original_name = target.name\n",
    "    match = regex_name.fullmatch(target.name)\n",
    "    if match.groups()[0] == 'H':\n",
    "        subunit = regex_subunit.search(target.description).groups()[0]\n",
    "        casp_id = target.name.replace('H', 'T') + 's' + subunit\n",
    "        target.description = (\n",
    "            target.description.replace(target.name, casp_id, 1)\n",
    "            + f' (original target name {original_name})'\n",
    "        )\n",
    "        target.name = casp_id\n",
    "        target.id = casp_id\n",
    "    return target, original_name\n",
    "\n",
    "df_sequences = []\n",
    "for casp_ed in [9, 10, 11, 12, 13]:\n",
    "    dest = Path(f'CASP{casp_ed}/sequences.fasta')\n",
    "    dest.parent.mkdir(exist_ok=True, parents=True)\n",
    "    \n",
    "    response = requests_get(f'https://predictioncenter.org/download_area/'\n",
    "                            f'CASP{casp_ed}/sequences/casp{casp_ed}.seq.txt')\n",
    "    sequences = Bio.SeqIO.parse(io.StringIO(response.text), format='fasta')\n",
    "    sequences = filter(is_qa_target, sequences)\n",
    "    sequences = map(rename, sequences)\n",
    "    \n",
    "    with dest.open('w') as f:\n",
    "        for seq, original_name in sequences:\n",
    "            df_sequences.append({\n",
    "                'casp_ed': casp_ed,\n",
    "                'target_id': seq.id,\n",
    "                'target_id_orig': original_name,\n",
    "                'length': len(seq),\n",
    "            })\n",
    "            Bio.SeqIO.write(seq, f, format='fasta')\n",
    "\n",
    "df_sequences = pd.DataFrame(df_sequences)\n",
    "df_sequences.to_csv('sequences.csv', index=False)\n",
    "df_sequences.groupby('casp_ed').size().rename_axis('Edition').to_frame('Targets')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In some cases the same protein (`target_id_orig`) corresponds to more targets (`target_id`):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>casp_ed</th>\n",
       "      <th>target_id</th>\n",
       "      <th>target_id_orig</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>439</th>\n",
       "      <td>13</td>\n",
       "      <td>T0953s1</td>\n",
       "      <td>H0953</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>440</th>\n",
       "      <td>13</td>\n",
       "      <td>T0953s2</td>\n",
       "      <td>H0953</td>\n",
       "      <td>249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>441</th>\n",
       "      <td>13</td>\n",
       "      <td>T0957s1</td>\n",
       "      <td>H0957</td>\n",
       "      <td>163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>442</th>\n",
       "      <td>13</td>\n",
       "      <td>T0957s2</td>\n",
       "      <td>H0957</td>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>443</th>\n",
       "      <td>13</td>\n",
       "      <td>T0968s1</td>\n",
       "      <td>H0968</td>\n",
       "      <td>126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444</th>\n",
       "      <td>13</td>\n",
       "      <td>T0968s2</td>\n",
       "      <td>H0968</td>\n",
       "      <td>116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>445</th>\n",
       "      <td>13</td>\n",
       "      <td>T0974s1</td>\n",
       "      <td>H0974</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>446</th>\n",
       "      <td>13</td>\n",
       "      <td>T0974s2</td>\n",
       "      <td>H0974</td>\n",
       "      <td>95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>447</th>\n",
       "      <td>13</td>\n",
       "      <td>T0980s1</td>\n",
       "      <td>H0980</td>\n",
       "      <td>111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>448</th>\n",
       "      <td>13</td>\n",
       "      <td>T0980s2</td>\n",
       "      <td>H0980</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>449</th>\n",
       "      <td>13</td>\n",
       "      <td>T0986s1</td>\n",
       "      <td>H0986</td>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>450</th>\n",
       "      <td>13</td>\n",
       "      <td>T0986s2</td>\n",
       "      <td>H0986</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>451</th>\n",
       "      <td>13</td>\n",
       "      <td>T0993s1</td>\n",
       "      <td>H0993</td>\n",
       "      <td>269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>452</th>\n",
       "      <td>13</td>\n",
       "      <td>T0993s2</td>\n",
       "      <td>H0993</td>\n",
       "      <td>109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>453</th>\n",
       "      <td>13</td>\n",
       "      <td>T1015s1</td>\n",
       "      <td>H1015</td>\n",
       "      <td>89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>454</th>\n",
       "      <td>13</td>\n",
       "      <td>T1015s2</td>\n",
       "      <td>H1015</td>\n",
       "      <td>129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>455</th>\n",
       "      <td>13</td>\n",
       "      <td>T1017s1</td>\n",
       "      <td>H1017</td>\n",
       "      <td>111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>456</th>\n",
       "      <td>13</td>\n",
       "      <td>T1017s2</td>\n",
       "      <td>H1017</td>\n",
       "      <td>129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>457</th>\n",
       "      <td>13</td>\n",
       "      <td>T1019s1</td>\n",
       "      <td>H1019</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>458</th>\n",
       "      <td>13</td>\n",
       "      <td>T1019s2</td>\n",
       "      <td>H1019</td>\n",
       "      <td>88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>459</th>\n",
       "      <td>13</td>\n",
       "      <td>T1021s1</td>\n",
       "      <td>H1021</td>\n",
       "      <td>149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>460</th>\n",
       "      <td>13</td>\n",
       "      <td>T1021s2</td>\n",
       "      <td>H1021</td>\n",
       "      <td>354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>461</th>\n",
       "      <td>13</td>\n",
       "      <td>T1021s3</td>\n",
       "      <td>H1021</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>462</th>\n",
       "      <td>13</td>\n",
       "      <td>T1022s1</td>\n",
       "      <td>H1022</td>\n",
       "      <td>229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>463</th>\n",
       "      <td>13</td>\n",
       "      <td>T1022s2</td>\n",
       "      <td>H1022</td>\n",
       "      <td>529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>464</th>\n",
       "      <td>13</td>\n",
       "      <td>T1023s1</td>\n",
       "      <td>H1023</td>\n",
       "      <td>315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>465</th>\n",
       "      <td>13</td>\n",
       "      <td>T1023s2</td>\n",
       "      <td>H1023</td>\n",
       "      <td>333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>466</th>\n",
       "      <td>13</td>\n",
       "      <td>T1023s3</td>\n",
       "      <td>H1023</td>\n",
       "      <td>472</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     casp_ed target_id target_id_orig  length\n",
       "439       13   T0953s1          H0953      72\n",
       "440       13   T0953s2          H0953     249\n",
       "441       13   T0957s1          H0957     163\n",
       "442       13   T0957s2          H0957     164\n",
       "443       13   T0968s1          H0968     126\n",
       "444       13   T0968s2          H0968     116\n",
       "445       13   T0974s1          H0974      72\n",
       "446       13   T0974s2          H0974      95\n",
       "447       13   T0980s1          H0980     111\n",
       "448       13   T0980s2          H0980      52\n",
       "449       13   T0986s1          H0986      96\n",
       "450       13   T0986s2          H0986     155\n",
       "451       13   T0993s1          H0993     269\n",
       "452       13   T0993s2          H0993     109\n",
       "453       13   T1015s1          H1015      89\n",
       "454       13   T1015s2          H1015     129\n",
       "455       13   T1017s1          H1017     111\n",
       "456       13   T1017s2          H1017     129\n",
       "457       13   T1019s1          H1019      58\n",
       "458       13   T1019s2          H1019      88\n",
       "459       13   T1021s1          H1021     149\n",
       "460       13   T1021s2          H1021     354\n",
       "461       13   T1021s3          H1021     295\n",
       "462       13   T1022s1          H1022     229\n",
       "463       13   T1022s2          H1022     529\n",
       "464       13   T1023s1          H1023     315\n",
       "465       13   T1023s2          H1023     333\n",
       "466       13   T1023s3          H1023     472"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sequences.query('target_id != target_id_orig')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "incorrectly_encoded_metadata": "toc-hr-collapsed=true toc-nb-collapsed=true"
   },
   "source": [
    "## Tertiary structure"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Native structures"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Downloaded from the CASP download area\n",
    "\n",
    "These native structures are downloaded from the CASP download area.\n",
    "\n",
    "Metadata about these structures is saved in [`natives_casp.csv`](./natives_casp.csv).\n",
    "\n",
    "- all names start with `T` and end with an optional subunit `s`\n",
    "- some of the published primary sequences don't have an official native structure (maybe canceled?)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Targets</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Edition</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Targets\n",
       "Edition         \n",
       "9            117\n",
       "10           103\n",
       "11            85\n",
       "12            40\n",
       "13            82"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parser = Bio.PDB.PDBParser(QUIET=True)\n",
    "df_natives = []\n",
    "# Match files like T0759s1.pdb but not T0759-D1.pdb\n",
    "regex = re.compile(r'T\\d\\d\\d\\d(?:s\\d)?.pdb')\n",
    "\n",
    "native_urls = {\n",
    "    9: 'https://predictioncenter.org/download_area/CASP9/targets/casp9.targ_unsplit.tgz',\n",
    "    10: 'https://predictioncenter.org/download_area/CASP10/targets/casp10.targets_unsplitted.noT0695T0739.tgz',\n",
    "    11: 'https://predictioncenter.org/download_area/CASP11/targets/casp11.targets_unsplitted.release11242014.tgz',\n",
    "    12: 'https://predictioncenter.org/download_area/CASP12/targets/casp12.targets_T0.releaseDec022016.tgz',\n",
    "    13: 'https://predictioncenter.org/download_area/CASP13/targets/casp13.targets.T.4public.tar.gz',\n",
    "}\n",
    "\n",
    "for casp_ed, url in native_urls.items():\n",
    "    dest = Path(f'CASP{casp_ed}') / 'native'\n",
    "    dest.mkdir(exist_ok=True, parents=True)\n",
    "    ! curl -s {url} | tar xz --directory {dest.as_posix()}\n",
    "    \n",
    "    if casp_ed == 13:\n",
    "        try:\n",
    "            import casp13_secret\n",
    "            ! echo '--user {casp13_secret.user}:{casp13_secret.pwd}' | curl -s --config - {casp13_secret.url} | tar xz --directory {dest}\n",
    "        except ImportError:\n",
    "            print(\"Could not get limited-access CASP13 files\")\n",
    "\n",
    "    for f in dest.iterdir():\n",
    "        if not regex.fullmatch(f.name):\n",
    "            f.unlink()\n",
    "            continue\n",
    "        target_id = f.with_suffix('').name\n",
    "        structure = parser.get_structure(target_id, f)\n",
    "        df_natives.append({\n",
    "            'casp_ed': casp_ed,\n",
    "            'target_id': target_id,\n",
    "            'chains': len(list(structure.get_chains())),\n",
    "            'residues': len(list(structure.get_residues())),\n",
    "            'atoms': len(list(structure.get_atoms())),\n",
    "        })\n",
    "        \n",
    "df_natives = pd.DataFrame(df_natives).sort_values(['casp_ed', 'target_id'])\n",
    "df_natives.to_csv('natives_casp.csv', index=False)\n",
    "df_natives.groupby('casp_ed').size().rename_axis('Edition').to_frame('Targets')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A full outer join between the sequences in `.fasta` format \n",
    "and the sequences found in the native's `.pdb` file\n",
    "shows the some discrepancies.\n",
    "\n",
    "In the following:\n",
    "- **left:**  primary sequences published on the CASP website\n",
    "- **right:** native structures published on the CASP website"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Outer join counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>both</th>\n",
       "      <td>431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>left_only</th>\n",
       "      <td>98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>right_only</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Outer join counts\n",
       "both                      431\n",
       "left_only                  98\n",
       "right_only                  0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>casp_ed</th>\n",
       "      <th>target_id</th>\n",
       "      <th>target_id_orig</th>\n",
       "      <th>length</th>\n",
       "      <th>residues</th>\n",
       "      <th>_merge</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9</td>\n",
       "      <td>T0519</td>\n",
       "      <td>T0519</td>\n",
       "      <td>180</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>9</td>\n",
       "      <td>T0535</td>\n",
       "      <td>T0535</td>\n",
       "      <td>294</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>9</td>\n",
       "      <td>T0546</td>\n",
       "      <td>T0546</td>\n",
       "      <td>134</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>9</td>\n",
       "      <td>T0554</td>\n",
       "      <td>T0554</td>\n",
       "      <td>135</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>9</td>\n",
       "      <td>T0556</td>\n",
       "      <td>T0556</td>\n",
       "      <td>73</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>470</th>\n",
       "      <td>13</td>\n",
       "      <td>T0952</td>\n",
       "      <td>T0952</td>\n",
       "      <td>35</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>473</th>\n",
       "      <td>13</td>\n",
       "      <td>T0956</td>\n",
       "      <td>T0956</td>\n",
       "      <td>178</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>487</th>\n",
       "      <td>13</td>\n",
       "      <td>T0972</td>\n",
       "      <td>T0972</td>\n",
       "      <td>106</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>518</th>\n",
       "      <td>13</td>\n",
       "      <td>T1007</td>\n",
       "      <td>T1007</td>\n",
       "      <td>149</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>523</th>\n",
       "      <td>13</td>\n",
       "      <td>T1012</td>\n",
       "      <td>T1012</td>\n",
       "      <td>199</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>98 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     casp_ed target_id target_id_orig  length  residues     _merge\n",
       "4          9     T0519          T0519     180       NaN  left_only\n",
       "20         9     T0535          T0535     294       NaN  left_only\n",
       "31         9     T0546          T0546     134       NaN  left_only\n",
       "39         9     T0554          T0554     135       NaN  left_only\n",
       "41         9     T0556          T0556      73       NaN  left_only\n",
       "..       ...       ...            ...     ...       ...        ...\n",
       "470       13     T0952          T0952      35       NaN  left_only\n",
       "473       13     T0956          T0956     178       NaN  left_only\n",
       "487       13     T0972          T0972     106       NaN  left_only\n",
       "518       13     T1007          T1007     149       NaN  left_only\n",
       "523       13     T1012          T1012     199       NaN  left_only\n",
       "\n",
       "[98 rows x 6 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Same length?</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>False</th>\n",
       "      <td>469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Same length?\n",
       "False           469\n",
       "True             60"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>casp_ed</th>\n",
       "      <th>target_id</th>\n",
       "      <th>target_id_orig</th>\n",
       "      <th>length</th>\n",
       "      <th>residues</th>\n",
       "      <th>_merge</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9</td>\n",
       "      <td>T0515</td>\n",
       "      <td>T0515</td>\n",
       "      <td>365</td>\n",
       "      <td>348.0</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>T0516</td>\n",
       "      <td>T0516</td>\n",
       "      <td>229</td>\n",
       "      <td>227.0</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>T0518</td>\n",
       "      <td>T0518</td>\n",
       "      <td>288</td>\n",
       "      <td>256.0</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>9</td>\n",
       "      <td>T0520</td>\n",
       "      <td>T0520</td>\n",
       "      <td>189</td>\n",
       "      <td>173.0</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>9</td>\n",
       "      <td>T0521</td>\n",
       "      <td>T0521</td>\n",
       "      <td>179</td>\n",
       "      <td>168.0</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>522</th>\n",
       "      <td>13</td>\n",
       "      <td>T1011</td>\n",
       "      <td>T1011</td>\n",
       "      <td>534</td>\n",
       "      <td>444.0</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524</th>\n",
       "      <td>13</td>\n",
       "      <td>T1013</td>\n",
       "      <td>T1013</td>\n",
       "      <td>537</td>\n",
       "      <td>290.0</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>525</th>\n",
       "      <td>13</td>\n",
       "      <td>T1014</td>\n",
       "      <td>T1014</td>\n",
       "      <td>276</td>\n",
       "      <td>268.0</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>526</th>\n",
       "      <td>13</td>\n",
       "      <td>T1016</td>\n",
       "      <td>T1016</td>\n",
       "      <td>203</td>\n",
       "      <td>202.0</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>528</th>\n",
       "      <td>13</td>\n",
       "      <td>T1020</td>\n",
       "      <td>T1020</td>\n",
       "      <td>577</td>\n",
       "      <td>313.0</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>371 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     casp_ed target_id target_id_orig  length  residues _merge\n",
       "0          9     T0515          T0515     365     348.0   both\n",
       "1          9     T0516          T0516     229     227.0   both\n",
       "3          9     T0518          T0518     288     256.0   both\n",
       "5          9     T0520          T0520     189     173.0   both\n",
       "6          9     T0521          T0521     179     168.0   both\n",
       "..       ...       ...            ...     ...       ...    ...\n",
       "522       13     T1011          T1011     534     444.0   both\n",
       "524       13     T1013          T1013     537     290.0   both\n",
       "525       13     T1014          T1014     276     268.0   both\n",
       "526       13     T1016          T1016     203     202.0   both\n",
       "528       13     T1020          T1020     577     313.0   both\n",
       "\n",
       "[371 rows x 6 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_merge = pd.merge(\n",
    "    df_sequences, \n",
    "    df_natives.drop(columns=['chains', 'atoms']), \n",
    "    on=['casp_ed', 'target_id'], how='outer', indicator=True\n",
    ")\n",
    "\n",
    "# Some targets are present in the .fasta files but absent from the native .pdb files\n",
    "display(\n",
    "    df_merge['_merge']\n",
    "    .value_counts()\n",
    "    .to_frame('Outer join counts')\n",
    ")\n",
    "display(df_merge.query(\"_merge!='both'\"))\n",
    "\n",
    "# Some targets have different lenghts in the .fasta and .pdb files\n",
    "display(\n",
    "    (df_merge['length'] == df_merge['residues'])\n",
    "    .value_counts()\n",
    "    .to_frame('Same length?')\n",
    ")\n",
    "display(df_merge.query('length != residues and _merge==\"both\"'))\n",
    "\n",
    "del df_merge"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Downloaded from the Protein Data Bank\n",
    "\n",
    "These native structures are downloaded from the Protein Data Bank using \n",
    "[the mapping from CASP id to PDB code can](https://predictioncenter.org/casp11/targetlist.cgi?view_targets=all).\n",
    "\n",
    "Metadata about these structures is saved in [`natives_pdb.csv`](./natives_pdb.csv).\n",
    "\n",
    "- Some targets are listed using `H????` names and some others using `T????[s?]` names, we only download the latter\n",
    "- Some structures downloaded from PDB actually have more than one model in the same `.pdb` file (but CASP only cares about one, right?)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-09-21 14:52:17.657 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Non-SGI research Centers and others, Sin1, Homo Sapiens\n",
      "2020-09-21 14:53:20.855 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Northeast Structural Genomics Consortium, HR8011D,\n",
      "2020-09-21 14:53:45.986 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Midwest Center for Structural Genomics, APC100304,\n",
      "2020-09-21 14:53:45.989 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Midwest Center for Structural Genomics, APC100239,\n",
      "2020-09-21 14:54:23.255 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Non-SGI research Centers and others, EhV298 ,\n",
      "2020-09-21 14:54:23.256 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Non-SGI research Centers and others, Q6MLV0,\n",
      "2020-09-21 14:55:25.190 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Non-SGI research Centers and others, nitroreductase Rv3368\n",
      "2020-09-21 14:55:29.065 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: New York Structural Genomics Research Center, triosephosphate isomerase\n",
      "2020-09-21 14:55:36.537 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Non-SGI research Centers and others, VCID6010\n",
      "2020-09-21 14:55:40.175 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Non-SGI research Centers and others, WRAP-5\n",
      "2020-09-21 14:55:41.148 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Non-SGI research Centers and others, CC0948\n",
      "2020-09-21 14:55:50.413 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Midwest Center for Structural Genomics, hypothetical protein YPO2471\n",
      "2020-09-21 14:55:50.416 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Non-SGI research Centers and others, CC_1988\n",
      "2020-09-21 14:55:51.182 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Joint Center for Structural Genomics, GS13694 - BVU_0573\n",
      "2020-09-21 14:56:27.598 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: LAdV2 fibre 1 head domain\n",
      "2020-09-21 14:56:30.142 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: TtnM\n",
      "2020-09-21 14:56:32.415 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: HPeV1 2A\n",
      "2020-09-21 14:56:32.417 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: LV4 2A2\n",
      "2020-09-21 14:56:32.419 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: AiV2A\n",
      "2020-09-21 14:56:35.624 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: C-terminal domain of MAdV2 fibre\n",
      "2020-09-21 14:56:35.625 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: GAdV4 fibre 2 head domain\n",
      "2020-09-21 14:56:36.404 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: AHG59376.1|\n",
      "2020-09-21 14:56:40.918 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: LAdV2 fibre 2 head domain\n",
      "2020-09-21 14:56:41.719 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: AAEL003864\n",
      "2020-09-21 14:56:46.676 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Bd0886\n",
      "2020-09-21 14:56:46.678 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: ANL-KT-U1\n",
      "2020-09-21 14:56:46.679 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: ANL-KT-U2\n",
      "2020-09-21 14:56:46.681 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Bd0412\n",
      "2020-09-21 14:56:47.434 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Bd3099\n",
      "2020-09-21 14:56:52.731 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Bd1483\n",
      "2020-09-21 14:56:52.732 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Fructose biphosphatase\n",
      "2020-09-21 14:56:52.734 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: PorM_NB-SU1\n",
      "2020-09-21 14:56:53.733 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: PKG I\n",
      "2020-09-21 14:56:56.970 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: F4ZCI3\n",
      "2020-09-21 14:56:59.239 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Red Sea protein\n",
      "2020-09-21 14:57:03.290 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: CttA3\n",
      "2020-09-21 14:57:11.199 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Bd0886\n",
      "2020-09-21 14:57:11.203 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Q6MH56_BDEBA\n",
      "2020-09-21 14:57:14.644 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: C-terminal part of Gp20\n",
      "2020-09-21 14:57:14.645 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Bd0553\n",
      "2020-09-21 14:57:20.155 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: B7JAQ5_ACIF2\n",
      "2020-09-21 14:57:29.975 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Endolysin ARW58837.1\n",
      "2020-09-21 14:57:34.358 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: CBM56\n",
      "2020-09-21 14:57:41.528 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Endolysin KPP12\n",
      "2020-09-21 14:57:46.109 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: EXO5\n",
      "2020-09-21 14:57:47.071 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: YP_004957431.1\n",
      "2020-09-21 14:57:47.074 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: DpdA, WP_001542917.1\n",
      "2020-09-21 14:57:47.075 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: A0A0G1XU35_9BACT\n",
      "2020-09-21 14:57:49.152 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: gp146\n",
      "2020-09-21 14:57:49.156 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: DynU16\n",
      "2020-09-21 14:57:49.157 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Cals10\n",
      "2020-09-21 14:57:50.179 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: ACL_1061\n",
      "2020-09-21 14:57:52.810 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Q6XQB2.1\n",
      "2020-09-21 14:57:54.303 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Q6MKZ7\n",
      "2020-09-21 14:57:56.965 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: UniProtKB - B3GNT7 (B3GNT7_BACPU)\n",
      "2020-09-21 14:57:58.265 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Q6MN59\n",
      "2020-09-21 14:58:02.583 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Q6MIM9\n",
      "2020-09-21 14:58:02.585 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: Q184J8\n",
      "2020-09-21 14:58:03.753 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: YP_009041344.1\n",
      "2020-09-21 14:58:10.535 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: UNK2\n",
      "2020-09-21 14:58:11.494 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: CDI_213\n",
      "2020-09-21 14:58:11.495 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: CDI_213\n",
      "2020-09-21 14:58:12.644 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: 201_INDD4\n",
      "2020-09-21 14:58:12.645 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: 201_INDD4\n",
      "2020-09-21 14:58:13.754 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: CDI207t\n",
      "2020-09-21 14:58:13.756 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: CDI207t\n",
      "2020-09-21 14:58:13.757 | WARNING  | __main__:extract_targets:34 - Could not parse PDB code: SLAC1\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Targets</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Edition</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Targets\n",
       "Edition         \n",
       "9            116\n",
       "10            92\n",
       "11            82\n",
       "12            51\n",
       "13            53"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Match names like T0759s1\n",
    "regex_name = re.compile(r'T\\d\\d\\d\\d(?:s\\d)?')\n",
    "parser = Bio.PDB.PDBParser(QUIET=True)\n",
    "logger.disable('__main__:extract_targets')\n",
    "df_natives_pdb = []\n",
    "\n",
    "def extract_targets(soup):\n",
    "    for target_row in soup.select('tr.datarow'):\n",
    "        target_cols = target_row.select('td')\n",
    "        \n",
    "        try:\n",
    "            casp_id = target_cols[1].select_one('a').text.strip()\n",
    "            if not regex_name.match(casp_id):\n",
    "                continue\n",
    "        except Exception:\n",
    "            msg = re.sub('\\s+', ' ', target_cols[1].text).strip()\n",
    "            logger.exception('Could not parse target id: ' + msg)\n",
    "            continue\n",
    "            \n",
    "        try:\n",
    "            length = int(target_cols[3].text.strip())\n",
    "        except Exception:\n",
    "            msg = re.sub('\\s+', ' ', target_cols[3].text).strip()\n",
    "            logger.exception('Could not parse length: ' + msg)\n",
    "            continue\n",
    "                \n",
    "        try:\n",
    "            txt = target_cols[-1].text.lower()\n",
    "            if 'no structure' in txt or 'canceled' in txt:\n",
    "                continue            \n",
    "            pdb_code = target_cols[-1].select_one('a').text.strip()\n",
    "        except Exception:\n",
    "            msg = msg = re.sub('\\s+', ' ', target_cols[-1].text).strip()\n",
    "            logger.warning('Could not parse PDB code: ' + msg)\n",
    "            continue\n",
    "                        \n",
    "        yield {\n",
    "            'target_id': casp_id,\n",
    "            'length': length,\n",
    "            'pdb_id': pdb_code,\n",
    "        }\n",
    "        \n",
    "\n",
    "def download_native(target_dict, dest):\n",
    "    if not dest.is_file():\n",
    "        response = requests.get(f'https://files.rcsb.org/download/{target_dict[\"pdb_id\"]}.pdb')\n",
    "        with dest.open('w') as f:\n",
    "            f.write(response.text)\n",
    "\n",
    "for casp_ed in [9,10,11,12,13]:\n",
    "    with logger.contextualize(casp=casp_ed):\n",
    "        dest = Path(f'CASP{casp_ed}') / 'native_pdb'\n",
    "        dest.mkdir(exist_ok=True, parents=True)\n",
    "\n",
    "        response = requests_get(f'https://predictioncenter.org/casp{casp_ed}/targetlist.cgi?view_targets=all')\n",
    "        soup = bs4.BeautifulSoup(response.content)\n",
    "\n",
    "        for target_dict in extract_targets(soup):\n",
    "            dest_path = dest / f'{target_dict[\"target_id\"]}.pdb'\n",
    "            download_native(target_dict, dest_path)\n",
    "            structure = parser.get_structure(target_dict[\"target_id\"], dest_path)\n",
    "            df_natives_pdb.append({\n",
    "                'casp_ed': casp_ed,\n",
    "                'chains': len(list(structure.get_chains())),\n",
    "                'residues': len(list(structure.get_residues())),\n",
    "                'atoms': len(list(structure.get_atoms())),\n",
    "                **target_dict\n",
    "            })\n",
    "        \n",
    "df_natives_pdb = pd.DataFrame(df_natives_pdb)\n",
    "df_natives_pdb.to_csv('natives_pdb.csv', index=False)\n",
    "df_natives_pdb.groupby('casp_ed').size().rename_axis('Edition').to_frame('Targets')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Difference between:\n",
    "- **left:**  primary sequences published on the CASP website\n",
    "- **right:** native structures downloaded from PDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Outer join counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>both</th>\n",
       "      <td>399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>left_only</th>\n",
       "      <td>130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>right_only</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Outer join counts\n",
       "both                      399\n",
       "left_only                 130\n",
       "right_only                  0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>casp_ed</th>\n",
       "      <th>target_id</th>\n",
       "      <th>target_id_orig</th>\n",
       "      <th>length</th>\n",
       "      <th>residues</th>\n",
       "      <th>pdb_id</th>\n",
       "      <th>_merge</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9</td>\n",
       "      <td>T0519</td>\n",
       "      <td>T0519</td>\n",
       "      <td>180</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>9</td>\n",
       "      <td>T0546</td>\n",
       "      <td>T0546</td>\n",
       "      <td>134</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>9</td>\n",
       "      <td>T0549</td>\n",
       "      <td>T0549</td>\n",
       "      <td>84</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>9</td>\n",
       "      <td>T0554</td>\n",
       "      <td>T0554</td>\n",
       "      <td>135</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>9</td>\n",
       "      <td>T0556</td>\n",
       "      <td>T0556</td>\n",
       "      <td>73</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>515</th>\n",
       "      <td>13</td>\n",
       "      <td>T1004</td>\n",
       "      <td>T1004</td>\n",
       "      <td>458</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>518</th>\n",
       "      <td>13</td>\n",
       "      <td>T1007</td>\n",
       "      <td>T1007</td>\n",
       "      <td>149</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>523</th>\n",
       "      <td>13</td>\n",
       "      <td>T1012</td>\n",
       "      <td>T1012</td>\n",
       "      <td>199</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524</th>\n",
       "      <td>13</td>\n",
       "      <td>T1013</td>\n",
       "      <td>T1013</td>\n",
       "      <td>537</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>528</th>\n",
       "      <td>13</td>\n",
       "      <td>T1020</td>\n",
       "      <td>T1020</td>\n",
       "      <td>577</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>130 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     casp_ed target_id target_id_orig  length  residues pdb_id     _merge\n",
       "4          9     T0519          T0519     180       NaN    NaN  left_only\n",
       "31         9     T0546          T0546     134       NaN    NaN  left_only\n",
       "34         9     T0549          T0549      84       NaN    NaN  left_only\n",
       "39         9     T0554          T0554     135       NaN    NaN  left_only\n",
       "41         9     T0556          T0556      73       NaN    NaN  left_only\n",
       "..       ...       ...            ...     ...       ...    ...        ...\n",
       "515       13     T1004          T1004     458       NaN    NaN  left_only\n",
       "518       13     T1007          T1007     149       NaN    NaN  left_only\n",
       "523       13     T1012          T1012     199       NaN    NaN  left_only\n",
       "524       13     T1013          T1013     537       NaN    NaN  left_only\n",
       "528       13     T1020          T1020     577       NaN    NaN  left_only\n",
       "\n",
       "[130 rows x 7 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Same length?</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>False</th>\n",
       "      <td>528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Same length?\n",
       "False           528\n",
       "True              1"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>casp_ed</th>\n",
       "      <th>target_id</th>\n",
       "      <th>target_id_orig</th>\n",
       "      <th>length</th>\n",
       "      <th>residues</th>\n",
       "      <th>pdb_id</th>\n",
       "      <th>_merge</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9</td>\n",
       "      <td>T0515</td>\n",
       "      <td>T0515</td>\n",
       "      <td>365</td>\n",
       "      <td>757.0</td>\n",
       "      <td>3mt1</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>T0516</td>\n",
       "      <td>T0516</td>\n",
       "      <td>229</td>\n",
       "      <td>1715.0</td>\n",
       "      <td>3no6</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>T0517</td>\n",
       "      <td>T0517</td>\n",
       "      <td>159</td>\n",
       "      <td>1149.0</td>\n",
       "      <td>3pnx</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>T0518</td>\n",
       "      <td>T0518</td>\n",
       "      <td>288</td>\n",
       "      <td>464.0</td>\n",
       "      <td>3nmb</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9</td>\n",
       "      <td>T0519</td>\n",
       "      <td>T0519</td>\n",
       "      <td>180</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524</th>\n",
       "      <td>13</td>\n",
       "      <td>T1013</td>\n",
       "      <td>T1013</td>\n",
       "      <td>537</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>525</th>\n",
       "      <td>13</td>\n",
       "      <td>T1014</td>\n",
       "      <td>T1014</td>\n",
       "      <td>276</td>\n",
       "      <td>476.0</td>\n",
       "      <td>6qrj</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>526</th>\n",
       "      <td>13</td>\n",
       "      <td>T1016</td>\n",
       "      <td>T1016</td>\n",
       "      <td>203</td>\n",
       "      <td>882.0</td>\n",
       "      <td>6e4b</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>527</th>\n",
       "      <td>13</td>\n",
       "      <td>T1018</td>\n",
       "      <td>T1018</td>\n",
       "      <td>334</td>\n",
       "      <td>1069.0</td>\n",
       "      <td>6n91</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>528</th>\n",
       "      <td>13</td>\n",
       "      <td>T1020</td>\n",
       "      <td>T1020</td>\n",
       "      <td>577</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>528 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     casp_ed target_id target_id_orig  length  residues pdb_id     _merge\n",
       "0          9     T0515          T0515     365     757.0   3mt1       both\n",
       "1          9     T0516          T0516     229    1715.0   3no6       both\n",
       "2          9     T0517          T0517     159    1149.0   3pnx       both\n",
       "3          9     T0518          T0518     288     464.0   3nmb       both\n",
       "4          9     T0519          T0519     180       NaN    NaN  left_only\n",
       "..       ...       ...            ...     ...       ...    ...        ...\n",
       "524       13     T1013          T1013     537       NaN    NaN  left_only\n",
       "525       13     T1014          T1014     276     476.0   6qrj       both\n",
       "526       13     T1016          T1016     203     882.0   6e4b       both\n",
       "527       13     T1018          T1018     334    1069.0   6n91       both\n",
       "528       13     T1020          T1020     577       NaN    NaN  left_only\n",
       "\n",
       "[528 rows x 7 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_merge = pd.merge(\n",
    "    df_sequences, \n",
    "    df_natives_pdb.drop(columns=['chains', 'atoms','length']),\n",
    "    suffixes=['_fasta', '_casp'],\n",
    "    on=['casp_ed', 'target_id'],\n",
    "    how='outer', indicator=True\n",
    ")\n",
    "\n",
    "display(\n",
    "    df_merge['_merge']\n",
    "    .value_counts()\n",
    "    .to_frame('Outer join counts')\n",
    ")\n",
    "display(df_merge.query(\"_merge!='both'\"))\n",
    "\n",
    "display(\n",
    "    (df_merge['length'] == df_merge['residues'])\n",
    "    .value_counts()\n",
    "    .to_frame('Same length?')\n",
    ")\n",
    "display(df_merge.query('length != residues'))\n",
    "\n",
    "del df_merge"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Server predictions\n",
    "\n",
    "These are the tertiary structures as predicted from the servers participating in CASP\n",
    "([submission file format](https://predictioncenter.org/casp13/index.cgi?page=format#TS)).\n",
    "\n",
    "The submission happens in two stages, the same target might get different names in the two stages.\n",
    "\n",
    "Each server can submit up to 5 models for each target, as indicated by the field `MODEL`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-09-21 15:17:38.627 | ERROR    | __main__:<module>:68 - Could not find stage 1 decoy T0649/server20_TS1 among all decoys of CASP10/T0649\n",
      "2020-09-21 15:18:13.970 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0653/Distill_TS1 among all decoys of CASP10/T0653\n",
      "2020-09-21 15:18:13.974 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0653/Distill_TS2 among all decoys of CASP10/T0653\n",
      "2020-09-21 15:18:13.977 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0653/Distill_TS3 among all decoys of CASP10/T0653\n",
      "2020-09-21 15:18:13.980 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0653/Distill_TS4 among all decoys of CASP10/T0653\n",
      "2020-09-21 15:18:13.983 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0653/Distill_TS5 among all decoys of CASP10/T0653\n",
      "2020-09-21 15:18:13.986 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0653/Distill_roll_TS1 among all decoys of CASP10/T0653\n",
      "2020-09-21 15:18:13.989 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0653/Distill_roll_TS2 among all decoys of CASP10/T0653\n",
      "2020-09-21 15:18:13.992 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0653/Distill_roll_TS3 among all decoys of CASP10/T0653\n",
      "2020-09-21 15:18:13.995 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0653/Distill_roll_TS4 among all decoys of CASP10/T0653\n",
      "2020-09-21 15:18:13.998 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0653/Distill_roll_TS5 among all decoys of CASP10/T0653\n",
      "2020-09-21 15:18:14.119 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0653/NewSerf_TS1 among all decoys of CASP10/T0653\n",
      "2020-09-21 15:18:20.867 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0654/Distill_TS1 among all decoys of CASP10/T0654\n",
      "2020-09-21 15:18:20.871 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0654/Distill_TS2 among all decoys of CASP10/T0654\n",
      "2020-09-21 15:18:20.874 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0654/Distill_TS3 among all decoys of CASP10/T0654\n",
      "2020-09-21 15:18:20.877 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0654/Distill_TS4 among all decoys of CASP10/T0654\n",
      "2020-09-21 15:18:20.880 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0654/Distill_TS5 among all decoys of CASP10/T0654\n",
      "2020-09-21 15:18:20.883 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0654/Distill_roll_TS1 among all decoys of CASP10/T0654\n",
      "2020-09-21 15:18:20.886 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0654/Distill_roll_TS2 among all decoys of CASP10/T0654\n",
      "2020-09-21 15:18:20.889 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0654/Distill_roll_TS3 among all decoys of CASP10/T0654\n",
      "2020-09-21 15:18:20.891 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0654/Distill_roll_TS4 among all decoys of CASP10/T0654\n",
      "2020-09-21 15:18:20.894 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0654/Distill_roll_TS5 among all decoys of CASP10/T0654\n",
      "2020-09-21 15:18:20.899 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0654/FFAS03c_TS1 among all decoys of CASP10/T0654\n",
      "2020-09-21 15:18:20.906 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0654/FFAS03mt_TS1 among all decoys of CASP10/T0654\n",
      "2020-09-21 15:18:20.969 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0654/NewSerf_TS1 among all decoys of CASP10/T0654\n",
      "2020-09-21 15:32:20.811 | WARNING  | __main__:<module>:54 - https://predictioncenter.org/download_area/CASP11/server_predictions/T0778.stage2.3D.srv.tar.gz 404\n",
      "2020-09-21 15:33:46.846 | WARNING  | __main__:<module>:54 - https://predictioncenter.org/download_area/CASP11/server_predictions/T0791.stage1.3D.srv.tar.gz 404\n",
      "2020-09-21 15:33:47.614 | WARNING  | __main__:<module>:54 - https://predictioncenter.org/download_area/CASP11/server_predictions/T0791.stage2.3D.srv.tar.gz 404\n",
      "2020-09-21 15:46:25.939 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0957s1/AWSEM-Suite_TS1 among all decoys of CASP13/T0957s1\n",
      "2020-09-21 15:46:25.943 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0957s1/AWSEM-Suite_TS2 among all decoys of CASP13/T0957s1\n",
      "2020-09-21 15:46:25.945 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0957s1/AWSEM-Suite_TS3 among all decoys of CASP13/T0957s1\n",
      "2020-09-21 15:46:25.947 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0957s1/AWSEM-Suite_TS4 among all decoys of CASP13/T0957s1\n",
      "2020-09-21 15:46:25.948 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0957s1/AWSEM-Suite_TS5 among all decoys of CASP13/T0957s1\n",
      "2020-09-21 15:46:32.675 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0957s2/AWSEM-Suite_TS1 among all decoys of CASP13/T0957s2\n",
      "2020-09-21 15:46:32.677 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0957s2/AWSEM-Suite_TS2 among all decoys of CASP13/T0957s2\n",
      "2020-09-21 15:46:32.679 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0957s2/AWSEM-Suite_TS3 among all decoys of CASP13/T0957s2\n",
      "2020-09-21 15:46:32.681 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0957s2/AWSEM-Suite_TS4 among all decoys of CASP13/T0957s2\n",
      "2020-09-21 15:46:32.683 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0957s2/AWSEM-Suite_TS5 among all decoys of CASP13/T0957s2\n",
      "2020-09-21 15:46:38.725 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0958/AWSEM-Suite_TS1 among all decoys of CASP13/T0958\n",
      "2020-09-21 15:46:38.728 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0958/AWSEM-Suite_TS2 among all decoys of CASP13/T0958\n",
      "2020-09-21 15:46:38.730 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0958/AWSEM-Suite_TS3 among all decoys of CASP13/T0958\n",
      "2020-09-21 15:46:38.733 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0958/AWSEM-Suite_TS4 among all decoys of CASP13/T0958\n",
      "2020-09-21 15:46:38.736 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0958/AWSEM-Suite_TS5 among all decoys of CASP13/T0958\n",
      "2020-09-21 15:46:45.572 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0959/AWSEM-Suite_TS1 among all decoys of CASP13/T0959\n",
      "2020-09-21 15:46:45.574 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0959/AWSEM-Suite_TS2 among all decoys of CASP13/T0959\n",
      "2020-09-21 15:46:45.576 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0959/AWSEM-Suite_TS3 among all decoys of CASP13/T0959\n",
      "2020-09-21 15:46:45.578 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0959/AWSEM-Suite_TS4 among all decoys of CASP13/T0959\n",
      "2020-09-21 15:46:45.580 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0959/AWSEM-Suite_TS5 among all decoys of CASP13/T0959\n",
      "2020-09-21 15:46:53.761 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0960/AWSEM-Suite_TS1 among all decoys of CASP13/T0960\n",
      "2020-09-21 15:46:53.766 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0960/AWSEM-Suite_TS2 among all decoys of CASP13/T0960\n",
      "2020-09-21 15:46:53.770 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0960/AWSEM-Suite_TS3 among all decoys of CASP13/T0960\n",
      "2020-09-21 15:46:53.773 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0960/AWSEM-Suite_TS4 among all decoys of CASP13/T0960\n",
      "2020-09-21 15:46:53.777 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0960/AWSEM-Suite_TS5 among all decoys of CASP13/T0960\n",
      "2020-09-21 15:47:18.268 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0963/AWSEM-Suite_TS1 among all decoys of CASP13/T0963\n",
      "2020-09-21 15:47:18.272 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0963/AWSEM-Suite_TS2 among all decoys of CASP13/T0963\n",
      "2020-09-21 15:47:18.275 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0963/AWSEM-Suite_TS3 among all decoys of CASP13/T0963\n",
      "2020-09-21 15:47:18.277 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0963/AWSEM-Suite_TS4 among all decoys of CASP13/T0963\n",
      "2020-09-21 15:47:18.280 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0963/AWSEM-Suite_TS5 among all decoys of CASP13/T0963\n",
      "2020-09-21 15:48:09.360 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0969/QUARK_TS1 among all decoys of CASP13/T0969\n",
      "2020-09-21 15:48:09.366 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0969/QUARK_TS3 among all decoys of CASP13/T0969\n",
      "2020-09-21 15:48:09.376 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0969/QUARK_TS2 among all decoys of CASP13/T0969\n",
      "2020-09-21 15:48:09.393 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0969/Zhang-Server_TS5 among all decoys of CASP13/T0969\n",
      "2020-09-21 15:48:09.398 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0969/QUARK_TS5 among all decoys of CASP13/T0969\n",
      "2020-09-21 15:48:09.408 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0969/Zhang-Server_TS2 among all decoys of CASP13/T0969\n",
      "2020-09-21 15:48:09.480 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0969/Zhang-Server_TS3 among all decoys of CASP13/T0969\n",
      "2020-09-21 15:48:09.642 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0969/QUARK_TS4 among all decoys of CASP13/T0969\n",
      "2020-09-21 15:48:09.648 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0969/README among all decoys of CASP13/T0969\n",
      "2020-09-21 15:48:09.652 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0969/Zhang-Server_TS4 among all decoys of CASP13/T0969\n",
      "2020-09-21 15:48:09.657 | ERROR    | __main__:<module>:68 - Could not find stage 2 decoy T0969/Zhang-Server_TS1 among all decoys of CASP13/T0969\n"
     ]
    }
   ],
   "source": [
    "@contextlib.contextmanager\n",
    "def read_archive(response):\n",
    "    with io.BytesIO(response.content) as fileobj:\n",
    "        with tarfile.open(fileobj=fileobj, mode='r') as archive:\n",
    "            yield archive\n",
    "\n",
    "df_decoys = {}\n",
    "for target in df_natives.itertuples():\n",
    "    dest_dir = Path(f'CASP{target.casp_ed}/decoys/{target.target_id}')    \n",
    "    dest_dir.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "    # Download all decoys for this target, compute md5sum and save them to disk\n",
    "    response = requests_get(\n",
    "        f'https://predictioncenter.org/download_area/'\n",
    "        f'CASP{target.casp_ed}/server_predictions/'\n",
    "        f'{target.target_id}.3D.srv.tar.gz'\n",
    "    )\n",
    "    if response.status_code != 200:\n",
    "        logger.warning(f'{response.url} {response.status_code}')\n",
    "        continue\n",
    "    with read_archive(response) as archive:\n",
    "        for member in archive.getmembers():\n",
    "            if not member.isfile():\n",
    "                continue\n",
    "            if Path(member.name).is_absolute():\n",
    "                logger.warning(f'Invalid path in tarfile: {response.url} {member.name}')\n",
    "                continue\n",
    "\n",
    "            decoy_id = Path(member.name).with_suffix('').name\n",
    "            with archive.extractfile(member) as fileobj:\n",
    "                content = fileobj.read()\n",
    "            with dest_dir.joinpath(decoy_id).with_suffix('.pdb').open('wb') as out:\n",
    "                out.write(content)\n",
    "            df_decoys[hashlib.md5(content).digest()] = {\n",
    "                'casp_ed': target.casp_ed,\n",
    "                'target_id': target.target_id,\n",
    "                'decoy_id': decoy_id,\n",
    "                'stage1': False,\n",
    "                'stage2': False,\n",
    "            }\n",
    "\n",
    "    if target.casp_ed == 9:\n",
    "        continue\n",
    "\n",
    "    # Download all stage1 and stage2 decoys for this target,\n",
    "    # don't save to disk, just compute md5sum and update the dict\n",
    "    for s in ['stage1', 'stage2']:\n",
    "        response = requests_get(\n",
    "            f'https://predictioncenter.org/download_area/'\n",
    "            f'CASP{target.casp_ed}/server_predictions/'\n",
    "            f'{target.target_id}.{s}.3D.srv.tar.gz'\n",
    "        )\n",
    "        if response.status_code != 200:\n",
    "            logger.warning(f'{response.url} {response.status_code}')\n",
    "            continue\n",
    "        with read_archive(response) as archive:\n",
    "            for member in archive.getmembers():\n",
    "                if not member.isfile():\n",
    "                    continue\n",
    "                if Path(member.name).is_absolute():\n",
    "                    logger.warning(f'Invalid path in tarfile: {response.url} {member.name}')\n",
    "                    continue\n",
    "                with archive.extractfile(member) as fileobj:\n",
    "                    content = fileobj.read()\n",
    "                try:\n",
    "                    df_decoys[hashlib.md5(content).digest()][s] = True\n",
    "                except KeyError:\n",
    "                    logger.error(f'Could not find stage {s[-1]} decoy {member.name} '\n",
    "                                 f'among all decoys of CASP{target.casp_ed}/{target.target_id}')\n",
    "    \n",
    "df_decoys = pd.DataFrame(df_decoys.values())\n",
    "df_decoys.to_csv('decoys.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.7G\tCASP10/decoys\n",
      "2.5G\tCASP11/decoys\n",
      "1.2G\tCASP12/decoys\n",
      "2.6G\tCASP13/decoys\n",
      "4.8G\tCASP9/decoys\n",
      "15G\ttotal\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unique targets</th>\n",
       "      <th>Total decoys</th>\n",
       "      <th>Decoys in stage 1</th>\n",
       "      <th>Decoys in stage 2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Edition</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>117</td>\n",
       "      <td>33737</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>103</td>\n",
       "      <td>24526</td>\n",
       "      <td>2050</td>\n",
       "      <td>14551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>85</td>\n",
       "      <td>16012</td>\n",
       "      <td>1680</td>\n",
       "      <td>12448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>40</td>\n",
       "      <td>6689</td>\n",
       "      <td>800</td>\n",
       "      <td>5791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>82</td>\n",
       "      <td>14621</td>\n",
       "      <td>1640</td>\n",
       "      <td>12139</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Unique targets  Total decoys  Decoys in stage 1  Decoys in stage 2\n",
       "Edition                                                                    \n",
       "9                   117         33737                  0                  0\n",
       "10                  103         24526               2050              14551\n",
       "11                   85         16012               1680              12448\n",
       "12                   40          6689                800               5791\n",
       "13                   82         14621               1640              12139"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Stage 1</th>\n",
       "      <th>Stage 2</th>\n",
       "      <th>Total</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Edition</th>\n",
       "      <th>Target</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">9</th>\n",
       "      <th>T0515</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0516</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0517</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0518</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0520</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">13</th>\n",
       "      <th>T1021s1</th>\n",
       "      <td>20</td>\n",
       "      <td>148</td>\n",
       "      <td>183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1021s2</th>\n",
       "      <td>20</td>\n",
       "      <td>148</td>\n",
       "      <td>174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1021s3</th>\n",
       "      <td>20</td>\n",
       "      <td>149</td>\n",
       "      <td>179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1022s1</th>\n",
       "      <td>20</td>\n",
       "      <td>149</td>\n",
       "      <td>177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1022s2</th>\n",
       "      <td>20</td>\n",
       "      <td>149</td>\n",
       "      <td>184</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>427 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Stage 1  Stage 2  Total\n",
       "Edition Target                          \n",
       "9       T0515          0        0    286\n",
       "        T0516          0        0    303\n",
       "        T0517          0        0    280\n",
       "        T0518          0        0    279\n",
       "        T0520          0        0    305\n",
       "...                  ...      ...    ...\n",
       "13      T1021s1       20      148    183\n",
       "        T1021s2       20      148    174\n",
       "        T1021s3       20      149    179\n",
       "        T1022s1       20      149    177\n",
       "        T1022s2       20      149    184\n",
       "\n",
       "[427 rows x 3 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "! du -shc CASP*/decoys\n",
    "\n",
    "df_decoys = pd.read_csv('decoys.csv')\n",
    "display(\n",
    "    df_decoys.groupby('casp_ed')\n",
    "    .agg({'target_id': 'nunique', 'decoy_id': 'size', 'stage1': 'sum', 'stage2': 'sum'})\n",
    "    .rename(columns={'target_id': 'Unique targets', 'decoy_id': 'Total decoys', \n",
    "                     'stage1': 'Decoys in stage 1', 'stage2': 'Decoys in stage 2'})\n",
    "    .rename_axis('Edition')\n",
    "    .astype(int)\n",
    ")\n",
    "display(\n",
    "    df_decoys.groupby(['casp_ed', 'target_id'])\n",
    "        .agg({'stage1': 'sum', 'stage2': 'sum', 'decoy_id': 'size'})\n",
    "        .rename(columns={'decoy_id': 'Total', 'stage1': 'Stage 1', 'stage2': 'Stage 2'})\n",
    "        .rename_axis(['Edition', 'Target'])\n",
    "        .astype(int)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualization\n",
    "\n",
    "Focus on `CASP11/T0759` and its decoy `T0759/3D-Jigsaw-V5_1_TS1` as an example.\n",
    "\n",
    "For `CASP11/T0759` we have the following decoys:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Stage 1</th>\n",
       "      <th>Stage 2</th>\n",
       "      <th>Total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>T0759</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "      <td>179</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Stage 1  Stage 2  Total\n",
       "T0759       20      150    179"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(\n",
    "    df_decoys.query('target_id == \"T0759\"')\n",
    "    .agg({'stage1': 'sum', 'stage2': 'sum', 'decoy_id': 'size'})\n",
    "    .rename({'decoy_id': 'Total', 'stage1': 'Stage 1', 'stage2': 'Stage 2'})\n",
    "    .to_frame('T0759')\n",
    "    .transpose()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Draw the coordinates of a residue's carbon alpha and direction `CA->CB`.\n",
    "If the residue is `GLY`, the atom `CB` is \"virtual\".\n",
    "\n",
    "<img src=\"https://vignette.wikia.nocookie.net/foldit/images/8/86/Backbone_overview.stickpolarh.png/revision/latest?cb=20180101214816\" style=\"background-color:white;width:15%;\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Native structure:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<video src=\"T0759.mp4\" controls  >\n",
       "      Your browser does not support the <code>video</code> element.\n",
       "    </video>"
      ],
      "text/plain": [
       "<IPython.core.display.Video object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "parser = Bio.PDB.PDBParser(QUIET=True)\n",
    "model = parser.get_structure('T0759', 'CASP11/native/T0759.pdb')[0]\n",
    "\n",
    "fig = plt.figure(figsize=(12, 6), facecolor='white', tight_layout=True)\n",
    "ax = fig.add_subplot(1, 1, 1, projection='3d')\n",
    "\n",
    "for chain_idx, chain in enumerate(model):\n",
    "    aa = [r.get_resname() for r in chain if Bio.PDB.is_aa(r)]\n",
    "    ca, orient = zip(*(ca_coord_and_orientation(r) for r in chain if Bio.PDB.is_aa(r)))\n",
    "    ca = np.stack(ca, axis=0)    \n",
    "    ax.plot(*ca.T, color=plt.get_cmap('tab10')(chain_idx))\n",
    "    for ca, aa, orient in zip(ca, aa, orient):\n",
    "        ax.plot(*zip(ca.T, (ca+orient).T), linewidth=3, color=plt.get_cmap('tab20')(aa_3_mapping[aa]))\n",
    "\n",
    "animate = lambda i: ax.view_init(30, i)\n",
    "ani = matplotlib.animation.FuncAnimation(fig, animate, frames=10 + 20 * np.sin(np.linspace(0, np.pi, num=100)), interval=50)\n",
    "ani.save('T0759.mp4')\n",
    "display(Video('T0759.mp4', embed=False))\n",
    "plt.close(fig)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Decoy structure:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PFRMAT TS\n",
      "TARGET T0759\n",
      "MODEL  1\n",
      "PARENT N/A\n",
      "ATOM      1  N   HIS    10      -0.109  -0.003   0.008  1.00  0.00           N  \n",
      "ATOM      2  CA  HIS    10       0.018   0.008   1.433  1.00  0.00           C  \n",
      "ATOM      3  C   HIS    10       0.057   1.463   1.816  1.00  0.00           C  \n",
      "ATOM      4  O   HIS    10       0.617   2.274   1.082  1.00  0.00           O  \n",
      "ATOM      5  CB  HIS    10       1.267  -0.762   1.866  1.00  0.00           C  \n",
      "\n",
      "Model ('T0759/3D-Jigsaw-V5_1_TS1', 0) (1 chains)\n",
      "  Chain ('T0759/3D-Jigsaw-V5_1_TS1', 0, ' ') (100 residues)\n",
      "    10 HIS  10 atoms\n",
      "    11 MET   8 atoms\n",
      "    12 VAL   7 atoms\n",
      "    13 VAL   7 atoms\n",
      "    14 ILE   8 atoms\n",
      "   ...\n",
      "   105 VAL   7 atoms\n",
      "   106 SER   6 atoms\n",
      "   107 GLY   4 atoms\n",
      "   108 GLN   9 atoms\n",
      "   109 LYS   9 atoms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<video src=\"T0759_3D-Jigsaw-V5_1_TS1.mp4\" controls  >\n",
       "      Your browser does not support the <code>video</code> element.\n",
       "    </video>"
      ],
      "text/plain": [
       "<IPython.core.display.Video object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "regex_target = re.compile(r'TARGET\\s+(T\\d\\d\\d\\d(:?s\\d+)?)')\n",
    "regex_model = re.compile(r'MODEL\\s+(\\d+)')\n",
    "\n",
    "with open('CASP11/decoys/T0759/3D-Jigsaw-V5_1_TS1.pdb') as f:\n",
    "    # Print the first lines of the pdb file\n",
    "    print(*f.readlines()[:9], sep='')\n",
    "\n",
    "    # The parser does not recognize TARGET and MODEL fields automatically\n",
    "    f.seek(0)\n",
    "    f.readline()\n",
    "    target_id = regex_target.match(f.readline()).groups()[0]\n",
    "    model_id = regex_model.match(f.readline()).groups()[0]\n",
    "\n",
    "    # Parse the structure\n",
    "    f.seek(0)\n",
    "    structure = parser.get_structure(f'T0759/3D-Jigsaw-V5_1_TS1', f)\n",
    "\n",
    "\n",
    "# Print parsed structure\n",
    "for model in structure:\n",
    "    print(f'Model {model.get_full_id()} ({len(model)} chains)')\n",
    "    for chain in model:\n",
    "        print(f'  Chain {chain.get_full_id()} ({len(chain)} residues)')\n",
    "        chain = list(chain)\n",
    "        for residue in chain[:5]:\n",
    "            print(f'   {residue.get_id()[1]:>3} {residue.get_resname()}  {len(residue):>2} atoms')\n",
    "        print('   ...')\n",
    "        for residue in chain[-5:]:\n",
    "            print(f'   {residue.get_id()[1]:>3} {residue.get_resname()}  {len(residue):>2} atoms')\n",
    "            \n",
    "\n",
    "# Show animation\n",
    "fig = plt.figure(figsize=(12, 6), facecolor='white', tight_layout=True)\n",
    "ax = fig.add_subplot(1, 1, 1, projection='3d')\n",
    "\n",
    "model = structure[0]\n",
    "for chain_idx, chain in enumerate(model):\n",
    "    aa = [r.get_resname() for r in chain]\n",
    "    ca, orient = zip(*(ca_coord_and_orientation(r) for r in chain))\n",
    "    ca = np.stack(ca, axis=0)    \n",
    "    ax.plot(*ca.T, c='gray')\n",
    "    for ca, aa, orient in zip(ca, aa, orient):\n",
    "        ax.plot(*zip(ca.T, (ca+orient).T), linewidth=3, color=plt.get_cmap('tab20')(aa_3_mapping[aa]))\n",
    "\n",
    "animate = lambda i: ax.view_init(30, i)\n",
    "ani = matplotlib.animation.FuncAnimation(fig, animate, frames=10 + 20 * np.sin(np.linspace(0, np.pi, num=100)), interval=50)\n",
    "ani.save('T0759_3D-Jigsaw-V5_1_TS1.mp4')\n",
    "display(Video('T0759_3D-Jigsaw-V5_1_TS1.mp4', embed=False))\n",
    "plt.close(fig)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "incorrectly_encoded_metadata": "toc-hr-collapsed=true toc-nb-collapsed=true"
   },
   "source": [
    "## Official QA global scores (summary tables)\n",
    "\n",
    "Summary tables contain official QA scores computed by CASP by comparison with the native structure for all tertiary structure predictions from all participants.\n",
    "QA metrics are only computed for global scores.\n",
    "\n",
    "A unique model `ACCESSION CODE` is composed from the number of the target, prediction format category, prediction group number, and model index. \n",
    "Example:\n",
    "```\n",
    "Accession code  T0444TS005_2  has the following components:\n",
    " T0044   target number\n",
    " TS      Tertiary Structure (PFRMAT TS)\n",
    " 005     prediction group 5\n",
    " 2       model index 2 \n",
    "```\n",
    "\n",
    "Summary tables are a bit different in each CASP edition.\n",
    "\n",
    "For each edition, the table is saved as `CASP*/QA_official/table.pkl.xz`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Match files like T0759s1.txt but not T0759-D1.txt\n",
    "regex = re.compile(r'T\\d\\d\\d\\d(?:s\\d)?.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CASP 9\n",
    "Single table with all targets and decoys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique targets: 14\n",
      "Unique groups: 34\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>GDT_HA</th>\n",
       "      <th>RMSD</th>\n",
       "      <th>GDT_SC</th>\n",
       "      <th>SphGr</th>\n",
       "      <th>MolProb</th>\n",
       "      <th>Exclude(missing too many atoms)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Target</th>\n",
       "      <th>Group</th>\n",
       "      <th>Model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0517</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <td>48.43</td>\n",
       "      <td>6.725</td>\n",
       "      <td>28.31</td>\n",
       "      <td>46.346</td>\n",
       "      <td>1.344</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>49.05</td>\n",
       "      <td>7.495</td>\n",
       "      <td>24.92</td>\n",
       "      <td>47.604</td>\n",
       "      <td>1.123</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>48.90</td>\n",
       "      <td>7.624</td>\n",
       "      <td>31.18</td>\n",
       "      <td>46.375</td>\n",
       "      <td>1.010</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>46.54</td>\n",
       "      <td>7.446</td>\n",
       "      <td>26.73</td>\n",
       "      <td>46.346</td>\n",
       "      <td>1.246</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>49.37</td>\n",
       "      <td>8.490</td>\n",
       "      <td>32.17</td>\n",
       "      <td>47.604</td>\n",
       "      <td>0.928</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0624</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">484</th>\n",
       "      <th>1</th>\n",
       "      <td>34.06</td>\n",
       "      <td>5.174</td>\n",
       "      <td>15.26</td>\n",
       "      <td>34.783</td>\n",
       "      <td>3.004</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>36.59</td>\n",
       "      <td>5.502</td>\n",
       "      <td>13.38</td>\n",
       "      <td>46.377</td>\n",
       "      <td>2.634</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>38.05</td>\n",
       "      <td>4.473</td>\n",
       "      <td>16.36</td>\n",
       "      <td>40.580</td>\n",
       "      <td>2.570</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>35.15</td>\n",
       "      <td>5.327</td>\n",
       "      <td>13.35</td>\n",
       "      <td>46.377</td>\n",
       "      <td>2.840</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>36.23</td>\n",
       "      <td>5.529</td>\n",
       "      <td>14.61</td>\n",
       "      <td>47.826</td>\n",
       "      <td>1.649</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1615 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    GDT_HA   RMSD  GDT_SC   SphGr  MolProb  \\\n",
       "Target Group Model                                           \n",
       "T0517  1     1       48.43  6.725   28.31  46.346    1.344   \n",
       "             2       49.05  7.495   24.92  47.604    1.123   \n",
       "             3       48.90  7.624   31.18  46.375    1.010   \n",
       "             4       46.54  7.446   26.73  46.346    1.246   \n",
       "             5       49.37  8.490   32.17  47.604    0.928   \n",
       "...                    ...    ...     ...     ...      ...   \n",
       "T0624  484   1       34.06  5.174   15.26  34.783    3.004   \n",
       "             2       36.59  5.502   13.38  46.377    2.634   \n",
       "             3       38.05  4.473   16.36  40.580    2.570   \n",
       "             4       35.15  5.327   13.35  46.377    2.840   \n",
       "             5       36.23  5.529   14.61  47.826    1.649   \n",
       "\n",
       "                    Exclude(missing too many atoms)  \n",
       "Target Group Model                                   \n",
       "T0517  1     1                                False  \n",
       "             2                                False  \n",
       "             3                                False  \n",
       "             4                                False  \n",
       "             5                                False  \n",
       "...                                             ...  \n",
       "T0624  484   1                                False  \n",
       "             2                                False  \n",
       "             3                                False  \n",
       "             4                                False  \n",
       "             5                                False  \n",
       "\n",
       "[1615 rows x 6 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'https://predictioncenter.org/download_area/CASP9/refinement_result_tables_assessor.txt'\n",
    "response = requests_get(url)\n",
    "\n",
    "df = pd.read_csv(io.BytesIO(response.content), sep='\\t')\n",
    "df.rename(columns={c: c.strip() for c in df.columns}, inplace=True)\n",
    "df.rename(columns={'GDT-HA': 'GDT_HA', 'GDC-SC': 'GDT_SC'}, inplace=True)\n",
    "df['Exclude(missing too many atoms)'] = (\n",
    "    df['Exclude(missing too many atoms)']\n",
    "    .str.strip()\n",
    "    .map({'True': True, 'False': False})\n",
    ")\n",
    "df['Target'] = df['Target'].map('T{:04d}'.format)\n",
    "df['Model'] = df['Model'].astype(str)\n",
    "df.set_index(['Target', 'Group', 'Model'], inplace=True)\n",
    "df.sort_index(inplace=True)\n",
    "\n",
    "Path('CASP9/QA_official').mkdir(exist_ok=True, parents=True)\n",
    "df.to_pickle('CASP9/QA_official/table.pkl.xz')\n",
    "\n",
    "print('Unique targets:', len(df.index.unique(level='Target')))\n",
    "print('Unique groups:', len(df.index.unique(level='Group')))\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CASP 10\n",
    "Single .tar.gz file with separate .txt files inside"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique targets: 96\n",
      "Unique groups: 150\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>#</th>\n",
       "      <th>GR#</th>\n",
       "      <th>GDT_TS</th>\n",
       "      <th>NP_P</th>\n",
       "      <th>RANK</th>\n",
       "      <th>Z-M1-GDT</th>\n",
       "      <th>Z-M1s-GDT</th>\n",
       "      <th>Z-MA-GDT</th>\n",
       "      <th>Z-MAs-GDT</th>\n",
       "      <th>GDT_HA</th>\n",
       "      <th>...</th>\n",
       "      <th>Z-Score[D]</th>\n",
       "      <th>Al.Res.</th>\n",
       "      <th>RMSD[D]</th>\n",
       "      <th>ProSA_Z-Score</th>\n",
       "      <th>MolPrb_Score</th>\n",
       "      <th>LDDT</th>\n",
       "      <th>SphGr</th>\n",
       "      <th>CAD_AA</th>\n",
       "      <th>RPF</th>\n",
       "      <th>SphGr_A</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Target</th>\n",
       "      <th>Group</th>\n",
       "      <th>Model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0644</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">24</th>\n",
       "      <th>1</th>\n",
       "      <td>255</td>\n",
       "      <td>024</td>\n",
       "      <td>75.36</td>\n",
       "      <td>97.16</td>\n",
       "      <td>65</td>\n",
       "      <td>0.51</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.58</td>\n",
       "      <td>NaN</td>\n",
       "      <td>55.32</td>\n",
       "      <td>...</td>\n",
       "      <td>18.2</td>\n",
       "      <td>134.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>-7.34</td>\n",
       "      <td>2.27</td>\n",
       "      <td>0.62</td>\n",
       "      <td>53.90</td>\n",
       "      <td>0.59</td>\n",
       "      <td>0.67</td>\n",
       "      <td>53.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>477</td>\n",
       "      <td>024</td>\n",
       "      <td>29.25</td>\n",
       "      <td>93.62</td>\n",
       "      <td>94</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.38</td>\n",
       "      <td>NaN</td>\n",
       "      <td>17.02</td>\n",
       "      <td>...</td>\n",
       "      <td>6.6</td>\n",
       "      <td>69.0</td>\n",
       "      <td>3.1</td>\n",
       "      <td>-2.38</td>\n",
       "      <td>2.50</td>\n",
       "      <td>0.33</td>\n",
       "      <td>12.06</td>\n",
       "      <td>0.41</td>\n",
       "      <td>0.31</td>\n",
       "      <td>12.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>282</td>\n",
       "      <td>024</td>\n",
       "      <td>73.76</td>\n",
       "      <td>97.87</td>\n",
       "      <td>53</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.51</td>\n",
       "      <td>NaN</td>\n",
       "      <td>53.19</td>\n",
       "      <td>...</td>\n",
       "      <td>17.9</td>\n",
       "      <td>132.0</td>\n",
       "      <td>1.6</td>\n",
       "      <td>-7.29</td>\n",
       "      <td>2.58</td>\n",
       "      <td>0.61</td>\n",
       "      <td>53.19</td>\n",
       "      <td>0.59</td>\n",
       "      <td>0.66</td>\n",
       "      <td>53.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">26</th>\n",
       "      <th>1</th>\n",
       "      <td>131</td>\n",
       "      <td>026</td>\n",
       "      <td>80.85</td>\n",
       "      <td>100.00</td>\n",
       "      <td>31</td>\n",
       "      <td>0.75</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.81</td>\n",
       "      <td>NaN</td>\n",
       "      <td>63.12</td>\n",
       "      <td>...</td>\n",
       "      <td>18.3</td>\n",
       "      <td>136.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>-6.99</td>\n",
       "      <td>3.18</td>\n",
       "      <td>0.68</td>\n",
       "      <td>64.54</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.71</td>\n",
       "      <td>64.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>189</td>\n",
       "      <td>026</td>\n",
       "      <td>78.19</td>\n",
       "      <td>100.00</td>\n",
       "      <td>39</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.69</td>\n",
       "      <td>NaN</td>\n",
       "      <td>62.06</td>\n",
       "      <td>...</td>\n",
       "      <td>19.9</td>\n",
       "      <td>138.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>-5.91</td>\n",
       "      <td>3.57</td>\n",
       "      <td>0.68</td>\n",
       "      <td>65.25</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.73</td>\n",
       "      <td>65.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0758</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">498</th>\n",
       "      <th>1</th>\n",
       "      <td>205</td>\n",
       "      <td>498s</td>\n",
       "      <td>65.50</td>\n",
       "      <td>100.00</td>\n",
       "      <td>55</td>\n",
       "      <td>-0.36</td>\n",
       "      <td>-0.36</td>\n",
       "      <td>-0.37</td>\n",
       "      <td>-0.37</td>\n",
       "      <td>43.92</td>\n",
       "      <td>...</td>\n",
       "      <td>38.5</td>\n",
       "      <td>348.0</td>\n",
       "      <td>2.7</td>\n",
       "      <td>-9.55</td>\n",
       "      <td>3.46</td>\n",
       "      <td>0.62</td>\n",
       "      <td>63.93</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.67</td>\n",
       "      <td>63.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>171</td>\n",
       "      <td>498s</td>\n",
       "      <td>67.62</td>\n",
       "      <td>100.00</td>\n",
       "      <td>38</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.04</td>\n",
       "      <td>46.58</td>\n",
       "      <td>...</td>\n",
       "      <td>39.2</td>\n",
       "      <td>349.0</td>\n",
       "      <td>2.6</td>\n",
       "      <td>-9.84</td>\n",
       "      <td>3.19</td>\n",
       "      <td>0.62</td>\n",
       "      <td>65.85</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.69</td>\n",
       "      <td>65.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>498s</td>\n",
       "      <td>70.22</td>\n",
       "      <td>100.00</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.54</td>\n",
       "      <td>0.54</td>\n",
       "      <td>48.98</td>\n",
       "      <td>...</td>\n",
       "      <td>39.8</td>\n",
       "      <td>355.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>-9.90</td>\n",
       "      <td>3.56</td>\n",
       "      <td>0.64</td>\n",
       "      <td>69.67</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.71</td>\n",
       "      <td>69.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>203</td>\n",
       "      <td>498s</td>\n",
       "      <td>65.57</td>\n",
       "      <td>100.00</td>\n",
       "      <td>34</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.36</td>\n",
       "      <td>-0.36</td>\n",
       "      <td>43.58</td>\n",
       "      <td>...</td>\n",
       "      <td>39.0</td>\n",
       "      <td>351.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>-10.13</td>\n",
       "      <td>3.34</td>\n",
       "      <td>0.61</td>\n",
       "      <td>65.30</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.68</td>\n",
       "      <td>65.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>24</td>\n",
       "      <td>498s</td>\n",
       "      <td>69.88</td>\n",
       "      <td>100.00</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.48</td>\n",
       "      <td>48.84</td>\n",
       "      <td>...</td>\n",
       "      <td>39.7</td>\n",
       "      <td>351.0</td>\n",
       "      <td>2.4</td>\n",
       "      <td>-9.71</td>\n",
       "      <td>3.57</td>\n",
       "      <td>0.64</td>\n",
       "      <td>69.13</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.71</td>\n",
       "      <td>69.13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>36625 rows × 39 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                      #   GR#  GDT_TS    NP_P  RANK  Z-M1-GDT  Z-M1s-GDT  \\\n",
       "Target Group Model                                                         \n",
       "T0644  24    1      255   024   75.36   97.16    65      0.51        NaN   \n",
       "             2      477   024   29.25   93.62    94       NaN        NaN   \n",
       "             3      282   024   73.76   97.87    53       NaN        NaN   \n",
       "       26    1      131   026   80.85  100.00    31      0.75        NaN   \n",
       "             2      189   026   78.19  100.00    39       NaN        NaN   \n",
       "...                 ...   ...     ...     ...   ...       ...        ...   \n",
       "T0758  498   1      205  498s   65.50  100.00    55     -0.36      -0.36   \n",
       "             2      171  498s   67.62  100.00    38       NaN        NaN   \n",
       "             3       15  498s   70.22  100.00     4       NaN        NaN   \n",
       "             4      203  498s   65.57  100.00    34       NaN        NaN   \n",
       "             5       24  498s   69.88  100.00     7       NaN        NaN   \n",
       "\n",
       "                    Z-MA-GDT  Z-MAs-GDT  GDT_HA  ...  Z-Score[D]  Al.Res.  \\\n",
       "Target Group Model                               ...                        \n",
       "T0644  24    1          0.58        NaN   55.32  ...        18.2    134.0   \n",
       "             2         -1.38        NaN   17.02  ...         6.6     69.0   \n",
       "             3          0.51        NaN   53.19  ...        17.9    132.0   \n",
       "       26    1          0.81        NaN   63.12  ...        18.3    136.0   \n",
       "             2          0.69        NaN   62.06  ...        19.9    138.0   \n",
       "...                      ...        ...     ...  ...         ...      ...   \n",
       "T0758  498   1         -0.37      -0.37   43.92  ...        38.5    348.0   \n",
       "             2          0.04       0.04   46.58  ...        39.2    349.0   \n",
       "             3          0.54       0.54   48.98  ...        39.8    355.0   \n",
       "             4         -0.36      -0.36   43.58  ...        39.0    351.0   \n",
       "             5          0.48       0.48   48.84  ...        39.7    351.0   \n",
       "\n",
       "                    RMSD[D]  ProSA_Z-Score  MolPrb_Score  LDDT  SphGr  CAD_AA  \\\n",
       "Target Group Model                                                              \n",
       "T0644  24    1          2.0          -7.34          2.27  0.62  53.90    0.59   \n",
       "             2          3.1          -2.38          2.50  0.33  12.06    0.41   \n",
       "             3          1.6          -7.29          2.58  0.61  53.19    0.59   \n",
       "       26    1          2.0          -6.99          3.18  0.68  64.54    0.62   \n",
       "             2          2.5          -5.91          3.57  0.68  65.25    0.60   \n",
       "...                     ...            ...           ...   ...    ...     ...   \n",
       "T0758  498   1          2.7          -9.55          3.46  0.62  63.93    0.61   \n",
       "             2          2.6          -9.84          3.19  0.62  65.85    0.62   \n",
       "             3          2.5          -9.90          3.56  0.64  69.67    0.62   \n",
       "             4          2.5         -10.13          3.34  0.61  65.30    0.60   \n",
       "             5          2.4          -9.71          3.57  0.64  69.13    0.62   \n",
       "\n",
       "                     RPF  SphGr_A  \n",
       "Target Group Model                 \n",
       "T0644  24    1      0.67    53.85  \n",
       "             2      0.31    12.06  \n",
       "             3      0.66    53.14  \n",
       "       26    1      0.71    64.47  \n",
       "             2      0.73    65.18  \n",
       "...                  ...      ...  \n",
       "T0758  498   1      0.67    63.93  \n",
       "             2      0.69    65.85  \n",
       "             3      0.71    69.67  \n",
       "             4      0.68    65.30  \n",
       "             5      0.71    69.13  \n",
       "\n",
       "[36625 rows x 39 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'https://predictioncenter.org/download_area/CASP10/SUMMARY_TABLES/T0_all.tar.gz'\n",
    "response = requests_get(url)\n",
    "archive = tarfile.open(fileobj=io.BytesIO(response.content), mode='r')\n",
    "\n",
    "dfs = {}\n",
    "for member in archive.getmembers():\n",
    "    if member.isfile() and regex.fullmatch(member.name):\n",
    "        f = archive.extractfile(member)\n",
    "        df = pd.read_csv(f, sep='\\s+')               \n",
    "        dfs[member.name] = df\n",
    "df = pd.concat(dfs.values(), axis=0)\n",
    "\n",
    "# Split accession codes\n",
    "split = df['Model'].str.split('TS', expand=True)\n",
    "df['Target'] = split[0]\n",
    "split = split[1].str.split('_', expand=True, n=1)\n",
    "df['Group'] = split[0].astype(int)\n",
    "df['Model'] = split[1]\n",
    "df.set_index(['Target', 'Group', 'Model'], inplace=True)\n",
    "df.sort_index(inplace=True)\n",
    "\n",
    "Path('CASP10/QA_official').mkdir(exist_ok=True, parents=True)\n",
    "df.to_pickle('CASP10/QA_official/table.pkl.xz')\n",
    "\n",
    "print('Unique targets:', len(df.index.unique(level='Target')))\n",
    "print('Unique groups:', len(df.index.unique(level='Group')))\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CASP 11\n",
    "Single .tar.gz file with separate .txt files inside, \n",
    "but also has some additional targets that are not in the .tar.gz file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique targets: 93\n",
      "Unique groups: 143\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>#</th>\n",
       "      <th>GR#</th>\n",
       "      <th>GDT_TS</th>\n",
       "      <th>NP_P</th>\n",
       "      <th>RANK</th>\n",
       "      <th>Z-M1-GDT</th>\n",
       "      <th>Z-M1s-GDT</th>\n",
       "      <th>Z-MA-GDT</th>\n",
       "      <th>Z-MAs-GDT</th>\n",
       "      <th>GDT_HA</th>\n",
       "      <th>...</th>\n",
       "      <th>CODM</th>\n",
       "      <th>DFM</th>\n",
       "      <th>Handed.</th>\n",
       "      <th>SOV</th>\n",
       "      <th>CE</th>\n",
       "      <th>QCS</th>\n",
       "      <th>CONTS</th>\n",
       "      <th>TMscore</th>\n",
       "      <th>Dali(raw)</th>\n",
       "      <th>FlexE</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Target</th>\n",
       "      <th>Group</th>\n",
       "      <th>Model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0759</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">6</th>\n",
       "      <th>1</th>\n",
       "      <td>499</td>\n",
       "      <td>006</td>\n",
       "      <td>18.75</td>\n",
       "      <td>100.0</td>\n",
       "      <td>499</td>\n",
       "      <td>-2.29</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-2.26</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11.46</td>\n",
       "      <td>...</td>\n",
       "      <td>0.47</td>\n",
       "      <td>1.52</td>\n",
       "      <td>0.49</td>\n",
       "      <td>37.3</td>\n",
       "      <td>2.30</td>\n",
       "      <td>35.55</td>\n",
       "      <td>45.89</td>\n",
       "      <td>0.33</td>\n",
       "      <td>NaN</td>\n",
       "      <td>36.48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>475</td>\n",
       "      <td>006</td>\n",
       "      <td>21.35</td>\n",
       "      <td>100.0</td>\n",
       "      <td>473</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.90</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12.50</td>\n",
       "      <td>...</td>\n",
       "      <td>0.42</td>\n",
       "      <td>1.91</td>\n",
       "      <td>0.40</td>\n",
       "      <td>23.2</td>\n",
       "      <td>2.30</td>\n",
       "      <td>31.00</td>\n",
       "      <td>44.13</td>\n",
       "      <td>0.25</td>\n",
       "      <td>NaN</td>\n",
       "      <td>52.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>482</td>\n",
       "      <td>006</td>\n",
       "      <td>20.83</td>\n",
       "      <td>100.0</td>\n",
       "      <td>480</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.97</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12.24</td>\n",
       "      <td>...</td>\n",
       "      <td>0.37</td>\n",
       "      <td>1.70</td>\n",
       "      <td>0.41</td>\n",
       "      <td>35.1</td>\n",
       "      <td>2.30</td>\n",
       "      <td>32.40</td>\n",
       "      <td>43.06</td>\n",
       "      <td>0.26</td>\n",
       "      <td>NaN</td>\n",
       "      <td>65.66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>503</td>\n",
       "      <td>006</td>\n",
       "      <td>18.23</td>\n",
       "      <td>100.0</td>\n",
       "      <td>502</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-2.33</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12.50</td>\n",
       "      <td>...</td>\n",
       "      <td>0.43</td>\n",
       "      <td>1.73</td>\n",
       "      <td>0.49</td>\n",
       "      <td>36.2</td>\n",
       "      <td>2.58</td>\n",
       "      <td>32.87</td>\n",
       "      <td>46.09</td>\n",
       "      <td>0.24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>43.48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>507</td>\n",
       "      <td>006</td>\n",
       "      <td>17.71</td>\n",
       "      <td>100.0</td>\n",
       "      <td>506</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-2.40</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11.46</td>\n",
       "      <td>...</td>\n",
       "      <td>0.26</td>\n",
       "      <td>1.71</td>\n",
       "      <td>0.54</td>\n",
       "      <td>36.9</td>\n",
       "      <td>1.64</td>\n",
       "      <td>28.65</td>\n",
       "      <td>42.86</td>\n",
       "      <td>0.27</td>\n",
       "      <td>NaN</td>\n",
       "      <td>48.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0858</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">499</th>\n",
       "      <th>1</th>\n",
       "      <td>51</td>\n",
       "      <td>499s</td>\n",
       "      <td>75.22</td>\n",
       "      <td>100.0</td>\n",
       "      <td>51</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.39</td>\n",
       "      <td>54.16</td>\n",
       "      <td>...</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.94</td>\n",
       "      <td>80.1</td>\n",
       "      <td>7.84</td>\n",
       "      <td>90.28</td>\n",
       "      <td>81.19</td>\n",
       "      <td>0.92</td>\n",
       "      <td>6868.4</td>\n",
       "      <td>5.28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>63</td>\n",
       "      <td>499s</td>\n",
       "      <td>74.72</td>\n",
       "      <td>100.0</td>\n",
       "      <td>63</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.35</td>\n",
       "      <td>53.50</td>\n",
       "      <td>...</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.94</td>\n",
       "      <td>77.7</td>\n",
       "      <td>7.84</td>\n",
       "      <td>90.04</td>\n",
       "      <td>81.42</td>\n",
       "      <td>0.92</td>\n",
       "      <td>6911.2</td>\n",
       "      <td>4.79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>84</td>\n",
       "      <td>499s</td>\n",
       "      <td>73.39</td>\n",
       "      <td>100.0</td>\n",
       "      <td>84</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.25</td>\n",
       "      <td>52.33</td>\n",
       "      <td>...</td>\n",
       "      <td>0.94</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.93</td>\n",
       "      <td>69.0</td>\n",
       "      <td>7.84</td>\n",
       "      <td>89.30</td>\n",
       "      <td>79.37</td>\n",
       "      <td>0.91</td>\n",
       "      <td>6700.3</td>\n",
       "      <td>6.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>103</td>\n",
       "      <td>499s</td>\n",
       "      <td>72.61</td>\n",
       "      <td>100.0</td>\n",
       "      <td>103</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.19</td>\n",
       "      <td>0.19</td>\n",
       "      <td>51.61</td>\n",
       "      <td>...</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.19</td>\n",
       "      <td>0.93</td>\n",
       "      <td>73.9</td>\n",
       "      <td>7.74</td>\n",
       "      <td>89.44</td>\n",
       "      <td>80.58</td>\n",
       "      <td>0.91</td>\n",
       "      <td>6714.6</td>\n",
       "      <td>5.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>135</td>\n",
       "      <td>499s</td>\n",
       "      <td>70.50</td>\n",
       "      <td>100.0</td>\n",
       "      <td>135</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.02</td>\n",
       "      <td>49.33</td>\n",
       "      <td>...</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.93</td>\n",
       "      <td>76.4</td>\n",
       "      <td>7.64</td>\n",
       "      <td>88.86</td>\n",
       "      <td>78.51</td>\n",
       "      <td>0.90</td>\n",
       "      <td>6580.3</td>\n",
       "      <td>5.93</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>31291 rows × 47 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                      #   GR#  GDT_TS   NP_P  RANK  Z-M1-GDT  Z-M1s-GDT  \\\n",
       "Target Group Model                                                        \n",
       "T0759  6     1      499   006   18.75  100.0   499     -2.29        NaN   \n",
       "             2      475   006   21.35  100.0   473       NaN        NaN   \n",
       "             3      482   006   20.83  100.0   480       NaN        NaN   \n",
       "             4      503   006   18.23  100.0   502       NaN        NaN   \n",
       "             5      507   006   17.71  100.0   506       NaN        NaN   \n",
       "...                 ...   ...     ...    ...   ...       ...        ...   \n",
       "T0858  499   1       51  499s   75.22  100.0    51      0.31       0.31   \n",
       "             2       63  499s   74.72  100.0    63       NaN        NaN   \n",
       "             3       84  499s   73.39  100.0    84       NaN        NaN   \n",
       "             4      103  499s   72.61  100.0   103       NaN        NaN   \n",
       "             5      135  499s   70.50  100.0   135       NaN        NaN   \n",
       "\n",
       "                    Z-MA-GDT  Z-MAs-GDT  GDT_HA  ...  CODM   DFM  Handed.  \\\n",
       "Target Group Model                               ...                        \n",
       "T0759  6     1         -2.26        NaN   11.46  ...  0.47  1.52     0.49   \n",
       "             2         -1.90        NaN   12.50  ...  0.42  1.91     0.40   \n",
       "             3         -1.97        NaN   12.24  ...  0.37  1.70     0.41   \n",
       "             4         -2.33        NaN   12.50  ...  0.43  1.73     0.49   \n",
       "             5         -2.40        NaN   11.46  ...  0.26  1.71     0.54   \n",
       "...                      ...        ...     ...  ...   ...   ...      ...   \n",
       "T0858  499   1          0.39       0.39   54.16  ...  0.95  0.20     0.94   \n",
       "             2          0.35       0.35   53.50  ...  0.95  0.20     0.94   \n",
       "             3          0.25       0.25   52.33  ...  0.94  0.23     0.93   \n",
       "             4          0.19       0.19   51.61  ...  0.95  0.19     0.93   \n",
       "             5          0.02       0.02   49.33  ...  0.95  0.23     0.93   \n",
       "\n",
       "                     SOV    CE    QCS  CONTS  TMscore  Dali(raw)  FlexE  \n",
       "Target Group Model                                                       \n",
       "T0759  6     1      37.3  2.30  35.55  45.89     0.33        NaN  36.48  \n",
       "             2      23.2  2.30  31.00  44.13     0.25        NaN  52.01  \n",
       "             3      35.1  2.30  32.40  43.06     0.26        NaN  65.66  \n",
       "             4      36.2  2.58  32.87  46.09     0.24        NaN  43.48  \n",
       "             5      36.9  1.64  28.65  42.86     0.27        NaN  48.04  \n",
       "...                  ...   ...    ...    ...      ...        ...    ...  \n",
       "T0858  499   1      80.1  7.84  90.28  81.19     0.92     6868.4   5.28  \n",
       "             2      77.7  7.84  90.04  81.42     0.92     6911.2   4.79  \n",
       "             3      69.0  7.84  89.30  79.37     0.91     6700.3   6.36  \n",
       "             4      73.9  7.74  89.44  80.58     0.91     6714.6   5.02  \n",
       "             5      76.4  7.64  88.86  78.51     0.90     6580.3   5.93  \n",
       "\n",
       "[31291 rows x 47 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'https://predictioncenter.org/download_area/CASP11/SUMMARY_TABLES/T0xxx_09.05.tar.gz'\n",
    "response = requests_get(url)\n",
    "archive = tarfile.open(fileobj=io.BytesIO(response.content), mode='r')\n",
    "\n",
    "dfs = {}\n",
    "for member in archive.getmembers():\n",
    "    if member.isfile() and regex.fullmatch(member.name):\n",
    "        f = archive.extractfile(member)\n",
    "        df = pd.read_csv(f, sep='\\s+')\n",
    "        dfs[member.name] = df\n",
    "        \n",
    "# Additional results that in the .tar.gz file are missing or outdated\n",
    "url = 'https://predictioncenter.org/download_area/CASP11/SUMMARY_TABLES/'\n",
    "for f in ['T0774.txt', 'T0812.txt', 'T0837.txt', \n",
    "          'T0840.txt', 'T0841.txt', 'T0851.txt']:\n",
    "    response = requests_get(url + f)\n",
    "    df = pd.read_csv(io.BytesIO(response.content), sep='\\s+')\n",
    "    dfs[f] = df\n",
    "    \n",
    "df = pd.concat(dfs.values(), axis=0)\n",
    "\n",
    "# Split accession codes\n",
    "split = df['Model'].str.split('TS', expand=True)\n",
    "df['Target'] = split[0]\n",
    "split = split[1].str.split('_', expand=True, n=1)\n",
    "df['Group'] = split[0].astype(int)\n",
    "df['Model'] = split[1]\n",
    "df.set_index(['Target', 'Group', 'Model'], inplace=True)\n",
    "df.sort_index(inplace=True)\n",
    "\n",
    "Path('CASP11/QA_official').mkdir(exist_ok=True, parents=True)\n",
    "df.to_pickle('CASP11/QA_official/table.pkl.xz')\n",
    "\n",
    "print('Unique targets:', len(df.index.unique(level='Target')))\n",
    "print('Unique groups:', len(df.index.unique(level='Group')))\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CASP 12\n",
    "No .tar.gz file, only a list of .txt files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique targets: 77\n",
      "Unique groups: 128\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>#</th>\n",
       "      <th>GR#</th>\n",
       "      <th>GDT_TS</th>\n",
       "      <th>NP_P</th>\n",
       "      <th>RANK</th>\n",
       "      <th>Z-M1-GDT</th>\n",
       "      <th>Z-M1s-GDT</th>\n",
       "      <th>Z-MA-GDT</th>\n",
       "      <th>Z-MAs-GDT</th>\n",
       "      <th>GDT_HA</th>\n",
       "      <th>...</th>\n",
       "      <th>DFM</th>\n",
       "      <th>Handed.</th>\n",
       "      <th>SOV</th>\n",
       "      <th>CE</th>\n",
       "      <th>QCS</th>\n",
       "      <th>CONTS</th>\n",
       "      <th>TMscore</th>\n",
       "      <th>Dali(raw)</th>\n",
       "      <th>FlexE</th>\n",
       "      <th>QSE</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Target</th>\n",
       "      <th>Group</th>\n",
       "      <th>Model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0859</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>001</td>\n",
       "      <td>27.66</td>\n",
       "      <td>100.00</td>\n",
       "      <td>2</td>\n",
       "      <td>1.85</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.18</td>\n",
       "      <td>NaN</td>\n",
       "      <td>19.25</td>\n",
       "      <td>...</td>\n",
       "      <td>1.54</td>\n",
       "      <td>0.51</td>\n",
       "      <td>51.40</td>\n",
       "      <td>4.25</td>\n",
       "      <td>28.90</td>\n",
       "      <td>38.83</td>\n",
       "      <td>0.58</td>\n",
       "      <td>401.6</td>\n",
       "      <td>257.42</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">4</th>\n",
       "      <th>1</th>\n",
       "      <td>109</td>\n",
       "      <td>004</td>\n",
       "      <td>22.79</td>\n",
       "      <td>100.00</td>\n",
       "      <td>107</td>\n",
       "      <td>0.63</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.80</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16.15</td>\n",
       "      <td>...</td>\n",
       "      <td>1.47</td>\n",
       "      <td>0.51</td>\n",
       "      <td>42.10</td>\n",
       "      <td>3.70</td>\n",
       "      <td>33.08</td>\n",
       "      <td>38.28</td>\n",
       "      <td>0.32</td>\n",
       "      <td>129.5</td>\n",
       "      <td>187.42</td>\n",
       "      <td>62.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>148</td>\n",
       "      <td>004</td>\n",
       "      <td>21.90</td>\n",
       "      <td>100.00</td>\n",
       "      <td>144</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15.93</td>\n",
       "      <td>...</td>\n",
       "      <td>1.63</td>\n",
       "      <td>0.51</td>\n",
       "      <td>34.00</td>\n",
       "      <td>3.29</td>\n",
       "      <td>31.68</td>\n",
       "      <td>39.16</td>\n",
       "      <td>0.34</td>\n",
       "      <td>120.9</td>\n",
       "      <td>150.16</td>\n",
       "      <td>70.40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>34</td>\n",
       "      <td>004</td>\n",
       "      <td>25.22</td>\n",
       "      <td>100.00</td>\n",
       "      <td>32</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.49</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15.71</td>\n",
       "      <td>...</td>\n",
       "      <td>1.77</td>\n",
       "      <td>0.54</td>\n",
       "      <td>37.10</td>\n",
       "      <td>3.29</td>\n",
       "      <td>25.14</td>\n",
       "      <td>38.00</td>\n",
       "      <td>0.37</td>\n",
       "      <td>188.1</td>\n",
       "      <td>139.78</td>\n",
       "      <td>74.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>147</td>\n",
       "      <td>004</td>\n",
       "      <td>21.90</td>\n",
       "      <td>100.00</td>\n",
       "      <td>144</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15.71</td>\n",
       "      <td>...</td>\n",
       "      <td>1.78</td>\n",
       "      <td>0.48</td>\n",
       "      <td>39.10</td>\n",
       "      <td>3.50</td>\n",
       "      <td>26.82</td>\n",
       "      <td>36.88</td>\n",
       "      <td>0.32</td>\n",
       "      <td>162.4</td>\n",
       "      <td>165.13</td>\n",
       "      <td>74.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0948</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">498</th>\n",
       "      <th>1</th>\n",
       "      <td>8</td>\n",
       "      <td>498</td>\n",
       "      <td>75.50</td>\n",
       "      <td>100.00</td>\n",
       "      <td>6</td>\n",
       "      <td>1.01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>59.23</td>\n",
       "      <td>...</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.90</td>\n",
       "      <td>85.70</td>\n",
       "      <td>6.11</td>\n",
       "      <td>89.75</td>\n",
       "      <td>81.57</td>\n",
       "      <td>0.82</td>\n",
       "      <td>1188.2</td>\n",
       "      <td>7.02</td>\n",
       "      <td>52.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>220</td>\n",
       "      <td>498</td>\n",
       "      <td>64.43</td>\n",
       "      <td>100.00</td>\n",
       "      <td>220</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.62</td>\n",
       "      <td>NaN</td>\n",
       "      <td>46.98</td>\n",
       "      <td>...</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.82</td>\n",
       "      <td>79.60</td>\n",
       "      <td>5.73</td>\n",
       "      <td>83.88</td>\n",
       "      <td>74.51</td>\n",
       "      <td>0.76</td>\n",
       "      <td>1052.4</td>\n",
       "      <td>16.44</td>\n",
       "      <td>56.69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>103</td>\n",
       "      <td>498</td>\n",
       "      <td>71.48</td>\n",
       "      <td>100.00</td>\n",
       "      <td>102</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.91</td>\n",
       "      <td>NaN</td>\n",
       "      <td>52.85</td>\n",
       "      <td>...</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.87</td>\n",
       "      <td>84.80</td>\n",
       "      <td>5.99</td>\n",
       "      <td>88.80</td>\n",
       "      <td>79.78</td>\n",
       "      <td>0.82</td>\n",
       "      <td>1205.4</td>\n",
       "      <td>9.30</td>\n",
       "      <td>57.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>200</td>\n",
       "      <td>498</td>\n",
       "      <td>66.78</td>\n",
       "      <td>97.99</td>\n",
       "      <td>200</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.72</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50.00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.81</td>\n",
       "      <td>78.29</td>\n",
       "      <td>5.86</td>\n",
       "      <td>85.37</td>\n",
       "      <td>74.40</td>\n",
       "      <td>0.76</td>\n",
       "      <td>1073.6</td>\n",
       "      <td>21.06</td>\n",
       "      <td>34.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>214</td>\n",
       "      <td>498</td>\n",
       "      <td>65.27</td>\n",
       "      <td>100.00</td>\n",
       "      <td>213</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.65</td>\n",
       "      <td>NaN</td>\n",
       "      <td>48.99</td>\n",
       "      <td>...</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.82</td>\n",
       "      <td>80.70</td>\n",
       "      <td>5.86</td>\n",
       "      <td>85.32</td>\n",
       "      <td>75.33</td>\n",
       "      <td>0.77</td>\n",
       "      <td>1090.8</td>\n",
       "      <td>12.22</td>\n",
       "      <td>56.34</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>29189 rows × 48 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                      #  GR#  GDT_TS    NP_P  RANK  Z-M1-GDT  Z-M1s-GDT  \\\n",
       "Target Group Model                                                        \n",
       "T0859  1     1        4  001   27.66  100.00     2      1.85        NaN   \n",
       "       4     1      109  004   22.79  100.00   107      0.63        NaN   \n",
       "             2      148  004   21.90  100.00   144       NaN        NaN   \n",
       "             3       34  004   25.22  100.00    32       NaN        NaN   \n",
       "             4      147  004   21.90  100.00   144       NaN        NaN   \n",
       "...                 ...  ...     ...     ...   ...       ...        ...   \n",
       "T0948  498   1        8  498   75.50  100.00     6      1.01        NaN   \n",
       "             2      220  498   64.43  100.00   220       NaN        NaN   \n",
       "             3      103  498   71.48  100.00   102       NaN        NaN   \n",
       "             4      200  498   66.78   97.99   200       NaN        NaN   \n",
       "             5      214  498   65.27  100.00   213       NaN        NaN   \n",
       "\n",
       "                    Z-MA-GDT  Z-MAs-GDT  GDT_HA  ...   DFM  Handed.    SOV  \\\n",
       "Target Group Model                               ...                         \n",
       "T0859  1     1          2.18        NaN   19.25  ...  1.54     0.51  51.40   \n",
       "       4     1          0.80        NaN   16.15  ...  1.47     0.51  42.10   \n",
       "             2          0.55        NaN   15.93  ...  1.63     0.51  34.00   \n",
       "             3          1.49        NaN   15.71  ...  1.77     0.54  37.10   \n",
       "             4          0.55        NaN   15.71  ...  1.78     0.48  39.10   \n",
       "...                      ...        ...     ...  ...   ...      ...    ...   \n",
       "T0948  498   1          1.07        NaN   59.23  ...  0.33     0.90  85.70   \n",
       "             2          0.62        NaN   46.98  ...  0.47     0.82  79.60   \n",
       "             3          0.91        NaN   52.85  ...  0.32     0.87  84.80   \n",
       "             4          0.72        NaN   50.00  ...  0.48     0.81  78.29   \n",
       "             5          0.65        NaN   48.99  ...  0.46     0.82  80.70   \n",
       "\n",
       "                      CE    QCS  CONTS  TMscore  Dali(raw)   FlexE    QSE  \n",
       "Target Group Model                                                         \n",
       "T0859  1     1      4.25  28.90  38.83     0.58      401.6  257.42    NaN  \n",
       "       4     1      3.70  33.08  38.28     0.32      129.5  187.42  62.36  \n",
       "             2      3.29  31.68  39.16     0.34      120.9  150.16  70.40  \n",
       "             3      3.29  25.14  38.00     0.37      188.1  139.78  74.50  \n",
       "             4      3.50  26.82  36.88     0.32      162.4  165.13  74.22  \n",
       "...                  ...    ...    ...      ...        ...     ...    ...  \n",
       "T0948  498   1      6.11  89.75  81.57     0.82     1188.2    7.02  52.85  \n",
       "             2      5.73  83.88  74.51     0.76     1052.4   16.44  56.69  \n",
       "             3      5.99  88.80  79.78     0.82     1205.4    9.30  57.08  \n",
       "             4      5.86  85.37  74.40     0.76     1073.6   21.06  34.25  \n",
       "             5      5.86  85.32  75.33     0.77     1090.8   12.22  56.34  \n",
       "\n",
       "[29189 rows x 48 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_url = 'https://predictioncenter.org/download_area/CASP12/SUMMARY_TABLES/'\n",
    "request = requests_get(base_url)\n",
    "soup = bs4.BeautifulSoup(request.content)\n",
    "\n",
    "dfs = {}\n",
    "for a in soup.select('table tr a'):\n",
    "    href = a.attrs['href']\n",
    "    if regex.fullmatch(href):\n",
    "        response = requests_get(base_url + href)\n",
    "        df = pd.read_csv(io.BytesIO(response.content), sep='\\s+')                \n",
    "        dfs[href] = df\n",
    "df = pd.concat(dfs.values(), axis=0)\n",
    "\n",
    "# Split accession codes\n",
    "split = df['Model'].str.split('TS', expand=True)\n",
    "df['Target'] = split[0]\n",
    "split = split[1].str.split('_', expand=True, n=1)\n",
    "df['Group'] = split[0].astype(int)\n",
    "df['Model'] = split[1]\n",
    "df.set_index(['Target', 'Group', 'Model'], inplace=True)\n",
    "df.sort_index(inplace=True)\n",
    "\n",
    "Path('CASP12/QA_official').mkdir(exist_ok=True, parents=True)\n",
    "df.to_pickle('CASP12/QA_official/table.pkl.xz')\n",
    "\n",
    "print('Unique targets:', len(df.index.unique(level='Target')))\n",
    "print('Unique groups:', len(df.index.unique(level='Group')))\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CASP 13\n",
    "Single .tar.gz file with separate .txt files inside, \n",
    "but also has some additional targets that are not in the .tar.gz file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique targets: 20\n",
      "Unique groups: 98\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>#</th>\n",
       "      <th>GR#</th>\n",
       "      <th>GDT_TS</th>\n",
       "      <th>NP_P</th>\n",
       "      <th>RANK</th>\n",
       "      <th>Z-M1-GDT</th>\n",
       "      <th>Z-M1s-GDT</th>\n",
       "      <th>Z-MA-GDT</th>\n",
       "      <th>Z-MAs-GDT</th>\n",
       "      <th>GDT_HA</th>\n",
       "      <th>...</th>\n",
       "      <th>CONTS</th>\n",
       "      <th>TMscore</th>\n",
       "      <th>Dali(raw)</th>\n",
       "      <th>FlexE</th>\n",
       "      <th>QSE</th>\n",
       "      <th>CAD_SS</th>\n",
       "      <th>MolPrb_clash</th>\n",
       "      <th>MolPrb_rotout</th>\n",
       "      <th>MolPrb_ramout</th>\n",
       "      <th>MolPrb_ramfv</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Target</th>\n",
       "      <th>Group</th>\n",
       "      <th>Model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0950</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">4</th>\n",
       "      <th>1</th>\n",
       "      <td>144</td>\n",
       "      <td>004s</td>\n",
       "      <td>10.89</td>\n",
       "      <td>100.00</td>\n",
       "      <td>143</td>\n",
       "      <td>-0.74</td>\n",
       "      <td>-0.74</td>\n",
       "      <td>-0.71</td>\n",
       "      <td>-0.71</td>\n",
       "      <td>7.61</td>\n",
       "      <td>...</td>\n",
       "      <td>48.55</td>\n",
       "      <td>0.16</td>\n",
       "      <td>208.7</td>\n",
       "      <td>170.08</td>\n",
       "      <td>29.05</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2.50</td>\n",
       "      <td>0.28</td>\n",
       "      <td>94.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>46</td>\n",
       "      <td>004s</td>\n",
       "      <td>17.62</td>\n",
       "      <td>100.00</td>\n",
       "      <td>46</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.35</td>\n",
       "      <td>9.57</td>\n",
       "      <td>...</td>\n",
       "      <td>54.46</td>\n",
       "      <td>0.29</td>\n",
       "      <td>583.2</td>\n",
       "      <td>266.73</td>\n",
       "      <td>29.29</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.88</td>\n",
       "      <td>0.28</td>\n",
       "      <td>95.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>175</td>\n",
       "      <td>004s</td>\n",
       "      <td>8.63</td>\n",
       "      <td>100.00</td>\n",
       "      <td>175</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.07</td>\n",
       "      <td>-1.07</td>\n",
       "      <td>5.92</td>\n",
       "      <td>...</td>\n",
       "      <td>49.27</td>\n",
       "      <td>0.12</td>\n",
       "      <td>223.3</td>\n",
       "      <td>271.86</td>\n",
       "      <td>24.11</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.17</td>\n",
       "      <td>2.50</td>\n",
       "      <td>1.42</td>\n",
       "      <td>95.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>142</td>\n",
       "      <td>004s</td>\n",
       "      <td>10.96</td>\n",
       "      <td>56.14</td>\n",
       "      <td>142</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.70</td>\n",
       "      <td>-0.70</td>\n",
       "      <td>7.82</td>\n",
       "      <td>...</td>\n",
       "      <td>30.26</td>\n",
       "      <td>0.16</td>\n",
       "      <td>193.7</td>\n",
       "      <td>77.98</td>\n",
       "      <td>34.65</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>4.73</td>\n",
       "      <td>1.58</td>\n",
       "      <td>95.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>143</td>\n",
       "      <td>004s</td>\n",
       "      <td>10.89</td>\n",
       "      <td>100.00</td>\n",
       "      <td>143</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.71</td>\n",
       "      <td>-0.71</td>\n",
       "      <td>7.61</td>\n",
       "      <td>...</td>\n",
       "      <td>48.55</td>\n",
       "      <td>0.16</td>\n",
       "      <td>267.0</td>\n",
       "      <td>170.08</td>\n",
       "      <td>29.05</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2.50</td>\n",
       "      <td>0.28</td>\n",
       "      <td>94.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">T1022s1</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">498</th>\n",
       "      <th>1</th>\n",
       "      <td>74</td>\n",
       "      <td>498s</td>\n",
       "      <td>35.87</td>\n",
       "      <td>100.00</td>\n",
       "      <td>73</td>\n",
       "      <td>0.84</td>\n",
       "      <td>1.58</td>\n",
       "      <td>1.11</td>\n",
       "      <td>1.89</td>\n",
       "      <td>22.54</td>\n",
       "      <td>...</td>\n",
       "      <td>67.50</td>\n",
       "      <td>0.48</td>\n",
       "      <td>673.6</td>\n",
       "      <td>8.07</td>\n",
       "      <td>60.68</td>\n",
       "      <td>0.31</td>\n",
       "      <td>16.14</td>\n",
       "      <td>16.92</td>\n",
       "      <td>9.25</td>\n",
       "      <td>77.53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>60</td>\n",
       "      <td>498s</td>\n",
       "      <td>37.11</td>\n",
       "      <td>100.00</td>\n",
       "      <td>60</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.22</td>\n",
       "      <td>2.04</td>\n",
       "      <td>22.53</td>\n",
       "      <td>...</td>\n",
       "      <td>68.75</td>\n",
       "      <td>0.51</td>\n",
       "      <td>677.0</td>\n",
       "      <td>8.19</td>\n",
       "      <td>63.65</td>\n",
       "      <td>0.31</td>\n",
       "      <td>28.04</td>\n",
       "      <td>25.64</td>\n",
       "      <td>8.81</td>\n",
       "      <td>80.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>101</td>\n",
       "      <td>498s</td>\n",
       "      <td>33.86</td>\n",
       "      <td>100.00</td>\n",
       "      <td>98</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.92</td>\n",
       "      <td>1.66</td>\n",
       "      <td>22.31</td>\n",
       "      <td>...</td>\n",
       "      <td>68.13</td>\n",
       "      <td>0.45</td>\n",
       "      <td>684.3</td>\n",
       "      <td>9.02</td>\n",
       "      <td>60.23</td>\n",
       "      <td>0.32</td>\n",
       "      <td>16.71</td>\n",
       "      <td>21.03</td>\n",
       "      <td>8.81</td>\n",
       "      <td>79.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>116</td>\n",
       "      <td>498s</td>\n",
       "      <td>32.85</td>\n",
       "      <td>100.00</td>\n",
       "      <td>115</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.83</td>\n",
       "      <td>1.54</td>\n",
       "      <td>20.85</td>\n",
       "      <td>...</td>\n",
       "      <td>66.63</td>\n",
       "      <td>0.44</td>\n",
       "      <td>655.5</td>\n",
       "      <td>9.00</td>\n",
       "      <td>62.49</td>\n",
       "      <td>0.30</td>\n",
       "      <td>17.56</td>\n",
       "      <td>17.95</td>\n",
       "      <td>9.69</td>\n",
       "      <td>78.41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>70</td>\n",
       "      <td>498s</td>\n",
       "      <td>36.10</td>\n",
       "      <td>100.00</td>\n",
       "      <td>70</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.13</td>\n",
       "      <td>1.92</td>\n",
       "      <td>23.32</td>\n",
       "      <td>...</td>\n",
       "      <td>70.14</td>\n",
       "      <td>0.47</td>\n",
       "      <td>692.2</td>\n",
       "      <td>7.34</td>\n",
       "      <td>83.26</td>\n",
       "      <td>0.28</td>\n",
       "      <td>28.89</td>\n",
       "      <td>27.69</td>\n",
       "      <td>9.69</td>\n",
       "      <td>79.30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7659 rows × 53 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       #   GR#  GDT_TS    NP_P  RANK  Z-M1-GDT  Z-M1s-GDT  \\\n",
       "Target  Group Model                                                         \n",
       "T0950   4     1      144  004s   10.89  100.00   143     -0.74      -0.74   \n",
       "              2       46  004s   17.62  100.00    46       NaN        NaN   \n",
       "              3      175  004s    8.63  100.00   175       NaN        NaN   \n",
       "              4      142  004s   10.96   56.14   142       NaN        NaN   \n",
       "              5      143  004s   10.89  100.00   143       NaN        NaN   \n",
       "...                  ...   ...     ...     ...   ...       ...        ...   \n",
       "T1022s1 498   1       74  498s   35.87  100.00    73      0.84       1.58   \n",
       "              2       60  498s   37.11  100.00    60       NaN        NaN   \n",
       "              3      101  498s   33.86  100.00    98       NaN        NaN   \n",
       "              4      116  498s   32.85  100.00   115       NaN        NaN   \n",
       "              5       70  498s   36.10  100.00    70       NaN        NaN   \n",
       "\n",
       "                     Z-MA-GDT  Z-MAs-GDT  GDT_HA  ...  CONTS  TMscore  \\\n",
       "Target  Group Model                               ...                   \n",
       "T0950   4     1         -0.71      -0.71    7.61  ...  48.55     0.16   \n",
       "              2          0.35       0.35    9.57  ...  54.46     0.29   \n",
       "              3         -1.07      -1.07    5.92  ...  49.27     0.12   \n",
       "              4         -0.70      -0.70    7.82  ...  30.26     0.16   \n",
       "              5         -0.71      -0.71    7.61  ...  48.55     0.16   \n",
       "...                       ...        ...     ...  ...    ...      ...   \n",
       "T1022s1 498   1          1.11       1.89   22.54  ...  67.50     0.48   \n",
       "              2          1.22       2.04   22.53  ...  68.75     0.51   \n",
       "              3          0.92       1.66   22.31  ...  68.13     0.45   \n",
       "              4          0.83       1.54   20.85  ...  66.63     0.44   \n",
       "              5          1.13       1.92   23.32  ...  70.14     0.47   \n",
       "\n",
       "                     Dali(raw)   FlexE    QSE  CAD_SS  MolPrb_clash  \\\n",
       "Target  Group Model                                                   \n",
       "T0950   4     1          208.7  170.08  29.05    0.08          0.00   \n",
       "              2          583.2  266.73  29.29    0.10          0.00   \n",
       "              3          223.3  271.86  24.11    0.09          0.17   \n",
       "              4          193.7   77.98  34.65    0.05          0.00   \n",
       "              5          267.0  170.08  29.05    0.08          0.00   \n",
       "...                        ...     ...    ...     ...           ...   \n",
       "T1022s1 498   1          673.6    8.07  60.68    0.31         16.14   \n",
       "              2          677.0    8.19  63.65    0.31         28.04   \n",
       "              3          684.3    9.02  60.23    0.32         16.71   \n",
       "              4          655.5    9.00  62.49    0.30         17.56   \n",
       "              5          692.2    7.34  83.26    0.28         28.89   \n",
       "\n",
       "                     MolPrb_rotout  MolPrb_ramout  MolPrb_ramfv  \n",
       "Target  Group Model                                              \n",
       "T0950   4     1               2.50           0.28         94.59  \n",
       "              2               1.88           0.28         95.16  \n",
       "              3               2.50           1.42         95.44  \n",
       "              4               4.73           1.58         95.26  \n",
       "              5               2.50           0.28         94.30  \n",
       "...                            ...            ...           ...  \n",
       "T1022s1 498   1              16.92           9.25         77.53  \n",
       "              2              25.64           8.81         80.18  \n",
       "              3              21.03           8.81         79.74  \n",
       "              4              17.95           9.69         78.41  \n",
       "              5              27.69           9.69         79.30  \n",
       "\n",
       "[7659 rows x 53 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'https://predictioncenter.org/download_area/CASP13/results/tables/casp13.res_tables.T.tar.gz'\n",
    "response = requests_get(url)\n",
    "archive = tarfile.open(fileobj=io.BytesIO(response.content), mode='r')\n",
    "\n",
    "dfs = {}\n",
    "for member in archive.getmembers():\n",
    "    if member.isfile() and regex.fullmatch(member.name):\n",
    "        f = archive.extractfile(member)\n",
    "        df = pd.read_csv(f, sep='\\s+')                \n",
    "        dfs[member.name] = df\n",
    "df = pd.concat(dfs.values(), axis=0)\n",
    "\n",
    "# Split accession codes\n",
    "split = df['Model'].str.split('TS', expand=True)\n",
    "df['Target'] = split[0]\n",
    "split = split[1].str.split('_', expand=True, n=1)\n",
    "df['Group'] = split[0].astype(int)\n",
    "df['Model'] = split[1]\n",
    "df.set_index(['Target', 'Group', 'Model'], inplace=True)\n",
    "df.sort_index(inplace=True)\n",
    "\n",
    "Path('CASP13/QA_official').mkdir(exist_ok=True, parents=True)\n",
    "df.to_pickle('CASP13/QA_official/table.pkl.xz')\n",
    "\n",
    "print('Unique targets:', len(df.index.unique(level='Target')))\n",
    "print('Unique groups:', len(df.index.unique(level='Group')))\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### All CASPs together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique targets: 300\n",
      "Unique groups: 363\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>GDT_TS</th>\n",
       "      <th>GDT_HA</th>\n",
       "      <th>LDDT</th>\n",
       "      <th>CAD_AA</th>\n",
       "      <th>TMscore</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Edition</th>\n",
       "      <th>Target</th>\n",
       "      <th>Group</th>\n",
       "      <th>Model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">9</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0950</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">4</th>\n",
       "      <th>1</th>\n",
       "      <td>10.89</td>\n",
       "      <td>7.61</td>\n",
       "      <td>0.26</td>\n",
       "      <td>0.43</td>\n",
       "      <td>0.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17.62</td>\n",
       "      <td>9.57</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8.63</td>\n",
       "      <td>5.92</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10.96</td>\n",
       "      <td>7.82</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>10.89</td>\n",
       "      <td>7.61</td>\n",
       "      <td>0.26</td>\n",
       "      <td>0.43</td>\n",
       "      <td>0.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">13</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0624</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">484</th>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>34.06</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>36.59</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>38.05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>35.15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NaN</td>\n",
       "      <td>36.23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>106379 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                            GDT_TS  GDT_HA  LDDT  CAD_AA  TMscore\n",
       "Edition Target Group Model                                       \n",
       "9       T0950  4     1       10.89    7.61  0.26    0.43     0.16\n",
       "                     2       17.62    9.57  0.31    0.47     0.29\n",
       "                     3        8.63    5.92  0.24    0.44     0.12\n",
       "                     4       10.96    7.82  0.16    0.25     0.16\n",
       "                     5       10.89    7.61  0.26    0.43     0.16\n",
       "...                            ...     ...   ...     ...      ...\n",
       "13      T0624  484   1         NaN   34.06   NaN     NaN      NaN\n",
       "                     2         NaN   36.59   NaN     NaN      NaN\n",
       "                     3         NaN   38.05   NaN     NaN      NaN\n",
       "                     4         NaN   35.15   NaN     NaN      NaN\n",
       "                     5         NaN   36.23   NaN     NaN      NaN\n",
       "\n",
       "[106379 rows x 5 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Targets</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Edition</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Targets\n",
       "Edition         \n",
       "9             20\n",
       "10            96\n",
       "11            93\n",
       "12            77\n",
       "13            14"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = pd.concat([\n",
    "    pd.read_pickle(p)\n",
    "    for p in Path().glob('CASP*/QA_official/table.pkl.xz')\n",
    "], keys=[9,10,11,12,13], names=['Edition'])\n",
    "\n",
    "print('Unique targets:', len(df.index.unique(level='Target')))\n",
    "print('Unique groups:', len(df.index.unique(level='Group')))\n",
    "\n",
    "with pd.option_context('display.max_rows', 15):\n",
    "    cols_to_keep = ['GDT_TS', 'GDT_HA', 'LDDT', 'CAD_AA', 'TMscore']\n",
    "    display(df[cols_to_keep])\n",
    "    display(\n",
    "        df.groupby('Edition')\n",
    "            .apply(lambda df: df.index.get_level_values('Target').nunique())\n",
    "            .to_frame('Targets')\n",
    "    )\n",
    "del df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Other group's QA predictions\n",
    "\n",
    "These are the QA predictions submitted by other participants to the QA track in CASP \n",
    "([file format reference](https://predictioncenter.org/casp13/index.cgi?page=format#QA)).\n",
    "\n",
    "Start with `PFRMAT QA`\n",
    "\n",
    "Use `MODEL 1` for predictions submitted in the first stage </br>\n",
    "(i.e., estimating quality of the selected server models released 5 days after the initial target release)\n",
    "\n",
    "Use `MODEL 2` for predictions submitted on the second, larger set of TS models </br>\n",
    "(i.e., estimating quality of models released 7 days after the initial target release).\n",
    "\n",
    "Timeline example.\n",
    "- May 1, 9am PDT - target T0644 is released for prediction in non-QA categories.\n",
    "- May 4, noon - the deadline for submitting tertiary structure predictions by servers.\n",
    "- May 6, noon - the first set of server TS predictions (up to 20 models selected primarily to test single-model methods) is sent to the registered QA servers and posted on the casp14 archive page (https://predictioncenter.org/download_area/CASP14/server_predictions/). QA predictions (marked as MODEL 1) for this subset are accepted for two days.\n",
    "- May 8, noon - deadline for \"stage 1\" QA predictions. The second set of server TS predictions (150 models selected to test both, single-model and clustering methods) is sent to the registered QA servers and posted on the casp14 archive page. QA predictions (marked as MODEL 2) for this second subset of models are accepted for two more days.\n",
    "- May 10, noon - deadline for \"stage 2\" QA predictions. All server TS predictions are posted on the casp14 archive page. No further QA predictions (from servers or manual groups) are accepted for this target.\n",
    "\n",
    "Data are inserted between `MODEL` and `END` records of the submission file. </br>\n",
    "You may submit your quality assessment prediction in one of the two different modes:\n",
    "- `QMODE 1` :   global model quality score (MQS - one number per model)\n",
    "- `QMODE 2` :   MQS and error estimates on per-residue basis.\n",
    "\n",
    "In both modes, the first column in each line contains model identifier (file name of the accepted 3D prediction). </br>\n",
    "The second column contains the accuracy score for a model as a whole (MQS). The accuracy score is a real number between 0.0 and 1.0 (1.0 being a perfect model). </br>\n",
    "If you don't provide error estimates on per residue basis, your data table will consist of these two columns only (Example A).\n",
    "\n",
    "If you do additionally provide residue error estimates (QMODE 2), \n",
    "each consecutive column should contain error estimate in Angstroms for all the consecutive residues in the target \n",
    "(i.e., column 3 corresponds to residue 1 in the target, column 4 - to residue 2 and so on). </br>\n",
    "This way data constitute a table (Number_of_models_for_the_target) BY (Number_of_residues_in_the_target + 1). </br>\n",
    "Do not skip columns if you are not predicting error estimates for some residues - instead put \"X\" in the corresponding column (Example B).</br>\n",
    "Please specify in the REMARKS what you consider to be an error estimate for a residue (CA location error, geometrical center error, etc.).\n",
    "\n",
    "Note 1. Please, be advised that a QA record line may be very long and that some editors/mailing programs may force line wrap potentially causing unexpected parsing errors. </br>\n",
    "To avoid this problem we recommend that you split long lines into shorter sublines (50-100 columns of data) by yourself. </br>\n",
    "Our parser will consider consecutive sublines (starting with the line containing evaluated model name and ending with the line containing the next model name or tag END) a part of the same logical line.\n",
    "\n",
    "Note 2. Please, be advised that model quality predictions in CASP are evaluated by comparing submitted estimates of \n",
    "global reliability and per-residue accuracy of structural models with the values obtained from CASP model evaluation packages (LGA, LDDT, CAD-score and others). </br>\n",
    "Since the evaluation score that is used across the categories in CASP is GDT_TS, predictors should strive to predict this score in QMODE1 (QA1). </br>\n",
    "Predicted per-residue distances in QMODE2 should ideally reproduce those extracted from the LGA optimal model-target superpositions.\n",
    "\n",
    "Examples:\n",
    "- (A) Global Model Quality Score\n",
    "    ```\n",
    "    PFRMAT QA\n",
    "    TARGET T0999\n",
    "    AUTHOR 1234-5678-9000\n",
    "    METHOD Description of methods used\n",
    "    MODEL 1\n",
    "    QMODE 1\n",
    "    3D-JIGSAW_TS1 0.8 \n",
    "    FORTE1_AL1.pdb 0.7 \n",
    "    END\n",
    "    ```\n",
    "- (B) Residue-based Quality Assessment (fragment of the table). \n",
    "  Note, that this case includes case (A) and there is no need to submit QMODE 1 predictions additionally to QMODE 2.\n",
    "    ```\n",
    "    PFRMAT QA\n",
    "    TARGET T0999\n",
    "    AUTHOR 1234-5678-9000\n",
    "    REMARK Error estimate is CA-CA distance in Angstroms\n",
    "    METHOD Description of methods used\n",
    "    MODEL 1\n",
    "    QMODE 2\n",
    "    3D-JIGSAW_TS1 0.8 10.0 6.5 5.0 2.0 1.0  \n",
    "    5.0 4.3 4.6\n",
    "    FORTE1_AL1.pdb 0.7 8.0 5.5 4.5 X X \n",
    "    4.5 4.2 5.0 \n",
    "    END\n",
    "    ```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### QA group names for CASP 11, 12, 13\n",
    "\n",
    "Each group participating in the QA track is assigned an id like `QA014`, \n",
    "but we also need the name of the group, which can be easily reconnected to the QA method used by the group."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5d\" ><caption>CASP 13</caption><thead>    <tr>        <th class=\"col_heading level0 col0\" >qa_group_id</th>        <th class=\"col_heading level0 col1\" >qa_group_name</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow0_col0\" class=\"data row0 col0\" >QA014</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow0_col1\" class=\"data row0 col1\" >Bhattacharya-ClustQ</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow1_col0\" class=\"data row1 col0\" >QA022</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow1_col1\" class=\"data row1 col1\" >Pcons</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow2_col0\" class=\"data row2 col0\" >QA023</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow2_col1\" class=\"data row2 col1\" >MULTICOM-NOVEL</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow3_col0\" class=\"data row3 col0\" >QA027</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow3_col1\" class=\"data row3 col1\" >FaeNNz</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow4_col0\" class=\"data row4 col0\" >QA030</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow4_col1\" class=\"data row4 col1\" >VoroMQA-B</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow5_col0\" class=\"data row5 col0\" >QA044</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow5_col1\" class=\"data row5 col1\" >ProQ2</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow6_col0\" class=\"data row6 col0\" >QA058</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow6_col1\" class=\"data row6 col1\" >MULTICOM_CLUSTER</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow7_col0\" class=\"data row7 col0\" >QA065</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow7_col1\" class=\"data row7 col1\" >Jagodzinski-Cao-QA</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow8_col0\" class=\"data row8 col0\" >QA067</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow8_col1\" class=\"data row8 col1\" >LamoureuxLab</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow9_col0\" class=\"data row9 col0\" >QA083</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow9_col1\" class=\"data row9 col1\" >Pcomb</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow10_col0\" class=\"data row10 col0\" >QA102</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow10_col1\" class=\"data row10 col1\" >Bhattacharya-Server</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow11_col0\" class=\"data row11 col0\" >QA107</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow11_col1\" class=\"data row11 col1\" >MUfoldQA_S2</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow12_col0\" class=\"data row12 col0\" >QA113</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow12_col1\" class=\"data row12 col1\" >MUFoldQA_M</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow13_col0\" class=\"data row13 col0\" >QA134</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow13_col1\" class=\"data row13 col1\" >PLU-TopQA</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow14_col0\" class=\"data row14 col0\" >QA135</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow14_col1\" class=\"data row14 col1\" >SBROD</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow15_col0\" class=\"data row15 col0\" >QA139</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow15_col1\" class=\"data row15 col1\" >ProQ3D</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow16_col0\" class=\"data row16 col0\" >QA146</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow16_col1\" class=\"data row16 col1\" >MASS1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow17_col0\" class=\"data row17 col0\" >QA164</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow17_col1\" class=\"data row17 col1\" >Yang-Server</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow18_col0\" class=\"data row18 col0\" >QA170</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow18_col1\" class=\"data row18 col1\" >Bhattacharya-SingQ</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow19_col0\" class=\"data row19 col0\" >QA171</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow19_col1\" class=\"data row19 col1\" >Davis-EMAconsensusAL</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow20_col0\" class=\"data row20 col0\" >QA187</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow20_col1\" class=\"data row20 col1\" >ProQ3</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow21_col0\" class=\"data row21 col0\" >QA194</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow21_col1\" class=\"data row21 col1\" >UOSHAN</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow22_col0\" class=\"data row22 col0\" >QA196</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow22_col1\" class=\"data row22 col1\" >Grudinin</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow23_col0\" class=\"data row23 col0\" >QA197</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow23_col1\" class=\"data row23 col1\" >MESHI</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow24_col0\" class=\"data row24 col0\" >QA198</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow24_col1\" class=\"data row24 col1\" >ProQ3D-CAD</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow25_col0\" class=\"data row25 col0\" >QA207</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow25_col1\" class=\"data row25 col1\" >SBROD-plus</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow26_col0\" class=\"data row26 col0\" >QA209</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow26_col1\" class=\"data row26 col1\" >PLU-AngularQA</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow27_col0\" class=\"data row27 col0\" >QA211</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow27_col1\" class=\"data row27 col1\" >MUfoldQA_T</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow28_col0\" class=\"data row28 col0\" >QA213</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow28_col1\" class=\"data row28 col1\" >ModFOLD7_cor</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow29_col0\" class=\"data row29 col0\" >QA220</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow29_col1\" class=\"data row29 col1\" >SASHAN</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow30_col0\" class=\"data row30 col0\" >QA237</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow30_col1\" class=\"data row30 col1\" >MESHI-corr-server</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow31_col0\" class=\"data row31 col0\" >QA243</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow31_col1\" class=\"data row31 col1\" >MULTICOM-CONSTRUCT</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow32_col0\" class=\"data row32 col0\" >QA267</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow32_col1\" class=\"data row32 col1\" >ProQ3D-TM</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow33_col0\" class=\"data row33 col0\" >QA272</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow33_col1\" class=\"data row33 col1\" >ModFOLD7_rank</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow34_col0\" class=\"data row34 col0\" >QA275</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow34_col1\" class=\"data row34 col1\" >ModFOLD7</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow35_col0\" class=\"data row35 col0\" >QA289</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow35_col1\" class=\"data row35 col1\" >MESHI-enrich-server</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow36_col0\" class=\"data row36 col0\" >QA312</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow36_col1\" class=\"data row36 col1\" >MUFold_server</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow37_col0\" class=\"data row37 col0\" >QA334</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow37_col1\" class=\"data row37 col1\" >RaptorX-DeepQA</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow38_col0\" class=\"data row38 col0\" >QA339</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow38_col1\" class=\"data row38 col1\" >VoroMQA-A</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow39_col0\" class=\"data row39 col0\" >QA344</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow39_col1\" class=\"data row39 col1\" >Kiharalab</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow40_col0\" class=\"data row40 col0\" >QA347</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow40_col1\" class=\"data row40 col1\" >MESHI-server</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow41_col0\" class=\"data row41 col0\" >QA349</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow41_col1\" class=\"data row41 col1\" >Davis-EMAconsensus</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow42_col0\" class=\"data row42 col0\" >QA359</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow42_col1\" class=\"data row42 col1\" >3DCNN</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow43_col0\" class=\"data row43 col0\" >QA360</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow43_col1\" class=\"data row43 col1\" >ProQ3D-lDDT</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow44_col0\" class=\"data row44 col0\" >QA364</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow44_col1\" class=\"data row44 col1\" >SBROD-server</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow45_col0\" class=\"data row45 col0\" >QA373</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow45_col1\" class=\"data row45 col1\" >ModFOLDclust2</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow46_col0\" class=\"data row46 col0\" >QA413</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow46_col1\" class=\"data row46 col1\" >FALCON-QA</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow47_col0\" class=\"data row47 col0\" >QA415</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow47_col1\" class=\"data row47 col1\" >MASS2</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow48_col0\" class=\"data row48 col0\" >QA440</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow48_col1\" class=\"data row48 col1\" >ProQ4</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow49_col0\" class=\"data row49 col0\" >QA457</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow49_col1\" class=\"data row49 col1\" >Wallner</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow50_col0\" class=\"data row50 col0\" >QA463</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow50_col1\" class=\"data row50 col1\" >Eagle</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow51_col0\" class=\"data row51 col0\" >QA471</td>\n",
       "                        <td id=\"T_e3cabfdc_fc12_11ea_bbc4_6bef0c24ab5drow51_col1\" class=\"data row51 col1\" >CPClab</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f94656700a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def extract_groups(soup):\n",
    "    for tr in soup.select('tr'):\n",
    "        if tr.attrs != {'class': [], 'onmouseover': 'row_over(this)', 'onmouseout': 'row_out(this)'}:\n",
    "            continue\n",
    "        tds = tr.select('td')\n",
    "        if 'QA' not in tds[4].text:\n",
    "            continue\n",
    "        group_name = tds[0].text\n",
    "        group_id = 'QA' + tds[1].text\n",
    "        yield group_id, group_name\n",
    "\n",
    "for casp_ed in [11,12,13]:\n",
    "    response = requests_get(f'https://predictioncenter.org/casp{casp_ed}/docs.cgi?view=groupsbyname')\n",
    "    soup = bs4.BeautifulSoup(response.content)\n",
    "    \n",
    "    df_groups = pd.DataFrame(\n",
    "        list(extract_groups(soup)), \n",
    "        columns=['qa_group_id', 'qa_group_name']\n",
    "    ).sort_values('qa_group_id').reset_index(drop=True)\n",
    "    \n",
    "    dest = Path(f'CASP{casp_ed}/QA_groups.csv')\n",
    "    dest.parent.mkdir(exist_ok=True, parents=True)\n",
    "    df_groups.to_csv(dest, header=True, index=False)\n",
    "\n",
    "display(df_groups.style.set_caption('CASP 13').hide_index())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download all QA predictions\n",
    "\n",
    "Each CASP has a slightly different file structure for QA predictions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "129\n",
      "1.1G\tCASP9/QA_predictions\n"
     ]
    }
   ],
   "source": [
    "dest = 'CASP9/QA_predictions'\n",
    "base_url = 'https://predictioncenter.org/download_area/CASP9/predictions/'\n",
    "qa_urls = [\n",
    "    'QA_T0515-T0539.tar.gz',\n",
    "    'QA_T0540-T0569.tar.gz',\n",
    "    'QA_T0570-T0599.tar.gz',\n",
    "    'QA_T0600-T0629.tar.gz',\n",
    "    'QA_T0630-T0643.tar.gz',\n",
    "]\n",
    "\n",
    "if not Path(dest).is_dir():\n",
    "    Path(dest).mkdir(parents=True)\n",
    "    for qa_url in qa_urls:\n",
    "        ! curl -s {base_url}{qa_url} | tar xz --directory {dest}\n",
    "    for p in Path(dest).glob('T????QA???_?'):\n",
    "        Path(dest).joinpath(p.name[:5]).mkdir(exist_ok=True)\n",
    "        p.rename(Path(dest) / p.name[:5] / p.name)\n",
    "\n",
    "! ls {dest} | wc -l\n",
    "! du -sh {dest}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "113\n",
      "620M\tCASP10/QA_predictions\n"
     ]
    }
   ],
   "source": [
    "dest = 'CASP10/QA_predictions'\n",
    "base_url = 'https://predictioncenter.org/download_area/CASP10/predictions/'\n",
    "qa_urls = [\n",
    "    'QA_T0644-T0669.tar.gz',\n",
    "    'QA_T0670-T0699.tar.gz',\n",
    "    'QA_T0700-T0729.tar.gz',\n",
    "    'QA_T0730-T0758.tar.gz',\n",
    "]\n",
    "\n",
    "if not Path(dest).is_dir():\n",
    "    Path(dest).mkdir(parents=True)\n",
    "    for qa_url in qa_urls:\n",
    "        ! curl -s {base_url}{qa_url} | tar xz --strip 1 --directory {dest}\n",
    "\n",
    "! ls {dest} | wc -l\n",
    "! du -sh {dest}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "98\n",
      "660M\tCASP11/QA_predictions\n"
     ]
    }
   ],
   "source": [
    "dest = 'CASP11/QA_predictions'\n",
    "base_url = 'https://predictioncenter.org/download_area/CASP11/predictions/'\n",
    "qa_urls = [\n",
    "    'QA_T0759-799.tar.gz',\n",
    "    'QA_T0800-829.tar.gz',\n",
    "    'QA_T0830-858.tar.gz',\n",
    "]\n",
    "\n",
    "if not Path(dest).is_dir():\n",
    "    Path(dest).mkdir(parents=True)\n",
    "    for qa_url in qa_urls:\n",
    "        ! curl -s {base_url}{qa_url} | tar xz --strip 1 --directory {dest}\n",
    "        \n",
    "! ls {dest} | wc -l\n",
    "! du -sh {dest}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "70\n",
      "526M\tCASP12/QA_predictions\n"
     ]
    }
   ],
   "source": [
    "dest = 'CASP12/QA_predictions'\n",
    "base_url = 'https://predictioncenter.org/download_area/CASP12/predictions/'\n",
    "qa_urls = [\n",
    "    'CASP12_QA_T08x.tgz',\n",
    "    'CASP12_QA_T09x.tgz',\n",
    "]\n",
    "\n",
    "if not Path(dest).is_dir():\n",
    "    Path(dest).mkdir(parents=True)\n",
    "    for qa_url in qa_urls:\n",
    "        ! curl -s {base_url}{qa_url} | tar xz --directory {dest}\n",
    "        \n",
    "! ls {dest} | wc -l\n",
    "! du -sh {dest}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Map CASP 12 naming to decoy_id naming:\n",
    "# T0949TS145_1 -> QUARK_TS1\n",
    "# TODO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "86\n",
      "795M\tCASP13/QA_predictions\n"
     ]
    }
   ],
   "source": [
    "dest = 'CASP13/QA_predictions'\n",
    "base_url = 'https://predictioncenter.org/download_area/CASP13/predictions/QA/'\n",
    "qa_urls = [\n",
    "    # Stage 1\n",
    "    'QA1.all.tar.gz',\n",
    "    # Stage 2\n",
    "    'QA2.T095_.tar.gz',\n",
    "    'QA2.T096_.tar.gz',\n",
    "    'QA2.T097_.tar.gz',\n",
    "    'QA2.T098_.tar.gz',\n",
    "    'QA2.T099_.tar.gz',\n",
    "    'QA2.T100_.tar.gz',\n",
    "    'QA2.T101_.tar.gz',\n",
    "    'QA2.T102_.tar.gz',\n",
    "]\n",
    "\n",
    "if not Path(dest).is_dir():\n",
    "    Path(dest).mkdir(parents=True)\n",
    "    for qa_url in qa_urls:\n",
    "        ! curl -s {base_url}{qa_url} | tar xz --directory {dest}\n",
    "\n",
    "! ls {dest} | wc -l\n",
    "! du -sh {dest}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/tmp/casp13/T0953s1.TS_as_accepted/T0953s1TS401_1 'utf-8' codec can't decode byte 0x96 in position 1581: invalid start byte\n",
      "/tmp/casp13/T0953s2.TS_as_accepted/T0953s2TS214_2 'utf-8' codec can't decode byte 0xa0 in position 59: invalid start byte\n",
      "/tmp/casp13/T0953s2.TS_as_accepted/T0953s2TS214_1 'utf-8' codec can't decode byte 0xa0 in position 59: invalid start byte\n",
      "/tmp/casp13/T0988.TS_as_accepted/T0988TS452_1 'AUTHOR \\n' list index out of range\n"
     ]
    }
   ],
   "source": [
    "# Map CASP 13 naming to decoy_id naming:\n",
    "# T0949TS145_1 -> QUARK_TS1\n",
    "mapping = {}\n",
    "\n",
    "base_url = \"https://www.predictioncenter.org/download_area/CASP13/predictions/TS_as_submitted/\"\n",
    "urls = [\n",
    "   \"T0949.TS_as_accepted.tar.gz\",\n",
    "   \"T0950.TS_as_accepted.tar.gz\",\n",
    "   \"T0951.TS_as_accepted.tar.gz\",\n",
    "   \"T0953s1.TS_as_accepted.tar.gz\",\n",
    "   \"T0953s2.TS_as_accepted.tar.gz\",\n",
    "   \"T0954.TS_as_accepted.tar.gz\",\n",
    "   \"T0955.TS_as_accepted.tar.gz\",\n",
    "   \"T0956.TS_as_accepted.tar.gz\",\n",
    "   \"T0957s1.TS_as_accepted.tar.gz\",\n",
    "   \"T0957s2.TS_as_accepted.tar.gz\",\n",
    "   \"T0958.TS_as_accepted.tar.gz\",\n",
    "   \"T0959.TS_as_accepted.tar.gz\",\n",
    "   \"T0960.TS_as_accepted.tar.gz\",\n",
    "   \"T0961.TS_as_accepted.tar.gz\",\n",
    "   \"T0962.TS_as_accepted.tar.gz\",\n",
    "   \"T0963.TS_as_accepted.tar.gz\",\n",
    "   \"T0964.TS_as_accepted.tar.gz\",\n",
    "   \"T0965.TS_as_accepted.tar.gz\",\n",
    "   \"T0966.TS_as_accepted.tar.gz\",\n",
    "   \"T0967.TS_as_accepted.tar.gz\",\n",
    "   \"T0968s1.TS_as_accepted.tar.gz\",\n",
    "   \"T0968s2.TS_as_accepted.tar.gz\",\n",
    "   \"T0969.TS_as_accepted.tar.gz\",\n",
    "   \"T0970.TS_as_accepted.tar.gz\",\n",
    "   \"T0971.TS_as_accepted.tar.gz\",\n",
    "   \"T0972.TS_as_accepted.tar.gz\",\n",
    "   \"T0973.TS_as_accepted.tar.gz\",\n",
    "   \"T0974s1.TS_as_accepted.tar.gz\",\n",
    "   \"T0974s2.TS_as_accepted.tar.gz\",\n",
    "   \"T0975.TS_as_accepted.tar.gz\",\n",
    "   \"T0976.TS_as_accepted.tar.gz\",\n",
    "   \"T0977.TS_as_accepted.tar.gz\",\n",
    "   \"T0978.TS_as_accepted.tar.gz\",\n",
    "   \"T0979.TS_as_accepted.tar.gz\",\n",
    "   \"T0980s1.TS_as_accepted.tar.gz\",\n",
    "   \"T0980s2.TS_as_accepted.tar.gz\",\n",
    "   \"T0981.TS_as_accepted.tar.gz\",\n",
    "   \"T0982.TS_as_accepted.tar.gz\",\n",
    "   \"T0983.TS_as_accepted.tar.gz\",\n",
    "   \"T0984.TS_as_accepted.tar.gz\",\n",
    "   \"T0985.TS_as_accepted.tar.gz\",\n",
    "   \"T0986s1.TS_as_accepted.tar.gz\",\n",
    "   \"T0986s2.TS_as_accepted.tar.gz\",\n",
    "   \"T0987.TS_as_accepted.tar.gz\",\n",
    "   \"T0988.TS_as_accepted.tar.gz\",\n",
    "   \"T0989.TS_as_accepted.tar.gz\",\n",
    "   \"T0990.TS_as_accepted.tar.gz\",\n",
    "   \"T0991.TS_as_accepted.tar.gz\",\n",
    "   \"T0992.TS_as_accepted.tar.gz\",\n",
    "   \"T0993s1.TS_as_accepted.tar.gz\",\n",
    "   \"T0993s2.TS_as_accepted.tar.gz\",\n",
    "   \"T0994.TS_as_accepted.tar.gz\",\n",
    "   \"T0995.TS_as_accepted.tar.gz\",\n",
    "   \"T0996.TS_as_accepted.tar.gz\",\n",
    "   \"T0997.TS_as_accepted.tar.gz\",\n",
    "   \"T0998.TS_as_accepted.tar.gz\",\n",
    "   \"T0999.TS_as_accepted.tar.gz\",\n",
    "   \"T1000.TS_as_accepted.tar.gz\",\n",
    "   \"T1001.TS_as_accepted.tar.gz\",\n",
    "   \"T1002.TS_as_accepted.tar.gz\",\n",
    "   \"T1003.TS_as_accepted.tar.gz\",\n",
    "   \"T1004.TS_as_accepted.tar.gz\",\n",
    "   \"T1005.TS_as_accepted.tar.gz\",\n",
    "   \"T1006.TS_as_accepted.tar.gz\",\n",
    "   \"T1007.TS_as_accepted.tar.gz\",\n",
    "   \"T1008.TS_as_accepted.tar.gz\",\n",
    "   \"T1009.TS_as_accepted.tar.gz\",\n",
    "   \"T1010.TS_as_accepted.tar.gz\",\n",
    "   \"T1011.TS_as_accepted.tar.gz\",\n",
    "   \"T1012.TS_as_accepted.tar.gz\",\n",
    "   \"T1013.TS_as_accepted.tar.gz\",\n",
    "   \"T1014.TS_as_accepted.tar.gz\",\n",
    "   \"T1015s1.TS_as_accepted.tar.gz\",\n",
    "   \"T1015s2.TS_as_accepted.tar.gz\",\n",
    "   \"T1016.TS_as_accepted.tar.gz\",\n",
    "   \"T1017s1.TS_as_accepted.tar.gz\",\n",
    "   \"T1017s2.TS_as_accepted.tar.gz\",\n",
    "   \"T1018.TS_as_accepted.tar.gz\",\n",
    "   \"T1019s1.TS_as_accepted.tar.gz\",\n",
    "   \"T1019s2.TS_as_accepted.tar.gz\",\n",
    "   \"T1020.TS_as_accepted.tar.gz\",\n",
    "   \"T1021s1.TS_as_accepted.tar.gz\",\n",
    "   \"T1021s2.TS_as_accepted.tar.gz\",\n",
    "   \"T1021s3.TS_as_accepted.tar.gz\",\n",
    "   \"T1022s1.TS_as_accepted.tar.gz\",\n",
    "   \"T1022s2.TS_as_accepted.tar.gz\",\n",
    "   \"T1023s1.TS_as_accepted.tar.gz\",\n",
    "   \"T1023s2.TS_as_accepted.tar.gz\",\n",
    "   \"T1023s3.TS_as_accepted.tar.gz\",\n",
    "]\n",
    "\n",
    "for url in urls:\n",
    "    target_dir = Path('/tmp/casp13').joinpath(url[:-7])\n",
    "    if not target_dir.is_dir():\n",
    "        target_dir.mkdir(parents=True)\n",
    "        ! curl -s {base_url}{url} | tar xz --strip 2 --directory {target_dir.as_posix()}\n",
    "    \n",
    "    for p in target_dir.iterdir():\n",
    "        try:\n",
    "            with p.open() as f:\n",
    "                for l in f:\n",
    "                    if l.startswith('TARGET'):\n",
    "                        target_id = l.split()[1]\n",
    "                    if l.startswith('AUTHOR'):\n",
    "                        decoy_id = l.split()[1]\n",
    "                    if l.startswith('MODEL'):\n",
    "                        model = l.split()[1]\n",
    "                        break\n",
    "                decoy_id = f'{decoy_id}_TS{model}'\n",
    "                mapping[p.name] = (target_id, decoy_id)\n",
    "                del target_id, decoy_id, model\n",
    "        except UnicodeDecodeError as e:\n",
    "            print(p, e)\n",
    "        except IndexError as e:\n",
    "            print(p, repr(l), e)\n",
    "        except ValueError as e:\n",
    "            print(p, repr(l), e)\n",
    "            \n",
    "! rm -r '/tmp/casp13'\n",
    "with open('CASP13/decoy_name_mapping.pkl', 'wb') as f:\n",
    "    pickle.dump(mapping, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parsing QA submissions\n",
    "\n",
    "These are some QA submissions for target `T0759`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T0759QA008_1\n",
      "T0759QA008_2\n",
      "T0759QA020_1\n"
     ]
    }
   ],
   "source": [
    "! ls CASP11/QA_predictions/T0759 | head -n3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These are the predictions made by QA group `QA008` for all decoys of target `T0759` submitted in stage `1`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PFRMAT QA\n",
      "TARGET T0759\n",
      "MODEL 1\n",
      "QMODE 2\n",
      "server04_TS1 0.391 6.073 5.287 5.008 3.850 3.101 3.084 2.224 1.875 7.066 2.143 2.003 2.066 2.202 2.197 2.393 2.215 \n",
      "2.140 2.488 2.329 2.582 2.470 2.333 2.127 2.227 2.140 2.062 2.075 2.129 2.002 2.656 2.265 \n",
      "2.256 2.227 2.110 2.494 2.395 2.373 2.187 2.232 7.886 7.705 2.420 2.486 3.108 2.026 1.996 \n",
      "2.062 2.024 1.949 2.024 2.419 2.061 2.773 2.942 2.611 2.524 6.676 2.471 2.170 2.205 2.426 \n",
      "2.262 2.142 2.321 2.822 2.204 3.025 2.476 2.272 5.215 2.273 2.535 8.080 2.579 2.167 2.629 \n",
      "2.555 2.656 2.065 3.211 2.526 2.918 2.433 2.512 2.593 2.882 2.293 2.728 2.710 2.429 2.337 \n"
     ]
    }
   ],
   "source": [
    "! head -n 10 CASP11/QA_predictions/T0759/T0759QA008_1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These are the predictions made by QA group `QA008` for all decoys of target `T0759` submitted in stage `2`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PFRMAT QA\n",
      "TARGET T0759\n",
      "MODEL 2\n",
      "QMODE 2\n",
      "MUFOLD-Server_TS2 0.660 7.167 6.428 5.492 4.609 3.464 3.637 2.478 2.194 7.383 2.368 3.012 2.580 2.309 2.271 2.672 2.409 \n",
      "7.838 2.410 2.324 2.495 2.432 2.362 2.159 2.413 2.208 2.154 2.220 2.319 2.091 2.778 2.279 \n",
      "2.181 2.026 1.903 2.280 2.099 2.137 2.010 1.996 2.179 7.554 2.302 2.447 3.196 2.105 1.993 \n",
      "2.075 1.996 1.853 1.940 2.319 1.946 2.513 2.723 2.376 2.313 2.704 2.304 2.084 2.138 2.442 \n",
      "2.213 2.101 2.276 2.702 2.082 2.401 2.273 2.135 2.154 2.047 2.300 7.870 2.440 2.027 2.420 \n",
      "2.393 2.718 2.046 3.100 2.502 2.908 2.638 2.529 8.209 3.034 2.380 3.049 2.802 2.502 2.548 \n"
     ]
    }
   ],
   "source": [
    "! head -n 10 CASP11/QA_predictions/T0759/T0759QA008_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_float_score(score):\n",
    "    if score == 'X':\n",
    "        return float('NaN')\n",
    "    return float(score)\n",
    "\n",
    "def parse_filename(path):\n",
    "    target, rest = path.name.split('QA')\n",
    "    qa_group, stage = rest.split('_')\n",
    "    return target, f'QA{qa_group}', int(stage)\n",
    "\n",
    "@logger.catch(reraise=True)\n",
    "def parse_qa_submission(path):   \n",
    "    with open(path) as f:\n",
    "        pformat = f.readline().split()\n",
    "        if pformat != ['PFRMAT', 'QA']:\n",
    "            raise ValueError(pformat)\n",
    "        target = f.readline().split()[1]\n",
    "        stage = int(f.readline().split()[1])\n",
    "        qmode = int(f.readline().split()[1])\n",
    "\n",
    "        if qmode == 1:\n",
    "            split = f.readline().split()\n",
    "            while split[0] != 'END':\n",
    "                decoy, global_score = split\n",
    "                yield decoy, parse_float_score(global_score), None\n",
    "                split = f.readline().split()\n",
    "        elif qmode == 2:\n",
    "            decoy = None\n",
    "            global_score = None\n",
    "            local_scores = None\n",
    "\n",
    "            line = f.readline()\n",
    "            while line != '' and line.strip() != 'END':\n",
    "                split = line.split()\n",
    "                decoy = split[0]\n",
    "                global_score = parse_float_score(split[1])\n",
    "                local_scores = [parse_float_score(s) for s in split[2:]]\n",
    "\n",
    "                try:\n",
    "                    while line != '':\n",
    "                        line = f.readline()\n",
    "                        split = line.split()\n",
    "                        local_scores.extend(parse_float_score(s) for s in split)\n",
    "                except ValueError:\n",
    "                    pass\n",
    "                yield decoy, global_score, local_scores\n",
    "        else:\n",
    "            raise ValueError(qmode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d8eed02657ce492f8f370090f1b7277e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='CASP11', max=98.0, style=ProgressStyle(description_width=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Raw dataframe\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>pred</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>qa_group_id</th>\n",
       "      <th>target_id</th>\n",
       "      <th>decoy_id</th>\n",
       "      <th>stage</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">BITS</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0759</th>\n",
       "      <th>3D-Jigsaw-V5_1_TS1</th>\n",
       "      <th>2</th>\n",
       "      <td>0.4740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3D-Jigsaw-V5_1_TS2</th>\n",
       "      <th>2</th>\n",
       "      <td>0.4765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3D-Jigsaw-V5_1_TS3</th>\n",
       "      <th>2</th>\n",
       "      <td>0.5718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3D-Jigsaw-V5_1_TS4</th>\n",
       "      <th>2</th>\n",
       "      <td>0.6246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3D-Jigsaw-V5_1_TS5</th>\n",
       "      <th>2</th>\n",
       "      <td>0.4703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">raghavagps-qaspro</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0858</th>\n",
       "      <th>slbio_TS1</th>\n",
       "      <th>2</th>\n",
       "      <td>0.7600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>slbio_TS2</th>\n",
       "      <th>2</th>\n",
       "      <td>0.8200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>slbio_TS3</th>\n",
       "      <th>2</th>\n",
       "      <td>0.8300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>slbio_TS4</th>\n",
       "      <th>2</th>\n",
       "      <td>0.8400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>slbio_TS5</th>\n",
       "      <th>2</th>\n",
       "      <td>0.7800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>584627 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                        pred\n",
       "qa_group_id       target_id decoy_id           stage        \n",
       "BITS              T0759     3D-Jigsaw-V5_1_TS1 2      0.4740\n",
       "                            3D-Jigsaw-V5_1_TS2 2      0.4765\n",
       "                            3D-Jigsaw-V5_1_TS3 2      0.5718\n",
       "                            3D-Jigsaw-V5_1_TS4 2      0.6246\n",
       "                            3D-Jigsaw-V5_1_TS5 2      0.4703\n",
       "...                                                      ...\n",
       "raghavagps-qaspro T0858     slbio_TS1          2      0.7600\n",
       "                            slbio_TS2          2      0.8200\n",
       "                            slbio_TS3          2      0.8300\n",
       "                            slbio_TS4          2      0.8400\n",
       "                            slbio_TS5          2      0.7800\n",
       "\n",
       "[584627 rows x 1 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of decoys of each target scored by each group in each stage\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stage</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>qa_group_id</th>\n",
       "      <th>target_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">BITS</th>\n",
       "      <th>T0759</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0760</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0761</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0762</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0763</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">raghavagps-qaspro</th>\n",
       "      <th>T0854</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0855</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0856</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0857</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0858</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3538 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "stage                         1    2\n",
       "qa_group_id       target_id         \n",
       "BITS              T0759      20  150\n",
       "                  T0760      20  150\n",
       "                  T0761      20  150\n",
       "                  T0762      20  150\n",
       "                  T0763      20  150\n",
       "...                          ..  ...\n",
       "raghavagps-qaspro T0854      20  150\n",
       "                  T0855      20  150\n",
       "                  T0856      20  150\n",
       "                  T0857      20  150\n",
       "                  T0858      20  150\n",
       "\n",
       "[3538 rows x 2 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of targets considered by each group in each CASP\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">target_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stage</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>qa_group_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BITS</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ConsMQAPsingle</th>\n",
       "      <td>89</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DAVIS-QAconsensus</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DAVIS-QAconsensusALL</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DandekarLab</th>\n",
       "      <td>36</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FUSION</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LNCCUnB</th>\n",
       "      <td>72</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MQAPmulti</th>\n",
       "      <td>89</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MQAPsingle</th>\n",
       "      <td>89</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MQAPsingleA</th>\n",
       "      <td>89</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MQAPsingleB</th>\n",
       "      <td>89</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MUFOLD-DQA</th>\n",
       "      <td>18</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MUFOLD-QA</th>\n",
       "      <td>98</td>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MUFOLD-Server</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MULTICOM-CLUSTER</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MULTICOM-CONSTRUCT</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MULTICOM-NOVEL</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MULTICOM-REFINE</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ModFOLD5</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ModFOLD5_single</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ModFOLDclust2</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>OccuScore</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pcons-net</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PconsD</th>\n",
       "      <td>96</td>\n",
       "      <td>95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ProQ2</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ProQ2-refine</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Qpot</th>\n",
       "      <td>36</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Qpotclust</th>\n",
       "      <td>36</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Qpotfilt</th>\n",
       "      <td>36</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RFMQA</th>\n",
       "      <td>98</td>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VoroMQA</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wallner</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wang_SVM</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wang_deep_1</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wang_deep_2</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wang_deep_3</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>keasar</th>\n",
       "      <td>98</td>\n",
       "      <td>90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>myprotein-me</th>\n",
       "      <td>97</td>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nns</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>raghavagps-qaspro</th>\n",
       "      <td>98</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     target_id    \n",
       "stage                        1   2\n",
       "qa_group_id                       \n",
       "BITS                        98  97\n",
       "ConsMQAPsingle              89  82\n",
       "DAVIS-QAconsensus           98  97\n",
       "DAVIS-QAconsensusALL        98  97\n",
       "DandekarLab                 36  36\n",
       "FUSION                      98  97\n",
       "LNCCUnB                     72  77\n",
       "MQAPmulti                   89  82\n",
       "MQAPsingle                  89  82\n",
       "MQAPsingleA                 89  81\n",
       "MQAPsingleB                 89  82\n",
       "MUFOLD-DQA                  18  14\n",
       "MUFOLD-QA                   98  96\n",
       "MUFOLD-Server               98  97\n",
       "MULTICOM-CLUSTER            98  97\n",
       "MULTICOM-CONSTRUCT          98  97\n",
       "MULTICOM-NOVEL              98  97\n",
       "MULTICOM-REFINE             98  97\n",
       "ModFOLD5                    98  97\n",
       "ModFOLD5_single             98  97\n",
       "ModFOLDclust2               98  97\n",
       "OccuScore                   98  97\n",
       "Pcons-net                   98  97\n",
       "PconsD                      96  95\n",
       "ProQ2                       98  97\n",
       "ProQ2-refine                98  97\n",
       "Qpot                        36  33\n",
       "Qpotclust                   36  33\n",
       "Qpotfilt                    36  29\n",
       "RFMQA                       98  96\n",
       "VoroMQA                     98  97\n",
       "Wallner                     98  97\n",
       "Wang_SVM                    98  97\n",
       "Wang_deep_1                 98  97\n",
       "Wang_deep_2                 98  97\n",
       "Wang_deep_3                 98  97\n",
       "keasar                      98  90\n",
       "myprotein-me                97  96\n",
       "nns                         98  97\n",
       "raghavagps-qaspro           98  97"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Raw dataframe\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>pred</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>qa_group_id</th>\n",
       "      <th>target_id</th>\n",
       "      <th>decoy_id</th>\n",
       "      <th>stage</th>\n",
       "      <th>residue_idx</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">ConsMQAPsingle</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0759</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">server01_TS1</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">myprotein-me</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0858</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">slbio_TS5</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">2</th>\n",
       "      <th>489</th>\n",
       "      <td>0.3981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>490</th>\n",
       "      <td>0.7188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491</th>\n",
       "      <td>1.3557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>492</th>\n",
       "      <td>2.7639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>493</th>\n",
       "      <td>2.7273</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>112455338 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                           pred\n",
       "qa_group_id    target_id decoy_id     stage residue_idx        \n",
       "ConsMQAPsingle T0759     server01_TS1 1     0               NaN\n",
       "                                            1               NaN\n",
       "                                            2               NaN\n",
       "                                            3               NaN\n",
       "                                            4               NaN\n",
       "...                                                         ...\n",
       "myprotein-me   T0858     slbio_TS5    2     489          0.3981\n",
       "                                            490          0.7188\n",
       "                                            491          1.3557\n",
       "                                            492          2.7639\n",
       "                                            493          2.7273\n",
       "\n",
       "[112455338 rows x 1 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "By stage\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stage</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>qa_group_id</th>\n",
       "      <th>target_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">ConsMQAPsingle</th>\n",
       "      <th>T0759</th>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0760</th>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0761</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0762</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0763</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">myprotein-me</th>\n",
       "      <th>T0854</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0855</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0856</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0857</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0858</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2591 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "stage                      1    2\n",
       "qa_group_id    target_id         \n",
       "ConsMQAPsingle T0759      20    0\n",
       "               T0760      19    0\n",
       "               T0761      20  150\n",
       "               T0762      20  150\n",
       "               T0763      20  150\n",
       "...                       ..  ...\n",
       "myprotein-me   T0854      20  150\n",
       "               T0855      20  150\n",
       "               T0856      20  150\n",
       "               T0857      20  150\n",
       "               T0858      20  150\n",
       "\n",
       "[2591 rows x 2 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d6253d0351ca48c8a23c9e9774b90574",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='CASP12', max=70.0, style=ProgressStyle(description_width=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Raw dataframe\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>pred</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>qa_group_id</th>\n",
       "      <th>target_id</th>\n",
       "      <th>decoy_id</th>\n",
       "      <th>stage</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">COFOLD_QA</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0859</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">T0859TS005_1</th>\n",
       "      <th>1</th>\n",
       "      <td>0.1809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.1960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0859TS005_2</th>\n",
       "      <th>2</th>\n",
       "      <td>0.2052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0859TS005_3</th>\n",
       "      <th>2</th>\n",
       "      <td>0.2176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0859TS005_4</th>\n",
       "      <th>2</th>\n",
       "      <td>0.2226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">qSVMQA</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0948</th>\n",
       "      <th>T0948TS479_1</th>\n",
       "      <th>2</th>\n",
       "      <td>0.7091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0948TS479_2</th>\n",
       "      <th>2</th>\n",
       "      <td>0.6680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0948TS479_3</th>\n",
       "      <th>2</th>\n",
       "      <td>0.7037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0948TS479_4</th>\n",
       "      <th>2</th>\n",
       "      <td>0.6374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0948TS479_5</th>\n",
       "      <th>2</th>\n",
       "      <td>0.6440</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>503847 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            pred\n",
       "qa_group_id target_id decoy_id     stage        \n",
       "COFOLD_QA   T0859     T0859TS005_1 1      0.1809\n",
       "                                   2      0.1960\n",
       "                      T0859TS005_2 2      0.2052\n",
       "                      T0859TS005_3 2      0.2176\n",
       "                      T0859TS005_4 2      0.2226\n",
       "...                                          ...\n",
       "qSVMQA      T0948     T0948TS479_1 2      0.7091\n",
       "                      T0948TS479_2 2      0.6680\n",
       "                      T0948TS479_3 2      0.7037\n",
       "                      T0948TS479_4 2      0.6374\n",
       "                      T0948TS479_5 2      0.6440\n",
       "\n",
       "[503847 rows x 1 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of decoys of each target scored by each group in each stage\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stage</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>qa_group_id</th>\n",
       "      <th>target_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">COFOLD_QA</th>\n",
       "      <th>T0859</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0860</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0861</th>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0862</th>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0863</th>\n",
       "      <td>20</td>\n",
       "      <td>132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">qSVMQA</th>\n",
       "      <th>T0944</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0945</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0946</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0947</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0948</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3020 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "stage                   1    2\n",
       "qa_group_id target_id         \n",
       "COFOLD_QA   T0859      20  150\n",
       "            T0860      20  150\n",
       "            T0861       0  150\n",
       "            T0862       0  150\n",
       "            T0863      20  132\n",
       "...                    ..  ...\n",
       "qSVMQA      T0944      20  150\n",
       "            T0945      20  150\n",
       "            T0946      20  150\n",
       "            T0947      20  150\n",
       "            T0948      20  150\n",
       "\n",
       "[3020 rows x 2 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of targets considered by each group in each CASP\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">target_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stage</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>qa_group_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>COFOLD_QA</th>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Davis-EMAconsensus</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Davis-EMAconsensusAL</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DeepFold-Boom</th>\n",
       "      <td>68</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Deepfold-Contact</th>\n",
       "      <td>68</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FDUBio</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GOAL</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Grudinin-DeepL</th>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Kiharalab</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MESHI_CON_SERVER</th>\n",
       "      <td>68</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MESHI_SERVER</th>\n",
       "      <td>69</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MULTICOM-CLUSTER</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MUfold2</th>\n",
       "      <td>69</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MUfoldQA_C</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MUfoldQA_S</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ModFOLD6</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ModFOLD6_cor</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ModFOLD6_rank</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ModFOLDclust2</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pcomb-domain</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pcons</th>\n",
       "      <td>66</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pcons-net</th>\n",
       "      <td>61</td>\n",
       "      <td>63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ProQ2</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ProQ3</th>\n",
       "      <td>69</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ProQ3_1</th>\n",
       "      <td>69</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ProQ3_1_diso</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ProQ3_2_diso</th>\n",
       "      <td>69</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ProTSAV-Plus</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>QASproCL</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>QASproGP</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>QMEAN</th>\n",
       "      <td>68</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RSA_SS_CONS</th>\n",
       "      <td>68</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SVMQA</th>\n",
       "      <td>69</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Seok-server</th>\n",
       "      <td>69</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VoroMQA</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VoroMQAsr</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wallner</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wang1</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wang2</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wang3</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wang4</th>\n",
       "      <td>70</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ZHOU-SPARKS-X</th>\n",
       "      <td>64</td>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>iFold_2</th>\n",
       "      <td>68</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>myprotein-me</th>\n",
       "      <td>55</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>naive</th>\n",
       "      <td>68</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>qSVMQA</th>\n",
       "      <td>69</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     target_id    \n",
       "stage                        1   2\n",
       "qa_group_id                       \n",
       "COFOLD_QA                   10   5\n",
       "Davis-EMAconsensus          70  70\n",
       "Davis-EMAconsensusAL        70  70\n",
       "DeepFold-Boom               68  70\n",
       "Deepfold-Contact            68  70\n",
       "FDUBio                      70  70\n",
       "GOAL                         1   4\n",
       "Grudinin-DeepL               7   8\n",
       "Kiharalab                   70  70\n",
       "MESHI_CON_SERVER            68  53\n",
       "MESHI_SERVER                69  64\n",
       "MULTICOM-CLUSTER            70  70\n",
       "MUfold2                     69  70\n",
       "MUfoldQA_C                  70  70\n",
       "MUfoldQA_S                  70  70\n",
       "ModFOLD6                    70  70\n",
       "ModFOLD6_cor                70  70\n",
       "ModFOLD6_rank               70  70\n",
       "ModFOLDclust2               70  70\n",
       "Pcomb-domain                70  70\n",
       "Pcons                       66  70\n",
       "Pcons-net                   61  63\n",
       "ProQ2                       70  70\n",
       "ProQ3                       69  70\n",
       "ProQ3_1                     69  70\n",
       "ProQ3_1_diso                70  70\n",
       "ProQ3_2_diso                69  70\n",
       "ProTSAV-Plus                70  70\n",
       "QASproCL                    70  70\n",
       "QASproGP                    70  70\n",
       "QMEAN                       68  68\n",
       "RSA_SS_CONS                 68  70\n",
       "SVMQA                       69  70\n",
       "Seok-server                 69  70\n",
       "VoroMQA                     70  70\n",
       "VoroMQAsr                   70  70\n",
       "Wallner                     70  70\n",
       "Wang1                       70  70\n",
       "Wang2                       70  70\n",
       "Wang3                       70  70\n",
       "Wang4                       70  70\n",
       "ZHOU-SPARKS-X               64  67\n",
       "iFold_2                     68  70\n",
       "myprotein-me                55  57\n",
       "naive                       68  70\n",
       "qSVMQA                      69  70"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Raw dataframe\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>pred</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>qa_group_id</th>\n",
       "      <th>target_id</th>\n",
       "      <th>decoy_id</th>\n",
       "      <th>stage</th>\n",
       "      <th>residue_idx</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">Davis-EMAconsensus</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0859</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0859TS005_1</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>0</th>\n",
       "      <td>40.568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>36.560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>36.961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>39.828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>37.549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">ZHOU-SPARKS-X</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0948</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0948TS479_5</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">2</th>\n",
       "      <th>161</th>\n",
       "      <td>8.702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162</th>\n",
       "      <td>15.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163</th>\n",
       "      <td>15.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>15.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165</th>\n",
       "      <td>15.000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>78068587 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                               pred\n",
       "qa_group_id        target_id decoy_id     stage residue_idx        \n",
       "Davis-EMAconsensus T0859     T0859TS005_1 1     0            40.568\n",
       "                                                1            36.560\n",
       "                                                2            36.961\n",
       "                                                3            39.828\n",
       "                                                4            37.549\n",
       "...                                                             ...\n",
       "ZHOU-SPARKS-X      T0948     T0948TS479_5 2     161           8.702\n",
       "                                                162          15.000\n",
       "                                                163          15.000\n",
       "                                                164          15.000\n",
       "                                                165          15.000\n",
       "\n",
       "[78068587 rows x 1 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "By stage\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stage</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>qa_group_id</th>\n",
       "      <th>target_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">Davis-EMAconsensus</th>\n",
       "      <th>T0859</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0860</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0861</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0862</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0863</th>\n",
       "      <td>20</td>\n",
       "      <td>132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">ZHOU-SPARKS-X</th>\n",
       "      <th>T0944</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0945</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0946</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0947</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0948</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1674 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "stage                          1    2\n",
       "qa_group_id        target_id         \n",
       "Davis-EMAconsensus T0859      20  150\n",
       "                   T0860      20  150\n",
       "                   T0861      20  150\n",
       "                   T0862      20  150\n",
       "                   T0863      20  132\n",
       "...                           ..  ...\n",
       "ZHOU-SPARKS-X      T0944      20  150\n",
       "                   T0945      20  150\n",
       "                   T0946      20  150\n",
       "                   T0947      20  150\n",
       "                   T0948      20  150\n",
       "\n",
       "[1674 rows x 2 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "869098f18b574d39afb276cefbef4718",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='CASP13', max=86.0, style=ProgressStyle(description_width=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Raw dataframe\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>pred</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>qa_group_id</th>\n",
       "      <th>target_id</th>\n",
       "      <th>decoy_id</th>\n",
       "      <th>stage</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">3DCNN</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0962</th>\n",
       "      <th>1270-5526-8383_TS1</th>\n",
       "      <th>2</th>\n",
       "      <td>0.369067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1270-5526-8383_TS2</th>\n",
       "      <th>2</th>\n",
       "      <td>0.353299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1270-5526-8383_TS3</th>\n",
       "      <th>2</th>\n",
       "      <td>0.362075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1270-5526-8383_TS4</th>\n",
       "      <th>2</th>\n",
       "      <td>0.399744</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1312-6135-4624_TS1</th>\n",
       "      <th>2</th>\n",
       "      <td>0.655809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">Yang-Server</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0950</th>\n",
       "      <th>ACOMPMOD_TS5</th>\n",
       "      <th>1</th>\n",
       "      <td>0.332000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MULTICOM-CONSTRUCT_TS5</th>\n",
       "      <th>1</th>\n",
       "      <td>0.338000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RaptorX-DeepModeller_TS2</th>\n",
       "      <th>1</th>\n",
       "      <td>0.339000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>YASARA_TS1</th>\n",
       "      <th>1</th>\n",
       "      <td>0.335000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>YASARA_TS3</th>\n",
       "      <th>1</th>\n",
       "      <td>0.334000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>713462 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                          pred\n",
       "qa_group_id target_id decoy_id                 stage          \n",
       "3DCNN       T0962     1270-5526-8383_TS1       2      0.369067\n",
       "                      1270-5526-8383_TS2       2      0.353299\n",
       "                      1270-5526-8383_TS3       2      0.362075\n",
       "                      1270-5526-8383_TS4       2      0.399744\n",
       "                      1312-6135-4624_TS1       2      0.655809\n",
       "...                                                        ...\n",
       "Yang-Server T0950     ACOMPMOD_TS5             1      0.332000\n",
       "                      MULTICOM-CONSTRUCT_TS5   1      0.338000\n",
       "                      RaptorX-DeepModeller_TS2 1      0.339000\n",
       "                      YASARA_TS1               1      0.335000\n",
       "                      YASARA_TS3               1      0.334000\n",
       "\n",
       "[713462 rows x 1 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of decoys of each target scored by each group in each stage\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stage</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>qa_group_id</th>\n",
       "      <th>target_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">3DCNN</th>\n",
       "      <th>T0962</th>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0969</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0970</th>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0971</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0972</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">Wallner</th>\n",
       "      <th>T1021s3</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1022s1</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1022s2</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Yang-Server</th>\n",
       "      <th>T0949</th>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0950</th>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4273 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "stage                   1    2\n",
       "qa_group_id target_id         \n",
       "3DCNN       T0962       0  150\n",
       "            T0969      20  150\n",
       "            T0970       0  150\n",
       "            T0971      20  150\n",
       "            T0972      20  150\n",
       "...                    ..  ...\n",
       "Wallner     T1021s3    20  150\n",
       "            T1022s1    20  150\n",
       "            T1022s2    20  150\n",
       "Yang-Server T0949       0  150\n",
       "            T0950      20    0\n",
       "\n",
       "[4273 rows x 2 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of targets considered by each group in each CASP\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">target_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stage</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>qa_group_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3DCNN</th>\n",
       "      <td>62</td>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bhattacharya-ClustQ</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bhattacharya-Server</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bhattacharya-SingQ</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CPClab</th>\n",
       "      <td>83</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Davis-EMAconsensus</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Davis-EMAconsensusAL</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Eagle</th>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FALCON-QA</th>\n",
       "      <td>86</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FaeNNz</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Grudinin</th>\n",
       "      <td>81</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Jagodzinski-Cao-QA</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Kiharalab</th>\n",
       "      <td>84</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LamoureuxLab</th>\n",
       "      <td>79</td>\n",
       "      <td>83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MASS1</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MASS2</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MESHI</th>\n",
       "      <td>73</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MESHI-corr-server</th>\n",
       "      <td>85</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MESHI-enrich-server</th>\n",
       "      <td>85</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MESHI-server</th>\n",
       "      <td>79</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MUFoldQA_M</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MUFold_server</th>\n",
       "      <td>86</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MULTICOM-CONSTRUCT</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MULTICOM-NOVEL</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MULTICOM_CLUSTER</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MUfoldQA_S2</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MUfoldQA_T</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ModFOLD7</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ModFOLD7_cor</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ModFOLD7_rank</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ModFOLDclust2</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PLU-AngularQA</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PLU-TopQA</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pcomb</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pcons</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ProQ2</th>\n",
       "      <td>85</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ProQ3</th>\n",
       "      <td>82</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ProQ3D</th>\n",
       "      <td>81</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ProQ3D-CAD</th>\n",
       "      <td>81</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ProQ3D-TM</th>\n",
       "      <td>81</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ProQ3D-lDDT</th>\n",
       "      <td>81</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ProQ4</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RaptorX-DeepQA</th>\n",
       "      <td>84</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SASHAN</th>\n",
       "      <td>84</td>\n",
       "      <td>83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SBROD</th>\n",
       "      <td>81</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SBROD-plus</th>\n",
       "      <td>80</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SBROD-server</th>\n",
       "      <td>80</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UOSHAN</th>\n",
       "      <td>84</td>\n",
       "      <td>83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VoroMQA-A</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VoroMQA-B</th>\n",
       "      <td>86</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wallner</th>\n",
       "      <td>85</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Yang-Server</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     target_id    \n",
       "stage                        1   2\n",
       "qa_group_id                       \n",
       "3DCNN                       62  62\n",
       "Bhattacharya-ClustQ         86  86\n",
       "Bhattacharya-Server         86  86\n",
       "Bhattacharya-SingQ          86  86\n",
       "CPClab                      83  80\n",
       "Davis-EMAconsensus          86  86\n",
       "Davis-EMAconsensusAL        86  86\n",
       "Eagle                        6   6\n",
       "FALCON-QA                   86  80\n",
       "FaeNNz                      86  86\n",
       "Grudinin                    81  77\n",
       "Jagodzinski-Cao-QA          86  86\n",
       "Kiharalab                   84  86\n",
       "LamoureuxLab                79  83\n",
       "MASS1                       86  86\n",
       "MASS2                       86  86\n",
       "MESHI                       73  81\n",
       "MESHI-corr-server           85  86\n",
       "MESHI-enrich-server         85  86\n",
       "MESHI-server                79  81\n",
       "MUFoldQA_M                  86  86\n",
       "MUFold_server               86  85\n",
       "MULTICOM-CONSTRUCT          86  86\n",
       "MULTICOM-NOVEL              86  86\n",
       "MULTICOM_CLUSTER            86  86\n",
       "MUfoldQA_S2                 86  86\n",
       "MUfoldQA_T                  86  86\n",
       "ModFOLD7                    86  86\n",
       "ModFOLD7_cor                86  86\n",
       "ModFOLD7_rank               86  86\n",
       "ModFOLDclust2               86  86\n",
       "PLU-AngularQA               86  86\n",
       "PLU-TopQA                   86  86\n",
       "Pcomb                       86  86\n",
       "Pcons                       86  86\n",
       "ProQ2                       85  85\n",
       "ProQ3                       82  85\n",
       "ProQ3D                      81  85\n",
       "ProQ3D-CAD                  81  85\n",
       "ProQ3D-TM                   81  85\n",
       "ProQ3D-lDDT                 81  85\n",
       "ProQ4                       86  86\n",
       "RaptorX-DeepQA              84  81\n",
       "SASHAN                      84  83\n",
       "SBROD                       81  77\n",
       "SBROD-plus                  80  80\n",
       "SBROD-server                80  80\n",
       "UOSHAN                      84  83\n",
       "VoroMQA-A                   86  86\n",
       "VoroMQA-B                   86  86\n",
       "Wallner                     85  85\n",
       "Yang-Server                  1   1"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Raw dataframe\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>pred</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>qa_group_id</th>\n",
       "      <th>target_id</th>\n",
       "      <th>decoy_id</th>\n",
       "      <th>stage</th>\n",
       "      <th>residue_idx</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">3DCNN</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0962</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">1270-5526-8383_TS1</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">2</th>\n",
       "      <th>0</th>\n",
       "      <td>2.779862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.369109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.471484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.124728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.437495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">Yang-Server</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0950</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">YASARA_TS3</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>348</th>\n",
       "      <td>9.600000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>349</th>\n",
       "      <td>9.650000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>350</th>\n",
       "      <td>9.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>351</th>\n",
       "      <td>9.900000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352</th>\n",
       "      <td>9.690000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>112030404 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                pred\n",
       "qa_group_id target_id decoy_id           stage residue_idx          \n",
       "3DCNN       T0962     1270-5526-8383_TS1 2     0            2.779862\n",
       "                                               1            3.369109\n",
       "                                               2            1.471484\n",
       "                                               3            2.124728\n",
       "                                               4            1.437495\n",
       "...                                                              ...\n",
       "Yang-Server T0950     YASARA_TS3         1     348          9.600000\n",
       "                                               349          9.650000\n",
       "                                               350          9.750000\n",
       "                                               351          9.900000\n",
       "                                               352          9.690000\n",
       "\n",
       "[112030404 rows x 1 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "By stage\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stage</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>qa_group_id</th>\n",
       "      <th>target_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">3DCNN</th>\n",
       "      <th>T0962</th>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0969</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0970</th>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0971</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0972</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">Wallner</th>\n",
       "      <th>T1021s3</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1022s1</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1022s2</th>\n",
       "      <td>20</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Yang-Server</th>\n",
       "      <th>T0949</th>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T0950</th>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2311 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "stage                   1    2\n",
       "qa_group_id target_id         \n",
       "3DCNN       T0962       0  150\n",
       "            T0969      20  150\n",
       "            T0970       0  150\n",
       "            T0971      20  150\n",
       "            T0972      20  150\n",
       "...                    ..  ...\n",
       "Wallner     T1021s3    20  150\n",
       "            T1022s1    20  150\n",
       "            T1022s2    20  150\n",
       "Yang-Server T0949       0  150\n",
       "            T0950      20    0\n",
       "\n",
       "[2311 rows x 2 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for casp_ed in [11,12,13]: # [9,10,11,12,13]\n",
    "    df_global = []\n",
    "    df_local = {}\n",
    "    qa_group_names = pd.read_csv(f'CASP{casp_ed}/QA_groups.csv').set_index('qa_group_id')['qa_group_name'].to_dict()\n",
    "    \n",
    "    if casp_ed == 13:\n",
    "        with open('CASP13/decoy_name_mapping.pkl', 'rb') as f:\n",
    "            casp_13_decoy_mapping = pickle.load(f)\n",
    "    \n",
    "    parsed_decoys = 0\n",
    "    targets = list(Path(f'CASP{casp_ed}/QA_predictions').glob('T*'))\n",
    "    casp_bar = tqdm.tqdm(targets, desc=f'CASP{casp_ed}', unit='targets')\n",
    "    for target_path in casp_bar:\n",
    "        for path in target_path.glob('T*QA*'):\n",
    "            target_id, qa_group_id, stage = parse_filename(path)\n",
    "            qa_group_id = qa_group_names[qa_group_id]\n",
    "            \n",
    "            for decoy_id, global_score, local_scores in parse_qa_submission(path):\n",
    "                if casp_ed == 13:\n",
    "                    _, decoy_id = casp_13_decoy_mapping[decoy_id]\n",
    "                parsed_decoys += 1\n",
    "                df_global.append((qa_group_id, target_id, decoy_id, stage, global_score))\n",
    "                if local_scores is not None:\n",
    "                    local_scores = pd.Series(local_scores, name='pred').rename_axis('residue_idx')\n",
    "                    df_local[(qa_group_id, target_id, decoy_id, stage)] = local_scores\n",
    "        \n",
    "        casp_bar.set_postfix({'parsed decoys': parsed_decoys})\n",
    "    casp_bar.close()\n",
    "    \n",
    "    # Global scores\n",
    "    df_global = pd.DataFrame(df_global, columns=['qa_group_id', 'target_id', 'decoy_id', 'stage', 'pred'])\n",
    "    df_global.sort_values(['qa_group_id', 'target_id', 'decoy_id', 'stage', 'pred'], inplace=True)\n",
    "    (\n",
    "        df_global\n",
    "        .set_index(['qa_group_id', 'target_id', 'decoy_id', 'stage'])\n",
    "        .to_pickle(f'CASP{casp_ed}/QA_predictions/global.pkl')\n",
    "    )\n",
    "    \n",
    "    print('Raw dataframe')\n",
    "    display(df_global.set_index(['qa_group_id', 'target_id', 'decoy_id', 'stage']))\n",
    "\n",
    "    print('Number of decoys of each target scored by each group in each stage')\n",
    "    display(\n",
    "        df_global.groupby(['qa_group_id', 'target_id', 'stage'])\n",
    "        .size()\n",
    "        .unstack('stage', fill_value=0)\n",
    "    )\n",
    "\n",
    "    print('Number of targets considered by each group in each CASP')\n",
    "    display(\n",
    "        df_global\n",
    "        .groupby(['qa_group_id', 'stage'])\n",
    "        .agg({'target_id': 'nunique'})\n",
    "        .unstack('stage', fill_value=0)\n",
    "    )\n",
    "    del df_global\n",
    "    \n",
    "    # Local scores\n",
    "    df_local = {k: df_local[k] for k in sorted(df_local.keys())}\n",
    "    df_local = pd.concat(\n",
    "        df_local.values(), \n",
    "        keys=df_local.keys(), \n",
    "        names=['qa_group_id', 'target_id', 'decoy_id', 'stage']\n",
    "    ).to_frame('pred')\n",
    "    df_local.to_pickle(f'CASP{casp_ed}/QA_predictions/local.pkl')\n",
    "    \n",
    "    print('Raw dataframe')\n",
    "    display(df_local)\n",
    "    \n",
    "    print('By stage')\n",
    "    display(\n",
    "        df_local\n",
    "        .groupby(['qa_group_id', 'target_id', 'decoy_id', 'stage'])\n",
    "        .first()\n",
    "        .groupby(['qa_group_id', 'target_id', 'stage'])\n",
    "        .size()\n",
    "        .unstack('stage', fill_value=0)\n",
    "    )\n",
    "    del df_local"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.3M\tCASP11/QA_predictions/global.pkl\n",
      "6.5M\tCASP12/QA_predictions/global.pkl\n",
      "8.9M\tCASP13/QA_predictions/global.pkl\n",
      "23M\ttotal\n",
      "\n",
      "1.6G\tCASP11/QA_predictions/local.pkl\n",
      "1.1G\tCASP12/QA_predictions/local.pkl\n",
      "1.6G\tCASP13/QA_predictions/local.pkl\n",
      "4.3G\ttotal\n",
      "\n",
      "175M\tCASP11/QA_predictions/local.pkl.xz\n",
      "154M\tCASP12/QA_predictions/local.pkl.xz\n",
      "229M\tCASP13/QA_predictions/local.pkl.xz\n",
      "556M\ttotal\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "# Global QA predictions are small enough\n",
    "du -hsc CASP*/QA_predictions/global.pkl\n",
    "echo\n",
    "\n",
    "# Local QA predictions should be compressed\n",
    "du -hsc CASP*/QA_predictions/local.pkl\n",
    "for f in CASP*/QA_predictions/local.pkl; do\n",
    "  xz -1 --compress \"$f\"\n",
    "done\n",
    "echo\n",
    "\n",
    "du -hsc CASP*/QA_predictions/local.pkl.xz"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Official QA local scores\n",
    "\n",
    "Offical local scores (per-residue) are only avaliable for CASP 13."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CASP 13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11G\tCASP13/QA_official/\n"
     ]
    }
   ],
   "source": [
    "base_url = 'https://predictioncenter.org/download_area/CASP13/results/sda/'\n",
    "response = requests_get(base_url)\n",
    "soup = bs4.BeautifulSoup(response.content)\n",
    "\n",
    "links = [\n",
    "    a.attrs['href']\n",
    "    for a in soup.select('tr td:nth-child(2) a')\n",
    "    if re.search(r'T\\d{4}(?:s\\d)?.*\\.tgz', a.attrs['href'])\n",
    "]\n",
    "with tempfile.NamedTemporaryFile(mode='w') as f:\n",
    "    f.write('\\n'.join([base_url + l for l in links]))\n",
    "    ! wget --quiet --input-file {f.name} --directory-prefix='CASP13/QA_official'\n",
    "\n",
    "for l in links:\n",
    "    ! tar xf \"CASP13/QA_official/{l}\" --directory 'CASP13/QA_official/' && rm \"CASP13/QA_official/{l}\"\n",
    "\n",
    "! du -sh CASP13/QA_official/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('CASP13/decoy_name_mapping.pkl', 'rb') as f:\n",
    "    decoy_name_mapping = pickle.load(f)\n",
    "\n",
    "def parse_lga(lga_path):\n",
    "    residue_dist_mapping = {}\n",
    "    with open(lga_path) as f:\n",
    "        for l in filter(lambda l: l.startswith('LGA '), f):\n",
    "            residue_idx = int(l.split()[2]) - 1\n",
    "            distance = float(l.split()[5])\n",
    "            residue_dist_mapping[residue_idx] = distance\n",
    "    return residue_dist_mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-09-21 21:04:52.576 | WARNING  | __main__:<module>:43 - T0953s1TS401_1-D1 not found in decoy_name_mapping\n",
      "2020-09-21 21:04:53.076 | WARNING  | __main__:<module>:27 - T0953s2TS214_2 not found in decoy_name_mapping\n",
      "2020-09-21 21:04:53.440 | WARNING  | __main__:<module>:27 - T0953s2TS214_1 not found in decoy_name_mapping\n",
      "2020-09-21 21:05:01.260 | WARNING  | __main__:<module>:47 - Check CASP13/QA_official/T0964-D1/T0964TS397_1-D1.lga (T0964 3711-3160-9340_TS1)\n",
      "2020-09-21 21:05:01.425 | WARNING  | __main__:<module>:47 - Check CASP13/QA_official/T0964-D1/T0964TS397_2-D1.lga (T0964 3711-3160-9340_TS2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "144M\tCASP13/QA_official/distances_true.pkl\n"
     ]
    }
   ],
   "source": [
    "target_links_mapping = {}\n",
    "for l in links:\n",
    "    target_id = l.split('.')[0].split('-')[0]\n",
    "    target_links_mapping.setdefault(target_id, []).append(l)\n",
    "\n",
    "for target_id in target_links_mapping:\n",
    "    if f'{target_id}.tgz' in target_links_mapping[target_id]:\n",
    "        # T0950.tgz is avalable, ignore T0950-D?.tgz files\n",
    "        target_links_mapping[target_id] = f'{target_id}.tgz'\n",
    "        \n",
    "distances_true = {\n",
    "    # target_id -> {\n",
    "    #     decoy_id -> {\n",
    "    #        residue_idx -> distance\n",
    "    #     }\n",
    "    # }\n",
    "}\n",
    "        \n",
    "for target_id in target_links_mapping:\n",
    "    target_series = {}\n",
    "    if isinstance(target_links_mapping[target_id], str):\n",
    "        # There is a single T0950.tgz file, load all ground-truth distances from it\n",
    "        for lga_file in Path(f'CASP13/QA_official/{target_links_mapping[target_id]}').with_suffix('').glob('*.lga'):\n",
    "            try:\n",
    "                _, decoy_id = decoy_name_mapping[lga_file.with_suffix('').name]\n",
    "            except KeyError:\n",
    "                logger.warning(f'{lga_file.with_suffix(\"\").name} not found in decoy_name_mapping')\n",
    "                continue\n",
    "            lga_dict = parse_lga(lga_file)\n",
    "            if len(lga_dict) == 0:\n",
    "                logger.warning(f'Check {lga_file} ({target_id} {decoy_id})')\n",
    "                continue\n",
    "            decoy_series = pd.Series(lga_dict).rename_axis('residue_idx').rename('true')\n",
    "            target_series[decoy_id] = decoy_series\n",
    "    else:\n",
    "        # The target has been split into domains, e.g. T0984-D1.tgz T0984-D2.tgz, \n",
    "        # must merge individual files\n",
    "        for domain_folder in target_links_mapping[target_id]:\n",
    "            for lga_file in Path(f'CASP13/QA_official/{domain_folder}').with_suffix('').glob('*.lga'):\n",
    "                try:\n",
    "                    _, decoy_id = decoy_name_mapping[lga_file.with_suffix('').name.split('-')[0]]\n",
    "                except KeyError:\n",
    "                    logger.warning(f'{lga_file.with_suffix(\"\").name} not found in decoy_name_mapping')\n",
    "                    continue\n",
    "                lga_dict = parse_lga(lga_file)\n",
    "                if len(lga_dict) == 0:\n",
    "                    logger.warning(f'Check {lga_file} ({target_id} {decoy_id})')\n",
    "                    continue\n",
    "                target_series.setdefault(decoy_id, {}).update(lga_dict)\n",
    "        for decoy_id in target_series:\n",
    "            target_series[decoy_id] = pd.Series(target_series[decoy_id]).rename_axis('residue_idx').rename('true')\n",
    "    \n",
    "    # Concat all decoys of a target into a single series\n",
    "    distances_true[target_id] = pd.concat(\n",
    "        [v for v in target_series.values() if isinstance(v, pd.Series)], \n",
    "        keys=[k for k in target_series if isinstance(target_series[k], pd.Series)], \n",
    "        names=['decoy_id']\n",
    "    )\n",
    "\n",
    "# Concat all targets of CASP13 into a single series    \n",
    "distances_true = pd.concat(distances_true.values(), keys=distances_true.keys(), names=['target_id'])\n",
    "                \n",
    "distances_true.to_pickle('CASP13/QA_official/distances_true.pkl')\n",
    "! du -h 'CASP13/QA_official/distances_true.pkl'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>true</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>target_id</th>\n",
       "      <th>decoy_id</th>\n",
       "      <th>residue_idx</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">T0949</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">7457-1968-5898_TS4</th>\n",
       "      <th>42</th>\n",
       "      <td>3.551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>1.831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>2.165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>1.554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>1.056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">T1022s2</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">3833-6713-1321_TS5</th>\n",
       "      <th>522</th>\n",
       "      <td>25.738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>523</th>\n",
       "      <td>25.879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524</th>\n",
       "      <td>24.289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>525</th>\n",
       "      <td>21.033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>526</th>\n",
       "      <td>18.785</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8362690 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            true\n",
       "target_id decoy_id           residue_idx        \n",
       "T0949     7457-1968-5898_TS4 42            3.551\n",
       "                             43            1.831\n",
       "                             44            2.165\n",
       "                             45            1.554\n",
       "                             46            1.056\n",
       "...                                          ...\n",
       "T1022s2   3833-6713-1321_TS5 522          25.738\n",
       "                             523          25.879\n",
       "                             524          24.289\n",
       "                             525          21.033\n",
       "                             526          18.785\n",
       "\n",
       "[8362690 rows x 1 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Example dataframe\n",
    "distances_true.to_frame()"
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "formats": "ipynb,py:percent"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc-autonumbering": true,
  "toc-showmarkdowntxt": false,
  "toc-showtags": false
 },
 "nbformat": 4,
 "nbformat_minor": 4
}