20b_external_benchmarking_gencode.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Abstract / Conclusion\n",
    "\n",
    "|                      | Accuracy  |\n",
    "|----------------------|-----------|\n",
    "| ION (Standard mode)  | **0.913** |\n",
    "| ION (Strict   mode)  | 0.149     |\n",
    "| SPLAM<sup>1</sup>    | 0.762     |\n",
    "\n",
    "We can see that overall ION (Standard) mode performs the best in all category except for the False Positive Rate (FPR), the Strict mode of ION performs the best in terms of FPR but sacrifices significantly the overall predictive performance of the model."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.metrics import accuracy_score\n",
    "import warnings\n",
    "from pyfaidx import Fasta\n",
    "from Bio.Seq import Seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "warnings.filterwarnings(\"ignore\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "data": {
      "text/plain": "       chr     start       end strand\n0     chr1  23367168  23369365      -\n1     chr1  23441845  23442185      -\n2     chr1  23452730  23455880      -\n3     chr1  23441217  23441386      -\n4     chr1  23456194  23484471      -\n...    ...       ...       ...    ...\n2269  chrY  57211569  57211760      +\n2270  chrY  57213125  57213203      -\n2271  chrY  57213357  57213525      -\n2272  chrY  57213602  57213879      -\n2273  chrY  57213964  57214349      -\n\n[2274 rows x 4 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>strand</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>23367168</td>\n      <td>23369365</td>\n      <td>-</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>23441845</td>\n      <td>23442185</td>\n      <td>-</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>23452730</td>\n      <td>23455880</td>\n      <td>-</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>23441217</td>\n      <td>23441386</td>\n      <td>-</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>23456194</td>\n      <td>23484471</td>\n      <td>-</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2269</th>\n      <td>chrY</td>\n      <td>57211569</td>\n      <td>57211760</td>\n      <td>+</td>\n    </tr>\n    <tr>\n      <th>2270</th>\n      <td>chrY</td>\n      <td>57213125</td>\n      <td>57213203</td>\n      <td>-</td>\n    </tr>\n    <tr>\n      <th>2271</th>\n      <td>chrY</td>\n      <td>57213357</td>\n      <td>57213525</td>\n      <td>-</td>\n    </tr>\n    <tr>\n      <th>2272</th>\n      <td>chrY</td>\n      <td>57213602</td>\n      <td>57213879</td>\n      <td>-</td>\n    </tr>\n    <tr>\n      <th>2273</th>\n      <td>chrY</td>\n      <td>57213964</td>\n      <td>57214349</td>\n      <td>-</td>\n    </tr>\n  </tbody>\n</table>\n<p>2274 rows × 4 columns</p>\n</div>"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"benchmarking/gencode_v46_new_introns.tsv\", sep=\"\\t\")\n",
    "df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "data": {
      "text/plain": "       chr     start       end  u1  class strand\n0     chr1  23367168  23369365   0      1      -\n1     chr1  23441845  23442185   0      1      -\n2     chr1  23452730  23455880   0      1      -\n3     chr1  23441217  23441386   0      1      -\n4     chr1  23456194  23484471   0      1      -\n...    ...       ...       ...  ..    ...    ...\n2269  chrY  57211569  57211760   0      1      +\n2270  chrY  57213125  57213203   0      1      -\n2271  chrY  57213357  57213525   0      1      -\n2272  chrY  57213602  57213879   0      1      -\n2273  chrY  57213964  57214349   0      1      -\n\n[2274 rows x 6 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>u1</th>\n      <th>class</th>\n      <th>strand</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>23367168</td>\n      <td>23369365</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>23441845</td>\n      <td>23442185</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>23452730</td>\n      <td>23455880</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>23441217</td>\n      <td>23441386</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>23456194</td>\n      <td>23484471</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2269</th>\n      <td>chrY</td>\n      <td>57211569</td>\n      <td>57211760</td>\n      <td>0</td>\n      <td>1</td>\n      <td>+</td>\n    </tr>\n    <tr>\n      <th>2270</th>\n      <td>chrY</td>\n      <td>57213125</td>\n      <td>57213203</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n    </tr>\n    <tr>\n      <th>2271</th>\n      <td>chrY</td>\n      <td>57213357</td>\n      <td>57213525</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n    </tr>\n    <tr>\n      <th>2272</th>\n      <td>chrY</td>\n      <td>57213602</td>\n      <td>57213879</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n    </tr>\n    <tr>\n      <th>2273</th>\n      <td>chrY</td>\n      <td>57213964</td>\n      <td>57214349</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n    </tr>\n  </tbody>\n</table>\n<p>2274 rows × 6 columns</p>\n</div>"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"u1\"] = 0\n",
    "df[\"class\"] = 1\n",
    "df[\"strand\"] = df.pop(\"strand\")\n",
    "df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "data": {
      "text/plain": "       chr     start       end  u1  class strand  \\\n0     chr1  23367168  23369365   0      1      -   \n1     chr1  23441845  23442185   0      1      -   \n2     chr1  23452730  23455880   0      1      -   \n3     chr1  23441217  23441386   0      1      -   \n4     chr1  23456194  23484471   0      1      -   \n...    ...       ...       ...  ..    ...    ...   \n2269  chrY  57211569  57211760   0      1      +   \n2270  chrY  57213125  57213203   0      1      -   \n2271  chrY  57213357  57213525   0      1      -   \n2272  chrY  57213602  57213879   0      1      -   \n2273  chrY  57213964  57214349   0      1      -   \n\n                                               sequence  \n0     GTAAGTCTGTTCGTCCCCCACTCGGGTTTTCGGCTTCCTACCTTCA...  \n1     GTAGGTATCTTCCCGCGTCCCTGCCACTAACCCTTCCAGTGTCACA...  \n2     GTGAGGCCCCTGTCTCTTCCTTGCCCAGACTCAGGGTAAATCTAGT...  \n3     GTCAGCCCCTGAACTATCCCAAAAGGAATGCCCTCCCCCAAGGCCC...  \n4     GTGGGGCATCGCGCCAGGGGGCGGGGTCTGAGGCGGTGAGGGGCTG...  \n...                                                 ...  \n2269  GTGGGCACTTGATGTCGGATCTCTTCAACAAGCTGGTCATGAGGCG...  \n2270  GTTCACTCCTGCCTTTTCCTTTCCCTAGAGCCTCCACCACCCCGAG...  \n2271  GCAAGCCTGGCTGCCTCCAGCTGGGTGGACAGACAGGGGCTGGAGA...  \n2272  GTGAGTGTCCCCAGTGTTGCAGAGGTGAGAGGAGAGTAGACAGTGA...  \n2273  GTAAGTAGTGCTTGTGCTCATCTCCTTGGCTGTGATACGTGGCCGG...  \n\n[2274 rows x 7 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>u1</th>\n      <th>class</th>\n      <th>strand</th>\n      <th>sequence</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>23367168</td>\n      <td>23369365</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTAAGTCTGTTCGTCCCCCACTCGGGTTTTCGGCTTCCTACCTTCA...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>23441845</td>\n      <td>23442185</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTAGGTATCTTCCCGCGTCCCTGCCACTAACCCTTCCAGTGTCACA...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>23452730</td>\n      <td>23455880</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTGAGGCCCCTGTCTCTTCCTTGCCCAGACTCAGGGTAAATCTAGT...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>23441217</td>\n      <td>23441386</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTCAGCCCCTGAACTATCCCAAAAGGAATGCCCTCCCCCAAGGCCC...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>23456194</td>\n      <td>23484471</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTGGGGCATCGCGCCAGGGGGCGGGGTCTGAGGCGGTGAGGGGCTG...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2269</th>\n      <td>chrY</td>\n      <td>57211569</td>\n      <td>57211760</td>\n      <td>0</td>\n      <td>1</td>\n      <td>+</td>\n      <td>GTGGGCACTTGATGTCGGATCTCTTCAACAAGCTGGTCATGAGGCG...</td>\n    </tr>\n    <tr>\n      <th>2270</th>\n      <td>chrY</td>\n      <td>57213125</td>\n      <td>57213203</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTTCACTCCTGCCTTTTCCTTTCCCTAGAGCCTCCACCACCCCGAG...</td>\n    </tr>\n    <tr>\n      <th>2271</th>\n      <td>chrY</td>\n      <td>57213357</td>\n      <td>57213525</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GCAAGCCTGGCTGCCTCCAGCTGGGTGGACAGACAGGGGCTGGAGA...</td>\n    </tr>\n    <tr>\n      <th>2272</th>\n      <td>chrY</td>\n      <td>57213602</td>\n      <td>57213879</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTGAGTGTCCCCAGTGTTGCAGAGGTGAGAGGAGAGTAGACAGTGA...</td>\n    </tr>\n    <tr>\n      <th>2273</th>\n      <td>chrY</td>\n      <td>57213964</td>\n      <td>57214349</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTAAGTAGTGCTTGTGCTCATCTCCTTGGCTGTGATACGTGGCCGG...</td>\n    </tr>\n  </tbody>\n</table>\n<p>2274 rows × 7 columns</p>\n</div>"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This is downloaded in https://hgdownload.soe.ucsc.edu/goldenPath/hg38/bigZips/\n",
    "genome = Fasta('data/human_ref_hg38_109/GRCh38.primary_assembly.genome.fa', sequence_always_upper=True)\n",
    "\n",
    "# A simple lambda function for matching the chromosome, start and end\n",
    "coords_to_dna = lambda start_c, end_c, chr_c: genome[chr_c][start_c - 1:end_c]\n",
    "\n",
    "def find_sequence_maxentscan(row):\n",
    "\tchromosome = row[\"chr\"]\n",
    "\tstart = row[\"start\"]\n",
    "\tstop = row[\"end\"]\n",
    "\tstrand = row[\"strand\"]\n",
    "\n",
    "\tseq_find_sequence = str(coords_to_dna(int(start) + 1 - 3, int(stop) + 3, chromosome))\n",
    "\n",
    "\tif strand == \"-\":  #strand\n",
    "\t\tseq_find_sequence = Seq(seq_find_sequence)  # Encode the sequence into\n",
    "\t\tseq_find_sequence = seq_find_sequence.reverse_complement()\n",
    "\n",
    "\tseq_find_sequence = str(seq_find_sequence)\n",
    "\tmiddle_part = seq_find_sequence[3:-3].upper()\n",
    "\n",
    "\treturn middle_part\n",
    "\n",
    "\n",
    "df[\"sequence\"] = df.apply(find_sequence_maxentscan, axis=1)\n",
    "df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "data": {
      "text/plain": "       chr     start       end  u1  class strand  \\\n0     chr1  23367168  23369365   0      1      -   \n1     chr1  23441845  23442185   0      1      -   \n2     chr1  23452730  23455880   0      1      -   \n3     chr1  23441217  23441386   0      1      -   \n4     chr1  23456194  23484471   0      1      -   \n...    ...       ...       ...  ..    ...    ...   \n2269  chrY  57211569  57211760   0      1      +   \n2270  chrY  57213125  57213203   0      1      -   \n2271  chrY  57213357  57213525   0      1      -   \n2272  chrY  57213602  57213879   0      1      -   \n2273  chrY  57213964  57214349   0      1      -   \n\n                                               sequence splice-site  \n0     GTAAGTCTGTTCGTCCCCCACTCGGGTTTTCGGCTTCCTACCTTCA...       GT:AG  \n1     GTAGGTATCTTCCCGCGTCCCTGCCACTAACCCTTCCAGTGTCACA...       GT:AG  \n2     GTGAGGCCCCTGTCTCTTCCTTGCCCAGACTCAGGGTAAATCTAGT...       GT:AG  \n3     GTCAGCCCCTGAACTATCCCAAAAGGAATGCCCTCCCCCAAGGCCC...       GT:AG  \n4     GTGGGGCATCGCGCCAGGGGGCGGGGTCTGAGGCGGTGAGGGGCTG...       GT:AG  \n...                                                 ...         ...  \n2269  GTGGGCACTTGATGTCGGATCTCTTCAACAAGCTGGTCATGAGGCG...       GT:AG  \n2270  GTTCACTCCTGCCTTTTCCTTTCCCTAGAGCCTCCACCACCCCGAG...       GT:AG  \n2271  GCAAGCCTGGCTGCCTCCAGCTGGGTGGACAGACAGGGGCTGGAGA...       GC:AG  \n2272  GTGAGTGTCCCCAGTGTTGCAGAGGTGAGAGGAGAGTAGACAGTGA...       GT:AG  \n2273  GTAAGTAGTGCTTGTGCTCATCTCCTTGGCTGTGATACGTGGCCGG...       GT:AG  \n\n[2274 rows x 8 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>u1</th>\n      <th>class</th>\n      <th>strand</th>\n      <th>sequence</th>\n      <th>splice-site</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>23367168</td>\n      <td>23369365</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTAAGTCTGTTCGTCCCCCACTCGGGTTTTCGGCTTCCTACCTTCA...</td>\n      <td>GT:AG</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>23441845</td>\n      <td>23442185</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTAGGTATCTTCCCGCGTCCCTGCCACTAACCCTTCCAGTGTCACA...</td>\n      <td>GT:AG</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>23452730</td>\n      <td>23455880</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTGAGGCCCCTGTCTCTTCCTTGCCCAGACTCAGGGTAAATCTAGT...</td>\n      <td>GT:AG</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>23441217</td>\n      <td>23441386</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTCAGCCCCTGAACTATCCCAAAAGGAATGCCCTCCCCCAAGGCCC...</td>\n      <td>GT:AG</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>23456194</td>\n      <td>23484471</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTGGGGCATCGCGCCAGGGGGCGGGGTCTGAGGCGGTGAGGGGCTG...</td>\n      <td>GT:AG</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2269</th>\n      <td>chrY</td>\n      <td>57211569</td>\n      <td>57211760</td>\n      <td>0</td>\n      <td>1</td>\n      <td>+</td>\n      <td>GTGGGCACTTGATGTCGGATCTCTTCAACAAGCTGGTCATGAGGCG...</td>\n      <td>GT:AG</td>\n    </tr>\n    <tr>\n      <th>2270</th>\n      <td>chrY</td>\n      <td>57213125</td>\n      <td>57213203</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTTCACTCCTGCCTTTTCCTTTCCCTAGAGCCTCCACCACCCCGAG...</td>\n      <td>GT:AG</td>\n    </tr>\n    <tr>\n      <th>2271</th>\n      <td>chrY</td>\n      <td>57213357</td>\n      <td>57213525</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GCAAGCCTGGCTGCCTCCAGCTGGGTGGACAGACAGGGGCTGGAGA...</td>\n      <td>GC:AG</td>\n    </tr>\n    <tr>\n      <th>2272</th>\n      <td>chrY</td>\n      <td>57213602</td>\n      <td>57213879</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTGAGTGTCCCCAGTGTTGCAGAGGTGAGAGGAGAGTAGACAGTGA...</td>\n      <td>GT:AG</td>\n    </tr>\n    <tr>\n      <th>2273</th>\n      <td>chrY</td>\n      <td>57213964</td>\n      <td>57214349</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTAAGTAGTGCTTGTGCTCATCTCCTTGGCTGTGATACGTGGCCGG...</td>\n      <td>GT:AG</td>\n    </tr>\n  </tbody>\n</table>\n<p>2274 rows × 8 columns</p>\n</div>"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"splice-site\"] = df[\"sequence\"].str[:2] + \":\" + df[\"sequence\"].str[-2:]\n",
    "df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "data": {
      "text/plain": "       chr     start       end  u1  class strand  \\\n0     chr1  23367168  23369365   0      1      -   \n1     chr1  23441845  23442185   0      1      -   \n2     chr1  23452730  23455880   0      1      -   \n3     chr1  23441217  23441386   0      1      -   \n4     chr1  23456194  23484471   0      1      -   \n...    ...       ...       ...  ..    ...    ...   \n2269  chrY  57211569  57211760   0      1      +   \n2270  chrY  57213125  57213203   0      1      -   \n2271  chrY  57213357  57213525   0      1      -   \n2272  chrY  57213602  57213879   0      1      -   \n2273  chrY  57213964  57214349   0      1      -   \n\n                                               sequence splice-site  \n0     GTAAGTCTGTTCGTCCCCCACTCGGGTTTTCGGCTTCCTACCTTCA...       GT:AG  \n1     GTAGGTATCTTCCCGCGTCCCTGCCACTAACCCTTCCAGTGTCACA...       GT:AG  \n2     GTGAGGCCCCTGTCTCTTCCTTGCCCAGACTCAGGGTAAATCTAGT...       GT:AG  \n3     GTCAGCCCCTGAACTATCCCAAAAGGAATGCCCTCCCCCAAGGCCC...       GT:AG  \n4     GTGGGGCATCGCGCCAGGGGGCGGGGTCTGAGGCGGTGAGGGGCTG...       GT:AG  \n...                                                 ...         ...  \n2269  GTGGGCACTTGATGTCGGATCTCTTCAACAAGCTGGTCATGAGGCG...       GT:AG  \n2270  GTTCACTCCTGCCTTTTCCTTTCCCTAGAGCCTCCACCACCCCGAG...       GT:AG  \n2271  GCAAGCCTGGCTGCCTCCAGCTGGGTGGACAGACAGGGGCTGGAGA...       GC:AG  \n2272  GTGAGTGTCCCCAGTGTTGCAGAGGTGAGAGGAGAGTAGACAGTGA...       GT:AG  \n2273  GTAAGTAGTGCTTGTGCTCATCTCCTTGGCTGTGATACGTGGCCGG...       GT:AG  \n\n[2208 rows x 8 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>u1</th>\n      <th>class</th>\n      <th>strand</th>\n      <th>sequence</th>\n      <th>splice-site</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>23367168</td>\n      <td>23369365</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTAAGTCTGTTCGTCCCCCACTCGGGTTTTCGGCTTCCTACCTTCA...</td>\n      <td>GT:AG</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>23441845</td>\n      <td>23442185</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTAGGTATCTTCCCGCGTCCCTGCCACTAACCCTTCCAGTGTCACA...</td>\n      <td>GT:AG</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>23452730</td>\n      <td>23455880</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTGAGGCCCCTGTCTCTTCCTTGCCCAGACTCAGGGTAAATCTAGT...</td>\n      <td>GT:AG</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>23441217</td>\n      <td>23441386</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTCAGCCCCTGAACTATCCCAAAAGGAATGCCCTCCCCCAAGGCCC...</td>\n      <td>GT:AG</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>23456194</td>\n      <td>23484471</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTGGGGCATCGCGCCAGGGGGCGGGGTCTGAGGCGGTGAGGGGCTG...</td>\n      <td>GT:AG</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2269</th>\n      <td>chrY</td>\n      <td>57211569</td>\n      <td>57211760</td>\n      <td>0</td>\n      <td>1</td>\n      <td>+</td>\n      <td>GTGGGCACTTGATGTCGGATCTCTTCAACAAGCTGGTCATGAGGCG...</td>\n      <td>GT:AG</td>\n    </tr>\n    <tr>\n      <th>2270</th>\n      <td>chrY</td>\n      <td>57213125</td>\n      <td>57213203</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTTCACTCCTGCCTTTTCCTTTCCCTAGAGCCTCCACCACCCCGAG...</td>\n      <td>GT:AG</td>\n    </tr>\n    <tr>\n      <th>2271</th>\n      <td>chrY</td>\n      <td>57213357</td>\n      <td>57213525</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GCAAGCCTGGCTGCCTCCAGCTGGGTGGACAGACAGGGGCTGGAGA...</td>\n      <td>GC:AG</td>\n    </tr>\n    <tr>\n      <th>2272</th>\n      <td>chrY</td>\n      <td>57213602</td>\n      <td>57213879</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTGAGTGTCCCCAGTGTTGCAGAGGTGAGAGGAGAGTAGACAGTGA...</td>\n      <td>GT:AG</td>\n    </tr>\n    <tr>\n      <th>2273</th>\n      <td>chrY</td>\n      <td>57213964</td>\n      <td>57214349</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>GTAAGTAGTGCTTGTGCTCATCTCCTTGGCTGTGATACGTGGCCGG...</td>\n      <td>GT:AG</td>\n    </tr>\n  </tbody>\n</table>\n<p>2208 rows × 8 columns</p>\n</div>"
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df[(df[\"splice-site\"] == \"GT:AG\") | (df[\"splice-site\"] == \"GC:AG\") | (df[\"splice-site\"] == \"AT:AC\")]\n",
    "df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [
    "df = df.drop(columns=[\"splice-site\", \"sequence\"])\n",
    "df.to_csv(\"benchmarking/gencode_v46_new_introns.bed\", sep=\"\\t\", header=False, index=False)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Benchmarking using Splam: a deep-learning-based splice site predictor that improves spliced alignments\n",
    "bioRxiv. 2023 Jul 29;2023.07.27.550754. doi: 10.1101/2023.07.27.550754. Preprint"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=====================================================================\r\n",
      " An accurate spliced alignment pruner and splice junction predictor. \r\n",
      "=====================================================================\r\n",
      "\r\n",
      "\r\n",
      "  ███████╗██████╗ ██╗      █████╗ ███╗   ███╗\r\n",
      "  ██╔════╝██╔══██╗██║     ██╔══██╗████╗ ████║\r\n",
      "  ███████╗██████╔╝██║     ███████║██╔████╔██║\r\n",
      "  ╚════██║██╔═══╝ ██║     ██╔══██║██║╚██╔╝██║\r\n",
      "  ███████║██║     ███████╗██║  ██║██║ ╚═╝ ██║\r\n",
      "  ╚══════╝╚═╝     ╚══════╝╚═╝  ╚═╝╚═╝     ╚═╝\r\n",
      "    \r\n",
      "[Info] Chromosomes in the annotation file is in 'chr*' style\r\n",
      "[Info] Running model in \"cuda\" mode\r\n",
      "[Info] Loading model ... (./data/splam/splam_script.pt)\r\n",
      "model = torch.load(model_path)!!\r\n",
      "[Info] Done loading model\r\n",
      "[Info] Loading data ...\r\n",
      "\t 2208  junctions loaded.\r\n",
      "[Info] Done loading data\r\n",
      "[Info] SPLAM!  |################################| 221/221\r\n",
      "\u001B[?25h"
     ]
    }
   ],
   "source": [
    "!splam score -G ./data/human_ref_hg38_109/GRCh38.primary_assembly.genome.fa -m ./data/splam/splam_script.pt -o splam_out benchmarking/gencode_v46_new_introns.bed"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [
    {
     "data": {
      "text/plain": "       chr     start       end  rc3_score  class strand  donor_score  \\\n0     chr1  23367168  23369365          0      1      -     1.000000   \n1     chr1  23441845  23442185          0      1      -     0.999741   \n2     chr1  23452730  23455880          0      1      -     1.000000   \n3     chr1  23441217  23441386          0      1      -     0.999990   \n4     chr1  23456194  23484471          0      1      -     0.999905   \n...    ...       ...       ...        ...    ...    ...          ...   \n2203  chrY  57211569  57211760          0      1      +     0.998500   \n2204  chrY  57213125  57213203          0      1      -     0.004676   \n2205  chrY  57213357  57213525          0      1      -     0.958597   \n2206  chrY  57213602  57213879          0      1      -     0.993605   \n2207  chrY  57213964  57214349          0      1      -     0.999961   \n\n      acceptor_score  \n0           1.000000  \n1           0.999877  \n2           1.000000  \n3           0.999966  \n4           0.999833  \n...              ...  \n2203        0.999796  \n2204        0.004959  \n2205        0.946810  \n2206        0.986091  \n2207        0.999953  \n\n[2208 rows x 8 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>rc3_score</th>\n      <th>class</th>\n      <th>strand</th>\n      <th>donor_score</th>\n      <th>acceptor_score</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>23367168</td>\n      <td>23369365</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>23441845</td>\n      <td>23442185</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>0.999741</td>\n      <td>0.999877</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>23452730</td>\n      <td>23455880</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>23441217</td>\n      <td>23441386</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>0.999990</td>\n      <td>0.999966</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>23456194</td>\n      <td>23484471</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>0.999905</td>\n      <td>0.999833</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2203</th>\n      <td>chrY</td>\n      <td>57211569</td>\n      <td>57211760</td>\n      <td>0</td>\n      <td>1</td>\n      <td>+</td>\n      <td>0.998500</td>\n      <td>0.999796</td>\n    </tr>\n    <tr>\n      <th>2204</th>\n      <td>chrY</td>\n      <td>57213125</td>\n      <td>57213203</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>0.004676</td>\n      <td>0.004959</td>\n    </tr>\n    <tr>\n      <th>2205</th>\n      <td>chrY</td>\n      <td>57213357</td>\n      <td>57213525</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>0.958597</td>\n      <td>0.946810</td>\n    </tr>\n    <tr>\n      <th>2206</th>\n      <td>chrY</td>\n      <td>57213602</td>\n      <td>57213879</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>0.993605</td>\n      <td>0.986091</td>\n    </tr>\n    <tr>\n      <th>2207</th>\n      <td>chrY</td>\n      <td>57213964</td>\n      <td>57214349</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>0.999961</td>\n      <td>0.999953</td>\n    </tr>\n  </tbody>\n</table>\n<p>2208 rows × 8 columns</p>\n</div>"
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_splam_pred = pd.read_csv(\"benchmarking/splam_out_gencode_v46/junction_score.bed\", sep=\"\\t\", names=[\"chr\",\"start\",\"end\",'rc3_score','class','strand','donor_score','acceptor_score','u']).drop(columns=\"u\")\n",
    "df_splam_pred"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [
    {
     "data": {
      "text/plain": "       chr     start       end  rc3_score  class strand  donor_score  \\\n0     chr1  23367168  23369365          0      1      -     1.000000   \n1     chr1  23441845  23442185          0      1      -     0.999741   \n2     chr1  23452730  23455880          0      1      -     1.000000   \n3     chr1  23441217  23441386          0      1      -     0.999990   \n4     chr1  23456194  23484471          0      1      -     0.999905   \n...    ...       ...       ...        ...    ...    ...          ...   \n2203  chrY  57211569  57211760          0      1      +     0.998500   \n2204  chrY  57213125  57213203          0      1      -     0.004676   \n2205  chrY  57213357  57213525          0      1      -     0.958597   \n2206  chrY  57213602  57213879          0      1      -     0.993605   \n2207  chrY  57213964  57214349          0      1      -     0.999961   \n\n      acceptor_score  splam_pred  \n0           1.000000           1  \n1           0.999877           1  \n2           1.000000           1  \n3           0.999966           1  \n4           0.999833           1  \n...              ...         ...  \n2203        0.999796           1  \n2204        0.004959           0  \n2205        0.946810           1  \n2206        0.986091           1  \n2207        0.999953           1  \n\n[2208 rows x 9 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>rc3_score</th>\n      <th>class</th>\n      <th>strand</th>\n      <th>donor_score</th>\n      <th>acceptor_score</th>\n      <th>splam_pred</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>23367168</td>\n      <td>23369365</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>23441845</td>\n      <td>23442185</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>0.999741</td>\n      <td>0.999877</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>23452730</td>\n      <td>23455880</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>23441217</td>\n      <td>23441386</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>0.999990</td>\n      <td>0.999966</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>23456194</td>\n      <td>23484471</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>0.999905</td>\n      <td>0.999833</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2203</th>\n      <td>chrY</td>\n      <td>57211569</td>\n      <td>57211760</td>\n      <td>0</td>\n      <td>1</td>\n      <td>+</td>\n      <td>0.998500</td>\n      <td>0.999796</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2204</th>\n      <td>chrY</td>\n      <td>57213125</td>\n      <td>57213203</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>0.004676</td>\n      <td>0.004959</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2205</th>\n      <td>chrY</td>\n      <td>57213357</td>\n      <td>57213525</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>0.958597</td>\n      <td>0.946810</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2206</th>\n      <td>chrY</td>\n      <td>57213602</td>\n      <td>57213879</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>0.993605</td>\n      <td>0.986091</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2207</th>\n      <td>chrY</td>\n      <td>57213964</td>\n      <td>57214349</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>0.999961</td>\n      <td>0.999953</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n<p>2208 rows × 9 columns</p>\n</div>"
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_splam_pred[\"splam_pred\"] = ((df_splam_pred['donor_score'] >= 0.5) & (df_splam_pred['acceptor_score'] >= 0.5)).astype(int)\n",
    "df_splam_pred"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [
    {
     "data": {
      "text/plain": "       chr     start       end  rc3_score  class strand  donor_score  \\\n0     chr1  23367168  23369365          0      1      -     1.000000   \n1     chr1  23441845  23442185          0      1      -     0.999741   \n2     chr1  23452730  23455880          0      1      -     1.000000   \n3     chr1  23441217  23441386          0      1      -     0.999990   \n4     chr1  23456194  23484471          0      1      -     0.999905   \n...    ...       ...       ...        ...    ...    ...          ...   \n2203  chrY  57211569  57211760          0      1      +     0.998500   \n2204  chrY  57213125  57213203          0      1      -     0.004676   \n2205  chrY  57213357  57213525          0      1      -     0.958597   \n2206  chrY  57213602  57213879          0      1      -     0.993605   \n2207  chrY  57213964  57214349          0      1      -     0.999961   \n\n      acceptor_score  splam_pred  splam_pred_avg  \n0           1.000000           1        1.000000  \n1           0.999877           1        0.999741  \n2           1.000000           1        1.000000  \n3           0.999966           1        0.999966  \n4           0.999833           1        0.999833  \n...              ...         ...             ...  \n2203        0.999796           1        0.998500  \n2204        0.004959           0        0.004676  \n2205        0.946810           1        0.946810  \n2206        0.986091           1        0.986091  \n2207        0.999953           1        0.999953  \n\n[2208 rows x 10 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>rc3_score</th>\n      <th>class</th>\n      <th>strand</th>\n      <th>donor_score</th>\n      <th>acceptor_score</th>\n      <th>splam_pred</th>\n      <th>splam_pred_avg</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>23367168</td>\n      <td>23369365</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n      <td>1</td>\n      <td>1.000000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>23441845</td>\n      <td>23442185</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>0.999741</td>\n      <td>0.999877</td>\n      <td>1</td>\n      <td>0.999741</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>23452730</td>\n      <td>23455880</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n      <td>1</td>\n      <td>1.000000</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>23441217</td>\n      <td>23441386</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>0.999990</td>\n      <td>0.999966</td>\n      <td>1</td>\n      <td>0.999966</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>23456194</td>\n      <td>23484471</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>0.999905</td>\n      <td>0.999833</td>\n      <td>1</td>\n      <td>0.999833</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2203</th>\n      <td>chrY</td>\n      <td>57211569</td>\n      <td>57211760</td>\n      <td>0</td>\n      <td>1</td>\n      <td>+</td>\n      <td>0.998500</td>\n      <td>0.999796</td>\n      <td>1</td>\n      <td>0.998500</td>\n    </tr>\n    <tr>\n      <th>2204</th>\n      <td>chrY</td>\n      <td>57213125</td>\n      <td>57213203</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>0.004676</td>\n      <td>0.004959</td>\n      <td>0</td>\n      <td>0.004676</td>\n    </tr>\n    <tr>\n      <th>2205</th>\n      <td>chrY</td>\n      <td>57213357</td>\n      <td>57213525</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>0.958597</td>\n      <td>0.946810</td>\n      <td>1</td>\n      <td>0.946810</td>\n    </tr>\n    <tr>\n      <th>2206</th>\n      <td>chrY</td>\n      <td>57213602</td>\n      <td>57213879</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>0.993605</td>\n      <td>0.986091</td>\n      <td>1</td>\n      <td>0.986091</td>\n    </tr>\n    <tr>\n      <th>2207</th>\n      <td>chrY</td>\n      <td>57213964</td>\n      <td>57214349</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-</td>\n      <td>0.999961</td>\n      <td>0.999953</td>\n      <td>1</td>\n      <td>0.999953</td>\n    </tr>\n  </tbody>\n</table>\n<p>2208 rows × 10 columns</p>\n</div>"
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_splam_pred[\"splam_pred_avg\"] = df_splam_pred[['donor_score', 'acceptor_score']].min(axis=1)\n",
    "df_splam_pred"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metrics for SPLAM\n",
      "Accuracy:  0.7721920289855072\n"
     ]
    }
   ],
   "source": [
    "acc = accuracy_score(df_splam_pred[\"class\"], df_splam_pred[\"splam_pred\"])\n",
    "\n",
    "print(\"Metrics for SPLAM\")\n",
    "print(\"Accuracy: \", acc)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# ION (High-Recall Mode)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metrics for ION (standard mode)\n",
      "Accuracy:  0.8442028985507246\n"
     ]
    }
   ],
   "source": [
    "df_ion1 = pd.read_csv(\"benchmarking/ION_out_gencode_v46/output_high_recall_mode.bed\", sep=\"\\t\")\n",
    "df_ion1[\"prediction_class\"] = (df_ion1[\"prediction\"] >= 0.5).astype(int)\n",
    "df_ion1[\"class\"] = df_ion1[\"u2\"]\n",
    "acc = accuracy_score(df_ion1[\"class\"], df_ion1[\"prediction_class\"])\n",
    "\n",
    "print(\"Metrics for ION (standard mode)\")\n",
    "print(\"Accuracy: \", acc)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# ION (Standard Mode)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metrics for ION (standard mode)\n",
      "Accuracy:  0.7078804347826086\n"
     ]
    }
   ],
   "source": [
    "df_ion2 = pd.read_csv(\"benchmarking/ION_out_gencode_v46/output_standard_mode.bed\", sep=\"\\t\")\n",
    "df_ion2[\"prediction_class\"] = (df_ion2[\"prediction\"] >= 0.5).astype(int)\n",
    "df_ion2[\"class\"] = df_ion2[\"u2\"]\n",
    "acc = accuracy_score(df_ion2[\"class\"], df_ion2[\"prediction_class\"])\n",
    "\n",
    "print(\"Metrics for ION (standard mode)\")\n",
    "print(\"Accuracy: \", acc)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# ION (High-Precision Mode)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metrics for ION (standard mode)\n",
      "Accuracy:  0.6019021739130435\n"
     ]
    }
   ],
   "source": [
    "df_ion3 = pd.read_csv(\"benchmarking/ION_out_gencode_v46/output_high_precision_mode.bed\", sep=\"\\t\")\n",
    "df_ion3[\"prediction_class\"] = (df_ion3[\"prediction\"] >= 0.5).astype(int)\n",
    "df_ion3[\"class\"] = df_ion3[\"u2\"]\n",
    "acc = accuracy_score(df_ion3[\"class\"], df_ion3[\"prediction_class\"])\n",
    "\n",
    "print(\"Metrics for ION (standard mode)\")\n",
    "print(\"Accuracy: \", acc)"
   ],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}