04_combining_positive_and_negative_train_set.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "data": {
      "text/plain": "         chr     start       end strand  class\n0       chr1     12227     12612      +      1\n1       chr1     12721     13220      +      1\n2       chr1     12057     12178      +      1\n3       chr1     12697     12974      +      1\n4       chr1     13052     13220      +      1\n...      ...       ...       ...    ...    ...\n402449  chrY  57211569  57211760      +      1\n402450  chrY  57213125  57213203      -      1\n402451  chrY  57213357  57213525      -      1\n402452  chrY  57213602  57213879      -      1\n402453  chrY  57213964  57214349      -      1\n\n[402454 rows x 5 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>strand</th>\n      <th>class</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>12227</td>\n      <td>12612</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>12721</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>12057</td>\n      <td>12178</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>12697</td>\n      <td>12974</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>13052</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>402449</th>\n      <td>chrY</td>\n      <td>57211569</td>\n      <td>57211760</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>402450</th>\n      <td>chrY</td>\n      <td>57213125</td>\n      <td>57213203</td>\n      <td>-</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>402451</th>\n      <td>chrY</td>\n      <td>57213357</td>\n      <td>57213525</td>\n      <td>-</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>402452</th>\n      <td>chrY</td>\n      <td>57213602</td>\n      <td>57213879</td>\n      <td>-</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>402453</th>\n      <td>chrY</td>\n      <td>57213964</td>\n      <td>57214349</td>\n      <td>-</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n<p>402454 rows × 5 columns</p>\n</div>"
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_gencode = pd.read_csv(\"data/01_gencode_introns.tsv\", sep=\"\\t\")\n",
    "df_gencode[\"class\"] = 1\n",
    "df_gencode"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [
    {
     "data": {
      "text/plain": "      chr      start        end strand  class\n0    chr1     732207     739802      -      0\n1    chr1    6634784    6634990      -      0\n2    chr1    9036720    9088663      +      0\n3    chr1    9088686    9100902      +      0\n4    chr1   11277018   11277484      -      0\n..    ...        ...        ...    ...    ...\n398  chrX  151403679  151404937      +      0\n399  chrX  151409210  151456968      +      0\n400  chrX  153906577  153906694      -      0\n401  chrX  153906409  153906520      -      0\n402  chrY   20579691   20582591      +      0\n\n[403 rows x 5 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>strand</th>\n      <th>class</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>732207</td>\n      <td>739802</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>6634784</td>\n      <td>6634990</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>9036720</td>\n      <td>9088663</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>9088686</td>\n      <td>9100902</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>11277018</td>\n      <td>11277484</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>398</th>\n      <td>chrX</td>\n      <td>151403679</td>\n      <td>151404937</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>399</th>\n      <td>chrX</td>\n      <td>151409210</td>\n      <td>151456968</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>400</th>\n      <td>chrX</td>\n      <td>153906577</td>\n      <td>153906694</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>401</th>\n      <td>chrX</td>\n      <td>153906409</td>\n      <td>153906520</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>402</th>\n      <td>chrY</td>\n      <td>20579691</td>\n      <td>20582591</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>403 rows × 5 columns</p>\n</div>"
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_manually_rejected_introns = pd.read_csv(\"data/02_manually_rejected_introns.tsv\", sep=\"\\t\")\n",
    "df_manually_rejected_introns[\"class\"] = 0\n",
    "df_manually_rejected_introns = df_manually_rejected_introns.drop_duplicates(subset=[\"chr\", \"start\", \"end\", \"strand\"])\n",
    "df_manually_rejected_introns"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [
    {
     "data": {
      "text/plain": "         chr     start       end strand  class\n0       chr1     14829     14969      -      0\n1       chr1     15038     15795      -      0\n2       chr1     15947     16606      -      0\n3       chr1     16765     16853      -      0\n4       chr1     17055     17232      -      0\n...      ...       ...       ...    ...    ...\n960777  chrY  57209354  57209531      +      0\n960778  chrY  57209733  57209821      +      0\n960779  chrY  57209980  57210639      +      0\n960780  chrY  57210792  57211551      +      0\n960781  chrY  57211620  57211760      +      0\n\n[294424 rows x 5 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>strand</th>\n      <th>class</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>14829</td>\n      <td>14969</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>15038</td>\n      <td>15795</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>15947</td>\n      <td>16606</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>16765</td>\n      <td>16853</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>17055</td>\n      <td>17232</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>960777</th>\n      <td>chrY</td>\n      <td>57209354</td>\n      <td>57209531</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>960778</th>\n      <td>chrY</td>\n      <td>57209733</td>\n      <td>57209821</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>960779</th>\n      <td>chrY</td>\n      <td>57209980</td>\n      <td>57210639</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>960780</th>\n      <td>chrY</td>\n      <td>57210792</td>\n      <td>57211551</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>960781</th>\n      <td>chrY</td>\n      <td>57211620</td>\n      <td>57211760</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>294424 rows × 5 columns</p>\n</div>"
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_simulation_a_introns = pd.read_csv(\"./data/false_introns_simulated/simulation_a_introns.bed\", sep=\"\\t\",\n",
    "\t\t\t\t\t\t\t\t\tnames=[\"chr\", \"start\", \"end\", \"features\", \"score\", \"strand\"])\n",
    "df_simulation_a_introns = df_simulation_a_introns.loc[:, [\"chr\", \"start\", \"end\", \"strand\"]]\n",
    "df_simulation_a_introns[\"class\"] = 0\n",
    "df_simulation_a_introns = df_simulation_a_introns.drop_duplicates(subset=[\"chr\", \"start\", \"end\", \"strand\"])\n",
    "df_simulation_a_introns"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [
    {
     "data": {
      "text/plain": "          chr     start       end strand  class\n0        chr1     14829     14969      -      0\n1        chr1     15038     15795      -      0\n2        chr1     15947     16606      -      0\n3        chr1     16765     16853      -      0\n4        chr1     17055     17232      -      0\n...       ...       ...       ...    ...    ...\n2879937  chrY  57209354  57209531      +      0\n2879938  chrY  57209733  57209821      +      0\n2879939  chrY  57209980  57210639      +      0\n2879940  chrY  57210792  57211551      +      0\n2879941  chrY  57211620  57211760      +      0\n\n[341481 rows x 5 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>strand</th>\n      <th>class</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>14829</td>\n      <td>14969</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>15038</td>\n      <td>15795</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>15947</td>\n      <td>16606</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>16765</td>\n      <td>16853</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>17055</td>\n      <td>17232</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2879937</th>\n      <td>chrY</td>\n      <td>57209354</td>\n      <td>57209531</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2879938</th>\n      <td>chrY</td>\n      <td>57209733</td>\n      <td>57209821</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2879939</th>\n      <td>chrY</td>\n      <td>57209980</td>\n      <td>57210639</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2879940</th>\n      <td>chrY</td>\n      <td>57210792</td>\n      <td>57211551</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2879941</th>\n      <td>chrY</td>\n      <td>57211620</td>\n      <td>57211760</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>341481 rows × 5 columns</p>\n</div>"
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_simulation_b_introns = pd.read_csv(\"./data/false_introns_simulated/simulation_b_introns.bed\", sep=\"\\t\",\n",
    "\t\t\t\t\t\t\t\t\tnames=[\"chr\", \"start\", \"end\", \"features\", \"score\", \"strand\"])\n",
    "df_simulation_b_introns = df_simulation_b_introns.loc[:, [\"chr\", \"start\", \"end\", \"strand\"]]\n",
    "df_simulation_b_introns[\"class\"] = 0\n",
    "df_simulation_b_introns = df_simulation_b_introns.drop_duplicates(subset=[\"chr\", \"start\", \"end\", \"strand\"])\n",
    "df_simulation_b_introns"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [
    {
     "data": {
      "text/plain": "          chr     start       end strand  class\n0        chr1     14829     14969      -      0\n1        chr1     15038     15795      -      0\n2        chr1     15947     16606      -      0\n3        chr1     16765     16853      -      0\n4        chr1     17055     17232      -      0\n...       ...       ...       ...    ...    ...\n2962663  chrY  57209733  57209821      +      0\n2962664  chrY  57209980  57210639      +      0\n2962665  chrY  57210792  57211551      +      0\n2962666  chrY  57211620  57211760      +      0\n2962707  chrY  57209350  57209531      +      0\n\n[316789 rows x 5 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>strand</th>\n      <th>class</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>14829</td>\n      <td>14969</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>15038</td>\n      <td>15795</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>15947</td>\n      <td>16606</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>16765</td>\n      <td>16853</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>17055</td>\n      <td>17232</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2962663</th>\n      <td>chrY</td>\n      <td>57209733</td>\n      <td>57209821</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2962664</th>\n      <td>chrY</td>\n      <td>57209980</td>\n      <td>57210639</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2962665</th>\n      <td>chrY</td>\n      <td>57210792</td>\n      <td>57211551</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2962666</th>\n      <td>chrY</td>\n      <td>57211620</td>\n      <td>57211760</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2962707</th>\n      <td>chrY</td>\n      <td>57209350</td>\n      <td>57209531</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>316789 rows × 5 columns</p>\n</div>"
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_simulation_c_introns = pd.read_csv(\"./data/false_introns_simulated/simulation_c_introns.bed\", sep=\"\\t\",\n",
    "\t\t\t\t\t\t\t\t\tnames=[\"chr\", \"start\", \"end\", \"features\", \"score\", \"strand\"])\n",
    "df_simulation_c_introns = df_simulation_c_introns.loc[:, [\"chr\", \"start\", \"end\", \"strand\"]]\n",
    "df_simulation_c_introns[\"class\"] = 0\n",
    "df_simulation_c_introns = df_simulation_c_introns.drop_duplicates(subset=[\"chr\", \"start\", \"end\", \"strand\"])\n",
    "df_simulation_c_introns"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [
    {
     "data": {
      "text/plain": "         chr     start       end strand  class\n0       chr1     12227     12612      +      0\n1       chr1     12721     13220      +      0\n6       chr1     12227     12594      +      0\n7       chr1     12721     13402      +      0\n8       chr1     14829     14969      -      0\n...      ...       ...       ...    ...    ...\n165932  chrY  57210792  57211551      +      0\n165935  chrY  57208979  57209218      +      0\n165940  chrY  57211620  57211760      +      0\n165941  chrY  57213357  57213855      -      0\n165942  chrY  57213964  57214349      -      0\n\n[78889 rows x 5 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>strand</th>\n      <th>class</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>12227</td>\n      <td>12612</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>12721</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>chr1</td>\n      <td>12227</td>\n      <td>12594</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>chr1</td>\n      <td>12721</td>\n      <td>13402</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>chr1</td>\n      <td>14829</td>\n      <td>14969</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>165932</th>\n      <td>chrY</td>\n      <td>57210792</td>\n      <td>57211551</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>165935</th>\n      <td>chrY</td>\n      <td>57208979</td>\n      <td>57209218</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>165940</th>\n      <td>chrY</td>\n      <td>57211620</td>\n      <td>57211760</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>165941</th>\n      <td>chrY</td>\n      <td>57213357</td>\n      <td>57213855</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>165942</th>\n      <td>chrY</td>\n      <td>57213964</td>\n      <td>57214349</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>78889 rows × 5 columns</p>\n</div>"
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_simulation_d_introns = pd.read_csv(\"./data/false_introns_simulated/simulation_d_introns.bed\", sep=\"\\t\",\n",
    "\t\t\t\t\t\t\t\t\tnames=[\"chr\", \"start\", \"end\", \"features\", \"score\", \"strand\"])\n",
    "df_simulation_d_introns = df_simulation_d_introns.loc[:, [\"chr\", \"start\", \"end\", \"strand\"]]\n",
    "df_simulation_d_introns[\"class\"] = 0\n",
    "df_simulation_d_introns = df_simulation_d_introns.drop_duplicates(subset=[\"chr\", \"start\", \"end\", \"strand\"])\n",
    "df_simulation_d_introns"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [],
   "source": [
    "df_trainset_combined = pd.concat([df_gencode, df_manually_rejected_introns, df_simulation_a_introns, df_simulation_b_introns,\n",
    "\t\t\t\t\t   df_simulation_c_introns, df_simulation_d_introns]).drop_duplicates(subset=[\"chr\", \"start\", \"end\", \"strand\"], keep='first')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [
    {
     "data": {
      "text/plain": "         chr     start       end strand  class\n0       chr1     12227     12612      +      1\n1       chr1     12721     13220      +      1\n2       chr1     12057     12178      +      1\n3       chr1     12697     12974      +      1\n4       chr1     13052     13220      +      1\n...      ...       ...       ...    ...    ...\n165903  chrY  25464577  25465486      +      0\n165908  chrY  25513173  25513588      -      0\n165909  chrY  25513745  25516715      -      0\n165911  chrY  25525288  25527646      -      0\n165941  chrY  57213357  57213855      -      0\n\n[519034 rows x 5 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>strand</th>\n      <th>class</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>12227</td>\n      <td>12612</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>12721</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>12057</td>\n      <td>12178</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>12697</td>\n      <td>12974</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>13052</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>165903</th>\n      <td>chrY</td>\n      <td>25464577</td>\n      <td>25465486</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>165908</th>\n      <td>chrY</td>\n      <td>25513173</td>\n      <td>25513588</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>165909</th>\n      <td>chrY</td>\n      <td>25513745</td>\n      <td>25516715</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>165911</th>\n      <td>chrY</td>\n      <td>25525288</td>\n      <td>25527646</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>165941</th>\n      <td>chrY</td>\n      <td>57213357</td>\n      <td>57213855</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>519034 rows × 5 columns</p>\n</div>"
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trainset_combined"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total amount of generated negative introns: 116580\n"
     ]
    }
   ],
   "source": [
    "print(\"Total amount of generated negative introns:\", sum(df_trainset_combined[\"class\"] == 0))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "outputs": [],
   "source": [
    "assert (len(df_trainset_combined[df_trainset_combined.start == 15038]) == 1)\n",
    "assert (df_trainset_combined[df_trainset_combined.start == 15038].iloc[0][\"class\"] == 1)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "outputs": [
    {
     "data": {
      "text/plain": "         chr     start       end strand  class\n0       chr1     12227     12612      +      1\n1       chr1     12721     13220      +      1\n2       chr1     12057     12178      +      1\n3       chr1     12697     12974      +      1\n4       chr1     13052     13220      +      1\n...      ...       ...       ...    ...    ...\n165903  chrY  25464577  25465486      +      0\n165908  chrY  25513173  25513588      -      0\n165909  chrY  25513745  25516715      -      0\n165911  chrY  25525288  25527646      -      0\n165941  chrY  57213357  57213855      -      0\n\n[519034 rows x 5 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>strand</th>\n      <th>class</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>12227</td>\n      <td>12612</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>12721</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>12057</td>\n      <td>12178</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>12697</td>\n      <td>12974</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>13052</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>165903</th>\n      <td>chrY</td>\n      <td>25464577</td>\n      <td>25465486</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>165908</th>\n      <td>chrY</td>\n      <td>25513173</td>\n      <td>25513588</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>165909</th>\n      <td>chrY</td>\n      <td>25513745</td>\n      <td>25516715</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>165911</th>\n      <td>chrY</td>\n      <td>25525288</td>\n      <td>25527646</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>165941</th>\n      <td>chrY</td>\n      <td>57213357</td>\n      <td>57213855</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>519034 rows × 5 columns</p>\n</div>"
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trainset_combined.to_csv(\"data/04_train_set.tsv\", sep=\"\\t\", index=False)\n",
    "df_trainset_combined"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5006dfaea420edf91d4e86a72bc2428a57ad3b184a8e3b9cc55ec048d17e001247a36322b18b48fa19b24a784b8bd7fe33eb2cfa074d4625138f4728b56d3324  data/04_train_set.tsv\r\n"
     ]
    }
   ],
   "source": [
    "!sha512sum data/04_train_set.tsv"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}