15_combining_all_features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2023-07-02T07:46:50.172318Z",
     "start_time": "2023-07-02T07:46:49.700667Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import ast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "data": {
      "text/plain": "         chr     start       end strand  class\n0       chr1     12227     12612      +      1\n1       chr1     12721     13220      +      1\n2       chr1     12057     12178      +      1\n3       chr1     12697     12974      +      1\n4       chr1     13052     13220      +      1\n...      ...       ...       ...    ...    ...\n519029  chrY  25464577  25465486      +      0\n519030  chrY  25513173  25513588      -      0\n519031  chrY  25513745  25516715      -      0\n519032  chrY  25525288  25527646      -      0\n519033  chrY  57213357  57213855      -      0\n\n[519034 rows x 5 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>strand</th>\n      <th>class</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>12227</td>\n      <td>12612</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>12721</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>12057</td>\n      <td>12178</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>12697</td>\n      <td>12974</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>13052</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>519029</th>\n      <td>chrY</td>\n      <td>25464577</td>\n      <td>25465486</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>519030</th>\n      <td>chrY</td>\n      <td>25513173</td>\n      <td>25513588</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>519031</th>\n      <td>chrY</td>\n      <td>25513745</td>\n      <td>25516715</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>519032</th>\n      <td>chrY</td>\n      <td>25525288</td>\n      <td>25527646</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>519033</th>\n      <td>chrY</td>\n      <td>57213357</td>\n      <td>57213855</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>519034 rows × 5 columns</p>\n</div>"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trainset = pd.read_csv(\"data/04_train_set.tsv\", sep=\"\\t\")\n",
    "df_trainset"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-02T07:43:22.621433Z",
     "start_time": "2023-07-02T07:43:21.825426Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "df_features_dict = {\"5\": pd.read_csv(\"data/05_trainset_recount3_feature.tsv\", sep=\"\\t\"),\n",
    "                    \"6\": pd.read_csv(\"data/06_trainset_repeat_feature.tsv\", sep=\"\\t\"),\n",
    "                    \"7\": pd.read_csv(\"data/07_trainset_antisense_feature.tsv\", sep=\"\\t\"),\n",
    "                    \"8\": pd.read_csv(\"data/08_trainset_with_seq.tsv\", sep=\"\\t\"),\n",
    "                    \"9\": pd.read_csv(\"data/09_trainset_nearest_alt_ss_dist_feature\", sep=\"\\t\"),\n",
    "                    \"10\": pd.read_csv(\"data/10_trainset_MaxEntScan_feature.tsv\", sep=\"\\t\"),\n",
    "                    \"11\": pd.read_csv(\"data/11_trainset_splam_feature.tsv\", sep=\"\\t\"),\n",
    "                    \"12\": pd.read_csv(\"data/12_trainset_conservation_scores_feature.tsv\", sep=\"\\t\"),\n",
    "                    \"13\": pd.read_csv(\"data/13_recount3_near_ss_with_better_score_feature.tsv\", sep=\"\\t\"),\n",
    "                    \"14\": pd.read_csv(\"data/14_trainset_BPP_score_feature.tsv\", sep=\"\\t\")}"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-02T07:43:58.238371Z",
     "start_time": "2023-07-02T07:43:57.435296Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "data": {
      "text/plain": "         chr     start       end strand  class\n0       chr1     12227     12612      +      1\n1       chr1     12721     13220      +      1\n2       chr1     12057     12178      +      1\n3       chr1     12697     12974      +      1\n4       chr1     13052     13220      +      1\n...      ...       ...       ...    ...    ...\n519029  chrY  25464577  25465486      +      0\n519030  chrY  25513173  25513588      -      0\n519031  chrY  25513745  25516715      -      0\n519032  chrY  25525288  25527646      -      0\n519033  chrY  57213357  57213855      -      0\n\n[519034 rows x 5 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>strand</th>\n      <th>class</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>12227</td>\n      <td>12612</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>12721</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>12057</td>\n      <td>12178</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>12697</td>\n      <td>12974</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>13052</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>519029</th>\n      <td>chrY</td>\n      <td>25464577</td>\n      <td>25465486</td>\n      <td>+</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>519030</th>\n      <td>chrY</td>\n      <td>25513173</td>\n      <td>25513588</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>519031</th>\n      <td>chrY</td>\n      <td>25513745</td>\n      <td>25516715</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>519032</th>\n      <td>chrY</td>\n      <td>25525288</td>\n      <td>25527646</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>519033</th>\n      <td>chrY</td>\n      <td>57213357</td>\n      <td>57213855</td>\n      <td>-</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>519034 rows × 5 columns</p>\n</div>"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merged = df_trainset.copy()\n",
    "df_merged"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "data": {
      "text/plain": "         chr     start       end strand  class  recount3_score  \\\n0       chr1     12227     12612      +      1           59151   \n1       chr1     12721     13220      +      1           61021   \n2       chr1     12057     12178      +      1               0   \n3       chr1     12697     12974      +      1              21   \n4       chr1     13052     13220      +      1             526   \n...      ...       ...       ...    ...    ...             ...   \n519029  chrY  25464577  25465486      +      0               0   \n519030  chrY  25513173  25513588      -      0           31374   \n519031  chrY  25513745  25516715      -      0              67   \n519032  chrY  25525288  25527646      -      0           18958   \n519033  chrY  57213357  57213855      -      0               0   \n\n       repeat_features_start_site repeat_features_end_site  \\\n0                              []                       []   \n1                              []                       []   \n2                              []                       []   \n3                              []                       []   \n4                              []                       []   \n...                           ...                      ...   \n519029                         []                       []   \n519030                         []                       []   \n519031                         []                       []   \n519032                         []                       []   \n519033                         []                       []   \n\n        antisense_exon_start_ss  antisense_exon_end_ss  ...  \\\n0                         False                  False  ...   \n1                         False                  False  ...   \n2                         False                  False  ...   \n3                         False                  False  ...   \n4                         False                  False  ...   \n...                         ...                    ...  ...   \n519029                    False                  False  ...   \n519030                    False                  False  ...   \n519031                    False                  False  ...   \n519032                    False                  False  ...   \n519033                    False                  False  ...   \n\n       recount3_near_start_ss_with_better_score  \\\n0                                         False   \n1                                          True   \n2                                         False   \n3                                          True   \n4                                         False   \n...                                         ...   \n519029                                    False   \n519030                                    False   \n519031                                     True   \n519032                                    False   \n519033                                    False   \n\n       recount3_near_end_ss_with_better_score      bps  bp_pos    sc_bps  \\\n0                                       False  GCCCCAG    20.0  0.000232   \n1                                       False  GCCTTAG    19.0  0.005862   \n2                                       False  CCACAAC    19.0  0.003559   \n3                                       False  GGGTCAT    17.0  0.032313   \n4                                       False  GCCTTAG    19.0  0.005862   \n...                                       ...      ...     ...       ...   \n519029                                  False  GCTTAAA    21.0  0.037471   \n519030                                  False  AGATAAT    30.0  0.132502   \n519031                                  False  TTCTAAT    24.0  0.217305   \n519032                                  False  TGGTAAA    22.0  0.048905   \n519033                                  False  GCCTTAG    19.0  0.005862   \n\n              sc_ppt            sc   zsc_bps   zsc_ppt       zsc  \n0       7.304590e-03  1.694310e-06  4.582576  0.986842  4.582576  \n1       5.257510e-03  3.082192e-05  4.210821  1.121772  4.429667  \n2       7.527838e-07  2.679516e-09  4.050680  1.012740  4.026648  \n3       1.904623e-04  6.154421e-06  3.187796 -1.023767  5.414235  \n4       5.257510e-03  3.082192e-05  4.210821  1.121772  4.429667  \n...              ...           ...       ...       ...       ...  \n519029  3.814200e-04  1.429227e-05  4.181171  0.768307  4.144208  \n519030  1.618332e-01  2.144329e-02  5.943683  1.104011  5.992466  \n519031  1.371833e-05  2.981062e-06  4.472136  1.160504  4.472136  \n519032  3.656019e-03  1.787993e-04  4.123106  1.018607  4.123106  \n519033  5.257510e-03  3.082192e-05  4.210821  1.121772  4.429667  \n\n[519034 rows x 35 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>strand</th>\n      <th>class</th>\n      <th>recount3_score</th>\n      <th>repeat_features_start_site</th>\n      <th>repeat_features_end_site</th>\n      <th>antisense_exon_start_ss</th>\n      <th>antisense_exon_end_ss</th>\n      <th>...</th>\n      <th>recount3_near_start_ss_with_better_score</th>\n      <th>recount3_near_end_ss_with_better_score</th>\n      <th>bps</th>\n      <th>bp_pos</th>\n      <th>sc_bps</th>\n      <th>sc_ppt</th>\n      <th>sc</th>\n      <th>zsc_bps</th>\n      <th>zsc_ppt</th>\n      <th>zsc</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>12227</td>\n      <td>12612</td>\n      <td>+</td>\n      <td>1</td>\n      <td>59151</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>GCCCCAG</td>\n      <td>20.0</td>\n      <td>0.000232</td>\n      <td>7.304590e-03</td>\n      <td>1.694310e-06</td>\n      <td>4.582576</td>\n      <td>0.986842</td>\n      <td>4.582576</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>12721</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>1</td>\n      <td>61021</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>GCCTTAG</td>\n      <td>19.0</td>\n      <td>0.005862</td>\n      <td>5.257510e-03</td>\n      <td>3.082192e-05</td>\n      <td>4.210821</td>\n      <td>1.121772</td>\n      <td>4.429667</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>12057</td>\n      <td>12178</td>\n      <td>+</td>\n      <td>1</td>\n      <td>0</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>CCACAAC</td>\n      <td>19.0</td>\n      <td>0.003559</td>\n      <td>7.527838e-07</td>\n      <td>2.679516e-09</td>\n      <td>4.050680</td>\n      <td>1.012740</td>\n      <td>4.026648</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>12697</td>\n      <td>12974</td>\n      <td>+</td>\n      <td>1</td>\n      <td>21</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>GGGTCAT</td>\n      <td>17.0</td>\n      <td>0.032313</td>\n      <td>1.904623e-04</td>\n      <td>6.154421e-06</td>\n      <td>3.187796</td>\n      <td>-1.023767</td>\n      <td>5.414235</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>13052</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>1</td>\n      <td>526</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>GCCTTAG</td>\n      <td>19.0</td>\n      <td>0.005862</td>\n      <td>5.257510e-03</td>\n      <td>3.082192e-05</td>\n      <td>4.210821</td>\n      <td>1.121772</td>\n      <td>4.429667</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>519029</th>\n      <td>chrY</td>\n      <td>25464577</td>\n      <td>25465486</td>\n      <td>+</td>\n      <td>0</td>\n      <td>0</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>GCTTAAA</td>\n      <td>21.0</td>\n      <td>0.037471</td>\n      <td>3.814200e-04</td>\n      <td>1.429227e-05</td>\n      <td>4.181171</td>\n      <td>0.768307</td>\n      <td>4.144208</td>\n    </tr>\n    <tr>\n      <th>519030</th>\n      <td>chrY</td>\n      <td>25513173</td>\n      <td>25513588</td>\n      <td>-</td>\n      <td>0</td>\n      <td>31374</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>AGATAAT</td>\n      <td>30.0</td>\n      <td>0.132502</td>\n      <td>1.618332e-01</td>\n      <td>2.144329e-02</td>\n      <td>5.943683</td>\n      <td>1.104011</td>\n      <td>5.992466</td>\n    </tr>\n    <tr>\n      <th>519031</th>\n      <td>chrY</td>\n      <td>25513745</td>\n      <td>25516715</td>\n      <td>-</td>\n      <td>0</td>\n      <td>67</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>True</td>\n      <td>False</td>\n      <td>TTCTAAT</td>\n      <td>24.0</td>\n      <td>0.217305</td>\n      <td>1.371833e-05</td>\n      <td>2.981062e-06</td>\n      <td>4.472136</td>\n      <td>1.160504</td>\n      <td>4.472136</td>\n    </tr>\n    <tr>\n      <th>519032</th>\n      <td>chrY</td>\n      <td>25525288</td>\n      <td>25527646</td>\n      <td>-</td>\n      <td>0</td>\n      <td>18958</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>TGGTAAA</td>\n      <td>22.0</td>\n      <td>0.048905</td>\n      <td>3.656019e-03</td>\n      <td>1.787993e-04</td>\n      <td>4.123106</td>\n      <td>1.018607</td>\n      <td>4.123106</td>\n    </tr>\n    <tr>\n      <th>519033</th>\n      <td>chrY</td>\n      <td>57213357</td>\n      <td>57213855</td>\n      <td>-</td>\n      <td>0</td>\n      <td>0</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>GCCTTAG</td>\n      <td>19.0</td>\n      <td>0.005862</td>\n      <td>5.257510e-03</td>\n      <td>3.082192e-05</td>\n      <td>4.210821</td>\n      <td>1.121772</td>\n      <td>4.429667</td>\n    </tr>\n  </tbody>\n</table>\n<p>519034 rows × 35 columns</p>\n</div>"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Loop over the dictionary and concatenating it to the merged df\n",
    "for key in df_features_dict.keys():\n",
    "    if \"class\" in str(df_features_dict[key].columns):\n",
    "        df_features_dict[key] = df_features_dict[key].drop(columns=\"class\", axis=1)\n",
    "    if len(df_features_dict[key]) == len(df_trainset):\n",
    "        if key != \"10\":\n",
    "                df_merged = pd.concat([df_merged, df_features_dict[key].drop(columns=[\"chr\", \"start\", \"end\", \"strand\"])], axis=1)\n",
    "        else:\n",
    "            df_merged = pd.concat([df_merged, df_features_dict[key]], axis=1)\n",
    "    else:\n",
    "        df_merged = df_merged.merge(df_features_dict[key], how=\"left\", on=[\"chr\", \"start\", \"end\", \"strand\"])\n",
    "\n",
    "df_merged"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Populate the repeat_features into one-hots"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "data": {
      "text/plain": "         chr     start       end strand  class  recount3_score  \\\n0       chr1     12227     12612      +      1           59151   \n1       chr1     12721     13220      +      1           61021   \n2       chr1     12057     12178      +      1               0   \n3       chr1     12697     12974      +      1              21   \n4       chr1     13052     13220      +      1             526   \n...      ...       ...       ...    ...    ...             ...   \n519029  chrY  25464577  25465486      +      0               0   \n519030  chrY  25513173  25513588      -      0           31374   \n519031  chrY  25513745  25516715      -      0              67   \n519032  chrY  25525288  25527646      -      0           18958   \n519033  chrY  57213357  57213855      -      0               0   \n\n       repeat_features_start_site repeat_features_end_site  \\\n0                              []                       []   \n1                              []                       []   \n2                              []                       []   \n3                              []                       []   \n4                              []                       []   \n...                           ...                      ...   \n519029                         []                       []   \n519030                         []                       []   \n519031                         []                       []   \n519032                         []                       []   \n519033                         []                       []   \n\n        antisense_exon_start_ss  antisense_exon_end_ss  ...      bps bp_pos  \\\n0                         False                  False  ...  GCCCCAG   20.0   \n1                         False                  False  ...  GCCTTAG   19.0   \n2                         False                  False  ...  CCACAAC   19.0   \n3                         False                  False  ...  GGGTCAT   17.0   \n4                         False                  False  ...  GCCTTAG   19.0   \n...                         ...                    ...  ...      ...    ...   \n519029                    False                  False  ...  GCTTAAA   21.0   \n519030                    False                  False  ...  AGATAAT   30.0   \n519031                    False                  False  ...  TTCTAAT   24.0   \n519032                    False                  False  ...  TGGTAAA   22.0   \n519033                    False                  False  ...  GCCTTAG   19.0   \n\n          sc_bps        sc_ppt            sc   zsc_bps   zsc_ppt       zsc  \\\n0       0.000232  7.304590e-03  1.694310e-06  4.582576  0.986842  4.582576   \n1       0.005862  5.257510e-03  3.082192e-05  4.210821  1.121772  4.429667   \n2       0.003559  7.527838e-07  2.679516e-09  4.050680  1.012740  4.026648   \n3       0.032313  1.904623e-04  6.154421e-06  3.187796 -1.023767  5.414235   \n4       0.005862  5.257510e-03  3.082192e-05  4.210821  1.121772  4.429667   \n...          ...           ...           ...       ...       ...       ...   \n519029  0.037471  3.814200e-04  1.429227e-05  4.181171  0.768307  4.144208   \n519030  0.132502  1.618332e-01  2.144329e-02  5.943683  1.104011  5.992466   \n519031  0.217305  1.371833e-05  2.981062e-06  4.472136  1.160504  4.472136   \n519032  0.048905  3.656019e-03  1.787993e-04  4.123106  1.018607  4.123106   \n519033  0.005862  5.257510e-03  3.082192e-05  4.210821  1.121772  4.429667   \n\n        lst_repeat_features_start_site lst_repeat_features_end_site  \n0                                   []                           []  \n1                                   []                           []  \n2                                   []                           []  \n3                                   []                           []  \n4                                   []                           []  \n...                                ...                          ...  \n519029                              []                           []  \n519030                              []                           []  \n519031                              []                           []  \n519032                              []                           []  \n519033                              []                           []  \n\n[519034 rows x 37 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>strand</th>\n      <th>class</th>\n      <th>recount3_score</th>\n      <th>repeat_features_start_site</th>\n      <th>repeat_features_end_site</th>\n      <th>antisense_exon_start_ss</th>\n      <th>antisense_exon_end_ss</th>\n      <th>...</th>\n      <th>bps</th>\n      <th>bp_pos</th>\n      <th>sc_bps</th>\n      <th>sc_ppt</th>\n      <th>sc</th>\n      <th>zsc_bps</th>\n      <th>zsc_ppt</th>\n      <th>zsc</th>\n      <th>lst_repeat_features_start_site</th>\n      <th>lst_repeat_features_end_site</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>12227</td>\n      <td>12612</td>\n      <td>+</td>\n      <td>1</td>\n      <td>59151</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>GCCCCAG</td>\n      <td>20.0</td>\n      <td>0.000232</td>\n      <td>7.304590e-03</td>\n      <td>1.694310e-06</td>\n      <td>4.582576</td>\n      <td>0.986842</td>\n      <td>4.582576</td>\n      <td>[]</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>12721</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>1</td>\n      <td>61021</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>GCCTTAG</td>\n      <td>19.0</td>\n      <td>0.005862</td>\n      <td>5.257510e-03</td>\n      <td>3.082192e-05</td>\n      <td>4.210821</td>\n      <td>1.121772</td>\n      <td>4.429667</td>\n      <td>[]</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>12057</td>\n      <td>12178</td>\n      <td>+</td>\n      <td>1</td>\n      <td>0</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>CCACAAC</td>\n      <td>19.0</td>\n      <td>0.003559</td>\n      <td>7.527838e-07</td>\n      <td>2.679516e-09</td>\n      <td>4.050680</td>\n      <td>1.012740</td>\n      <td>4.026648</td>\n      <td>[]</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>12697</td>\n      <td>12974</td>\n      <td>+</td>\n      <td>1</td>\n      <td>21</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>GGGTCAT</td>\n      <td>17.0</td>\n      <td>0.032313</td>\n      <td>1.904623e-04</td>\n      <td>6.154421e-06</td>\n      <td>3.187796</td>\n      <td>-1.023767</td>\n      <td>5.414235</td>\n      <td>[]</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>13052</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>1</td>\n      <td>526</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>GCCTTAG</td>\n      <td>19.0</td>\n      <td>0.005862</td>\n      <td>5.257510e-03</td>\n      <td>3.082192e-05</td>\n      <td>4.210821</td>\n      <td>1.121772</td>\n      <td>4.429667</td>\n      <td>[]</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>519029</th>\n      <td>chrY</td>\n      <td>25464577</td>\n      <td>25465486</td>\n      <td>+</td>\n      <td>0</td>\n      <td>0</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>GCTTAAA</td>\n      <td>21.0</td>\n      <td>0.037471</td>\n      <td>3.814200e-04</td>\n      <td>1.429227e-05</td>\n      <td>4.181171</td>\n      <td>0.768307</td>\n      <td>4.144208</td>\n      <td>[]</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>519030</th>\n      <td>chrY</td>\n      <td>25513173</td>\n      <td>25513588</td>\n      <td>-</td>\n      <td>0</td>\n      <td>31374</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>AGATAAT</td>\n      <td>30.0</td>\n      <td>0.132502</td>\n      <td>1.618332e-01</td>\n      <td>2.144329e-02</td>\n      <td>5.943683</td>\n      <td>1.104011</td>\n      <td>5.992466</td>\n      <td>[]</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>519031</th>\n      <td>chrY</td>\n      <td>25513745</td>\n      <td>25516715</td>\n      <td>-</td>\n      <td>0</td>\n      <td>67</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>TTCTAAT</td>\n      <td>24.0</td>\n      <td>0.217305</td>\n      <td>1.371833e-05</td>\n      <td>2.981062e-06</td>\n      <td>4.472136</td>\n      <td>1.160504</td>\n      <td>4.472136</td>\n      <td>[]</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>519032</th>\n      <td>chrY</td>\n      <td>25525288</td>\n      <td>25527646</td>\n      <td>-</td>\n      <td>0</td>\n      <td>18958</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>TGGTAAA</td>\n      <td>22.0</td>\n      <td>0.048905</td>\n      <td>3.656019e-03</td>\n      <td>1.787993e-04</td>\n      <td>4.123106</td>\n      <td>1.018607</td>\n      <td>4.123106</td>\n      <td>[]</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>519033</th>\n      <td>chrY</td>\n      <td>57213357</td>\n      <td>57213855</td>\n      <td>-</td>\n      <td>0</td>\n      <td>0</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>GCCTTAG</td>\n      <td>19.0</td>\n      <td>0.005862</td>\n      <td>5.257510e-03</td>\n      <td>3.082192e-05</td>\n      <td>4.210821</td>\n      <td>1.121772</td>\n      <td>4.429667</td>\n      <td>[]</td>\n      <td>[]</td>\n    </tr>\n  </tbody>\n</table>\n<p>519034 rows × 37 columns</p>\n</div>"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merged['lst_repeat_features_start_site'] = df_merged['repeat_features_start_site'].apply(lambda x: ast.literal_eval(x) if len(x) > 2 else [])\n",
    "df_merged['lst_repeat_features_end_site'] = df_merged['repeat_features_end_site'].apply(lambda x: ast.literal_eval(x) if len(x) > 2 else [])\n",
    "df_merged"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-02T07:58:43.301009Z",
     "start_time": "2023-07-02T07:58:37.261338Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "data": {
      "text/plain": "{'Centromere',\n 'Dust',\n 'LTRs',\n 'Low complexity regions',\n 'RNA repeats',\n 'Satellite repeats',\n 'Simple repeats',\n 'Tandem repeats',\n 'Type I Transposons/LINE',\n 'Type I Transposons/SINE',\n 'Type II Transposons',\n 'Unknown'}"
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unique_repeat_features_values_start_site = set()\n",
    "unique_repeat_features_values_end_site = set()\n",
    "for idx, row in df_merged.iterrows():\n",
    "    unique_repeat_features_values_start_site.update(row['lst_repeat_features_start_site'])\n",
    "    unique_repeat_features_values_end_site.update(row['lst_repeat_features_end_site'])\n",
    "\n",
    "unique_repeat_features_values_start_site"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-02T08:01:59.876653Z",
     "start_time": "2023-07-02T08:01:41.295812Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "data": {
      "text/plain": "         chr     start       end strand  class  recount3_score  \\\n0       chr1     12227     12612      +      1           59151   \n1       chr1     12721     13220      +      1           61021   \n2       chr1     12057     12178      +      1               0   \n3       chr1     12697     12974      +      1              21   \n4       chr1     13052     13220      +      1             526   \n...      ...       ...       ...    ...    ...             ...   \n519029  chrY  25464577  25465486      +      0               0   \n519030  chrY  25513173  25513588      -      0           31374   \n519031  chrY  25513745  25516715      -      0              67   \n519032  chrY  25525288  25527646      -      0           18958   \n519033  chrY  57213357  57213855      -      0               0   \n\n       repeat_features_start_site repeat_features_end_site  \\\n0                              []                       []   \n1                              []                       []   \n2                              []                       []   \n3                              []                       []   \n4                              []                       []   \n...                           ...                      ...   \n519029                         []                       []   \n519030                         []                       []   \n519031                         []                       []   \n519032                         []                       []   \n519033                         []                       []   \n\n        antisense_exon_start_ss  antisense_exon_end_ss  ...  \\\n0                         False                  False  ...   \n1                         False                  False  ...   \n2                         False                  False  ...   \n3                         False                  False  ...   \n4                         False                  False  ...   \n...                         ...                    ...  ...   \n519029                    False                  False  ...   \n519030                    False                  False  ...   \n519031                    False                  False  ...   \n519032                    False                  False  ...   \n519033                    False                  False  ...   \n\n       repeat_features_end_site_Type I Transposons/SINE  \\\n0                                                     0   \n1                                                     0   \n2                                                     0   \n3                                                     0   \n4                                                     0   \n...                                                 ...   \n519029                                                0   \n519030                                                0   \n519031                                                0   \n519032                                                0   \n519033                                                0   \n\n       repeat_features_end_site_RNA repeats repeat_features_end_site_Unknown  \\\n0                                         0                                0   \n1                                         0                                0   \n2                                         0                                0   \n3                                         0                                0   \n4                                         0                                0   \n...                                     ...                              ...   \n519029                                    0                                0   \n519030                                    0                                0   \n519031                                    0                                0   \n519032                                    0                                0   \n519033                                    0                                0   \n\n        repeat_features_end_site_Low complexity regions  \\\n0                                                     0   \n1                                                     0   \n2                                                     0   \n3                                                     0   \n4                                                     0   \n...                                                 ...   \n519029                                                0   \n519030                                                0   \n519031                                                0   \n519032                                                0   \n519033                                                0   \n\n        repeat_features_end_site_Type I Transposons/LINE  \\\n0                                                      0   \n1                                                      0   \n2                                                      0   \n3                                                      0   \n4                                                      0   \n...                                                  ...   \n519029                                                 0   \n519030                                                 0   \n519031                                                 0   \n519032                                                 0   \n519033                                                 0   \n\n       repeat_features_end_site_Centromere repeat_features_end_site_Dust  \\\n0                                        0                             0   \n1                                        0                             0   \n2                                        0                             0   \n3                                        0                             0   \n4                                        0                             0   \n...                                    ...                           ...   \n519029                                   0                             0   \n519030                                   0                             0   \n519031                                   0                             0   \n519032                                   0                             0   \n519033                                   0                             0   \n\n        repeat_features_end_site_Satellite repeats  \\\n0                                                0   \n1                                                0   \n2                                                0   \n3                                                0   \n4                                                0   \n...                                            ...   \n519029                                           0   \n519030                                           0   \n519031                                           0   \n519032                                           0   \n519033                                           0   \n\n        repeat_features_end_site_Tandem repeats repeat_features_end_site_LTRs  \n0                                             0                             0  \n1                                             0                             0  \n2                                             0                             0  \n3                                             0                             0  \n4                                             0                             0  \n...                                         ...                           ...  \n519029                                        0                             0  \n519030                                        0                             0  \n519031                                        0                             0  \n519032                                        0                             0  \n519033                                        0                             0  \n\n[519034 rows x 61 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>strand</th>\n      <th>class</th>\n      <th>recount3_score</th>\n      <th>repeat_features_start_site</th>\n      <th>repeat_features_end_site</th>\n      <th>antisense_exon_start_ss</th>\n      <th>antisense_exon_end_ss</th>\n      <th>...</th>\n      <th>repeat_features_end_site_Type I Transposons/SINE</th>\n      <th>repeat_features_end_site_RNA repeats</th>\n      <th>repeat_features_end_site_Unknown</th>\n      <th>repeat_features_end_site_Low complexity regions</th>\n      <th>repeat_features_end_site_Type I Transposons/LINE</th>\n      <th>repeat_features_end_site_Centromere</th>\n      <th>repeat_features_end_site_Dust</th>\n      <th>repeat_features_end_site_Satellite repeats</th>\n      <th>repeat_features_end_site_Tandem repeats</th>\n      <th>repeat_features_end_site_LTRs</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>12227</td>\n      <td>12612</td>\n      <td>+</td>\n      <td>1</td>\n      <td>59151</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>12721</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>1</td>\n      <td>61021</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>12057</td>\n      <td>12178</td>\n      <td>+</td>\n      <td>1</td>\n      <td>0</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>12697</td>\n      <td>12974</td>\n      <td>+</td>\n      <td>1</td>\n      <td>21</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>13052</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>1</td>\n      <td>526</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>519029</th>\n      <td>chrY</td>\n      <td>25464577</td>\n      <td>25465486</td>\n      <td>+</td>\n      <td>0</td>\n      <td>0</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>519030</th>\n      <td>chrY</td>\n      <td>25513173</td>\n      <td>25513588</td>\n      <td>-</td>\n      <td>0</td>\n      <td>31374</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>519031</th>\n      <td>chrY</td>\n      <td>25513745</td>\n      <td>25516715</td>\n      <td>-</td>\n      <td>0</td>\n      <td>67</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>519032</th>\n      <td>chrY</td>\n      <td>25525288</td>\n      <td>25527646</td>\n      <td>-</td>\n      <td>0</td>\n      <td>18958</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>519033</th>\n      <td>chrY</td>\n      <td>57213357</td>\n      <td>57213855</td>\n      <td>-</td>\n      <td>0</td>\n      <td>0</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>519034 rows × 61 columns</p>\n</div>"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for repeat_feature in unique_repeat_features_values_start_site:\n",
    "    df_merged[\"repeat_features_start_site_\"+repeat_feature] = df_merged[\"repeat_features_start_site\"].str.count(\"'\"+repeat_feature+\"'\").astype(int)\n",
    "\n",
    "\n",
    "for repeat_feature in unique_repeat_features_values_start_site:\n",
    "    df_merged[\"repeat_features_end_site_\"+repeat_feature] = df_merged[\"repeat_features_end_site\"].str.count(\"'\"+repeat_feature+\"'\").astype(int)\n",
    "\n",
    "df_merged"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-02T08:03:55.749700Z",
     "start_time": "2023-07-02T08:03:53.340585Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "data": {
      "text/plain": "         chr     start       end strand  class  recount3_score  \\\n0       chr1     12227     12612      +      1           59151   \n1       chr1     12721     13220      +      1           61021   \n2       chr1     12057     12178      +      1               0   \n3       chr1     12697     12974      +      1              21   \n4       chr1     13052     13220      +      1             526   \n...      ...       ...       ...    ...    ...             ...   \n519029  chrY  25464577  25465486      +      0               0   \n519030  chrY  25513173  25513588      -      0           31374   \n519031  chrY  25513745  25516715      -      0              67   \n519032  chrY  25525288  25527646      -      0           18958   \n519033  chrY  57213357  57213855      -      0               0   \n\n        antisense_exon_start_ss  antisense_exon_end_ss  \\\n0                         False                  False   \n1                         False                  False   \n2                         False                  False   \n3                         False                  False   \n4                         False                  False   \n...                         ...                    ...   \n519029                    False                  False   \n519030                    False                  False   \n519031                    False                  False   \n519032                    False                  False   \n519033                    False                  False   \n\n                                      maxentscan_sequence start_ss  ...  \\\n0       ccaGTAAGTAGTGCTTGTGCTCATCTCCTTGGCTGTGATACGTGGC...       GT  ...   \n1       gagGTGAGAGGAGAGTAGACAGTGAGTGGGAGTGGCGTCGCCCCTA...       GT  ...   \n2       gagCACTGGAGTGGAGTTTTCCTGTGGAGAGGAGCCATGCCTAGAG...       CA  ...   \n3       cttGTGAGTGTCCCCAGTGTTGCAGAGGTGAGAGGAGAGTAGACAG...       GT  ...   \n4       tagGCAAGCCTGGCTGCCTCCAGCTGGGTCGACAGACAGGGGCTGG...       GC  ...   \n...                                                   ...      ...  ...   \n519029  cacGTTGAGGCGCCCAGTGGCGGCCTCACGGGGCAGGGCGAGGGCG...       GT  ...   \n519030  aggGTAAGATCAGTGCTATTGTCAGAGGAAAAACTCCTGGCCATCA...       GT  ...   \n519031  gctGTAAGTTCCACATTGATTATCATAGGCTAACCATGGGCCAGGC...       GT  ...   \n519032  tgtGTGTATAAATATCTGGACTTTTTGGTTAAGTAATTATAGTTAA...       GT  ...   \n519033  gagGTGAGAGGAGAGTAGACAGTGAGTGGGAGTGGCGTCGCCCCTA...       GT  ...   \n\n       repeat_features_end_site_Type I Transposons/SINE  \\\n0                                                     0   \n1                                                     0   \n2                                                     0   \n3                                                     0   \n4                                                     0   \n...                                                 ...   \n519029                                                0   \n519030                                                0   \n519031                                                0   \n519032                                                0   \n519033                                                0   \n\n        repeat_features_end_site_RNA repeats  \\\n0                                          0   \n1                                          0   \n2                                          0   \n3                                          0   \n4                                          0   \n...                                      ...   \n519029                                     0   \n519030                                     0   \n519031                                     0   \n519032                                     0   \n519033                                     0   \n\n        repeat_features_end_site_Unknown  \\\n0                                      0   \n1                                      0   \n2                                      0   \n3                                      0   \n4                                      0   \n...                                  ...   \n519029                                 0   \n519030                                 0   \n519031                                 0   \n519032                                 0   \n519033                                 0   \n\n       repeat_features_end_site_Low complexity regions  \\\n0                                                    0   \n1                                                    0   \n2                                                    0   \n3                                                    0   \n4                                                    0   \n...                                                ...   \n519029                                               0   \n519030                                               0   \n519031                                               0   \n519032                                               0   \n519033                                               0   \n\n       repeat_features_end_site_Type I Transposons/LINE  \\\n0                                                     0   \n1                                                     0   \n2                                                     0   \n3                                                     0   \n4                                                     0   \n...                                                 ...   \n519029                                                0   \n519030                                                0   \n519031                                                0   \n519032                                                0   \n519033                                                0   \n\n        repeat_features_end_site_Centromere  repeat_features_end_site_Dust  \\\n0                                         0                              0   \n1                                         0                              0   \n2                                         0                              0   \n3                                         0                              0   \n4                                         0                              0   \n...                                     ...                            ...   \n519029                                    0                              0   \n519030                                    0                              0   \n519031                                    0                              0   \n519032                                    0                              0   \n519033                                    0                              0   \n\n       repeat_features_end_site_Satellite repeats  \\\n0                                               0   \n1                                               0   \n2                                               0   \n3                                               0   \n4                                               0   \n...                                           ...   \n519029                                          0   \n519030                                          0   \n519031                                          0   \n519032                                          0   \n519033                                          0   \n\n        repeat_features_end_site_Tandem repeats  repeat_features_end_site_LTRs  \n0                                             0                              0  \n1                                             0                              0  \n2                                             0                              0  \n3                                             0                              0  \n4                                             0                              0  \n...                                         ...                            ...  \n519029                                        0                              0  \n519030                                        0                              0  \n519031                                        0                              0  \n519032                                        0                              0  \n519033                                        0                              0  \n\n[519034 rows x 57 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>chr</th>\n      <th>start</th>\n      <th>end</th>\n      <th>strand</th>\n      <th>class</th>\n      <th>recount3_score</th>\n      <th>antisense_exon_start_ss</th>\n      <th>antisense_exon_end_ss</th>\n      <th>maxentscan_sequence</th>\n      <th>start_ss</th>\n      <th>...</th>\n      <th>repeat_features_end_site_Type I Transposons/SINE</th>\n      <th>repeat_features_end_site_RNA repeats</th>\n      <th>repeat_features_end_site_Unknown</th>\n      <th>repeat_features_end_site_Low complexity regions</th>\n      <th>repeat_features_end_site_Type I Transposons/LINE</th>\n      <th>repeat_features_end_site_Centromere</th>\n      <th>repeat_features_end_site_Dust</th>\n      <th>repeat_features_end_site_Satellite repeats</th>\n      <th>repeat_features_end_site_Tandem repeats</th>\n      <th>repeat_features_end_site_LTRs</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>chr1</td>\n      <td>12227</td>\n      <td>12612</td>\n      <td>+</td>\n      <td>1</td>\n      <td>59151</td>\n      <td>False</td>\n      <td>False</td>\n      <td>ccaGTAAGTAGTGCTTGTGCTCATCTCCTTGGCTGTGATACGTGGC...</td>\n      <td>GT</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>chr1</td>\n      <td>12721</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>1</td>\n      <td>61021</td>\n      <td>False</td>\n      <td>False</td>\n      <td>gagGTGAGAGGAGAGTAGACAGTGAGTGGGAGTGGCGTCGCCCCTA...</td>\n      <td>GT</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>chr1</td>\n      <td>12057</td>\n      <td>12178</td>\n      <td>+</td>\n      <td>1</td>\n      <td>0</td>\n      <td>False</td>\n      <td>False</td>\n      <td>gagCACTGGAGTGGAGTTTTCCTGTGGAGAGGAGCCATGCCTAGAG...</td>\n      <td>CA</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>chr1</td>\n      <td>12697</td>\n      <td>12974</td>\n      <td>+</td>\n      <td>1</td>\n      <td>21</td>\n      <td>False</td>\n      <td>False</td>\n      <td>cttGTGAGTGTCCCCAGTGTTGCAGAGGTGAGAGGAGAGTAGACAG...</td>\n      <td>GT</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>chr1</td>\n      <td>13052</td>\n      <td>13220</td>\n      <td>+</td>\n      <td>1</td>\n      <td>526</td>\n      <td>False</td>\n      <td>False</td>\n      <td>tagGCAAGCCTGGCTGCCTCCAGCTGGGTCGACAGACAGGGGCTGG...</td>\n      <td>GC</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>519029</th>\n      <td>chrY</td>\n      <td>25464577</td>\n      <td>25465486</td>\n      <td>+</td>\n      <td>0</td>\n      <td>0</td>\n      <td>False</td>\n      <td>False</td>\n      <td>cacGTTGAGGCGCCCAGTGGCGGCCTCACGGGGCAGGGCGAGGGCG...</td>\n      <td>GT</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>519030</th>\n      <td>chrY</td>\n      <td>25513173</td>\n      <td>25513588</td>\n      <td>-</td>\n      <td>0</td>\n      <td>31374</td>\n      <td>False</td>\n      <td>False</td>\n      <td>aggGTAAGATCAGTGCTATTGTCAGAGGAAAAACTCCTGGCCATCA...</td>\n      <td>GT</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>519031</th>\n      <td>chrY</td>\n      <td>25513745</td>\n      <td>25516715</td>\n      <td>-</td>\n      <td>0</td>\n      <td>67</td>\n      <td>False</td>\n      <td>False</td>\n      <td>gctGTAAGTTCCACATTGATTATCATAGGCTAACCATGGGCCAGGC...</td>\n      <td>GT</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>519032</th>\n      <td>chrY</td>\n      <td>25525288</td>\n      <td>25527646</td>\n      <td>-</td>\n      <td>0</td>\n      <td>18958</td>\n      <td>False</td>\n      <td>False</td>\n      <td>tgtGTGTATAAATATCTGGACTTTTTGGTTAAGTAATTATAGTTAA...</td>\n      <td>GT</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>519033</th>\n      <td>chrY</td>\n      <td>57213357</td>\n      <td>57213855</td>\n      <td>-</td>\n      <td>0</td>\n      <td>0</td>\n      <td>False</td>\n      <td>False</td>\n      <td>gagGTGAGAGGAGAGTAGACAGTGAGTGGGAGTGGCGTCGCCCCTA...</td>\n      <td>GT</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>519034 rows × 57 columns</p>\n</div>"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merged = df_merged.drop(columns=[\"repeat_features_start_site\", \"repeat_features_end_site\", \"lst_repeat_features_start_site\", \"lst_repeat_features_end_site\"])\n",
    "df_merged"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-02T08:06:24.687934Z",
     "start_time": "2023-07-02T08:06:24.231368Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [],
   "source": [
    "df_merged.to_csv(\"data/15_trainset_populated.tsv\", sep=\"\\t\", index=False)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9e624882d56b040072ec23939df03c88  data/15_trainset_populated.tsv\r\n"
     ]
    }
   ],
   "source": [
    "!md5sum data/15_trainset_populated.tsv"
   ],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}