Project1_starter_code.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pyBigWig==0.2.7\n",
      "  Downloading pyBigWig-0.2.7.tar.gz (49 kB)\n",
      "\u001b[K     |████████████████████████████████| 49 kB 4.0 MB/s eta 0:00:011\n",
      "\u001b[?25hBuilding wheels for collected packages: pyBigWig\n",
      "  Building wheel for pyBigWig (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for pyBigWig: filename=pyBigWig-0.2.7-cp39-cp39-macosx_10_9_x86_64.whl size=37408 sha256=fbcd0aaadb9605da070327a9d8947d2c75ec2459b9452601e25ca3d281e67c48\n",
      "  Stored in directory: /Users/sidhu/Library/Caches/pip/wheels/20/42/5a/bc4c4f11ffd7e253fbaa62b9afdfff0eee272ac5817542f36d\n",
      "Successfully built pyBigWig\n",
      "Installing collected packages: pyBigWig\n",
      "Successfully installed pyBigWig-0.2.7\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install pyBigWig==0.2.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import libraries that are required to run your project\n",
    "# You are allowed to add more libraries as you need\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy.stats import spearmanr\n",
    "import pyBigWig as pbw"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Work Package 1.1 - Modeling Choices & Data Pre-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: \n",
    "# Load your feature (bed and/or bigwig and/or fasta) and target files (tsv) here.\n",
    "# Decide which features to use for training. Feel free to process them however you need.\n",
    "\n",
    "# NOTE: \n",
    "# bed and bigwig files contain signals of all chromosomes (including sex chromosomes).\n",
    "# Training and validation split based on chromosomes has been done for you. \n",
    "# However, you can resplit the data in any way you want.\n",
    "\n",
    "#path_data = \"/path/to/your/data/files\"  # TODO\n",
    "path_data = \"/Users/sidhu/Documents/GENOMICS/GENOMICS/Data/\"\n",
    "path_test = \"/path/to/test/info/file\"   # X3_test_info.tsv ; TODO\n",
    "test_genes = pd.read_csv(path_test, sep='\\t')\n",
    "# ---------------------------INSERT CODE HERE---------------------------\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# ---------------------------------------------------------------------- "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1.1 DATASET DESCRIPTION"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CAGE-train folder\n",
    "\n",
    "We are given train_info and labels y for 3 different DNA strands X1, X2 & X3\n",
    "\n",
    "### For X1 and X2\n",
    "\n",
    "#### TRAIN DATA\n",
    "\n",
    "##### train_info: \n",
    "contains the 14'310 entries each containing 7 values:\n",
    "\n",
    "1. gene_name\n",
    "2. chr number: contain all numbers [2, 22] \\ {14, 19}\n",
    "3. gene_start\n",
    "4. gene_end\n",
    "5. TSS_start\n",
    "6. TSS_end\n",
    "7. strand: if + we read from gene_start to gene_end, for - we read from gene_end to gene_start\n",
    "\n",
    "##### train_y: \n",
    "contains also 14'310 entries with the label y representing GEX (gene expression)\n",
    "\n",
    "#### VALIDATION DATA\n",
    "\n",
    "The val_info and val_y follow the same procedure except that they only have 1974 entries and they only contain chromosom 19 & 14\n",
    "\n",
    "### X3\n",
    "\n",
    "For X3 we have 1984 entries again with 7 values given as in train_info except we only have chr 1.\n",
    "\n",
    "\n",
    "## TODO:\n",
    "1. What is difference between gene_starte/end and TSS_start/end.\n",
    "2. What is strand +/- and do we need it?\n",
    "3. You could have potentially multiple TSS in one gene, why not the case in the dataset?\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tsv_path = \"/Users/sidhu/Documents/GENOMICS/GENOMICS/Data/CAGE-train/\"\n",
    "X1_train_info = pd.read_csv(tsv_path + \"X1_train_info.tsv\", sep= '\\t')\n",
    "X1_train_y = pd.read_csv(tsv_path + \"X1_train_y.tsv\", sep= '\\t')\n",
    "X1_val_info = pd.read_csv(tsv_path + \"X1_val_info.tsv\", sep= '\\t')\n",
    "X1_val_y = pd.read_csv(tsv_path + \"X1_val_y.tsv\", sep= '\\t')\n",
    "X3_test_info = pd.read_csv(tsv_path + \"X3_test_info.tsv\", sep= '\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(14310, 7)\n",
      "<bound method NDFrame.describe of       gene_name    chr  gene_start   gene_end  TSS_start    TSS_end strand\n",
      "0       SLC20A1   chr2   112645939  112663825  112658362  112658412      +\n",
      "1      C11orf58  chr11    16613132   16758340   16738643   16738693      +\n",
      "2        ZSCAN9   chr6    28224886   28233487   28225263   28225313      +\n",
      "3          CD19  chr16    28931965   28939342   28931956   28932006      +\n",
      "4       TMEM123  chr11   102396332  102470384  102452789  102452839      -\n",
      "...         ...    ...         ...        ...        ...        ...    ...\n",
      "14305     ACOX1  chr17    75941507   75979177   75955565   75955589      -\n",
      "14306     MLXIP  chr12   122078756  122147344  122127887  122127937      +\n",
      "14307     ASGR2  chr17     7101322    7115700    7114614    7114653      -\n",
      "14308     OR5A1  chr11    59436469   59451380   59443143   59443193      +\n",
      "14309      TLR3   chr4   186069155  186088073  186069154  186069204      +\n",
      "\n",
      "[14310 rows x 7 columns]>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gene_name</th>\n",
       "      <th>chr</th>\n",
       "      <th>gene_start</th>\n",
       "      <th>gene_end</th>\n",
       "      <th>TSS_start</th>\n",
       "      <th>TSS_end</th>\n",
       "      <th>strand</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SLC20A1</td>\n",
       "      <td>chr2</td>\n",
       "      <td>112645939</td>\n",
       "      <td>112663825</td>\n",
       "      <td>112658362</td>\n",
       "      <td>112658412</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C11orf58</td>\n",
       "      <td>chr11</td>\n",
       "      <td>16613132</td>\n",
       "      <td>16758340</td>\n",
       "      <td>16738643</td>\n",
       "      <td>16738693</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ZSCAN9</td>\n",
       "      <td>chr6</td>\n",
       "      <td>28224886</td>\n",
       "      <td>28233487</td>\n",
       "      <td>28225263</td>\n",
       "      <td>28225313</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CD19</td>\n",
       "      <td>chr16</td>\n",
       "      <td>28931965</td>\n",
       "      <td>28939342</td>\n",
       "      <td>28931956</td>\n",
       "      <td>28932006</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TMEM123</td>\n",
       "      <td>chr11</td>\n",
       "      <td>102396332</td>\n",
       "      <td>102470384</td>\n",
       "      <td>102452789</td>\n",
       "      <td>102452839</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14305</th>\n",
       "      <td>ACOX1</td>\n",
       "      <td>chr17</td>\n",
       "      <td>75941507</td>\n",
       "      <td>75979177</td>\n",
       "      <td>75955565</td>\n",
       "      <td>75955589</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14306</th>\n",
       "      <td>MLXIP</td>\n",
       "      <td>chr12</td>\n",
       "      <td>122078756</td>\n",
       "      <td>122147344</td>\n",
       "      <td>122127887</td>\n",
       "      <td>122127937</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14307</th>\n",
       "      <td>ASGR2</td>\n",
       "      <td>chr17</td>\n",
       "      <td>7101322</td>\n",
       "      <td>7115700</td>\n",
       "      <td>7114614</td>\n",
       "      <td>7114653</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14308</th>\n",
       "      <td>OR5A1</td>\n",
       "      <td>chr11</td>\n",
       "      <td>59436469</td>\n",
       "      <td>59451380</td>\n",
       "      <td>59443143</td>\n",
       "      <td>59443193</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14309</th>\n",
       "      <td>TLR3</td>\n",
       "      <td>chr4</td>\n",
       "      <td>186069155</td>\n",
       "      <td>186088073</td>\n",
       "      <td>186069154</td>\n",
       "      <td>186069204</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14310 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      gene_name    chr  gene_start   gene_end  TSS_start    TSS_end strand\n",
       "0       SLC20A1   chr2   112645939  112663825  112658362  112658412      +\n",
       "1      C11orf58  chr11    16613132   16758340   16738643   16738693      +\n",
       "2        ZSCAN9   chr6    28224886   28233487   28225263   28225313      +\n",
       "3          CD19  chr16    28931965   28939342   28931956   28932006      +\n",
       "4       TMEM123  chr11   102396332  102470384  102452789  102452839      -\n",
       "...         ...    ...         ...        ...        ...        ...    ...\n",
       "14305     ACOX1  chr17    75941507   75979177   75955565   75955589      -\n",
       "14306     MLXIP  chr12   122078756  122147344  122127887  122127937      +\n",
       "14307     ASGR2  chr17     7101322    7115700    7114614    7114653      -\n",
       "14308     OR5A1  chr11    59436469   59451380   59443143   59443193      +\n",
       "14309      TLR3   chr4   186069155  186088073  186069154  186069204      +\n",
       "\n",
       "[14310 rows x 7 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(X1_train_info.shape)\n",
    "print(X1_train_info.describe)\n",
    "X1_train_info"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### HOW Long are the TSS sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:ylabel='Frequency'>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAD5CAYAAAANxrPXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWNUlEQVR4nO3dfbAd9X3f8ffHkoMhDjYPgqiSiHCtOgbGT1xTtXZb28RBqR1EWlMrExdNSqOG0sZukrElJ1PSzmjGniS2Q1NIiHERtmMs4wfUxCSWRRzaGUC++KFCPBQ1ELiWguRnnDg4wt/+cX5KDldHV4e7Oudydd+vmTNn97v72/3t/nE/d/e355xUFZIkzdaz5roDkqT5zSCRJHVikEiSOjFIJEmdGCSSpE4MEklSJ4tHteEkHwDeCOyvqvOmLftl4NeBJVX11VbbBFwOPAn8QlX9caufD9wAnAh8GnhrVVWSE4AbgfOBrwFvrqqHj9av008/vVauXHksDlGSFoy77777q1W1ZNCykQUJvT/+v03vj/3fSrICeD3wSF/tHGAdcC7w94DPJvkHVfUkcC2wAbiTXpCsAW6lFzrfqKoXJlkHvBt489E6tXLlSiYnJzsfnCQtJEn+/EjLRnZrq6puB74+YNF7gbcD/Z+EXAvcVFVPVNVDwB7ggiRLgZOr6o7qfXLyRuCSvjZb2vTNwIVJcuyPRJI0k7GOkSS5GPhKVX152qJlwKN981OttqxNT68/pU1VHQS+BZw2gm5LkmYwyltbT5HkJOBXgB8ftHhArWaoz9Rm0L430Ls9xllnnXXUvkqShjfOK5K/D5wNfDnJw8By4AtJfpjelcaKvnWXA3tbffmAOv1tkiwGnsfgW2lU1XVVNVFVE0uWDBwrkiTN0tiCpKp2VdUZVbWyqlbSC4JXVNVfANuAdUlOSHI2sArYWVX7gMeTrG7jH5cBt7RNbgPWt+k3AbeV30ApSWM3siBJ8hHgDuBFSaaSXH6kdatqN7AVuBf4I+DK9sQWwBXA++kNwP8/ek9sAVwPnJZkD/CLwMaRHIgkaUZZaP/ET0xMlI//StLTk+TuqpoYtMxPtkuSOjFIJEmdjO3xX0kSrNz4h3O274ff9YaRbNcrEklSJwaJJKkTg0SS1IlBIknqxCCRJHVikEiSOjFIJEmdGCSSpE4MEklSJwaJJKkTg0SS1IlBIknqxCCRJHVikEiSOjFIJEmdGCSSpE4MEklSJwaJJKkTg0SS1MnIgiTJB5LsT3JPX+3Xk9yf5P8k+WSS5/ct25RkT5IHklzUVz8/ya627OokafUTkny01e9KsnJUxyJJOrJRXpHcAKyZVtsOnFdVLwH+L7AJIMk5wDrg3NbmmiSLWptrgQ3AqvY6tM3LgW9U1QuB9wLvHtmRSJKOaGRBUlW3A1+fVvtMVR1ss3cCy9v0WuCmqnqiqh4C9gAXJFkKnFxVd1RVATcCl/S12dKmbwYuPHS1Ikkan7kcI/k3wK1tehnwaN+yqVZb1qan15/SpoXTt4DTBu0oyYYkk0kmDxw4cMwOQJI0R0GS5FeAg8CHD5UGrFYz1Gdqc3ix6rqqmqiqiSVLljzd7kqSZjD2IEmyHngj8DPtdhX0rjRW9K22HNjb6ssH1J/SJsli4HlMu5UmSRq9sQZJkjXAO4CLq+qv+hZtA9a1J7HOpjeovrOq9gGPJ1ndxj8uA27pa7O+Tb8JuK0vmCRJY7J4VBtO8hHgNcDpSaaAq+g9pXUCsL2Ni99ZVT9fVbuTbAXupXfL68qqerJt6gp6T4CdSG9M5dC4yvXAB5PsoXclsm5UxyJJOrKRBUlV/fSA8vUzrL8Z2DygPgmcN6D+18ClXfooSerOT7ZLkjoxSCRJnRgkkqRODBJJUicGiSSpE4NEktSJQSJJ6sQgkSR1YpBIkjoxSCRJnRgkkqRODBJJUicGiSSpE4NEktSJQSJJ6sQgkSR1YpBIkjoxSCRJnRgkkqRODBJJUicGiSSpk5EFSZIPJNmf5J6+2qlJtid5sL2f0rdsU5I9SR5IclFf/fwku9qyq5Ok1U9I8tFWvyvJylEdiyTpyEZ5RXIDsGZabSOwo6pWATvaPEnOAdYB57Y21yRZ1NpcC2wAVrXXoW1eDnyjql4IvBd498iORJJ0RCMLkqq6Hfj6tPJaYEub3gJc0le/qaqeqKqHgD3ABUmWAidX1R1VVcCN09oc2tbNwIWHrlYkSeMz7jGSM6tqH0B7P6PVlwGP9q031WrL2vT0+lPaVNVB4FvAaSPruSRpoGfKYPugK4maoT5Tm8M3nmxIMplk8sCBA7PsoiRpkHEHyWPtdhXtfX+rTwEr+tZbDuxt9eUD6k9pk2Qx8DwOv5UGQFVdV1UTVTWxZMmSY3QokiQYf5BsA9a36fXALX31de1JrLPpDarvbLe/Hk+yuo1/XDatzaFtvQm4rY2jSJLGaPGoNpzkI8BrgNOTTAFXAe8Ctia5HHgEuBSgqnYn2QrcCxwErqyqJ9umrqD3BNiJwK3tBXA98MEke+hdiawb1bFIko5sZEFSVT99hEUXHmH9zcDmAfVJ4LwB9b+mBZEkae48UwbbJUnzlEEiSerEIJEkdWKQSJI6MUgkSZ0YJJKkTgwSSVInBokkqRODRJLUiUEiSerEIJEkdWKQSJI6MUgkSZ0YJJKkTgwSSVInQwVJksN+D0SSJBj+iuR3kuxM8u+TPH+UHZIkzS9DBUlVvRr4GWAFMJnk95O8fqQ9kyTNC0OPkVTVg8CvAu8A/hlwdZL7k/yLUXVOkvTMN+wYyUuSvBe4D3gd8JNV9eI2/d4R9k+S9Ay3eMj1fhv4PeCdVfXdQ8Wq2pvkV0fSM0nSvDBskPxz4LtV9SRAkmcBz6mqv6qqD46sd5KkZ7xhx0g+C5zYN39Sq81Kkv+UZHeSe5J8JMlzkpyaZHuSB9v7KX3rb0qyJ8kDSS7qq5+fZFdbdnWSzLZPkqTZGTZInlNV3zk006ZPms0OkywDfgGYqKrzgEXAOmAjsKOqVgE72jxJzmnLzwXWANckWdQ2dy2wAVjVXmtm0ydJ0uwNGyR/meQVh2aSnA98d4b1j2YxcGKSxfQCaS+wFtjSlm8BLmnTa4GbquqJqnoI2ANckGQpcHJV3VFVBdzY10aSNCbDjpG8DfhYkr1tfinw5tnssKq+kuQ3gEfohdFnquozSc6sqn1tnX1JzmhNlgF39m1iqtX+pk1Prx8myQZ6Vy6cddZZs+m2JOkIhgqSqvp8kh8FXgQEuL+q/mY2O2xjH2uBs4Fv0guot8zUZFCXZqgfXqy6DrgOYGJiYuA6kqTZGfaKBOCVwMrW5uVJqKobZ7HPHwMeqqoDAEk+Afxj4LEkS9vVyFJgf1t/it4n6g9ZTu9W2FSbnl6XJI3RsB9I/CDwG8Cr6QXKK4GJWe7zEWB1kpPaU1YX0vug4zZgfVtnPXBLm94GrEtyQpKz6Q2q72y3wR5Psrpt57K+NpKkMRn2imQCOKcNandSVXcluRn4AnAQ+CK9207PBbYmuZxe2Fza1t+dZCtwb1v/ykOfZwGuAG6g92jyre0lSRqjYYPkHuCHgX3HYqdVdRVw1bTyE/SuTgatvxnYPKA+CfgV95I0h4YNktOBe5PspPcHH4CqungkvZIkzRvDBsmvjbITkqT5a9jHf/80yY8Aq6rqs0lOoveJdEnSAjfsU1s/B9wM/G4rLQM+NaI+SZLmkWG/IuVK4FXAt+Fvf+TqjBlbSJIWhGGD5Imq+t6hmfYdWX5CXJI0dJD8aZJ30vuixdcDHwP+5+i6JUmaL4YNko3AAWAX8O+AT9P7/XZJ0gI37FNb36f3U7u/N9ruSJLmm6GCJMlDDBgTqaoXHPMeSZLmlafzXVuHPIfe92Cdeuy7I0mab4YaI6mqr/W9vlJV7wNeN9quSZLmg2Fvbb2ib/ZZ9K5QfmgkPZIkzSvD3tr6zb7pg8DDwL865r2RJM07wz619dpRd0SSND8Ne2vrF2daXlXvOTbdkSTNN0/nqa1X0vvZW4CfBG4HHh1FpyRJ88fT+WGrV1TV4wBJfg34WFX921F1TJI0Pwz7FSlnAd/rm/8esPKY90aSNO8Me0XyQWBnkk/S+4T7TwE3jqxXkqR5Y9intjYnuRX4J630s1X1xdF1S5I0Xwx7awvgJODbVfVbwFSSs0fUJ0nSPDLsT+1eBbwD2NRKzwY+NNudJnl+kpuT3J/kviT/KMmpSbYnebC9n9K3/qYke5I8kOSivvr5SXa1ZVcnyWz7JEmanWGvSH4KuBj4S4Cq2ku3r0j5LeCPqupHgZcC99H7zZMdVbUK2NHmSXIOsA44F1gDXJNkUdvOtcAGYFV7renQJ0nSLAwbJN+rqqJ9lXySH5ztDpOcDPxT4HqAqvpeVX0TWAtsaattAS5p02uBm6rqiap6CNgDXJBkKXByVd3R+nZjXxtJ0pgMGyRbk/wu8PwkPwd8ltn/yNUL6P3a4v9I8sUk72/BdGZV7QNo72e09Zfx1A8+TrXasjY9vX6YJBuSTCaZPHDgwCy7LUka5KhB0sYdPgrcDHwceBHwn6vqv81yn4uBVwDXVtXL6d0u2zhTFwbUaob64cWq66pqoqomlixZ8nT7K0mawVEf/62qSvKpqjof2H4M9jkFTFXVXW3+ZnpB8liSpVW1r9222t+3/oq+9suBva2+fEBdkjRGw97aujPJK4/FDqvqL4BHk7yolS4E7qX3PV7rW209cEub3gasS3JCe+R4FbCz3f56PMnqdtV0WV8bSdKYDPvJ9tcCP5/kYXq3okLvYuUls9zvfwQ+nOQHgD8DfpZeqG1NcjnwCL2f86WqdifZSi9sDgJXVtWTbTtXADcAJwK3tpckaYxmDJIkZ1XVI8BPHMudVtWXeOrvwB9y4RHW3wxsHlCfBM47ln2TJD09R7si+RS9b/398yQfr6p/OYY+SZLmkaONkfQ/GfWCUXZEkjQ/HS1I6gjTkiQBR7+19dIk36Z3ZXJim4a/G2w/eaS9kyQ9480YJFW1aKblkiQ9na+RlyTpMAaJJKkTg0SS1IlBIknqxCCRJHVikEiSOjFIJEmdGCSSpE4MEklSJwaJJKkTg0SS1IlBIknqxCCRJHVikEiSOjFIJEmdGCSSpE7mLEiSLEryxSR/0OZPTbI9yYPt/ZS+dTcl2ZPkgSQX9dXPT7KrLbs6SQbtS5I0OnN5RfJW4L6++Y3AjqpaBexo8yQ5B1gHnAusAa5JcuiXG68FNgCr2mvNeLouSTpkToIkyXLgDcD7+8prgS1tegtwSV/9pqp6oqoeAvYAFyRZCpxcVXdUVQE39rWRJI3JXF2RvA94O/D9vtqZVbUPoL2f0erLgEf71ptqtWVtenpdkjRGYw+SJG8E9lfV3cM2GVCrGeqD9rkhyWSSyQMHDgy5W0nSMObiiuRVwMVJHgZuAl6X5EPAY+12Fe19f1t/CljR1345sLfVlw+oH6aqrquqiaqaWLJkybE8Fkla8MYeJFW1qaqWV9VKeoPot1XVW4BtwPq22nrglja9DViX5IQkZ9MbVN/Zbn89nmR1e1rrsr42kqQxWTzXHejzLmBrksuBR4BLAapqd5KtwL3AQeDKqnqytbkCuAE4Ebi1vSRJYzSnQVJVnwM+16a/Blx4hPU2A5sH1CeB80bXQ0nS0fjJdklSJwaJJKkTg0SS1IlBIknqxCCRJHVikEiSOjFIJEmdGCSSpE4MEklSJwaJJKkTg0SS1IlBIknqxCCRJHVikEiSOjFIJEmdGCSSpE4MEklSJwaJJKkTg0SS1IlBIknqxCCRJHVikEiSOhl7kCRZkeRPktyXZHeSt7b6qUm2J3mwvZ/S12ZTkj1JHkhyUV/9/CS72rKrk2TcxyNJC91cXJEcBH6pql4MrAauTHIOsBHYUVWrgB1tnrZsHXAusAa4Jsmitq1rgQ3AqvZaM84DkSTNQZBU1b6q+kKbfhy4D1gGrAW2tNW2AJe06bXATVX1RFU9BOwBLkiyFDi5qu6oqgJu7GsjSRqTOR0jSbISeDlwF3BmVe2DXtgAZ7TVlgGP9jWbarVlbXp6fdB+NiSZTDJ54MCBY3oMkrTQzVmQJHku8HHgbVX17ZlWHVCrGeqHF6uuq6qJqppYsmTJ0++sJOmI5iRIkjybXoh8uKo+0cqPtdtVtPf9rT4FrOhrvhzY2+rLB9QlSWM0F09tBbgeuK+q3tO3aBuwvk2vB27pq69LckKSs+kNqu9st78eT7K6bfOyvjaSpDFZPAf7fBXwr4FdSb7Uau8E3gVsTXI58AhwKUBV7U6yFbiX3hNfV1bVk63dFcANwInAre0lSRqjsQdJVf1vBo9vAFx4hDabgc0D6pPAeceud5Kkp8tPtkuSOjFIJEmdGCSSpE4MEklSJwaJJKkTg0SS1IlBIknqxCCRJHVikEiSOjFIJEmdGCSSpE4MEklSJwaJJKkTg0SS1IlBIknqxCCRJHVikEiSOjFIJEmdGCSSpE4MEklSJwaJJKmTeR8kSdYkeSDJniQb57o/krTQLJ7rDnSRZBHw34HXA1PA55Nsq6p757Znkp7pVm78w7nuwnFjXgcJcAGwp6r+DCDJTcBawCDRvOMfNs1X8z1IlgGP9s1PAf9wjvpyXPKPm6Sjme9BkgG1OmylZAOwoc1+J8kDI+3VM8/pwFfnuhNzaKEfP3gOwHNA3t3pHPzIkRbM9yCZAlb0zS8H9k5fqaquA64bV6eeaZJMVtXEXPdjriz04wfPAXgOYHTnYL4/tfV5YFWSs5P8ALAO2DbHfZKkBWVeX5FU1cEk/wH4Y2AR8IGq2j3H3ZKkBWVeBwlAVX0a+PRc9+MZbsHe1msW+vGD5wA8BzCic5Cqw8amJUka2nwfI5EkzTGD5DiS5ANJ9ie5p692apLtSR5s76fMZR9HLcmKJH+S5L4ku5O8tdUXxHlI8pwkO5N8uR3/f2n1BXH8/ZIsSvLFJH/Q5hfUOUjycJJdSb6UZLLVRnIODJLjyw3Ammm1jcCOqloF7Gjzx7ODwC9V1YuB1cCVSc5h4ZyHJ4DXVdVLgZcBa5KsZuEcf7+3Avf1zS/Ec/DaqnpZ3yO/IzkHBslxpKpuB74+rbwW2NKmtwCXjLNP41ZV+6rqC236cXp/SJaxQM5D9XynzT67vYoFcvyHJFkOvAF4f195QZ2DIxjJOTBIjn9nVtU+6P2RBc6Y4/6MTZKVwMuBu1hA56Hd0vkSsB/YXlUL6vib9wFvB77fV1to56CAzyS5u327B4zoHMz7x3+lQZI8F/g48Laq+nYy6Nt0jk9V9STwsiTPBz6Z5Lw57tJYJXkjsL+q7k7ymjnuzlx6VVXtTXIGsD3J/aPakVckx7/HkiwFaO/757g/I5fk2fRC5MNV9YlWXnDnoaq+CXyO3rjZQjr+VwEXJ3kYuAl4XZIPsbDOAVW1t73vBz5J79vSR3IODJLj3zZgfZteD9wyh30ZufQuPa4H7quq9/QtWhDnIcmSdiVCkhOBHwPuZ4EcP0BVbaqq5VW1kt7XJt1WVW9hAZ2DJD+Y5IcOTQM/DtzDiM6BH0g8jiT5CPAaet9y+hhwFfApYCtwFvAIcGlVTR+QP24keTXwv4Bd/N398XfSGyc57s9DkpfQG0RdRO8fxa1V9V+TnMYCOP7p2q2tX66qNy6kc5DkBfSuQqA3hPH7VbV5VOfAIJEkdeKtLUlSJwaJJKkTg0SS1IlBIknqxCCRJHVikEiSOjFIJEmdGCSSpE7+P4r2y4S5NqdTAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "abs(X1_train_info['TSS_start'] - X1_train_info['TSS_end']).plot.hist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### HOW Long are the gene sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:ylabel='Frequency'>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEICAYAAAB1f3LfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAXS0lEQVR4nO3df7RlZX3f8fdHRhFqUJFBZ82AgzrRAEsrjJTE1qrUMtXEIa2k4zJhlqVOpTTVpiaCyQr+M2vpaisJK4GEiAWMiiMamSZigmhi2sUPL/4IDEiYiIErE2f8BZgYcMi3f5znxsOdc+89M/ueczn3vl9rnXX3/u797P08bp2P+8fZJ1WFJEmH6klL3QFJ0mQzSCRJnRgkkqRODBJJUicGiSSpE4NEktTJyIIkyQeS7E1yx4Bl70hSSY7pq12YZHeSu5Oc2Vc/NcntbdklSdLqhyf5aKvfkmT9qMYiSZrbKM9IrgQ2zS4mOQ54DXBfX+1EYAtwUmtzaZLD2uLLgG3AhvaZ2ea5wHer6gXAxcB7RzIKSdK8Vo1qw1X1+TnOEi4GfgW4rq+2Gbimqh4B7k2yGzgtydeBo6rqJoAkVwNnAde3Nu9u7a8FfitJaoFvWB5zzDG1fv2gbkmS5nLbbbd9q6pWD1o2siAZJMnrgW9U1VfaFaoZa4Gb++anW+2HbXp2fabN/QBVtT/Jg8CzgG/N14f169czNTXVZRiStOIk+eu5lo0tSJIcCfwq8K8HLR5Qq3nq87UZtO9t9C6Pcfzxxy/YV0nS8Mb51NbzgROAr7RLVuuALyZ5Dr0zjeP61l0HPNDq6wbU6W+TZBXwdOA7g3ZcVZdX1caq2rh69cAzM0nSIRpbkFTV7VV1bFWtr6r19ILglKr6G2AnsKU9iXUCvZvqt1bVHuDhJKe3p7XO4Uf3VnYCW9v0G4DPLnR/RJK0+Eb5+O9HgJuAFyaZTnLuXOtW1S5gB3An8Gng/Kp6rC0+D3g/sBv4K3o32gGuAJ7Vbsz/EnDBSAYiSZpXVtr/id+4cWN5s12SDk6S26pq46BlfrNdktSJQSJJ6sQgkSR1YpBIkjoZ6zfbJ936C/5oyfb99fe8bsn2LUnz8YxEktSJQSJJ6sQgkSR1YpBIkjoxSCRJnRgkkqRODBJJUicGiSSpE4NEktSJQSJJ6sQgkSR1YpBIkjoxSCRJnRgkkqRODBJJUicGiSSpE4NEktTJyIIkyQeS7E1yR1/tfyT5apK/SPIHSZ7Rt+zCJLuT3J3kzL76qUlub8suSZJWPzzJR1v9liTrRzUWSdLcRnlGciWwaVbtBuDkqnox8JfAhQBJTgS2ACe1NpcmOay1uQzYBmxon5ltngt8t6peAFwMvHdkI5EkzWlkQVJVnwe+M6v2J1W1v83eDKxr05uBa6rqkaq6F9gNnJZkDXBUVd1UVQVcDZzV1+aqNn0tcMbM2YokaXyW8h7JfwCub9Nrgfv7lk232to2Pbv+uDYtnB4EnjXC/kqSBliSIEnyq8B+4EMzpQGr1Tz1+doM2t+2JFNJpvbt23ew3ZUkzWPsQZJkK/DTwJva5SronWkc17faOuCBVl83oP64NklWAU9n1qW0GVV1eVVtrKqNq1evXqyhSJIYc5Ak2QS8E3h9Vf1d36KdwJb2JNYJ9G6q31pVe4CHk5ze7n+cA1zX12Zrm34D8Nm+YJIkjcmqUW04yUeAVwLHJJkGLqL3lNbhwA3tvvjNVfXWqtqVZAdwJ71LXudX1WNtU+fRewLsCHr3VGbuq1wBfDDJbnpnIltGNRZJ0txGFiRV9cYB5SvmWX87sH1AfQo4eUD974Gzu/RRktSd32yXJHVikEiSOjFIJEmdGCSSpE4MEklSJwaJJKkTg0SS1IlBIknqxCCRJHVikEiSOjFIJEmdGCSSpE4MEklSJwaJJKkTg0SS1IlBIknqxCCRJHVikEiSOjFIJEmdGCSSpE4MEklSJwaJJKmTkQVJkg8k2Zvkjr7a0UluSHJP+/vMvmUXJtmd5O4kZ/bVT01ye1t2SZK0+uFJPtrqtyRZP6qxSJLmNsozkiuBTbNqFwA3VtUG4MY2T5ITgS3ASa3NpUkOa20uA7YBG9pnZpvnAt+tqhcAFwPvHdlIJElzGlmQVNXnge/MKm8GrmrTVwFn9dWvqapHqupeYDdwWpI1wFFVdVNVFXD1rDYz27oWOGPmbEWSND7jvkfy7KraA9D+Htvqa4H7+9abbrW1bXp2/XFtqmo/8CDwrEE7TbItyVSSqX379i3SUCRJ8MS52T7oTKLmqc/X5sBi1eVVtbGqNq5evfoQuyhJGmTcQfLNdrmK9ndvq08Dx/Wttw54oNXXDag/rk2SVcDTOfBSmiRpxMYdJDuBrW16K3BdX31LexLrBHo31W9tl78eTnJ6u/9xzqw2M9t6A/DZdh9FkjRGq0a14SQfAV4JHJNkGrgIeA+wI8m5wH3A2QBVtSvJDuBOYD9wflU91jZ1Hr0nwI4Arm8fgCuADybZTe9MZMuoxiJJmtvIgqSq3jjHojPmWH87sH1AfQo4eUD972lBJElaOk+Um+2SpAllkEiSOjFIJEmdGCSSpE4MEklSJwaJJKkTg0SS1IlBIknqxCCRJHVikEiSOjFIJEmdGCSSpE4MEklSJwaJJKkTg0SS1IlBIknqZKggSXLAD0tJkgTDn5H8TpJbk/znJM8YZYckSZNlqCCpqn8OvAk4DphK8uEkrxlpzyRJE2HoeyRVdQ/wa8A7gX8JXJLkq0n+7ag6J0l64hv2HsmLk1wM3AW8GviZqvqJNn3xCPsnSXqCWzXker8F/B7wrqr6wUyxqh5I8msj6ZkkaSIMe2nrtcCHZ0IkyZOSHAlQVR882J0m+W9JdiW5I8lHkjw1ydFJbkhyT/v7zL71L0yyO8ndSc7sq5+a5Pa27JIkOdi+SJK6GTZIPgMc0Td/ZKsdtCRrgf8KbKyqk4HDgC3ABcCNVbUBuLHNk+TEtvwkYBNwaZLD2uYuA7YBG9pn06H0SZJ06IYNkqdW1fdnZtr0kR32uwo4Ismqtp0HgM3AVW35VcBZbXozcE1VPVJV9wK7gdOSrAGOqqqbqqqAq/vaSJLGZNgg+dskp8zMJDkV+ME868+pqr4B/E/gPmAP8GBV/Qnw7Kra09bZAxzbmqwF7u/bxHSrrW3Ts+sHSLItyVSSqX379h1KtyVJcxj2ZvvbgY8leaDNrwH+/aHssN372AycAHyvbffn52syoFbz1A8sVl0OXA6wcePGgetIkg7NUEFSVV9I8iLghfT+Af9qVf3wEPf5r4B7q2ofQJJPAD8FfDPJmqra0y5b7W3rT9P7IuSMdfQuhU236dl1SdIYHcxLG18GvBh4KfDGJOcc4j7vA05PcmR7yuoMet9P2QlsbetsBa5r0zuBLUkOT3ICvZvqt7bLXw8nOb1t55y+NpKkMRnqjCTJB4HnA18GHmvlmRvcB6WqbklyLfBFYD/wJXqXnZ4G7EhyLr2wObutvyvJDuDOtv75VTXTh/OAK+k9UXZ9+0iSxmjYeyQbgRPb01GdVdVFwEWzyo/QOzsZtP52YPuA+hTgm4klaQkNe2nrDuA5o+yIJGkyDXtGcgxwZ5Jb6Z05AFBVrx9JryRJE2PYIHn3KDshSZpcwz7++2dJngtsqKrPtPdsHbZQO0nS8jfsa+TfAlwL/G4rrQU+OaI+SZImyLA3288HXg48BP/4I1fHzttCkrQiDBskj1TVozMz7WWLvmpEkjR0kPxZknfRe2Pva4CPAf9ndN2SJE2KYYPkAmAfcDvwn4BP0fv9dknSCjfsU1v/QO+ndn9vtN2RJE2aYd+1dS8D7olU1fMWvUeSpIlyMO/amvFUei9UPHrxuyNJmjRD3SOpqm/3fb5RVb8BvHq0XZMkTYJhL22d0jf7JHpnKD82kh5JkibKsJe2/lff9H7g68DPLXpvJEkTZ9intl416o5IkibTsJe2fmm+5VX1vsXpjiRp0hzMU1svo/f76QA/A3weuH8UnZIkTY6D+WGrU6rqYYAk7wY+VlX/cVQdkyRNhmFfkXI88Gjf/KPA+kXvjSRp4gx7RvJB4NYkf0DvG+4/C1w9sl5JkibGsE9tbU9yPfAvWunNVfWl0XVLkjQphr20BXAk8FBV/SYwneSEQ91pkmckuTbJV5PcleQnkxyd5IYk97S/z+xb/8Iku5PcneTMvvqpSW5vyy5JkkPtkyTp0Az7U7sXAe8ELmylJwO/32G/vwl8uqpeBLwEuIveq+pvrKoNwI1tniQnAluAk4BNwKVJZn4v/jJgG7ChfTZ16JMk6RAMe0bys8Drgb8FqKoHOMRXpCQ5CngFcEXb1qNV9T1gM3BVW+0q4Kw2vRm4pqoeqap7gd3AaUnWAEdV1U1VVfTu2cy0kSSNybBB8mj7x7oAkvyTDvt8Hr0fyfrfSb6U5P1te8+uqj0A7e/Mb8Kv5fHfV5lutbVtenb9AEm2JZlKMrVv374OXZckzTZskOxI8rvAM5K8BfgMh/4jV6uAU4DLquql9M5yLphn/UH3PWqe+oHFqsuramNVbVy9evXB9leSNI8Fn9pqN7A/CrwIeAh4IfDrVXXDIe5zGpiuqlva/LX0guSbSdZU1Z522Wpv3/rH9bVfBzzQ6usG1CVJY7TgGUm7pPXJqrqhqn65qt7RIUSoqr8B7k/ywlY6A7iT3utXtrbaVuC6Nr0T2JLk8Pak2Abg1nb56+Ekp7ewO6evjSRpTIb9QuLNSV5WVV9YpP3+IvChJE8Bvga8mV6o7UhyLnAfvV9hpKp2JdlBL2z2A+dX1WNtO+cBVwJHANe3jyRpjIYNklcBb03ydXr3NELvZOXFh7LTqvoyj//53hlnzLH+dmD7gPoUcPKh9EGStDjmDZIkx1fVfcC/GVN/JEkTZqEzkk/Se+vvXyf5eFX9uzH0SZI0QRa62d7/iO3zRtkRSdJkWihIao5pSZKAhS9tvSTJQ/TOTI5o0/Cjm+1HjbR3kqQnvHmDpKoOm2+5JEkH8xp5SZIOYJBIkjoxSCRJnRgkkqRODBJJUicGiSSpE4NEktSJQSJJ6sQgkSR1YpBIkjoxSCRJnRgkkqRODBJJUicGiSSpE4NEktSJQSJJ6mTJgiTJYUm+lOQP2/zRSW5Ick/7+8y+dS9MsjvJ3UnO7KufmuT2tuySJBm0L0nS6CzlGcnbgLv65i8AbqyqDcCNbZ4kJwJbgJOATcClSWZ+ufEyYBuwoX02jafrkqQZSxIkSdYBrwPe31feDFzVpq8CzuqrX1NVj1TVvcBu4LQka4Cjquqmqirg6r42kqQxWaozkt8AfgX4h77as6tqD0D7e2yrrwXu71tvutXWtunZ9QMk2ZZkKsnUvn37FmUAkqSesQdJkp8G9lbVbcM2GVCreeoHFqsur6qNVbVx9erVQ+5WkjSMVUuwz5cDr0/yWuCpwFFJfh/4ZpI1VbWnXbba29afBo7ra78OeKDV1w2oS5LGaOxnJFV1YVWtq6r19G6if7aqfh7YCWxtq20FrmvTO4EtSQ5PcgK9m+q3tstfDyc5vT2tdU5fG0nSmCzFGclc3gPsSHIucB9wNkBV7UqyA7gT2A+cX1WPtTbnAVcCRwDXt48kaYyWNEiq6k+BP23T3wbOmGO97cD2AfUp4OTR9VCStBC/2S5J6sQgkSR1YpBIkjoxSCRJnRgkkqRODBJJUicGiSSpE4NEktSJQSJJ6sQgkSR1YpBIkjoxSCRJnRgkkqRODBJJUicGiSSpE4NEktSJQSJJ6sQgkSR1YpBIkjoxSCRJnRgkkqROxh4kSY5L8rkkdyXZleRtrX50khuS3NP+PrOvzYVJdie5O8mZffVTk9zell2SJOMejyStdEtxRrIf+O9V9RPA6cD5SU4ELgBurKoNwI1tnrZsC3ASsAm4NMlhbVuXAduADe2zaZwDkSQtQZBU1Z6q+mKbfhi4C1gLbAauaqtdBZzVpjcD11TVI1V1L7AbOC3JGuCoqrqpqgq4uq+NJGlMlvQeSZL1wEuBW4BnV9Ue6IUNcGxbbS1wf1+z6VZb26Zn1yVJY7RkQZLkacDHgbdX1UPzrTqgVvPUB+1rW5KpJFP79u07+M5Kkua0JEGS5Mn0QuRDVfWJVv5mu1xF+7u31aeB4/qarwMeaPV1A+oHqKrLq2pjVW1cvXr14g1EkrQkT20FuAK4q6re17doJ7C1TW8Fruurb0lyeJIT6N1Uv7Vd/no4yeltm+f0tZEkjcmqJdjny4FfAG5P8uVWexfwHmBHknOB+4CzAapqV5IdwJ30nvg6v6oea+3OA64EjgCubx9J0hiNPUiq6v8y+P4GwBlztNkObB9QnwJOXrzeSZIOlt9slyR1YpBIkjoxSCRJnRgkkqRODBJJUicGiSSpE4NEktSJQSJJ6sQgkSR1YpBIkjoxSCRJnRgkkqRODBJJUicGiSSpE4NEktSJQSJJ6mQpfiFRh2D9BX+0JPv9+ntetyT7lTQ5PCORJHVikEiSOjFIJEmdGCSSpE4MEklSJxMfJEk2Jbk7ye4kFyx1fyRppZnox3+THAb8NvAaYBr4QpKdVXXn0vZs+Viqx47BR4+lSTHpZySnAbur6mtV9ShwDbB5ifskSSvKRJ+RAGuB+/vmp4F/tkR90SJbyrOhpeAZmCbVpAdJBtTqgJWSbcC2Nvv9JHcf4v6OAb51iG0n2Uoc99jHnPeOc29z8livHAc77ufOtWDSg2QaOK5vfh3wwOyVqupy4PKuO0syVVUbu25n0qzEca/EMcPKHPdKHDMs7rgn/R7JF4ANSU5I8hRgC7BzifskSSvKRJ+RVNX+JP8F+GPgMOADVbVribslSSvKRAcJQFV9CvjUmHbX+fLYhFqJ416JY4aVOe6VOGZYxHGn6oB705IkDW3S75FIkpaYQTLAQq9dSc8lbflfJDllKfq5mIYY8yuTPJjky+3z60vRz8WU5ANJ9ia5Y47ly+44w1DjXo7H+rgkn0tyV5JdSd42YJ1ld7yHHHf3411Vfvo+9G7a/xXwPOApwFeAE2et81rgenrfYzkduGWp+z2GMb8S+MOl7usij/sVwCnAHXMsX1bH+SDGvRyP9RrglDb9Y8BfLvf/XR/EuDsfb89IDjTMa1c2A1dXz83AM5KsGXdHF9GKfNVMVX0e+M48qyy34wwMNe5lp6r2VNUX2/TDwF303ozRb9kd7yHH3ZlBcqBBr12Z/R/8MOtMkmHH85NJvpLk+iQnjadrS2q5HeeDsWyPdZL1wEuBW2YtWtbHe55xQ8fjPfGP/47AMK9dGerVLBNkmPF8EXhuVX0/yWuBTwIbRt2xJbbcjvOwlu2xTvI04OPA26vqodmLBzRZFsd7gXF3Pt6ekRxomNeuDPVqlgmy4Hiq6qGq+n6b/hTw5CTHjK+LS2K5HeehLNdjneTJ9P4x/VBVfWLAKsvyeC807sU43gbJgYZ57cpO4Jz2lMfpwINVtWfcHV1EC445yXOSpE2fRu+/O98ee0/Ha7kd56Esx2PdxnMFcFdVvW+O1Zbd8R5m3ItxvL20NUvN8dqVJG9ty3+H3jfpXwvsBv4OePNS9XcxDDnmNwDnJdkP/ADYUu2Rj0mV5CP0nlg5Jsk0cBHwZFiex3nGEONedscaeDnwC8DtSb7cau8CjodlfbyHGXfn4+032yVJnXhpS5LUiUEiSerEIJEkdWKQSJI6MUgkaRlb6CWdA9b/uSR3tpc8fnioNj61JUnLV5JXAN+n9x6xkxdYdwOwA3h1VX03ybFVtXehfXhGIknL2KCXdCZ5fpJPJ7ktyZ8neVFb9Bbgt6vqu63tgiECBokkrUSXA79YVacC7wAubfUfB348yf9LcnOSTcNszG+2S9IK0l7g+FPAx9qbUQAOb39X0Xth4yvpvWvsz5OcXFXfm2+bBokkrSxPAr5XVf90wLJp4Oaq+iFwb5K76QXLFxbaoCRphWivkb83ydnwjz8x/JK2+JPAq1r9GHqXur620DYNEklaxtpLOm8CXphkOsm5wJuAc5N8BdjFj34R9Y+Bbye5E/gc8MtVteCbgH38V5LUiWckkqRODBJJUicGiSSpE4NEktSJQSJJ6sQgkSR1YpBIkjoxSCRJnfx/SYV+77Iih5oAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "abs(X1_train_info['gene_start'] - X1_train_info['gene_end']).plot.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method NDFrame.describe of       gene_name          gex\n",
       "0       SLC20A1     0.000000\n",
       "1      C11orf58  2239.103328\n",
       "2        ZSCAN9    19.798064\n",
       "3          CD19   411.530623\n",
       "4       TMEM123    34.214129\n",
       "...         ...          ...\n",
       "14305     ACOX1     0.000000\n",
       "14306     MLXIP     0.000000\n",
       "14307     ASGR2     0.000000\n",
       "14308     OR5A1     0.000000\n",
       "14309      TLR3     0.480536\n",
       "\n",
       "[14310 rows x 2 columns]>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(X1_train_y.shape)\n",
    "print(X1_train_y.describe)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## All the different GEX which ranges from [0, 20'000] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='gene_name', ylabel='gex'>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaUAAAEGCAYAAADFWoruAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAxwklEQVR4nO3de5wcVZ338c+vZyaTkHCJISpkAgETxISFaGaRNciD4AKKArtcFrzAPrJm5Qm7qAjIuiyoqyuwyC4K7AZhAeUWgpqgoLIERRQCE5kEEgKEayaJJIYQEkgmM92/5486PanpqZnpmfSleub7fr36NT2nu6pOVVedX51LVZm7IyIikgaZamdAREQkT0FJRERSQ0FJRERSQ0FJRERSQ0FJRERSo77aGai0Pffc0ydNmlTtbIiI1JTFixf/yd3Hl3s5wy4oTZo0iZaWlmpnQ0SkppjZK5VYTtma78xsopk9ZGbPmNkyMzsvpL/DzB4ws+fD37GxaS42s5Vm9qyZHRtLn2FmT4XPrjEzC+mNZnZXSF9kZpPKtT4iIlJ+5exT6gTOd/f3AYcBs81sKvBV4EF3nwI8GP4nfHY6MA04DrjOzOrCvK4HZgFTwuu4kH42sNHdJwNXA5eXcX1ERKTMyhaU3H2tu/8hvN8MPANMAE4EbglfuwU4Kbw/EbjT3dvd/SVgJXCome0F7Obuj3p0+4lbC6bJz2secHS+FiUiIrWnIqPvQrPa+4FFwLvcfS1EgQt4Z/jaBGBVbLK2kDYhvC9M7zaNu3cCm4BxCcufZWYtZtayfv36Eq2ViIiUWtmDkpmNAe4Bvujub/b11YQ07yO9r2m6J7jPcfdmd28eP77sg0dERGSQyhqUzKyBKCDd5u4/DsmvhSY5wt91Ib0NmBibvAlYE9KbEtK7TWNm9cDuwOulXxORZBu2tLNk1Rts2NJe7ayIDAnlHH1nwI3AM+7+3dhHC4CzwvuzgPmx9NPDiLr9iAY0PB6a+Dab2WFhnmcWTJOf1ynAQtdtz6VC5reuZublC/nMDxYx8/KFLGhdXe0sidS8cl6nNBP4LPCUmbWGtH8CvgPMNbOzgVeBUwHcfZmZzQWWE43cm+3u2TDdOcDNwCjg/vCCKOj90MxWEtWQTi/j+oh02bClnYvuWcq2jhzbyAFw4T1LmTl5T8aNaaxy7kRqV9mCkrs/QnKfD8DRvUzzLeBbCektwEEJ6dsIQU2kkto2bqUhk+kKSAANmQxtG7cqKInsBN37TmQQmsaOoiOX65bWkcvRNHZUlXIkMjQoKIkMwrgxjVxx8sGMbMiwa2M9IxsyXHHywaolieykYXfvO5FSOWH6BGZO3pO2jVtpGjtKAUmkBBSURHbCuDGNCkYiJaTmOxERSQ0FJRERSQ0FJRERSQ0FJRERSQ0FJRERSQ0FJRERSQ0FJRERSQ0FJRERSQ0FJRERSQ0FJRERSQ0FJRERSQ0FJRERSQ0FJRERSY2yBSUzu8nM1pnZ07G0u8ysNbxezj8m3cwmmdnW2Gf/FZtmhpk9ZWYrzewaM7OQ3hjmt9LMFpnZpHKti4iIVEY5a0o3A8fFE9z9b9x9urtPB+4Bfhz7+IX8Z+7+hVj69cAsYEp45ed5NrDR3ScDVwOXl2UtRESkYsoWlNz9YeD1pM9Cbec04I6+5mFmewG7ufuj7u7ArcBJ4eMTgVvC+3nA0flalIiI1KZq9Sl9GHjN3Z+Ppe1nZk+a2W/M7MMhbQLQFvtOW0jLf7YKwN07gU3AuKSFmdksM2sxs5b169eXcj1ERKSEqhWUzqB7LWktsI+7vx/4MnC7me0GJNV8PPzt67Puie5z3L3Z3ZvHjx+/E9kWEZFyqvjj0M2sHvhrYEY+zd3bgfbwfrGZvQAcQFQzaopN3gSsCe/bgIlAW5jn7vTSXCgiIrWhGjWljwIr3L2rWc7MxptZXXi/P9GAhhfdfS2w2cwOC/1FZwLzw2QLgLPC+1OAhaHfSUREalQ5h4TfATwKvNfM2szs7PDR6fQc4HAEsNTMlhANWviCu+drPecAPwBWAi8A94f0G4FxZraSqMnvq+VaFxERqQwbbpWL5uZmb2lpqXY2RERqipktdvfmci9Hd3QQEZHUUFASEZHUUFASEZHUUFASEZHUUFASEZHUUFASEZHUUFASEZHUUFASEZHUUFASEZHUUFASEZHUUFASEZHUUFASEZHUUFASEZHUUFASEZHUUFASEZHUUFASEZHUKOeTZ28ys3Vm9nQs7TIzW21mreH18dhnF5vZSjN71syOjaXPMLOnwmfXhMeiY2aNZnZXSF9kZpPKtS4iIlIZ5awp3Qwcl5B+tbtPD6/7AMxsKtFj0qeFaa4zs7rw/euBWcCU8MrP82xgo7tPBq4GLi/XioiISGWULSi5+8PA60V+/UTgTndvd/eXgJXAoWa2F7Cbuz/q0XPbbwVOik1zS3g/Dzg6X4sSEZHaVI0+pXPNbGlo3hsb0iYAq2LfaQtpE8L7wvRu07h7J7AJGJe0QDObZWYtZtayfv360q2JiIiUVKWD0vXAe4DpwFrgqpCeVMPxPtL7mqZnovscd2929+bx48cPKMMiIlI5FQ1K7v6au2fdPQfcABwaPmoDJsa+2gSsCelNCendpjGzemB3im8uFBGRFKpoUAp9RHl/BeRH5i0ATg8j6vYjGtDwuLuvBTab2WGhv+hMYH5smrPC+1OAhaHfSUREalR9uWZsZncARwJ7mlkbcClwpJlNJ2pmexn4ewB3X2Zmc4HlQCcw292zYVbnEI3kGwXcH14ANwI/NLOVRDWk08u1LiIiUhk23CoXzc3N3tLSUu1siIjUFDNb7O7N5V6O7uggIiKpoaAkIiKpoaAkIiKpoaAkIiKpoaAkUkM2bGlnyao32LClvdpZESmLsg0JF5HSmt+6movuWUpDJkNHLscVJx/MCdMn9D+hSA1RTUmkBmzY0s5F9yxlW0eOze2dbOvIceE9S1VjkiFHQUmkBrRt3EpDpvvh2pDJ0LZxa5VyJFIeCkoiNaBp7Cg6crluaR25HE1jR1UpRyLloaAkUgPGjWnkipMPZmRDhl0b6xnZkOGKkw9m3JjGamdNpKQ00EGkRpwwfQIzJ+9J28atNI0dpYAkQ5KCkkgNGTemUcFIhjQ134mISGooKImISGooKImISGooKImISGqULSiZ2U1mts7Mno6lXWlmK8xsqZn9xMz2COmTzGyrmbWG13/FpplhZk+Z2UozuyY8Fp3w6PS7QvoiM5tUrnUREZHKKGdN6WbguIK0B4CD3P1g4Dng4thnL7j79PD6Qiz9emAWMCW88vM8G9jo7pOBq4HLS78KIiJSSWULSu7+MPB6Qdqv3L0z/PsY0NTXPMxsL2A3d3/Uo+e23wqcFD4+EbglvJ8HHJ2vRYmISG2qZp/S54D7Y//vZ2ZPmtlvzOzDIW0C0Bb7TltIy3+2CiAEuk3AuKQFmdksM2sxs5b169eXch1ERKSEqhKUzOxrQCdwW0haC+zj7u8Hvgzcbma7AUk1H8/Ppo/Puie6z3H3ZndvHj9+/M5lXkREyqbid3Qws7OATwBHhyY53L0daA/vF5vZC8ABRDWjeBNfE7AmvG8DJgJtZlYP7E5Bc6GIiNSWitaUzOw44CLgBHd/O5Y+3szqwvv9iQY0vOjua4HNZnZY6C86E5gfJlsAnBXenwIszAc5ERGpTWWrKZnZHcCRwJ5m1gZcSjTarhF4IIxJeCyMtDsC+IaZdQJZ4Avunq/1nEM0km8UUR9Uvh/qRuCHZraSqIZ0ernWRUREKsOGW+WiubnZW1paqp0NEZGaYmaL3b253MvRHR1ERCQ1FJRERCQ1FJRERCQ1FJRERCQ1FJRERCQ1FJRERCQ1FJRERCQ1FJRERCQ1FJRERCQ1FJRERCQ1FJRERCQ1FJRERCQ1FJRERCQ1FJRERCQ1FJRERCQ1FJRERCQ1igpKZvZNM6uP/b+bmf1PP9PcZGbrzOzpWNo7zOwBM3s+/B0b++xiM1tpZs+a2bGx9Blm9lT47JrwWHTMrNHM7grpi8xs0gDWW0REUqjYmlI9sMjMDjazY4AngMX9THMzcFxB2leBB919CvBg+B8zm0r0OPNpYZrrzKwuTHM9MAuYEl75eZ4NbHT3ycDVwOVFrouISEVt2NLOklVvsGFLe7Wzknr1/X8F3P1iM3sQWARsBI5w95X9TPNwQu3lRODI8P4W4NfARSH9TndvB14ys5XAoWb2MrCbuz8KYGa3AicB94dpLgvzmgd838zMh9vz3UUk1ea3ruaie5bSkMnQkctxxckHc8L0CdXOVmoV23x3BPCfwDeIAsn3zWzvQSzvXe6+FiD8fWdInwCsin2vLaRNCO8L07tN4+6dwCZg3CDyJCJSFhu2tHPRPUvZ1pFjc3sn2zpyXHjPUtWY+lBUTQn4d+BUd18OYGZ/DSwEDixRPiwhzftI72uanjM3m0XUBMg+++wzmPyJiAxY28atNGQybCPXldaQydC2cSvjxjRWMWfpVWyf0l/kAxKAu/8YmDmI5b1mZnsBhL/rQnobMDH2vSZgTUhvSkjvNk0YhLE78HrSQt19jrs3u3vz+PHjB5FtEZGBaxo7io5crltaRy5H09hRVcpR+hUblN5jZg/mR9KZ2cHAOYNY3gLgrPD+LGB+LP30MKJuP6IBDY+HJr7NZnZYGHV3ZsE0+XmdAixUf5KIpMm4MY1ccfLBjGzIsGtjPSMbMlxx8sGqJfWh2Oa7G4ALgP8GcPelZnY78K+9TWBmdxANatjTzNqAS4HvAHPN7GzgVeDUML9lZjYXWA50ArPdPRtmdQ7RSL5RRAMc7g/pNwI/DIMiXicavScikionTJ/AzMl70rZxK01jRykg9cOKqVyY2RPu/udm9qS7vz+ktbr79HJnsNSam5u9paWl2tkQEakpZrbY3ZvLvZxim+/+ZGbvIQwkMLNTgLVly5WIiAxLxTbfzQbmAAea2WrgJeDTZcuViIgMS8UGpZOA+4CHiGpXbwEfDdW51vJkTUREhptim++agS8AY4E9iK75ORK4wcwuLEvORERk2Cm2pjQO+IC7bwEws0uJbu1zBNE98K4oT/ZERGQ4KbamtA+wPfZ/B7Cvu28FdL8MEREpiWJrSrcDj5lZ/sLVTwJ3mNloomuLREREdlqxdwn/ppndBxxOdM+5L7h7/mIfjcITEZGSKLamhLsvpv9nKImIiAyaHocuIiKpoaAkkkJ6UqkMV0U334lIZehJpTKcqaYkkiJ6UqkMdwpKIimSf1JpXP5JpSLDgYKSSIroSaUy3CkoiaSInlQqw50GOoikjJ5UKsNZxWtKZvZeM2uNvd40sy+a2WVmtjqW/vHYNBeb2Uoze9bMjo2lzzCzp8Jn15iZVXp9RMph3JhGDpm4hwKSDDsVD0ru/qy7Tw+PUp8BvA38JHx8df4zd78PwMymAqcD04DjgOvMrC58/3qix2hMCa/jKrcmUkt03Y9Ibah2893RwAvu/koflZwTgTvdvR14ycxWAoea2cvAbu7+KICZ3Ur0MML7y55rqSm67kekdlR7oMPpwB2x/881s6VmdpOZjQ1pE4BVse+0hbQJ4X1heg9mNsvMWsysZf369aXLvaServsRqS1VC0pmNgI4Abg7JF0PvAeYDqwFrsp/NWFy7yO9Z6L7HHdvdvfm8ePH70y2pcbouh+R2lLNmtLHgD+4+2sA7v6au2fdPQfcABwavtcGTIxN1wSsCelNCekiXXTdj0htqWZQOoNY052Z7RX77K+Ap8P7BcDpZtZoZvsRDWh43N3XApvN7LAw6u5MYD4iMbruR6S2VGWgg5ntAvwl8Pex5CvMbDpRE9zL+c/cfZmZzSV6wm0nMNvds2Gac4CbgVFEAxw0yEF60HU/IrXD3BO7YYas5uZmb2lp6f+LIiLSxcwWu3tzuZdT7dF3IiIiXRSUREQkNRSURESqTHcc2aHad3QQERnWdMeR7lRTEhGpEt1xpCcFJSkZNUGIDIzuONKTmu+kJNQEITJwuuNIT6opyU5TE4TI4OiOIz2ppiQ7Ld8EsY0dZ3z5JojhfHBJ5W3Y0l5zd+7QHUe6U1CSnaYmCEmDWm5CHjemcdgHozw138lOUxOEVJuakIcO1ZSkJNQEIdWkJuShQ0FJSkZNEFItakIeOtR8J1ICukarutSEPHSopiSyk2q5g30oURPy0KCakshOUAd7dRXWUMeNaeSQiXsoINWwqgQlM3vZzJ4ys1Yzawlp7zCzB8zs+fB3bOz7F5vZSjN71syOjaXPCPNZaWbXhMeii1SMbhNTPfNbVzPz8oV85geLmHn5Qha0rq52lqQEqllT+oi7T489yfCrwIPuPgV4MPyPmU0FTgemAccB15lZXZjmemAWMCW8jqtg/kXUwV4lqqEOXWlqvjsRuCW8vwU4KZZ+p7u3u/tLwErgUDPbC9jN3R/16Jnut8amEakIdbBXh2qoQ1e1Bjo48Cszc+C/3X0O8C53Xwvg7mvN7J3huxOAx2LTtoW0jvC+ML0HM5tFVKNin332KeV6iKiDvQpUQx26qlVTmunuHwA+Bsw2syP6+G5SP5H3kd4z0X2Ouze7e/P48eMHnluRfqiDvbJUQx26qlJTcvc14e86M/sJcCjwmpntFWpJewHrwtfbgImxyZuANSG9KSFdRIYB1VCHporXlMxstJntmn8PHAM8DSwAzgpfOwuYH94vAE43s0Yz249oQMPjoalvs5kdFkbdnRmbRmJ0YacMVaqhDj3VqCm9C/hJGL1dD9zu7r8wsyeAuWZ2NvAqcCqAuy8zs7nAcqATmO3u2TCvc4CbgVHA/eElMbqwU2R4qsXHeABYNHBt+GhubvaWlpZqZ6MiNmxpZ+blC9nWsaNDeGRDht9ddFRN7aSSHrVa0A035TgZNbPFsUt4yka3GRrCdOdkKSXVumtD/Bqu/LF/4T1LmTl5z5o47tN0nZKUmIbNSqnoYtXaUevXcCkoDWEaNiulUs6CTgNxSqvWT0bVfDfEadislEK5Cjo1CZZe/mT0woLtWivHvgY6SNWp87w2LGhd3aOg25kAooE45VXq40oDHWRY0Jly7Sh1rVsDccqrVp8ErT6lFBvqbe3qPK89pbxYtdb7PqQ8FJRSajg8K6bWRwnJztFAHEmi5rsUqvXrDIpVi2fKlez/Gg59bRqII4UUlFJouLS119oooUr2fw2nvrZa7PsYDicM1aKglEK1WIMYrFo5U65k7XW41JRr1XA6YagG9Sml0HBra6+FOz1Xsv9LF6qmlwbnlJ9qSilVbA1CzQiVUcnaqy5UTa/h0rReTaopVchgzlD7q0EMhxF6aVHJ2ms5llXrZ/hpqeENp6b1alFNqQLKcYaqfofK66v2Wuoaqy5U3SFNNbxaG5xTixSUyqxcwaOWC5laljRSrFyFZilHpdXqGX4aT76qPThnqDfZV+Nx6BPN7CEze8bMlpnZeSH9MjNbbWat4fXx2DQXm9lKM3vWzI6Npc8ws6fCZ9eEx6KnSrk6rWu1kBlqaqVZrLBJsLHemH3k5AHPp9LNaGm9wLpag3OGQ5N9NfqUOoHz3f19wGHAbDObGj672t2nh9d9AOGz04FpwHHAdWZWF75/PTALmBJex1VwPYpSruAxmH6HtLTLDyVpLTSTnDB9Ar+76Cg+f8T+gDHn4RcHVLBVo0CspZOvDVvaefi59Tz83LqyHGO1cgK0syrefOfua4G14f1mM3sG6Kut40TgTndvB14ys5XAoWb2MrCbuz8KYGa3AicB95cx+0UprF6Xqw16IM0Itz32Cl//2XJG1BmdOdfIqxKppUIz77pfr6S9M0d7Z/HNYeVqRuuvKapW+nDmt67m/LmthE1KQ51x1amHlPQYGy5N9lXtUzKzScD7gUXATOBcMzsTaCGqTW0kCliPxSZrC2kd4X1helkUHjy9HUy99S+Uqw26mH6H2x57ha/99GkAtndGadVulx8qaqXQzBtswdbbdMvWvMnuoxoGtV8X2xdX7T6c/mzY0s6F85Z0BSSAjqxzwbzSHmO1eAI0GFULSmY2BrgH+KK7v2lm1wPfBDz8vQr4HJDUT+R9pCctaxZRMx/77LPPgPNaePCcNqOJuYvbehxM/Z1NVuNg2rClna/fu6xHel3GKnqGNZQ7ZytZaO7sdhxswZY03daOTj5/awsj6gY+wGOgNa/C46ca+1Nvy2zbuJU6ywDZbt8v9TFWaydAg1WVoGRmDUQB6TZ3/zGAu78W+/wG4Gfh3zZgYmzyJmBNSG9KSO/B3ecAcyB6yN9A8pp08Nz62KsAPQ6mNFav2zZupaEuw/Zs9wOmI+sDOsPamUIgTUN6y6USJx2l2I6DLdgKp9uezZJzBtwMmLczx0op96di9+u+ltk0dhRZz/WYJpsr/hgrNh9przWWQsWDUhghdyPwjLt/N5a+V+hvAvgr4OnwfgFwu5l9F9ibaEDD4+6eNbPNZnYYUfPfmcD3Sp3fpIOnUP5gSmP1OjpgesbhSz85tegdemcKgTQO6a22wQT4Um7HwRZs8ek2bd3O7NuepCPb2fX5QE7ABnuslHI7FLtfF9MCcuUph/Dlgj6lK08prhYz0OOrFm9gOxDVqCnNBD4LPGVmrSHtn4AzzGw6URPcy8DfA7j7MjObCywnGrk3293zp/3nADcDo4gGOJR8kEPSwVMofjDNPnIy339oZbcmjWruQPEz3DozOrI5Lv3kND79wX2Lmn5nC4E01h6rabABvtTbcbAFW366DVvad+oEbKA1tnwg37R1e0m2w0D262K2fT5gL1vzJuBM23v3xPwk9U3rpK27aoy+e4Tk/qD7+pjmW8C3EtJbgINKl7uekg6e05qbmNvSvU/pkZV/6ipswJl1xP586oP7pGLHGuyZ8YYt7Ty0Yh31me4/VyXOiIeinSmA0rYdk052Ljm++No3FL9fxgN5vtkwbjDbYSBBvthtP25MI0ccML7XZSadkOw7brRO2grojg5FSDp4zjv6gK7/AWZevrBbYXPtr1fysYPenZq234GeGecPoDoz3tpe0B9VxjPigeitGSwpPQ0DLforCPvKYxo7uU+YPoHN2zqjSw3qM3zz58vZdWT9gPp3+tsvkwJ5fQYa6zM71RoxkCCf1J+WdOFxX79fbyckPzv38FSdbKSBglKRCg+e+P9LVr3Ro7DxnPPx7z1C4yBGJkH5CtFi5hs/gOJGN9aRDdc49VXYF6aVo3O2t2awpHSHkg60GEgwjOurICymWS9tndwbtrTzzZ8vZ3tnrmyXGiQF8lEN9Vz76Q/0ORS91Nc/5bf9bYte5dqHVjLn4Re59tcr+9zv4r9fbyckb23Ppu5ko9oUlEogqbBpzzrgbB/EyKRyjVaL1366+pYO69m3lHQAjR5Rx9c/OY2PHPjOrnUYSADYmc7ZYtrhL5i3hIzBhfOW0t65I/38ua1kMpluaTtTcA4kGBb+Zr0VhIRtVkyzXqU6uePbHOh1KLQXtKV5znttehrMiVZvgXza3rsV1dxX6uufki48nrrXbv3+fn2dkBwycY9UnWxUm4JSCRS2r7d3ZqkLBWFenRkPrVjXrVDvrZZRrivnC2s/X/vp02D0GPSQdABl3XvkPSkwgJUsAEDx7fDtnc75dy/tts0BOnJAwboMts2+t9+mmEIpL14Qjh5Rx1vbsyxbsylV/Qrxbb61oxMzY2R9XY8CfvSIunDytUN71hk9oq7PeW7PZjn3I1OK6nPtr0azswMHBhLke6vttCa0lBT+fuPGNHLajKauy0kATmtu6vb5cA9GeQpKJRJvX2+sz/DW9u4F4Vvbs1x27zL+ef7TXTWKC+ctoc4yZD3HladEtyQp12i16AK/nuNLvn7vco6b9u4eTZP9NSkk5bPOMj2GsAwk78XWiK469RC2Z3uOiCwMSL3Z1plNbLPv70x+IIVS0klI3rgxjd0GxhTTeV+pPrGkbQ7eNfQ7XsC/tT1LY333k6+RDZkefZBJ87zqgef4/kMrufKU/lsBCms0EDWZP716E9/8+fKKDRzorbYzfeIe/fYLbdjSzu2Pv9rtO7cvepXzjj5AwaiAglKJdGtfj6XvMiLD2yFAbWmPDtYL5i2hM+tEJ5lR2pfntjJ1r93YtLWjR4E7mI7PwkKsaewoOhIK8oa65KvO+2vaSK5N5cC7R6Vi8z6QGtEF854im8vRUGc0ZDK83ZHtY849ecJ1W8U0+QykUCo8CYnPq7/O+8KO9EpcfNzXkOu4eAH/9OpNiScChb93b9f6tXfm+qzFFO7D48Y0Jg7AKXbgwM4G9nFjGjmtuYlbH+1e25n8rl17nMRdcvzUrpvyjhvTyLI1b1K4qTpzcO+SNXzykL0VmGIUlEqkt36Ysw/fj5t+91JXQALIYD0uaO3Mwceu+S0j6+u6Ctx4k8lgRs4VFmKXfnJa1z3w8vq66ryvJoW++kcG2mmbVEifP7eVb5x4UGKNaGsIQo31cMUpf9aj2W5EHWRzkO3l3h111j0QF9vk09s6xwuleGGZ/80L59VX533rqje6OtK//9DzfG7mftz0u5cH3SRazECU+a2ru2rtnbksnnjFRiRewF+WcPuqSz4xtcdIwr6u9eutFpO0D8+cvGfiAJw8z3mvAwfiNdPBBvYNW9qZ29LWLW1uSxvnHX1At5O4pBrcHrs0JM7zO/c/w7fve6bX/t3hSEGpRHrrhznhkL2Z89sXu6V3JtySBKJb/+SbSRrr4dpPv7/bRXj5Az3fF5E0+mvZmjd7dPbnCzGAjNHVVFSXgS9/9IBuZ3QD0VttqphO23ihlVRId+Tg4p88TcaiWkRSy1x9XYaJ7xjNlaf0PEu97N5lZHuJSoX9HslNkcnNb72tcz79oRXruOzeZd1OQgoL3t5qXHvvPpJZP+zekX79b7rvO0nz600xA1EuOX4qly54OmzfKM+GM6LO2J6w/fJB5+Hn1tGR8PnEMJKwsGn6ipMP5oJ5S2jv7D5NUk165WubuWDeUrbH9uGv3L2E7542vc9aXP53TWruK7xk44J5S7oF9mJqUcvWbCJjvV+zl5/ub+Y82uME50efO5T6THSH/rhtYXt87adP89b2Tg589270dfFtsdJwCcRgKSiVSLFn0R3ZHF855r1c+atnEw/qvBF1dew+akTXWecPfvsiNz7yEkZ08I1siJ7hUzj6KxMGGsR5Dq78xQruLDjLy+bg2/ev6DGvga73QEeIFRaWlxw/NbFGBFEANaCx3noUaNs7c2zaup2Zk/fkdxcd1RWwW1e9QWN9ho5scrNeYb9HUpDoq/mtt/UbN6aRjxz4Tv55fvfaaGHBWzgwJn/h6Vvbs/3e0ippfkmKHYhyWVdA2sGB846ewrW/foG3Y9tp9Ig6Dtp79/Bfcm1q4TPr+NGiV7oFuS/PbWXRP32U33/1aG5f9GqfdzyZ37qaC+5e0iMgbs86X5rb2uc6N9QZazZtY/K7du33ko32Tuf2Ra/yD0dPKap5NB9o+wuqSSc4nnPO+MGiri2WFJwAvn3fim7rMphHX2zY0t41bH0wN8pNAwWlEtmwpZ19x43mZ+ce3qMWU3iR4Xf/9znOOHQidzy+qtfA1N6ZZfSIOua3rubLd7X2aIrKN2EUjv5KnFc21yMg9TavUt/epJjBC5cueJqEbp4uWYdsZ/IXZt/2ZI9aQNIFv3Huzqat29mwpb2r8Cqm+Q2Sh0bn1zN/i5lLPjGVb/5seZ9NmEkXnl5y/NQ+b2m1y4g6cu79Nolu2NLOvUvW9AgbSQNR6kIeE7YS2YKCM+s7mnqn7b0bddaziXRHQNqhMwePvrCBie/YhU99cB8+9cF9Eh8DM3pEHRfdszSxhgZRS0J9Jiqwk46bjqzz+VtbugZPxOdbeENigO8/FF3gXrg/fuXuVjIGu41qYFoIwhfds7RHQGqs7/lgzWhEYtLlITvkct7rOsTX5fy7lwzoeCwMnIO5UW4aKCjthPxOn9SGfMjEPbp9r/Aiw7ktbdzxdx/kUzc+3nUtU1wmYxz/vd+SzXmvfSPQ++ivwchgLFvzZp+3Sonrr78iqR0/afBCkYPmABjVkKEzmwOzbs2d8VpAb0Y31tHeEY10ywezS46fykETdu+qbV31q+d6jJLKYPzgty/yP79/OfFsOukBb5d9choHTdi9zwD2jZ913ye+8bNlnP+XybXoxnrjvz7zgcTm3PgyCvMSlzQQJec56jLWIwBd/5sX++zbHDemkS9+9ACueuC5btPVZzJ0JgS58+5qZZeGum7bPJ/ffFNfRy5Lxvp+GPaIujou+cT7uPTe5YnHTX7wxOZtnd2OyQ9PGc//PrOu23c9l6N11Rs9RqVuz8K5d7QC0W/5j0dN6bHP7tJQx399dka3YyX/IE0LZ1gjGzJs78j1OCpzwGkfmMCdT/R+oghRYLrpkRf54P7j+m3O29Hk2bOwqDPbqedeVZoljUQaypqbm72lpWWn5hGvItcZvF1QQ2msN244s7lrR1qy6g0+84NFbG7fcUflXRvr+dHffZBXNrzV1eTX3tlJ1nsWEH1prM/w8384nOO/90j366IyUfPcQI2oM/7hqCl87KB3s2bTNvLt20CPzvHCC3GBrqfbbu/MkvPuzRQjGzL86HOH8qkfPN5rc11/6uuMzx++Hz967NVu23NkfYZMxro1N8WNHlHHBce+l3+7f0WPwDWqIUPO4V8+MZVv/GxZjzPiJCMbMvzuoqMA+NB3Hkw8i/79V4/qtQM/gyWOGqzPGI31GbZ1ZMlkLGqCSbjQOelJwjMn75mYl3x+/uUTU7m7ZRWtbZu60jMWVZ56O/HJ78t77z6q2/6Qr+F86DsLu23PDBR1ajR6RIbtnTk6c708BK0PjfUZ/ubPo/tPes571ERG1WfIOkXtYwdP2JWlqzf3+Z06i04S4ycKI+rgnz7+Pg6fPJ7J79q124M08xoy0bolnSCMqDPcnY5ccdusPgPfPW16YjPc/NbVfOXuJX3WvApPLgbTnGdmi929ecATDnQ5CkoD01vbcqGR9dFO/NnD9uUzh+3LJ77/SLfmtZENma6mvo7OLC9veJunVm/ilkdfGVB+/vYv9mXUiDrmPPxiV8FSn4H/+6H9uOGRlwa8fkkyFj2wbGR91AwSHxE2EI2hAI0fO8bAC6WGDF13aeiW3keTiAG3fO7PmX3bk92CWVyGqMDbWsR6jarPcO5RkxndWMflv3iWrR09A903TzyI6RP36CrM9959VI+Th77kC8N853q+WWrOb17g2/ev6PbdkQ0Z5nx2Bl/44R8Sg92HJ4/j0Rc3DKhWCtHJ0+eP2J9rHnwusSa46MUN/PsDz4F7r81u5TCiDs4/5kD+vZ++2VIb2ZChvSPXbZ89rXkCP31yTY/1H9mQwaDHvjFYdQaPf+2jPQY3HfZvDw5oG+RPqAZaY1JQKpPBBqV8k8vnb20ZcGFswMcPejf/u2JdV+dj/um12WyOjlxU0JZo3x226kLwNHfaE7blf5x2MF+Zt3TABfNg9TaCbbAM+MejJvOfC1f2+Gx0Yx3Xf3oGf3fLEyVdZr1FC07aZvl9dkRd1ORVafGRpJWS1I8GsEtDpkeLSUOdQagNlcqtnzu0q8kw33d42b3LBzSPfCtNvIuhGJUKSupTKkK+yQUv/q4BcQ78/Ok/Um9wzJ/tRfOksVx27/Ju81JA2nlZp9dh4ABfnLu0grmh5DUHh8SABPBWe5Zn1r5JZ4mX2en0WpXN77PVCEhQ+YAEvTdzJtWGDp88Dhweeu5PJVv+3CdeZdreuzFvcRtX/vJZbMDtDOm/C7lqSv3YsKW96xoHERkedqk33i6ib7EWnXDIXlxzxgcGPF2lakp9D3WRXu8ZJyJD11ANSAALlqzltkUD67uupJoPSmZ2nJk9a2YrzeyrpZ5/09hRbKtW+4SISBl8fcFyNmxpr3Y2EtV0UDKzOuBa4GPAVOAMM5taymWMG9OIQpKIDCXbs7mu24ulTU0HJeBQYKW7v+ju24E7gRNLuYCrf/lMKWcnIpIKC5evrXYWEtV6UJoArIr93xbSujGzWWbWYmYt69evH9AC5i/9487lUEQkhX61YmBlYaXUelBKGoHQo4fS3ee4e7O7N48fX9wtdPJOPPjdg82biEhqnXTwXtXOQqJaD0ptwMTY/03AmlIu4EvHvo8GDb4TkSFkVL3x9x+ZUu1sJKr1oPQEMMXM9jOzEcDpwIJSL+T5fzue8z6yf6lnKyJScRcfewDP/OvHq52NXtX0HR3cvdPMzgV+CdQBN7l7z8dhlsCXjn0fXzr2feWYtYiIBDUdlADc/T7gvmrnQ0REdl6tN9+JiMgQoqAkIiKpoaAkIiKpoaAkIiKpMeweXWFm64HB3iJ3RinzIiJSYQ78YZDT7uvuA7v7wCAMu6C0M8xMG0tEapq7p/p2AGq+ExGR1FBQEhGR1Kj5i2crLEfyTWBFRGpB6h8Ppz4lERFJDTXfiYhIaigoiYhIatRsn5KZ/RXwY+B9wDLg2fDeUb+PiEil5cLrT8AIYCzwErA/MB64Cdjf3Q/qaya1XFM6A3iE6BlKW4GPA+tIePKsiIiUVZYoIGWAFURPbmgH/pLoZgXHA1uKmVFNBiUzGwPMBM4mCkp5bwPbq5IpEZHhy4BtwAbg/cCFQNbdXwyfnQP8azEzqsmgBJwE/MLdnwNeZ8d6jCQKTCIiUjkZoppQA1E5/PPwHmAP4DqKLJtrNSidAdwZ3t9JtBHuA/YkasoTEZHKcaI+pD2I+pEOB+rNbBZRcCr6Qaw1d52SmY0D2tjRf1QHTADeAywENgMHocEOIiKV9BYwmqgZ7zVgH6La0UhgDVEl6J3A7939yN5mUos1pVOAW919X3ef5O4TiYJTE9FG0eg7EZHKqwt/7wImEw18+BuiSsQHiGpPz/UVkKA2a0q/Br7j7r+IpbUT1ZImAucDv0ieWkREyqidqLkuf0u2PwLvJmrZehg4qL8h4bj7sHwBHyIaqjijj+8cArwAdAIdRP1V24Bm4GWikX6/BX4U5vU/RMPUtxKNClwNjAEeAL4NvDfMby7RAI328KM5sBb4a6JnnawOyx8f5vETonbarUSdiauS8h1+/NawA7wZ5vu9kIfWMO3bwKvArDDN90L65rCea8N3/iKs35fDvDYSDbufDTwPLE7Ylo+H7fJ4yMtrwKb4tiYapLKa6OxpRtjGS0OeNoTtvAR4Lcz7O8BDQGP4znPA2PDZSWF5fwz/Pxa26YfDumwF1hX8ns+F7X8I8Hjss3eE9c0Rdco+RHStxXnhN/4TsD5sq23AtWEZ28NvmAu/6faQz2eI9pPWsE7Ojus4OoiaM5YRtcP/WcjrD4k6iF8Pv/ebYRmXhO/fC7wI7BvydXWYh4ft/Eb4bY4N338+bJ9Xwt99w7oeHab5TfitPkS0z7QBk4iu+esIv18L8Dui/WNdfHuGeV0dPttEtO+2Evat8PnssM3zefSwLf8D2Fwwr73CcreHbT0WOJBouPGpYdsvIWoR6Qzr+CbwyzDPFUTH2MthHt8P851EtF+sI9qP88fJnSHvL8a2zSFhmywNee0ErgGODO8fAJ4iqgmsJ9oXOoiOjfx8vxHmcSo7fvutIQ/5Y+AVov0mSzRc+mSi4+6s8N0/hnkvItoPG0Pel7Fj/59NVJ6sCvP9CfAvIZ//QxQE/hf4z7CtssCaMG1+6PaaMO3VwP8DDgu/07erVjZXOzik9QV8gaiA3hx2kHVhRzom9p0bw06T36FPCDvS8+EHP7OXea8NO8gSogP2FeDXRIXZq8CZYV4rwg6XLyzfILr4rLc87w0sB67t5fOngHuA+lhafjm/ISoInw879oqwo74Rdvx9Y9tleXw7hPTV4WB6BTiGqCB7G/hMP9t4efj+dWE98wHwzP7WK5b3M2PLuwpYGfL9ej6fYVnrw7p8K74ORMFjVfiN48u9MaQ9SXT5wR/Deuan+0ZYhhMVPh3AHfG8EgXafOG2rnD7F6zPy0T7298WrFvhfpb//6dEBd1Xe9l3u+2v4bNbwr70auHvGLbz6+HzLQXr2pWfgu3zUvjdtgIfSlin/wjz28qOwLwm/P+lvn5joiHE7UT7ZHv4TbeGbfRMSE/c1xP2kavj+ScK/tuAcxL2xzUhr6uJTgCuIxTo4f8Twr7wRuE2js1jYcjzLYX7TUIebwh5WRGm+W34/67Y/vkMcHEf6/kU8PvC3yi2DdpC/s8s3NbA/cCDwBcL94lqvGqu+U5ERIauWhzoICIiQ5SCkoiIpIaCkoiIpIaCkoiIpIaCkoiIpIaCkkgfzOwSM1thZg+Y2R1m9hUze4+Z/cLMFpvZb83swPDdm83sGjP7vZm9aGanxOZzgZk9YWZLzezrfSxvkpk9Y2Y3mNkyM/uVmY0Kn30+zGOJmd1jZrvElnu9mT0Ulvt/zOymMJ+bY/M+xsweNbM/mNnd4W77IqmioCTSCzNrJrqo8f1EFzY3h4/mAP/g7jOArxBdx5K3F9HtVD5BdOEvZnYMMAU4FJgOzDCzI/pY9BSia0imEV0Lc3JI/7G7/7m7H0J03crZsWnGAkcBXyK6yPZqYBrwZ2Y23cz2BP4Z+Ki7f4DootgvD2R7iFRCzT55VqQCDgfmu/tWADO7l+jmkh8C7jbrusViY2yan7p7DlhuZu8KaceE15Ph/zFEgefhXpb7kru3hveLie5IAHCQmf0r0Z2YxxDdySDvXnd3M3uK6G4YT4U8LwvTNwFTgd+FfI8AHi1mI4hUkoKSSO+SbuybAd5w9+m9TNOeML0B/+bu/13kcuPzyAKjwvubgZPcfYmZ/S3RrW8Kp8kVTJ8jOs6zwAPufkaReRCpCjXfifTuEeCTZjYy9L8cT3Qro5fM7FQAixzSz3x+CXwu34djZhPM7J2DyM+uwFozawA+PcBpHwNmmtnkkIddzOyAQeRBpKwUlER64e5PAAuI7lH4Y6J+mE1EAeFsM1tCdIPME/uZz6+A24FHQ/PaPKIAM1CXEN3z7gGie5wVzd3XE91T7w4zW0oUpA4cRB5Eykr3vhPpg5mNcfctYaTbw0R3wP5DtfMlMlSpT0mkb3PMbCrRAIdbFJBEyks1JZEqMLNxRI8LKHS0u2+odH5E0kJBSUREUkMDHUREJDUUlEREJDUUlEREJDUUlEREJDX+Pwuk45WV4MGcAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "X1_train_y.plot.scatter(x='gene_name', y='gex')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gene_name</th>\n",
       "      <th>gex</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SLC20A1</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C11orf58</td>\n",
       "      <td>2239.103328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ZSCAN9</td>\n",
       "      <td>19.798064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CD19</td>\n",
       "      <td>411.530623</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TMEM123</td>\n",
       "      <td>34.214129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14305</th>\n",
       "      <td>ACOX1</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14306</th>\n",
       "      <td>MLXIP</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14307</th>\n",
       "      <td>ASGR2</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14308</th>\n",
       "      <td>OR5A1</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14309</th>\n",
       "      <td>TLR3</td>\n",
       "      <td>0.480536</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14310 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      gene_name          gex\n",
       "0       SLC20A1     0.000000\n",
       "1      C11orf58  2239.103328\n",
       "2        ZSCAN9    19.798064\n",
       "3          CD19   411.530623\n",
       "4       TMEM123    34.214129\n",
       "...         ...          ...\n",
       "14305     ACOX1     0.000000\n",
       "14306     MLXIP     0.000000\n",
       "14307     ASGR2     0.000000\n",
       "14308     OR5A1     0.000000\n",
       "14309      TLR3     0.480536\n",
       "\n",
       "[14310 rows x 2 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X1_train_y.query"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DNase-bed Folder\n",
    "\n",
    "This is our chromatin accessibility dataset.\n",
    "\n",
    "\n",
    "The Dnase Folder has 3 Files X1.bed, X2.bed and X3.bed\n",
    "\n",
    "The X1.bed has 56'451 entries with 10 values each\n",
    "X2.bed has 181'145\n",
    "X3.bed has 71'811\n",
    "\n",
    "We have genderchromosome chrX and chrY also included in this data.\n",
    "\n",
    "\n",
    "1. chromosom name\n",
    "2. gene_start/TSS_start\n",
    "3. gene_end/TSS_end\n",
    "4. unknown\n",
    "5. unknown\n",
    "6. unknown\n",
    "7. unknown\n",
    "8. unknown\n",
    "9. unknown\n",
    "10. unknown\n",
    "\n",
    "## TODO:\n",
    "\n",
    "1. Determine what these unknown values are\n",
    "2. Do we need the gender chromosome chrX and chrY\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "Dnase_bed_path = \"/Users/sidhu/Documents/GENOMICS/GENOMICS/Data/DNase-bed/\"\n",
    "X1_dnase = pd.read_csv(Dnase_bed_path + \"X1.bed\", sep= '\\t')\n",
    "X2_dnase = pd.read_csv(Dnase_bed_path + \"X2.bed\", sep= '\\t')\n",
    "X3_dnase = pd.read_csv(Dnase_bed_path + \"X3.bed\", sep= '\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>chr1</th>\n",
       "      <th>181405</th>\n",
       "      <th>181555</th>\n",
       "      <th>.</th>\n",
       "      <th>0</th>\n",
       "      <th>..1</th>\n",
       "      <th>12.379542683931337</th>\n",
       "      <th>-1</th>\n",
       "      <th>-1.1</th>\n",
       "      <th>75</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>chr1</td>\n",
       "      <td>267990</td>\n",
       "      <td>268140</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>12.379543</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>chr1</td>\n",
       "      <td>629160</td>\n",
       "      <td>629310</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>22.263896</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>chr1</td>\n",
       "      <td>629494</td>\n",
       "      <td>629644</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>10.732151</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>chr1</td>\n",
       "      <td>629870</td>\n",
       "      <td>630020</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>3603.694491</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>chr1</td>\n",
       "      <td>630155</td>\n",
       "      <td>630305</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>15.674327</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56446</th>\n",
       "      <td>chrY</td>\n",
       "      <td>10627255</td>\n",
       "      <td>10627405</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>68.390877</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56447</th>\n",
       "      <td>chrY</td>\n",
       "      <td>10994543</td>\n",
       "      <td>10994693</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>8.261062</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56448</th>\n",
       "      <td>chrY</td>\n",
       "      <td>11302695</td>\n",
       "      <td>11302845</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>34.619337</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56449</th>\n",
       "      <td>chrY</td>\n",
       "      <td>11329834</td>\n",
       "      <td>11329984</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>25.558680</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56450</th>\n",
       "      <td>chrY</td>\n",
       "      <td>15355495</td>\n",
       "      <td>15355645</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>17.321719</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>56451 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       chr1    181405    181555  .  0 ..1  12.379542683931337  -1  -1.1  75\n",
       "0      chr1    267990    268140  .  0   .           12.379543  -1    -1  75\n",
       "1      chr1    629160    629310  .  0   .           22.263896  -1    -1  75\n",
       "2      chr1    629494    629644  .  0   .           10.732151  -1    -1  75\n",
       "3      chr1    629870    630020  .  0   .         3603.694491  -1    -1  75\n",
       "4      chr1    630155    630305  .  0   .           15.674327  -1    -1  75\n",
       "...     ...       ...       ... .. ..  ..                 ...  ..   ...  ..\n",
       "56446  chrY  10627255  10627405  .  0   .           68.390877  -1    -1  75\n",
       "56447  chrY  10994543  10994693  .  0   .            8.261062  -1    -1  75\n",
       "56448  chrY  11302695  11302845  .  0   .           34.619337  -1    -1  75\n",
       "56449  chrY  11329834  11329984  .  0   .           25.558680  -1    -1  75\n",
       "56450  chrY  15355495  15355645  .  0   .           17.321719  -1    -1  75\n",
       "\n",
       "[56451 rows x 10 columns]"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X1_dnase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "#X1_dnase.loc[-1] = ['chr1', '181405', '181555', '.', '0', '..1', '12.379542683931337', '-1', '-1.1', '75']\n",
    "\n",
    "# We add the first column to the dataframe\n",
    "X1_dnase.loc[-1] = X1_dnase.columns\n",
    "X1_dnase.index = X1_dnase.index + 1\n",
    "X1_dnase = X1_dnase.sort_index()\n",
    "# increase the index and sort to have the same config\n",
    "\n",
    "# Add a descriptive index on the top\n",
    "X1_dnase = X1_dnase.set_axis(['chr', 'gene_start', 'gene_end', 'unk', 'unk', 'unk', 'unk', 'unk', 'unk', 'unk' ], axis='columns')\n",
    "\n",
    "\n",
    "# We will do the for the rest of the data as well\n",
    "X2_dnase.loc[-1] = X2_dnase.columns\n",
    "X2_dnase.index = X2_dnase.index + 1\n",
    "X2_dnase = X2_dnase.sort_index()\n",
    "X2_dnase = X2_dnase.set_axis(['chr', 'gene_start', 'gene_end', 'unk', 'unk', 'unk', 'unk', 'unk', 'unk', 'unk' ], axis='columns')\n",
    "\n",
    "X3_dnase.loc[-1] = X3_dnase.columns\n",
    "X3_dnase.index = X3_dnase.index + 1\n",
    "X3_dnase = X3_dnase.sort_index()\n",
    "X3_dnase = X3_dnase.set_axis(['chr', 'gene_start', 'gene_end', 'unk', 'unk', 'unk', 'unk', 'unk', 'unk', 'unk' ], axis='columns')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chr                         chr1\n",
      "gene_start                181405\n",
      "gene_end                  181555\n",
      "unk                            .\n",
      "unk                            0\n",
      "unk                          ..1\n",
      "unk           12.379542683931337\n",
      "unk                           -1\n",
      "unk                         -1.1\n",
      "unk                           75\n",
      "Name: 0, dtype: object\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>chr</th>\n",
       "      <th>gene_start</th>\n",
       "      <th>gene_end</th>\n",
       "      <th>unk</th>\n",
       "      <th>unk</th>\n",
       "      <th>unk</th>\n",
       "      <th>unk</th>\n",
       "      <th>unk</th>\n",
       "      <th>unk</th>\n",
       "      <th>unk</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>chr1</td>\n",
       "      <td>267990</td>\n",
       "      <td>268140</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>12.379543</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>chr1</td>\n",
       "      <td>629160</td>\n",
       "      <td>629310</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>22.263896</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>chr1</td>\n",
       "      <td>629494</td>\n",
       "      <td>629644</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>10.732151</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>chr1</td>\n",
       "      <td>629870</td>\n",
       "      <td>630020</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>3603.694491</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>chr1</td>\n",
       "      <td>630155</td>\n",
       "      <td>630305</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>15.674327</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56448</th>\n",
       "      <td>chrY</td>\n",
       "      <td>10994543</td>\n",
       "      <td>10994693</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>8.261062</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56449</th>\n",
       "      <td>chrY</td>\n",
       "      <td>11302695</td>\n",
       "      <td>11302845</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>34.619337</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56450</th>\n",
       "      <td>chrY</td>\n",
       "      <td>11329834</td>\n",
       "      <td>11329984</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>25.55868</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56451</th>\n",
       "      <td>chrY</td>\n",
       "      <td>15355495</td>\n",
       "      <td>15355645</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>17.321719</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>chr1</td>\n",
       "      <td>181405</td>\n",
       "      <td>181555</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>..1</td>\n",
       "      <td>12.379542683931337</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1.1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>56452 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        chr gene_start  gene_end unk unk  unk                 unk unk   unk  \\\n",
       "1      chr1     267990    268140   .   0    .           12.379543  -1    -1   \n",
       "2      chr1     629160    629310   .   0    .           22.263896  -1    -1   \n",
       "3      chr1     629494    629644   .   0    .           10.732151  -1    -1   \n",
       "4      chr1     629870    630020   .   0    .         3603.694491  -1    -1   \n",
       "5      chr1     630155    630305   .   0    .           15.674327  -1    -1   \n",
       "...     ...        ...       ...  ..  ..  ...                 ...  ..   ...   \n",
       "56448  chrY   10994543  10994693   .   0    .            8.261062  -1    -1   \n",
       "56449  chrY   11302695  11302845   .   0    .           34.619337  -1    -1   \n",
       "56450  chrY   11329834  11329984   .   0    .            25.55868  -1    -1   \n",
       "56451  chrY   15355495  15355645   .   0    .           17.321719  -1    -1   \n",
       "0      chr1     181405    181555   .   0  ..1  12.379542683931337  -1  -1.1   \n",
       "\n",
       "      unk  \n",
       "1      75  \n",
       "2      75  \n",
       "3      75  \n",
       "4      75  \n",
       "5      75  \n",
       "...    ..  \n",
       "56448  75  \n",
       "56449  75  \n",
       "56450  75  \n",
       "56451  75  \n",
       "0      75  \n",
       "\n",
       "[56452 rows x 10 columns]"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(X1_dnase.loc[0])\n",
    "#print(X2_dnase.loc[0])\n",
    "#print(X3_dnase.loc[0])\n",
    "X1_dnase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>chr</th>\n",
       "      <th>gene_start</th>\n",
       "      <th>gene_end</th>\n",
       "      <th>unk</th>\n",
       "      <th>unk</th>\n",
       "      <th>unk</th>\n",
       "      <th>unk</th>\n",
       "      <th>unk</th>\n",
       "      <th>unk</th>\n",
       "      <th>unk</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>chr1</td>\n",
       "      <td>181415</td>\n",
       "      <td>181565</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>..1</td>\n",
       "      <td>49.88189434009729</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1.1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>chr1</td>\n",
       "      <td>191435</td>\n",
       "      <td>191585</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>52.050523</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>chr1</td>\n",
       "      <td>267981</td>\n",
       "      <td>268131</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>10.846587</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>chr1</td>\n",
       "      <td>778655</td>\n",
       "      <td>778805</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>179.999586</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>chr1</td>\n",
       "      <td>779135</td>\n",
       "      <td>779285</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>12.292339</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181141</th>\n",
       "      <td>chrY</td>\n",
       "      <td>26408676</td>\n",
       "      <td>26408826</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>9.400835</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181142</th>\n",
       "      <td>chrY</td>\n",
       "      <td>26457840</td>\n",
       "      <td>26457990</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>5.063578</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181143</th>\n",
       "      <td>chrY</td>\n",
       "      <td>26563057</td>\n",
       "      <td>26563207</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>7.232207</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181144</th>\n",
       "      <td>chrY</td>\n",
       "      <td>56839220</td>\n",
       "      <td>56839370</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>23.13548</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181145</th>\n",
       "      <td>chrY</td>\n",
       "      <td>56850055</td>\n",
       "      <td>56850205</td>\n",
       "      <td>.</td>\n",
       "      <td>0</td>\n",
       "      <td>.</td>\n",
       "      <td>23.858356</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>181146 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         chr gene_start  gene_end unk unk  unk                unk unk   unk  \\\n",
       "0       chr1     181415    181565   .   0  ..1  49.88189434009729  -1  -1.1   \n",
       "1       chr1     191435    191585   .   0    .          52.050523  -1    -1   \n",
       "2       chr1     267981    268131   .   0    .          10.846587  -1    -1   \n",
       "3       chr1     778655    778805   .   0    .         179.999586  -1    -1   \n",
       "4       chr1     779135    779285   .   0    .          12.292339  -1    -1   \n",
       "...      ...        ...       ...  ..  ..  ...                ...  ..   ...   \n",
       "181141  chrY   26408676  26408826   .   0    .           9.400835  -1    -1   \n",
       "181142  chrY   26457840  26457990   .   0    .           5.063578  -1    -1   \n",
       "181143  chrY   26563057  26563207   .   0    .           7.232207  -1    -1   \n",
       "181144  chrY   56839220  56839370   .   0    .           23.13548  -1    -1   \n",
       "181145  chrY   56850055  56850205   .   0    .          23.858356  -1    -1   \n",
       "\n",
       "       unk  \n",
       "0       75  \n",
       "1       75  \n",
       "2       75  \n",
       "3       75  \n",
       "4       75  \n",
       "...     ..  \n",
       "181141  75  \n",
       "181142  75  \n",
       "181143  75  \n",
       "181144  75  \n",
       "181145  75  \n",
       "\n",
       "[181146 rows x 10 columns]"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X2_dnase.sort_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## H3* Folders\n",
    "\n",
    "These are our Histone marks dataset\n",
    "\n",
    "We have 6 .bed folder for different histones. In particular we have H3K4me1, H3K4me3, H3K9me3, H3K27ac, H3K27me3 & H3K36me3\n",
    "\n",
    "H3K4me1 and H3K27ac are listed as active enhancers!\n",
    "\n",
    "![alt text](histone.png \"Title\")\n",
    "\n",
    "All the files have 3 .bed files for X1, X2 & X3 and are built the same for the expection of #entries\n",
    "\n",
    "Example: H3k37ac.bed --> X1.bed\n",
    "\n",
    "\n",
    "The X1_K27ac has 57'886, X2_K27ac has 76092, X3_K27ac has 54'473 entries each containing 10 values\n",
    "\n",
    "1. chromosom name\n",
    "2. gene_start/TSS_start\n",
    "3. gene_end/TSS_end\n",
    "4. Peak number\n",
    "5. unknown\n",
    "6. unknown\n",
    "7. unknown\n",
    "8. unknown\n",
    "9. unknown\n",
    "10. unknown\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "H3K27ac_bed_path = \"/Users/sidhu/Documents/GENOMICS/GENOMICS/Data/H3K27ac-bed/\"\n",
    "X1_K27ac = pd.read_csv(H3K27ac_bed_path + \"X1.bed\", sep= '\\t')\n",
    "X2_K27ac = pd.read_csv(H3K27ac_bed_path + \"X2.bed\", sep= '\\t')\n",
    "X3_K27ac = pd.read_csv(H3K27ac_bed_path + \"X3.bed\", sep= '\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "X1_K27ac.loc[-1] = X1_K27ac.columns\n",
    "X1_K27ac.index = X1_K27ac.index + 1\n",
    "X1_K27ac.sort_index()\n",
    "X1_K27ac = X1_K27ac.set_axis(['chr', 'gene_start', 'gene_end', 'unk', 'unk', 'unk', 'unk', 'unk', 'unk', 'unk' ], axis='columns')\n",
    "\n",
    "X2_K27ac.loc[-1] = X2_K27ac.columns\n",
    "X2_K27ac.index = X2_K27ac.index + 1\n",
    "X2_K27ac.sort_index()\n",
    "X2_K27ac = X2_K27ac.set_axis(['chr', 'gene_start', 'gene_end', 'unk', 'unk', 'unk', 'unk', 'unk', 'unk', 'unk' ], axis='columns')\n",
    "\n",
    "X3_K27ac.loc[-1] = X3_K27ac.columns\n",
    "X3_K27ac.index = X3_K27ac.index + 1\n",
    "X3_K27ac.sort_index()\n",
    "X3_K27ac = X3_K27ac.set_axis(['chr', 'gene_start', 'gene_end', 'unk', 'unk', 'unk', 'unk', 'unk', 'unk', 'unk' ], axis='columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "chr                       chr10\n",
       "gene_start            100020043\n",
       "gene_end              100020177\n",
       "unk                  Peak_82084\n",
       "unk                          21\n",
       "unk                           .\n",
       "unk           16.22943969610636\n",
       "unk                     3.82297\n",
       "unk                      1.5382\n",
       "unk                          36\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X2_K27ac.loc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "H3K27me3_bed_path = \"/Users/sidhu/Documents/GENOMICS/GENOMICS/Data/H3K27me3-bed/\"\n",
    "X1_K27me3 = pd.read_csv(H3K27me3_bed_path + \"X1.bed\", sep= '\\t')\n",
    "X2_K27me3 = pd.read_csv(H3K27me3_bed_path + \"X2.bed\", sep= '\\t')\n",
    "X3_K27me3 = pd.read_csv(H3K27me3_bed_path + \"X3.bed\", sep= '\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>chr10</th>\n",
       "      <th>100100915</th>\n",
       "      <th>100101123</th>\n",
       "      <th>Peak_17277</th>\n",
       "      <th>107</th>\n",
       "      <th>.</th>\n",
       "      <th>31.155124653739612</th>\n",
       "      <th>4.07966</th>\n",
       "      <th>0.66916</th>\n",
       "      <th>75</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>chr10</td>\n",
       "      <td>100140031</td>\n",
       "      <td>100140219</td>\n",
       "      <td>Peak_14127</td>\n",
       "      <td>127</td>\n",
       "      <td>.</td>\n",
       "      <td>34.003416</td>\n",
       "      <td>4.41054</td>\n",
       "      <td>0.88574</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>chr10</td>\n",
       "      <td>100566712</td>\n",
       "      <td>100566879</td>\n",
       "      <td>Peak_36184</td>\n",
       "      <td>53</td>\n",
       "      <td>.</td>\n",
       "      <td>25.393536</td>\n",
       "      <td>2.92767</td>\n",
       "      <td>0.12659</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>chr10</td>\n",
       "      <td>100567356</td>\n",
       "      <td>100567545</td>\n",
       "      <td>Peak_13190</td>\n",
       "      <td>131</td>\n",
       "      <td>.</td>\n",
       "      <td>36.565097</td>\n",
       "      <td>4.54974</td>\n",
       "      <td>0.98555</td>\n",
       "      <td>101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>chr10</td>\n",
       "      <td>100570317</td>\n",
       "      <td>100570551</td>\n",
       "      <td>Peak_29275</td>\n",
       "      <td>68</td>\n",
       "      <td>.</td>\n",
       "      <td>28.896399</td>\n",
       "      <td>3.22608</td>\n",
       "      <td>0.25694</td>\n",
       "      <td>216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>chr10</td>\n",
       "      <td>100571000</td>\n",
       "      <td>100571280</td>\n",
       "      <td>Peak_3317</td>\n",
       "      <td>205</td>\n",
       "      <td>.</td>\n",
       "      <td>44.566205</td>\n",
       "      <td>6.02366</td>\n",
       "      <td>1.73814</td>\n",
       "      <td>235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26894</th>\n",
       "      <td>chrX</td>\n",
       "      <td>74423063</td>\n",
       "      <td>74423242</td>\n",
       "      <td>Peak_24560</td>\n",
       "      <td>88</td>\n",
       "      <td>.</td>\n",
       "      <td>32.981163</td>\n",
       "      <td>3.68802</td>\n",
       "      <td>0.42361</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26895</th>\n",
       "      <td>chrX</td>\n",
       "      <td>8731525</td>\n",
       "      <td>8731685</td>\n",
       "      <td>Peak_24561</td>\n",
       "      <td>88</td>\n",
       "      <td>.</td>\n",
       "      <td>32.981163</td>\n",
       "      <td>3.68802</td>\n",
       "      <td>0.42361</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26896</th>\n",
       "      <td>chrX</td>\n",
       "      <td>8880394</td>\n",
       "      <td>8880658</td>\n",
       "      <td>Peak_5203</td>\n",
       "      <td>195</td>\n",
       "      <td>.</td>\n",
       "      <td>41.900185</td>\n",
       "      <td>5.88995</td>\n",
       "      <td>1.70396</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26897</th>\n",
       "      <td>chrX</td>\n",
       "      <td>9788557</td>\n",
       "      <td>9788731</td>\n",
       "      <td>Peak_20095</td>\n",
       "      <td>107</td>\n",
       "      <td>.</td>\n",
       "      <td>33.413850</td>\n",
       "      <td>4.07966</td>\n",
       "      <td>0.66916</td>\n",
       "      <td>83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26898</th>\n",
       "      <td>chrX</td>\n",
       "      <td>9880132</td>\n",
       "      <td>9880341</td>\n",
       "      <td>Peak_33113</td>\n",
       "      <td>68</td>\n",
       "      <td>.</td>\n",
       "      <td>26.637581</td>\n",
       "      <td>3.22608</td>\n",
       "      <td>0.25694</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>26899 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       chr10  100100915  100101123  Peak_17277  107  .  31.155124653739612  \\\n",
       "0      chr10  100140031  100140219  Peak_14127  127  .           34.003416   \n",
       "1      chr10  100566712  100566879  Peak_36184   53  .           25.393536   \n",
       "2      chr10  100567356  100567545  Peak_13190  131  .           36.565097   \n",
       "3      chr10  100570317  100570551  Peak_29275   68  .           28.896399   \n",
       "4      chr10  100571000  100571280   Peak_3317  205  .           44.566205   \n",
       "...      ...        ...        ...         ...  ... ..                 ...   \n",
       "26894   chrX   74423063   74423242  Peak_24560   88  .           32.981163   \n",
       "26895   chrX    8731525    8731685  Peak_24561   88  .           32.981163   \n",
       "26896   chrX    8880394    8880658   Peak_5203  195  .           41.900185   \n",
       "26897   chrX    9788557    9788731  Peak_20095  107  .           33.413850   \n",
       "26898   chrX    9880132    9880341  Peak_33113   68  .           26.637581   \n",
       "\n",
       "       4.07966  0.66916   75  \n",
       "0      4.41054  0.88574   77  \n",
       "1      2.92767  0.12659   59  \n",
       "2      4.54974  0.98555  101  \n",
       "3      3.22608  0.25694  216  \n",
       "4      6.02366  1.73814  235  \n",
       "...        ...      ...  ...  \n",
       "26894  3.68802  0.42361   43  \n",
       "26895  3.68802  0.42361   47  \n",
       "26896  5.88995  1.70396  100  \n",
       "26897  4.07966  0.66916   83  \n",
       "26898  3.22608  0.25694   97  \n",
       "\n",
       "[26899 rows x 10 columns]"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X1_K27me3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MODEL\n",
    "\n",
    "## INPUT\n",
    "\n",
    "Our input is the following: \n",
    "0       SLC20A1   chr2   112645939  112663825  112658362  112658412      +\n",
    "\n",
    "where we have the gene name, chr, gene start, end, TSS start, end, strand\n",
    "\n",
    "## DATA\n",
    "\n",
    "our data is the following:\n",
    "\n",
    "### Chromatin Accessibility (DNase)\n",
    "\n",
    "Here we have the following structure:\n",
    "chr1\t267990\t268140\t.\t0\t.\t12.379543\t-1\t-1\t75\n",
    "\n",
    "### H3* Histone marks\n",
    "\n",
    "chr10\t100140031\t100140219\tPeak_14127\t127\t.\t34.003416\t4.41054\t0.88574\t77\n",
    "\n",
    "## OUTPUT\n",
    "\n",
    "our output has to look like this:\n",
    "SLC20A1\t0.000000\n",
    "\n",
    "where we have gene name and GEX\n",
    "\n",
    "\n",
    "## METHOD\n",
    "\n",
    "### Preprocessing\n",
    "\n",
    "The data has to be preprocessed\n",
    "\n",
    "1. Remove Genderchromosom from Histone datasets\n",
    "2. Sort dataset after chromosoms (optional but better for following)\n",
    "3. Given our input, select a #features from the histone marks and chromatin accesibility datasets\n",
    "4. The histone marks can only be found between [gene_start, TSS_start] and [gene_end, TSS_end].\n",
    "5. The probability probably is multimodal\n",
    "6. Use these features to perform standard linear regression and work up to more sophisticated models\n",
    "\n",
    "\n",
    "### Bayesian Approach to preprocessing\n",
    "\n",
    "We try to fit a combination of Linear_kernels or RBF_kernel on a single histone mark dataset and then sample from the distribution and pick the following features. Then with the selected features we perform a bayesian linear regression or GP and with bayesian learning/optimization we can select new features\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Work Package 1.2 - Model Building"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: \n",
    "# Select the best model to predict gene expression from the obtained features in WP 1.1.\n",
    "\n",
    "# ---------------------------INSERT CODE HERE---------------------------\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# ----------------------------------------------------------------------\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Work Package 1.3 - Prediction on Test Data (Evaluation Metric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO:\n",
    "# Using the model trained in WP 1.2, make predictions on the test data (chr 1 of cell line X3).\n",
    "# Store predictions in a variable called \"pred\" which is a numpy array.\n",
    "\n",
    "pred = None\n",
    "# ---------------------------INSERT CODE HERE---------------------------\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# ----------------------------------------------------------------------\n",
    "\n",
    "# Check if \"pred\" meets the specified constrains\n",
    "assert isinstance(pred, np.ndarray), 'Prediction array must be a numpy array'\n",
    "assert np.issubdtype(pred.dtype, np.number), 'Prediction array must be numeric'\n",
    "assert pred.shape[0] == len(test_genes), 'Each gene should have a unique predicted expression'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Store Predictions in the Required Format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Store predictions in a ZIP. \n",
    "# Upload this zip on the project website under \"Your submission\".\n",
    "# Zip this notebook along with the conda environment (and README, optional) and upload this under \"Your code\".\n",
    "\n",
    "save_dir = 'path/to/save/output/file'  # TODO\n",
    "file_name = 'gex_predicted.csv'         # PLEASE DO NOT CHANGE THIS\n",
    "zip_name = \"Ramesh_Sidharth_Project1.zip\" # TODO\n",
    "save_path = f'{save_dir}/{zip_name}'\n",
    "compression_options = dict(method=\"zip\", archive_name=file_name)\n",
    "\n",
    "test_genes['gex_predicted'] = pred.tolist()\n",
    "test_genes[['gene_name', 'gex_predicted']].to_csv(save_path, compression=compression_options)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}