jaccard_sandbox.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Jaccard similarity between different configurations of oviIN modularity\n",
    "* full oviINr connectome\n",
    "* oviINr inputs\n",
    "* oviINr outputs\n",
    "* combined oviIN connectome\n",
    "* oviINr connectome without oviINr\n",
    "* combined oviIN connectome with oviINs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing with dummy data\n",
    "I'm going to do this with dummy data first. I will create a dataframe with 2 columns that each have 2 clusters. There is 50% overlap between the 2 columns. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0.0</th>\n",
       "      <th>0.05</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0.0  0.05\n",
       "0     1     1\n",
       "1     1     2\n",
       "2     1     1\n",
       "3     1     2\n",
       "4     1     1\n",
       "5     1     2\n",
       "6     1     1\n",
       "7     1     2\n",
       "8     1     1\n",
       "9     1     2\n",
       "10    2     1\n",
       "11    2     2\n",
       "12    2     1\n",
       "13    2     2\n",
       "14    2     1\n",
       "15    2     2\n",
       "16    2     1\n",
       "17    2     2\n",
       "18    2     1\n",
       "19    2     2"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create the dummy data for the columns\n",
    "column_0 = np.concatenate([np.full(10, 1), np.full(10, 2)])\n",
    "column_05 = np.tile([1, 2], 10)\n",
    "\n",
    "# Create the DataFrame\n",
    "df = pd.DataFrame({'0.0': column_0, '0.05': column_05})\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get the coarse modules from the data\n",
    "coarse_modules = df['0.0'].unique().tolist()\n",
    "coarse_modules"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "dict_zero gets the indices (which might be the body IDs depending on the df) of the rows that correspond to the 2 coarse clusters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],\n",
       " 2: [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get bodyIds at zero resolution for each coarse cluster\n",
    "dict_zero = {module: [] for module in coarse_modules}\n",
    "for key, value in dict_zero.items():\n",
    "    dict_zero[key] = df[df['0.0']==key].index.tolist()\n",
    "dict_zero"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The keys in dict are the coarse cluster numbers and the items are the chi cluster numbers that contain the members from the coarse cluster. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: array([1, 2]), 2: array([1, 2])}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Gets cluster numbers for chi resolution for each coarse cluster\n",
    "chi = '0.05'\n",
    "chi_values = [chi]\n",
    "dict = {}\n",
    "\n",
    "for i, x in enumerate(coarse_modules):\n",
    "    # grab a coarse cluster\n",
    "    df_test = df[df['0.0']==x]\n",
    "    for f, y in enumerate(chi_values):\n",
    "        # take new cluster numbers overlapping with coarse cluster\n",
    "        cluster = df_test[y].unique()\n",
    "        #cluster_all[i,f, :len(cluster)] = cluster\n",
    "    dict[coarse_modules[i]] = cluster\n",
    "dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The keys in dict_new are also the coarse cluster numbers but the items are the indices that correspond to all the chi clusters for that coarse cluster."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19],\n",
       " 2: [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19]}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get bodyIDs for each cluster number in the resolution\n",
    "dict_new = {module: [] for module in coarse_modules}\n",
    "for key, value in dict.items():\n",
    "    body_ids = []\n",
    "    for i, x in enumerate(value):\n",
    "        # appends all the body_ids from the chi clusters\n",
    "        body_ids.extend(df[df[chi]==x].index.tolist())\n",
    "    dict_new[key] = body_ids\n",
    "dict_new"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These dictionaries can then be used to compute the Jaccard similarity between a coarse cluster and the chi clusters that contain its members. Rhessa did something like this."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "set1 = set(dict_new[key])\n",
    "set2 = set(dict_zero[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{10, 11, 12, 13, 14, 15, 16, 17, 18, 19}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set1-set2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set2-set1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_1 = set1-set2\n",
    "unique_2 = set2-set1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "common = set1.intersection(set2)\n",
    "# total_unique is not actually the total unique count\n",
    "total_unique = len(unique_1) + len(unique_2)\n",
    "jaccard_sim = len(common) / (total_unique + len(common))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{10, 11, 12, 13, 14, 15, 16, 17, 18, 19}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "common"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_unique"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I will actually do the union between sets instead."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# the union of set1 and set2\n",
    "union_1_2 = set1 | set2\n",
    "union_1_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# jaccard similarity is intersection divided by union\n",
    "jaccard_sim = len(common) / len(union_1_2)\n",
    "jaccard_sim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can continue and make a dictionary to store jaccard values for the different coarse clusters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: [0.5], 2: [0.5]}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a dictionary to store the jaccard similarities\n",
    "jaccard_dict = {}\n",
    "for key, value in dict_new.items():\n",
    "    jaccard_dict[key] = []\n",
    "    unique_1, unique_2, common, jaccard_sim = calculate_difference(dict_new[key], dict_zero[key])\n",
    "    jaccard_dict[key].append(jaccard_sim)\n",
    "    \n",
    "jaccard_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     1    2\n",
       "0  0.5  0.5"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a dataframe to store the jaccard similarities\n",
    "df_jaccard = pd.DataFrame(jaccard_dict)\n",
    "df_jaccard"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I'm not sure why Rhessa wanted to shift the indexing but I will leave that alone in case there was a reason. Doesn't seem to hurt anything."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     1    2\n",
       "1  0.5  0.5"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# starts indexing at 1\n",
    "df_jaccard.index = np.arange(1, len(df_jaccard)+1)\n",
    "df_jaccard"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Updates to Rhessa's Jaccard functions\n",
    "The modified functions are below. This is first tested on the dummy data before overwriting df and testing on the oviIN connectome data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a function that takes in two lists of id numbers and returns the jaccard sim of the two lists\n",
    "def calculate_difference(list1, list2):\n",
    "    set1 = set(list1)\n",
    "    set2 = set(list2)\n",
    "\n",
    "    # the intersection of set1 and set2\n",
    "    common = set1.intersection(set2)\n",
    "\n",
    "    # the union of set1 and set2\n",
    "    union_1_2 = set1 | set2\n",
    "\n",
    "    # Jaccard similarity index for set1 and set2\n",
    "    jaccard_sim = len(common) / len(union_1_2)\n",
    "\n",
    "    return unique_1, unique_2, common, jaccard_sim\n",
    "\n",
    "# function that takes in a partition dataframe, the modules at the base resolution, and the chi value at which to compare\n",
    "def body_ids_by_cluster(df, ref_col, chi):\n",
    "    \"\"\" This function takes in a partition dataframe, the column name for modules at the base resolution (ref_col), and the chi value at which to compare. \n",
    "    It returns a dictionary of body IDs for each cluster number at the resolution and a dictionary of body IDs for each cluster \n",
    "    number at the base resolution.\"\"\"\n",
    "    # get the coarse modules from the data\n",
    "    coarse_modules = df[ref_col].unique().tolist()\n",
    "\n",
    "    # Get bodyIds at zero resolution for each coarse cluster\n",
    "    dict_zero = {module: [] for module in coarse_modules}\n",
    "    for key, value in dict_zero.items():\n",
    "        dict_zero[key] = df[df[ref_col]==key].index.tolist()\n",
    "\n",
    "    # Gets cluster numbers for chi resolution for each coarse cluster\n",
    "    chi_values = [chi]\n",
    "    dict = {}\n",
    "    for i, x in enumerate(coarse_modules):\n",
    "        # grab a coarse cluster\n",
    "        df_test = df[df[ref_col]==x]\n",
    "        for f, y in enumerate(chi_values):\n",
    "            # take new cluster numbers overlapping with coarse cluster\n",
    "            cluster = df_test[y].unique()\n",
    "        dict[coarse_modules[i]] = cluster\n",
    "    \n",
    "    # Get bodyIDs for each cluster number in the chi resolution clusters\n",
    "    dict_new = {module: [] for module in coarse_modules}\n",
    "    for key, value in dict.items():\n",
    "        body_ids = []\n",
    "        for i, x in enumerate(value):\n",
    "            # appends all the body_ids from the chi clusters\n",
    "            body_ids.extend(df[df[chi]==x].index.tolist())\n",
    "        dict_new[key] = body_ids\n",
    "\n",
    "    return dict_new, dict_zero\n",
    "    \n",
    "# Function that takes in partition dataframe, the modules at the base resolution, and the chi value at which to compare\n",
    "def main_jaccard(df, ref_col, chi):\n",
    "    \"\"\" This function takes in a partition dataframe, the modules at the base resolution, and the chi \n",
    "    value at which to compare. It returns a dataframe of the jaccard similarities between the resolutions at each cluster\"\"\"\n",
    "\n",
    "    # Get the body IDs for each cluster number at the resolution and the base resolution\n",
    "    dict_new, dict_zero = body_ids_by_cluster(df, ref_col, chi)\n",
    "\n",
    "    # Create a dictionary to store the jaccard similarities\n",
    "    jaccard_dict = {}\n",
    "    for key, value in dict_new.items():\n",
    "        jaccard_dict[key] = []\n",
    "        unique_1, unique_2, common, jaccard_sim = calculate_difference(dict_new[key], dict_zero[key])\n",
    "        jaccard_dict[key].append(jaccard_sim)\n",
    "    \n",
    "    # Create a dataframe to store the jaccard similarities\n",
    "    df_jaccard = pd.DataFrame(jaccard_dict)\n",
    "    \n",
    "    return df_jaccard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     1    2\n",
       "0  0.5  0.5"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coarse_col = '0.0'\n",
    "chi = '0.05'\n",
    "test = main_jaccard(df, coarse_col, chi)\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# file path for oviIN modularity data for full ovi connectome\n",
    "os.chdir('/Users/ggutierr/My Drive (ggutierr@barnard.edu)/GitHub/oviIN-analyses-gabrielle/ovi_preprocessed/preprocessed-v1.2.1')\n",
    "\n",
    "# load in the data\n",
    "ovi_HB_node_df = pd.read_csv('preprocessed_nodes.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = ovi_HB_node_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the coarse modules from the data\n",
    "coarse_modules = df['0.0'].unique().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.287668</td>\n",
       "      <td>0.221907</td>\n",
       "      <td>0.130645</td>\n",
       "      <td>0.259494</td>\n",
       "      <td>0.166893</td>\n",
       "      <td>0.179653</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          1         2         3         4         5         6\n",
       "0  0.287668  0.221907  0.130645  0.259494  0.166893  0.179653"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "real_test = main_jaccard(df, coarse_col, chi)\n",
    "real_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Co-clustering matrices\n",
    "Based on a suggestion from Alex. \n",
    "\"... just re-run the clustering like 100 times and compute the co-clustering matrix (C_ij = 1 if nodes i and j belong to the same cluster, 0 otherwise) each time, then take the average. If the distribution of values in the average is really strongly bimodal (a giant peak at 0 and at 1, with very little for values strictly in between), that means you can trust the clusters\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Co-clustering of full oviINr runs\n",
    "Testing this out on the only run that I have for now."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# file path for oviIN modularity data for full ovi connectome\n",
    "os.chdir('/Users/ggutierr/My Drive (ggutierr@barnard.edu)/GitHub/oviIN-analyses-gabrielle/ovi_preprocessed/preprocessed-v1.2.1')\n",
    "\n",
    "ovi_HB_node_df = pd.read_csv('preprocessed_nodes.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>0.0</th>\n",
       "      <th>0.05</th>\n",
       "      <th>0.1</th>\n",
       "      <th>0.5</th>\n",
       "      <th>0.75</th>\n",
       "      <th>1.0</th>\n",
       "      <th>instance</th>\n",
       "      <th>celltype</th>\n",
       "      <th>pre</th>\n",
       "      <th>...</th>\n",
       "      <th>status</th>\n",
       "      <th>cropped</th>\n",
       "      <th>statusLabel</th>\n",
       "      <th>cellBodyFiber</th>\n",
       "      <th>somaRadius</th>\n",
       "      <th>somaLocation</th>\n",
       "      <th>roiInfo</th>\n",
       "      <th>notes</th>\n",
       "      <th>inputRois</th>\n",
       "      <th>outputRois</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1003215282</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>CL229_R</td>\n",
       "      <td>CL229</td>\n",
       "      <td>100</td>\n",
       "      <td>...</td>\n",
       "      <td>Traced</td>\n",
       "      <td>False</td>\n",
       "      <td>Roughly traced</td>\n",
       "      <td>PDM19</td>\n",
       "      <td>301.0</td>\n",
       "      <td>[23044, 14981, 11600]</td>\n",
       "      <td>{'INP': {'pre': 87, 'post': 351, 'downstream':...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['EPA(R)', 'GOR(R)', 'IB', 'ICL(R)', 'INP', 'S...</td>\n",
       "      <td>['GOR(R)', 'IB', 'ICL(R)', 'INP', 'SCL(R)', 'S...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1005952640</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>IB058_R</td>\n",
       "      <td>IB058</td>\n",
       "      <td>664</td>\n",
       "      <td>...</td>\n",
       "      <td>Traced</td>\n",
       "      <td>False</td>\n",
       "      <td>Roughly traced</td>\n",
       "      <td>PVL20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'INP': {'pre': 464, 'post': 1327, 'downstream...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['ATL(R)', 'IB', 'ICL(R)', 'INP', 'PLP(R)', 'S...</td>\n",
       "      <td>['ATL(R)', 'IB', 'ICL(R)', 'INP', 'PLP(R)', 'S...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1006928515</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>CL300_R</td>\n",
       "      <td>CL300</td>\n",
       "      <td>86</td>\n",
       "      <td>...</td>\n",
       "      <td>Traced</td>\n",
       "      <td>False</td>\n",
       "      <td>Roughly traced</td>\n",
       "      <td>PVL13</td>\n",
       "      <td>236.0</td>\n",
       "      <td>[12083, 10523, 16816]</td>\n",
       "      <td>{'INP': {'pre': 79, 'post': 126, 'downstream':...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['ATL(R)', 'IB', 'ICL(R)', 'INP', 'SCL(R)', 'S...</td>\n",
       "      <td>['ATL(R)', 'IB', 'ICL(R)', 'INP', 'SCL(R)', 'S...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1007260806</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>CL301_R</td>\n",
       "      <td>CL301</td>\n",
       "      <td>119</td>\n",
       "      <td>...</td>\n",
       "      <td>Traced</td>\n",
       "      <td>False</td>\n",
       "      <td>Roughly traced</td>\n",
       "      <td>PVL13</td>\n",
       "      <td>236.0</td>\n",
       "      <td>[13524, 10108, 16480]</td>\n",
       "      <td>{'INP': {'pre': 40, 'post': 128, 'downstream':...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['GOR(R)', 'IB', 'ICL(R)', 'INP', 'PLP(R)', 'S...</td>\n",
       "      <td>['IB', 'ICL(R)', 'INP', 'PLP(R)', 'SCL(R)', 'S...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1007402796</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>PS119_R</td>\n",
       "      <td>PS119</td>\n",
       "      <td>245</td>\n",
       "      <td>...</td>\n",
       "      <td>Traced</td>\n",
       "      <td>False</td>\n",
       "      <td>Roughly traced</td>\n",
       "      <td>PDM16</td>\n",
       "      <td>301.0</td>\n",
       "      <td>[25364, 12010, 12544]</td>\n",
       "      <td>{'SNP(R)': {'pre': 100, 'post': 50, 'downstrea...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['CAN(R)', 'GOR(R)', 'IB', 'ICL(L)', 'ICL(R)',...</td>\n",
       "      <td>['AVLP(R)', 'CAN(R)', 'IB', 'ICL(L)', 'INP', '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>988269593</th>\n",
       "      <td>4545</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>95</td>\n",
       "      <td>13</td>\n",
       "      <td>1785</td>\n",
       "      <td>FB4E_L</td>\n",
       "      <td>FB4E</td>\n",
       "      <td>168</td>\n",
       "      <td>...</td>\n",
       "      <td>Traced</td>\n",
       "      <td>False</td>\n",
       "      <td>Roughly traced</td>\n",
       "      <td>AVM08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'SNP(L)': {'post': 25, 'upstream': 25, 'mito'...</td>\n",
       "      <td>CRELALFB4_1</td>\n",
       "      <td>['CRE(-RUB)(L)', 'CRE(L)', 'CX', 'FB', 'FB-col...</td>\n",
       "      <td>['CRE(-RUB)(L)', 'CRE(L)', 'CX', 'FB', 'FB-col...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>988291460</th>\n",
       "      <td>4546</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>263</td>\n",
       "      <td>1059</td>\n",
       "      <td>1501</td>\n",
       "      <td>1786</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>Assign</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.5assign</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'SNP(L)': {'pre': 2, 'post': 1, 'downstream':...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['SMP(L)', 'SNP(L)']</td>\n",
       "      <td>['SMP(L)', 'SNP(L)']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>988567837</th>\n",
       "      <td>4547</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>FB4G_R</td>\n",
       "      <td>FB4G</td>\n",
       "      <td>785</td>\n",
       "      <td>...</td>\n",
       "      <td>Traced</td>\n",
       "      <td>False</td>\n",
       "      <td>Roughly traced</td>\n",
       "      <td>AVM08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'SNP(R)': {'pre': 6, 'post': 73, 'downstream'...</td>\n",
       "      <td>CRELALFB4_3</td>\n",
       "      <td>['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...</td>\n",
       "      <td>['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>988909130</th>\n",
       "      <td>4548</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>27</td>\n",
       "      <td>56</td>\n",
       "      <td>436</td>\n",
       "      <td>FB5V_R</td>\n",
       "      <td>FB5V</td>\n",
       "      <td>269</td>\n",
       "      <td>...</td>\n",
       "      <td>Traced</td>\n",
       "      <td>False</td>\n",
       "      <td>Roughly traced</td>\n",
       "      <td>AVM10</td>\n",
       "      <td>296.5</td>\n",
       "      <td>[13226, 32024, 18600]</td>\n",
       "      <td>{'SNP(R)': {'pre': 1, 'post': 28, 'downstream'...</td>\n",
       "      <td>CRELALFB5</td>\n",
       "      <td>['AB(R)', 'CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX',...</td>\n",
       "      <td>['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>989228019</th>\n",
       "      <td>4549</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>27</td>\n",
       "      <td>56</td>\n",
       "      <td>241</td>\n",
       "      <td>FB5V_R</td>\n",
       "      <td>FB5V</td>\n",
       "      <td>322</td>\n",
       "      <td>...</td>\n",
       "      <td>Traced</td>\n",
       "      <td>False</td>\n",
       "      <td>Roughly traced</td>\n",
       "      <td>AVM10</td>\n",
       "      <td>296.5</td>\n",
       "      <td>[12516, 33587, 18648]</td>\n",
       "      <td>{'SNP(R)': {'pre': 1, 'post': 45, 'downstream'...</td>\n",
       "      <td>CRELALFB5</td>\n",
       "      <td>['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...</td>\n",
       "      <td>['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4549 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             key  0.0  0.05  0.1   0.5  0.75   1.0 instance celltype  pre  \\\n",
       "id                                                                          \n",
       "1003215282     1    1     1    1     1     1     1  CL229_R    CL229  100   \n",
       "1005952640     2    1     1    2     2     2     2  IB058_R    IB058  664   \n",
       "1006928515     3    1     1    1     3     3     3  CL300_R    CL300   86   \n",
       "1007260806     4    1     2    1     4     4     4  CL301_R    CL301  119   \n",
       "1007402796     5    1     1    2     5     5     5  PS119_R    PS119  245   \n",
       "...          ...  ...   ...  ...   ...   ...   ...      ...      ...  ...   \n",
       "988269593   4545    3     5    5    95    13  1785   FB4E_L     FB4E  168   \n",
       "988291460   4546    4     4  263  1059  1501  1786     None     None    2   \n",
       "988567837   4547    5     8    7    13    13    13   FB4G_R     FB4G  785   \n",
       "988909130   4548    5     8    7    27    56   436   FB5V_R     FB5V  269   \n",
       "989228019   4549    5     8    7    27    56   241   FB5V_R     FB5V  322   \n",
       "\n",
       "            ...  status  cropped     statusLabel  cellBodyFiber  somaRadius  \\\n",
       "id          ...                                                               \n",
       "1003215282  ...  Traced    False  Roughly traced          PDM19       301.0   \n",
       "1005952640  ...  Traced    False  Roughly traced          PVL20         NaN   \n",
       "1006928515  ...  Traced    False  Roughly traced          PVL13       236.0   \n",
       "1007260806  ...  Traced    False  Roughly traced          PVL13       236.0   \n",
       "1007402796  ...  Traced    False  Roughly traced          PDM16       301.0   \n",
       "...         ...     ...      ...             ...            ...         ...   \n",
       "988269593   ...  Traced    False  Roughly traced          AVM08         NaN   \n",
       "988291460   ...  Assign      NaN       0.5assign            NaN         NaN   \n",
       "988567837   ...  Traced    False  Roughly traced          AVM08         NaN   \n",
       "988909130   ...  Traced    False  Roughly traced          AVM10       296.5   \n",
       "989228019   ...  Traced    False  Roughly traced          AVM10       296.5   \n",
       "\n",
       "                     somaLocation  \\\n",
       "id                                  \n",
       "1003215282  [23044, 14981, 11600]   \n",
       "1005952640                    NaN   \n",
       "1006928515  [12083, 10523, 16816]   \n",
       "1007260806  [13524, 10108, 16480]   \n",
       "1007402796  [25364, 12010, 12544]   \n",
       "...                           ...   \n",
       "988269593                     NaN   \n",
       "988291460                     NaN   \n",
       "988567837                     NaN   \n",
       "988909130   [13226, 32024, 18600]   \n",
       "989228019   [12516, 33587, 18648]   \n",
       "\n",
       "                                                      roiInfo        notes  \\\n",
       "id                                                                           \n",
       "1003215282  {'INP': {'pre': 87, 'post': 351, 'downstream':...          NaN   \n",
       "1005952640  {'INP': {'pre': 464, 'post': 1327, 'downstream...          NaN   \n",
       "1006928515  {'INP': {'pre': 79, 'post': 126, 'downstream':...          NaN   \n",
       "1007260806  {'INP': {'pre': 40, 'post': 128, 'downstream':...          NaN   \n",
       "1007402796  {'SNP(R)': {'pre': 100, 'post': 50, 'downstrea...          NaN   \n",
       "...                                                       ...          ...   \n",
       "988269593   {'SNP(L)': {'post': 25, 'upstream': 25, 'mito'...  CRELALFB4_1   \n",
       "988291460   {'SNP(L)': {'pre': 2, 'post': 1, 'downstream':...          NaN   \n",
       "988567837   {'SNP(R)': {'pre': 6, 'post': 73, 'downstream'...  CRELALFB4_3   \n",
       "988909130   {'SNP(R)': {'pre': 1, 'post': 28, 'downstream'...    CRELALFB5   \n",
       "989228019   {'SNP(R)': {'pre': 1, 'post': 45, 'downstream'...    CRELALFB5   \n",
       "\n",
       "                                                    inputRois  \\\n",
       "id                                                              \n",
       "1003215282  ['EPA(R)', 'GOR(R)', 'IB', 'ICL(R)', 'INP', 'S...   \n",
       "1005952640  ['ATL(R)', 'IB', 'ICL(R)', 'INP', 'PLP(R)', 'S...   \n",
       "1006928515  ['ATL(R)', 'IB', 'ICL(R)', 'INP', 'SCL(R)', 'S...   \n",
       "1007260806  ['GOR(R)', 'IB', 'ICL(R)', 'INP', 'PLP(R)', 'S...   \n",
       "1007402796  ['CAN(R)', 'GOR(R)', 'IB', 'ICL(L)', 'ICL(R)',...   \n",
       "...                                                       ...   \n",
       "988269593   ['CRE(-RUB)(L)', 'CRE(L)', 'CX', 'FB', 'FB-col...   \n",
       "988291460                                ['SMP(L)', 'SNP(L)']   \n",
       "988567837   ['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...   \n",
       "988909130   ['AB(R)', 'CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX',...   \n",
       "989228019   ['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...   \n",
       "\n",
       "                                                   outputRois  \n",
       "id                                                             \n",
       "1003215282  ['GOR(R)', 'IB', 'ICL(R)', 'INP', 'SCL(R)', 'S...  \n",
       "1005952640  ['ATL(R)', 'IB', 'ICL(R)', 'INP', 'PLP(R)', 'S...  \n",
       "1006928515  ['ATL(R)', 'IB', 'ICL(R)', 'INP', 'SCL(R)', 'S...  \n",
       "1007260806  ['IB', 'ICL(R)', 'INP', 'PLP(R)', 'SCL(R)', 'S...  \n",
       "1007402796  ['AVLP(R)', 'CAN(R)', 'IB', 'ICL(L)', 'INP', '...  \n",
       "...                                                       ...  \n",
       "988269593   ['CRE(-RUB)(L)', 'CRE(L)', 'CX', 'FB', 'FB-col...  \n",
       "988291460                                ['SMP(L)', 'SNP(L)']  \n",
       "988567837   ['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...  \n",
       "988909130   ['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...  \n",
       "989228019   ['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...  \n",
       "\n",
       "[4549 rows x 25 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ovi_HB_node_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = ovi_HB_node_df[['id','0.0']].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>0.0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4544</th>\n",
       "      <td>988269593</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4545</th>\n",
       "      <td>988291460</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4546</th>\n",
       "      <td>988567837</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4547</th>\n",
       "      <td>988909130</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4548</th>\n",
       "      <td>989228019</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             id  0.0\n",
       "4544  988269593    3\n",
       "4545  988291460    4\n",
       "4546  988567837    5\n",
       "4547  988909130    5\n",
       "4548  989228019    5"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = test[4544:]\n",
    "test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Co-pilot helped me out with a nested list comprehension for creating the co-clustering matrix. All I need to do is then create a co-clustering matrix for each run and average them. I plan to put this into a for loop where I iterate over each run, create its co-clustering matrix and add it to an accumulating matrix. After the loop, I will divide all elements in the matrix by the number of runs and thus obtain the average. I need to ensure that bodyIDs are always in the same order. They seem to be sorted, but make sure.\n",
    "\n",
    "There is one potentially big problem with this approach. The clusters might be numbered differently in different runs even if the same nodes stick together. For this reason, I think Jaccard similarity will be the way to go."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>id</th>\n",
       "      <th>988269593</th>\n",
       "      <th>988291460</th>\n",
       "      <th>988567837</th>\n",
       "      <th>988909130</th>\n",
       "      <th>989228019</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>988269593</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>988291460</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>988567837</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>988909130</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>989228019</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "id         988269593  988291460  988567837  988909130  989228019\n",
       "id                                                              \n",
       "988269593          1          0          0          0          0\n",
       "988291460          0          1          0          0          0\n",
       "988567837          0          0          1          1          1\n",
       "988909130          0          0          1          1          1\n",
       "989228019          0          0          1          1          1"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create the co-clustering matrix\n",
    "coclust_matrix = pd.DataFrame([[1 if test['0.0'].iloc[i] == test['0.0'].iloc[j] else 0 for j in range(len(test))] for i in range(len(test))], \n",
    "                      index=test['id'], columns=test['id'])\n",
    "\n",
    "coclust_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Jaccard similarity of full oviINr runs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, make a for loop and load each df. Attach the coarse column to a df that will collect the coarse columns of each run. Exit loop. Then nested (triangular) loop to compute similarity between pairs of columns.\n",
    "The end result can be a matrix where runs are rows and columns and entries are similarity."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Jaccard similarities between connectome configurations\n",
    "Now I will compute similarities between columns from different dataframes. Because there might be different sets of body IDs for the different configurations, we may need a new function that compares 2 dataframes. \n",
    "\n",
    "Ultimately, when comparing different configurations, perhaps the most fair thing to do is to only do so with intersecting groups of neurons. For example, if I want to compare the full oviINr clustering to the oviINr inputs, I really only want to work with nodes that are inputs to oviINr. This would effectively be a quantification of the Sankeys that I did in sankey_modularity.ipynb. In which case, I may not need to create new functions, I just have to do some extra pre-processing steps.\n",
    "\n",
    "What I'll do here is to create a new dataframe that combines neurons in common between 2 partition datasets. Then we can simply use the same functions for computing Jaccard similarity that we already have."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# file path for oviIN modularity data for full ovi connectome\n",
    "os.chdir('/Users/ggutierr/My Drive (ggutierr@barnard.edu)/GitHub/oviIN-analyses-gabrielle/hemibrain_preprocessed/preprocessed-v1.2')\n",
    "\n",
    "# load in the data\n",
    "HB_node_df = pd.read_csv('preprocessed_nodes.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>key</th>\n",
       "      <th>0.0</th>\n",
       "      <th>0.05</th>\n",
       "      <th>0.1</th>\n",
       "      <th>0.5</th>\n",
       "      <th>0.75</th>\n",
       "      <th>1.0</th>\n",
       "      <th>instance</th>\n",
       "      <th>celltype</th>\n",
       "      <th>...</th>\n",
       "      <th>status</th>\n",
       "      <th>cropped</th>\n",
       "      <th>statusLabel</th>\n",
       "      <th>cellBodyFiber</th>\n",
       "      <th>somaRadius</th>\n",
       "      <th>somaLocation</th>\n",
       "      <th>roiInfo</th>\n",
       "      <th>notes</th>\n",
       "      <th>inputRois</th>\n",
       "      <th>outputRois</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1003215282</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>CL229_R</td>\n",
       "      <td>CL229</td>\n",
       "      <td>...</td>\n",
       "      <td>Traced</td>\n",
       "      <td>False</td>\n",
       "      <td>Roughly traced</td>\n",
       "      <td>PDM19</td>\n",
       "      <td>301.0</td>\n",
       "      <td>[23044, 14981, 11600]</td>\n",
       "      <td>{'INP': {'pre': 87, 'post': 351, 'downstream':...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['EPA(R)', 'GOR(R)', 'IB', 'ICL(R)', 'INP', 'S...</td>\n",
       "      <td>['GOR(R)', 'IB', 'ICL(R)', 'INP', 'SCL(R)', 'S...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1005952640</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>IB058_R</td>\n",
       "      <td>IB058</td>\n",
       "      <td>...</td>\n",
       "      <td>Traced</td>\n",
       "      <td>False</td>\n",
       "      <td>Roughly traced</td>\n",
       "      <td>PVL20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'INP': {'pre': 464, 'post': 1327, 'downstream...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['ATL(R)', 'IB', 'ICL(R)', 'INP', 'PLP(R)', 'S...</td>\n",
       "      <td>['ATL(R)', 'IB', 'ICL(R)', 'INP', 'PLP(R)', 'S...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1006928515</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>CL300_R</td>\n",
       "      <td>CL300</td>\n",
       "      <td>...</td>\n",
       "      <td>Traced</td>\n",
       "      <td>False</td>\n",
       "      <td>Roughly traced</td>\n",
       "      <td>PVL13</td>\n",
       "      <td>236.0</td>\n",
       "      <td>[12083, 10523, 16816]</td>\n",
       "      <td>{'INP': {'pre': 79, 'post': 126, 'downstream':...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['ATL(R)', 'IB', 'ICL(R)', 'INP', 'SCL(R)', 'S...</td>\n",
       "      <td>['ATL(R)', 'IB', 'ICL(R)', 'INP', 'SCL(R)', 'S...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1007260806</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>CL301_R</td>\n",
       "      <td>CL301</td>\n",
       "      <td>...</td>\n",
       "      <td>Traced</td>\n",
       "      <td>False</td>\n",
       "      <td>Roughly traced</td>\n",
       "      <td>PVL13</td>\n",
       "      <td>236.0</td>\n",
       "      <td>[13524, 10108, 16480]</td>\n",
       "      <td>{'INP': {'pre': 40, 'post': 128, 'downstream':...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['GOR(R)', 'IB', 'ICL(R)', 'INP', 'PLP(R)', 'S...</td>\n",
       "      <td>['IB', 'ICL(R)', 'INP', 'PLP(R)', 'SCL(R)', 'S...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1008024276</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>FB5N_R</td>\n",
       "      <td>FB5N</td>\n",
       "      <td>...</td>\n",
       "      <td>Traced</td>\n",
       "      <td>False</td>\n",
       "      <td>Roughly traced</td>\n",
       "      <td>AVM08</td>\n",
       "      <td>472.5</td>\n",
       "      <td>[19178, 29711, 37312]</td>\n",
       "      <td>{'SNP(L)': {'post': 5, 'upstream': 5, 'mito': ...</td>\n",
       "      <td>SMPCREFB5_4</td>\n",
       "      <td>['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...</td>\n",
       "      <td>['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2506</th>\n",
       "      <td>987273073</td>\n",
       "      <td>2507</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>409</td>\n",
       "      <td>604</td>\n",
       "      <td>629</td>\n",
       "      <td>(PVL05)_L</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>Traced</td>\n",
       "      <td>False</td>\n",
       "      <td>Roughly traced</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'SNP(R)': {'pre': 65, 'post': 52, 'downstream...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['CRE(-ROB,-RUB)(R)', 'CRE(-RUB)(L)', 'CRE(L)'...</td>\n",
       "      <td>['CRE(-ROB,-RUB)(R)', 'CRE(-RUB)(L)', 'CRE(L)'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2507</th>\n",
       "      <td>987842109</td>\n",
       "      <td>2508</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>23</td>\n",
       "      <td>533</td>\n",
       "      <td>780</td>\n",
       "      <td>815</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>Orphan</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Orphan hotknife</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'SNP(R)': {'pre': 2, 'post': 13, 'downstream'...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['SMP(R)', 'SNP(R)']</td>\n",
       "      <td>['SMP(R)', 'SNP(R)']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2508</th>\n",
       "      <td>988567837</td>\n",
       "      <td>2509</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>58</td>\n",
       "      <td>63</td>\n",
       "      <td>FB4G_R</td>\n",
       "      <td>FB4G</td>\n",
       "      <td>...</td>\n",
       "      <td>Traced</td>\n",
       "      <td>False</td>\n",
       "      <td>Roughly traced</td>\n",
       "      <td>AVM08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'SNP(R)': {'pre': 6, 'post': 73, 'downstream'...</td>\n",
       "      <td>CRELALFB4_3</td>\n",
       "      <td>['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...</td>\n",
       "      <td>['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2509</th>\n",
       "      <td>988909130</td>\n",
       "      <td>2510</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>389</td>\n",
       "      <td>559</td>\n",
       "      <td>572</td>\n",
       "      <td>FB5V_R</td>\n",
       "      <td>FB5V</td>\n",
       "      <td>...</td>\n",
       "      <td>Traced</td>\n",
       "      <td>False</td>\n",
       "      <td>Roughly traced</td>\n",
       "      <td>AVM10</td>\n",
       "      <td>296.5</td>\n",
       "      <td>[13226, 32024, 18600]</td>\n",
       "      <td>{'SNP(R)': {'pre': 1, 'post': 28, 'downstream'...</td>\n",
       "      <td>CRELALFB5</td>\n",
       "      <td>['AB(R)', 'CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX',...</td>\n",
       "      <td>['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2510</th>\n",
       "      <td>989228019</td>\n",
       "      <td>2511</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>389</td>\n",
       "      <td>559</td>\n",
       "      <td>572</td>\n",
       "      <td>FB5V_R</td>\n",
       "      <td>FB5V</td>\n",
       "      <td>...</td>\n",
       "      <td>Traced</td>\n",
       "      <td>False</td>\n",
       "      <td>Roughly traced</td>\n",
       "      <td>AVM10</td>\n",
       "      <td>296.5</td>\n",
       "      <td>[12516, 33587, 18648]</td>\n",
       "      <td>{'SNP(R)': {'pre': 1, 'post': 45, 'downstream'...</td>\n",
       "      <td>CRELALFB5</td>\n",
       "      <td>['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...</td>\n",
       "      <td>['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2511 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              id   key  0.0  0.05  0.1  0.5  0.75  1.0   instance celltype  \\\n",
       "0     1003215282     1    1     1    1    1     1    1    CL229_R    CL229   \n",
       "1     1005952640     2    2     1    1    2     2    2    IB058_R    IB058   \n",
       "2     1006928515     3    1     1    1    3     3    3    CL300_R    CL300   \n",
       "3     1007260806     4    2     1    1    4     4    4    CL301_R    CL301   \n",
       "4     1008024276     5    3     2    2    5     5    5     FB5N_R     FB5N   \n",
       "...          ...   ...  ...   ...  ...  ...   ...  ...        ...      ...   \n",
       "2506   987273073  2507    3     8    8  409   604  629  (PVL05)_L     None   \n",
       "2507   987842109  2508    3     9   23  533   780  815       None     None   \n",
       "2508   988567837  2509    2     3    4   16    58   63     FB4G_R     FB4G   \n",
       "2509   988909130  2510    2     3    4  389   559  572     FB5V_R     FB5V   \n",
       "2510   989228019  2511    2     3    4  389   559  572     FB5V_R     FB5V   \n",
       "\n",
       "      ...  status  cropped      statusLabel  cellBodyFiber  somaRadius  \\\n",
       "0     ...  Traced    False   Roughly traced          PDM19       301.0   \n",
       "1     ...  Traced    False   Roughly traced          PVL20         NaN   \n",
       "2     ...  Traced    False   Roughly traced          PVL13       236.0   \n",
       "3     ...  Traced    False   Roughly traced          PVL13       236.0   \n",
       "4     ...  Traced    False   Roughly traced          AVM08       472.5   \n",
       "...   ...     ...      ...              ...            ...         ...   \n",
       "2506  ...  Traced    False   Roughly traced            NaN         NaN   \n",
       "2507  ...  Orphan      NaN  Orphan hotknife            NaN         NaN   \n",
       "2508  ...  Traced    False   Roughly traced          AVM08         NaN   \n",
       "2509  ...  Traced    False   Roughly traced          AVM10       296.5   \n",
       "2510  ...  Traced    False   Roughly traced          AVM10       296.5   \n",
       "\n",
       "               somaLocation  \\\n",
       "0     [23044, 14981, 11600]   \n",
       "1                       NaN   \n",
       "2     [12083, 10523, 16816]   \n",
       "3     [13524, 10108, 16480]   \n",
       "4     [19178, 29711, 37312]   \n",
       "...                     ...   \n",
       "2506                    NaN   \n",
       "2507                    NaN   \n",
       "2508                    NaN   \n",
       "2509  [13226, 32024, 18600]   \n",
       "2510  [12516, 33587, 18648]   \n",
       "\n",
       "                                                roiInfo        notes  \\\n",
       "0     {'INP': {'pre': 87, 'post': 351, 'downstream':...          NaN   \n",
       "1     {'INP': {'pre': 464, 'post': 1327, 'downstream...          NaN   \n",
       "2     {'INP': {'pre': 79, 'post': 126, 'downstream':...          NaN   \n",
       "3     {'INP': {'pre': 40, 'post': 128, 'downstream':...          NaN   \n",
       "4     {'SNP(L)': {'post': 5, 'upstream': 5, 'mito': ...  SMPCREFB5_4   \n",
       "...                                                 ...          ...   \n",
       "2506  {'SNP(R)': {'pre': 65, 'post': 52, 'downstream...          NaN   \n",
       "2507  {'SNP(R)': {'pre': 2, 'post': 13, 'downstream'...          NaN   \n",
       "2508  {'SNP(R)': {'pre': 6, 'post': 73, 'downstream'...  CRELALFB4_3   \n",
       "2509  {'SNP(R)': {'pre': 1, 'post': 28, 'downstream'...    CRELALFB5   \n",
       "2510  {'SNP(R)': {'pre': 1, 'post': 45, 'downstream'...    CRELALFB5   \n",
       "\n",
       "                                              inputRois  \\\n",
       "0     ['EPA(R)', 'GOR(R)', 'IB', 'ICL(R)', 'INP', 'S...   \n",
       "1     ['ATL(R)', 'IB', 'ICL(R)', 'INP', 'PLP(R)', 'S...   \n",
       "2     ['ATL(R)', 'IB', 'ICL(R)', 'INP', 'SCL(R)', 'S...   \n",
       "3     ['GOR(R)', 'IB', 'ICL(R)', 'INP', 'PLP(R)', 'S...   \n",
       "4     ['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...   \n",
       "...                                                 ...   \n",
       "2506  ['CRE(-ROB,-RUB)(R)', 'CRE(-RUB)(L)', 'CRE(L)'...   \n",
       "2507                               ['SMP(R)', 'SNP(R)']   \n",
       "2508  ['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...   \n",
       "2509  ['AB(R)', 'CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX',...   \n",
       "2510  ['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...   \n",
       "\n",
       "                                             outputRois  \n",
       "0     ['GOR(R)', 'IB', 'ICL(R)', 'INP', 'SCL(R)', 'S...  \n",
       "1     ['ATL(R)', 'IB', 'ICL(R)', 'INP', 'PLP(R)', 'S...  \n",
       "2     ['ATL(R)', 'IB', 'ICL(R)', 'INP', 'SCL(R)', 'S...  \n",
       "3     ['IB', 'ICL(R)', 'INP', 'PLP(R)', 'SCL(R)', 'S...  \n",
       "4     ['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...  \n",
       "...                                                 ...  \n",
       "2506  ['CRE(-ROB,-RUB)(R)', 'CRE(-RUB)(L)', 'CRE(L)'...  \n",
       "2507                               ['SMP(R)', 'SNP(R)']  \n",
       "2508  ['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...  \n",
       "2509  ['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...  \n",
       "2510  ['CRE(-ROB,-RUB)(R)', 'CRE(R)', 'CX', 'FB', 'F...  \n",
       "\n",
       "[2511 rows x 26 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "# file path for oviIN modularity data for ovi input connectome\n",
    "os.chdir('/Users/ggutierr/My Drive (ggutierr@barnard.edu)/GitHub/oviIN-analyses-gabrielle/ovi_preprocessed/preprocessed_inputs-v1.2.1')\n",
    "\n",
    "ovi_in_node_df = pd.read_csv('preprocessed_nodes.csv')\n",
    "ovi_in_node_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# return to cd\n",
    "os.chdir('/Users/ggutierr/My Drive (ggutierr@barnard.edu)/GitHub/oviIN-analyses-gabrielle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "test1 = HB_node_df[['id','0.0','celltype','instance']].copy()\n",
    "test2 = ovi_in_node_df[['id','0.0']].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>0.0_x</th>\n",
       "      <th>celltype</th>\n",
       "      <th>instance</th>\n",
       "      <th>0.0_y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>263674097</td>\n",
       "      <td>3</td>\n",
       "      <td>LHPD2a5_a</td>\n",
       "      <td>LHPD2a5_a_R</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>266187480</td>\n",
       "      <td>3</td>\n",
       "      <td>SMP349</td>\n",
       "      <td>SMP349_R</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>266187559</td>\n",
       "      <td>3</td>\n",
       "      <td>SLP399</td>\n",
       "      <td>SLP399_R</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>267214250</td>\n",
       "      <td>3</td>\n",
       "      <td>pC1b</td>\n",
       "      <td>pC1b_R</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>267223104</td>\n",
       "      <td>4</td>\n",
       "      <td>SMP025</td>\n",
       "      <td>SMP025_R</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1827</th>\n",
       "      <td>5901225755</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1828</th>\n",
       "      <td>5901227238</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1829</th>\n",
       "      <td>5901232053</td>\n",
       "      <td>3</td>\n",
       "      <td>SMP272</td>\n",
       "      <td>SMP272(PDL21)_L</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1830</th>\n",
       "      <td>6400000773</td>\n",
       "      <td>3</td>\n",
       "      <td>SMP411</td>\n",
       "      <td>SMP411_R</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1831</th>\n",
       "      <td>7112622044</td>\n",
       "      <td>3</td>\n",
       "      <td>LAL137</td>\n",
       "      <td>LAL137(PVL05)_L</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1832 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              id  0.0_x   celltype         instance  0.0_y\n",
       "0      263674097      3  LHPD2a5_a      LHPD2a5_a_R      5\n",
       "1      266187480      3     SMP349         SMP349_R      5\n",
       "2      266187559      3     SLP399         SLP399_R      5\n",
       "3      267214250      3       pC1b           pC1b_R      5\n",
       "4      267223104      4     SMP025         SMP025_R      5\n",
       "...          ...    ...        ...              ...    ...\n",
       "1827  5901225755      1       None             None      2\n",
       "1828  5901227238      1       None             None      2\n",
       "1829  5901232053      3     SMP272  SMP272(PDL21)_L      1\n",
       "1830  6400000773      3     SMP411         SMP411_R      5\n",
       "1831  7112622044      3     LAL137  LAL137(PVL05)_L      2\n",
       "\n",
       "[1832 rows x 5 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test3 = pd.merge(test1, test2, on='id', how='inner')\n",
    "test3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'unique_1' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[26], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m coarse_col \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m0.0_x\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m      2\u001b[0m chi \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m0.0_y\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[0;32m----> 3\u001b[0m test4 \u001b[38;5;241m=\u001b[39m \u001b[43mmain_jaccard\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtest3\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcoarse_col\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mchi\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m      4\u001b[0m test4\n",
      "Cell \u001b[0;32mIn[25], line 64\u001b[0m, in \u001b[0;36mmain_jaccard\u001b[0;34m(df, ref_col, chi)\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m dict_new\u001b[38;5;241m.\u001b[39mitems():\n\u001b[1;32m     63\u001b[0m     jaccard_dict[key] \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m---> 64\u001b[0m     unique_1, unique_2, common, jaccard_sim \u001b[38;5;241m=\u001b[39m \u001b[43mcalculate_difference\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdict_new\u001b[49m\u001b[43m[\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdict_zero\u001b[49m\u001b[43m[\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     65\u001b[0m     jaccard_dict[key]\u001b[38;5;241m.\u001b[39mappend(jaccard_sim)\n\u001b[1;32m     67\u001b[0m \u001b[38;5;66;03m# Create a dataframe to store the jaccard similarities\u001b[39;00m\n",
      "Cell \u001b[0;32mIn[25], line 15\u001b[0m, in \u001b[0;36mcalculate_difference\u001b[0;34m(list1, list2)\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[38;5;66;03m# Jaccard similarity index for set1 and set2\u001b[39;00m\n\u001b[1;32m     13\u001b[0m jaccard_sim \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(common) \u001b[38;5;241m/\u001b[39m \u001b[38;5;28mlen\u001b[39m(union_1_2)\n\u001b[0;32m---> 15\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43munique_1\u001b[49m, unique_2, common, jaccard_sim\n",
      "\u001b[0;31mNameError\u001b[0m: name 'unique_1' is not defined"
     ]
    }
   ],
   "source": [
    "coarse_col = '0.0_x'\n",
    "chi = '0.0_y'\n",
    "test4 = main_jaccard(test3, coarse_col, chi)\n",
    "test4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id\n",
       "1003215282    1\n",
       "1005952640    1\n",
       "1006928515    1\n",
       "1007260806    1\n",
       "1007402796    1\n",
       "             ..\n",
       "988269593     3\n",
       "988291460     4\n",
       "988567837     5\n",
       "988909130     5\n",
       "989228019     5\n",
       "Name: 0.0, Length: 4549, dtype: int64"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a new df from the reference column\n",
    "configs_df = df['0.0'].copy()\n",
    "\n",
    "# append column\n",
    "configs_df['0.05'] = df['0.05']\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "modularity",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}