12-image-groups.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "training = pd.read_csv(\"matched_training_images.csv\")\n",
    "\n",
    "testing = pd.read_csv(\"matched_testing_images.csv\")\n",
    "\n",
    "validation = pd.read_csv(\"matched_validation_images.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "841"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training.shape[0] + testing.shape[0] + validation.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "testing_groups = pd.read_csv(\"/storage/data/DSB2018/metadata/image_groups_defined_by_beth.csv\")\n",
    "test_meta = pd.read_csv(\"/storage/data/DSB2018/metadata/dsb_summary_metadata_stage2testdata_v2.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "testing[\"Image\"] = testing[\"crop\"].str.split(\"/\", expand=True)[5].str.replace(\".png\",\"\")\n",
    "testing = pd.merge(testing, testing_groups, on=\"Image\")\n",
    "merged_test = pd.merge(test_meta, testing, left_on=\"hash_name\", right_on=\"Image\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Image</th>\n",
       "      <th>fluorescence</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>project_id</th>\n",
       "      <th>Group</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>p_16ure_pig</th>\n",
       "      <th>Default</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_17Fraunhofer_carp</th>\n",
       "      <th>Default</th>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_17Fraunhofer_koi</th>\n",
       "      <th>Default</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_17Fraunhofer_sturgeon</th>\n",
       "      <th>Default</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_17Fraunhofer_trout</th>\n",
       "      <th>Default</th>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_17Marinaccio</th>\n",
       "      <th>Purple</th>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_17Moscow</th>\n",
       "      <th>Default</th>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_17_forum_cham</th>\n",
       "      <th>Default</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_17_forum_florian</th>\n",
       "      <th>Default</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_17_forum_michesonn</th>\n",
       "      <th>Default</th>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_17_forum_montel</th>\n",
       "      <th>Superbig</th>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_17_forum_nelson_diff</th>\n",
       "      <th>Default</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_17annedm</th>\n",
       "      <th>Default</th>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">p_17frias</th>\n",
       "      <th>Default</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Superbig</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">p_18_cc0_public_set1</th>\n",
       "      <th>PinkAndPurple</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TissueBW</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Image  fluorescence\n",
       "project_id              Group                             \n",
       "p_16ure_pig             Default            4             4\n",
       "p_17Fraunhofer_carp     Default            8             8\n",
       "p_17Fraunhofer_koi      Default            4             4\n",
       "p_17Fraunhofer_sturgeon Default            1             1\n",
       "p_17Fraunhofer_trout    Default            8             8\n",
       "p_17Marinaccio          Purple            16            16\n",
       "p_17Moscow              Default           16            16\n",
       "p_17_forum_cham         Default            1             1\n",
       "p_17_forum_florian      Default            4             4\n",
       "p_17_forum_michesonn    Default           12            12\n",
       "p_17_forum_montel       Superbig           8             8\n",
       "p_17_forum_nelson_diff  Default            2             2\n",
       "p_17annedm              Default            8             8\n",
       "p_17frias               Default            4             4\n",
       "                        Superbig           4             4\n",
       "p_18_cc0_public_set1    PinkAndPurple      5             5\n",
       "                        TissueBW           1             1"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_test.groupby([\"project_id\", \"Group\"])[\"Image\",\"fluorescence\"].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_test.to_csv(\"test_metadata.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "pink_purple = os.listdir(\"/data1/image-segmentation/dsb_eval/groups/my_groups/Pink-Purple/\")\n",
    "super_big = os.listdir(\"/data1/image-segmentation/dsb_eval/groups/my_groups/SuperBig/\")\n",
    "tissue_bw = os.listdir(\"/data1/image-segmentation/dsb_eval/groups/my_groups/TissueBW/\")\n",
    "purple = os.listdir(\"/data1/image-segmentation/dsb_eval/groups/my_groups/Purple/\")\n",
    "default = os.listdir(\"/data1/image-segmentation/dsb_eval/groups/my_groups/Default/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "default = list( set(default).difference(set(super_big)) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "670"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pink_purple) + len(super_big) + len(tissue_bw) + len(purple) + len(default)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "groups = [\"PinkAndPurple\"]*len(pink_purple) + [\"SuperBig\"]*len(super_big) + [\"TissueBW\"]*len(tissue_bw) + [\"Purple\"]*len(purple) + [\"Default\"]*len(default)\n",
    "images = pink_purple + super_big + tissue_bw + purple + default\n",
    "data = [{\"Image\":i, \"Group\":g} for i,g in zip(images, groups)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_groups = pd.DataFrame(data=data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "training[\"Image\"] = training[\"crop\"].str.split(\"/\", expand=True)[7]\n",
    "training[\"project\"] = training.match.str.split(\"/\", expand=True)[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "training = pd.merge(training, training_groups, on=\"Image\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>coord_x</th>\n",
       "      <th>coord_y</th>\n",
       "      <th>crop</th>\n",
       "      <th>exact</th>\n",
       "      <th>match</th>\n",
       "      <th>Image</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>project</th>\n",
       "      <th>Group</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>bbbc002</th>\n",
       "      <th>Default</th>\n",
       "      <td>72</td>\n",
       "      <td>72</td>\n",
       "      <td>72</td>\n",
       "      <td>72</td>\n",
       "      <td>72</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bbbc018</th>\n",
       "      <th>Default</th>\n",
       "      <td>144</td>\n",
       "      <td>144</td>\n",
       "      <td>144</td>\n",
       "      <td>144</td>\n",
       "      <td>144</td>\n",
       "      <td>144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bbbc022</th>\n",
       "      <th>Default</th>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_06Eggert</th>\n",
       "      <th>Default</th>\n",
       "      <td>49</td>\n",
       "      <td>49</td>\n",
       "      <td>49</td>\n",
       "      <td>49</td>\n",
       "      <td>49</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_07mito</th>\n",
       "      <th>Default</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_07mito_fibro</th>\n",
       "      <th>Default</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_14ISBI</th>\n",
       "      <th>TissueBW</th>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_15Stachler_bw</th>\n",
       "      <th>Default</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_15Stachler_c1</th>\n",
       "      <th>Purple</th>\n",
       "      <td>66</td>\n",
       "      <td>66</td>\n",
       "      <td>66</td>\n",
       "      <td>66</td>\n",
       "      <td>66</td>\n",
       "      <td>66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_15Stachler_c2</th>\n",
       "      <th>PinkAndPurple</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_15jager</th>\n",
       "      <th>Default</th>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_16ure_human</th>\n",
       "      <th>Default</th>\n",
       "      <td>84</td>\n",
       "      <td>84</td>\n",
       "      <td>84</td>\n",
       "      <td>84</td>\n",
       "      <td>84</td>\n",
       "      <td>84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_16ure_pig</th>\n",
       "      <th>Default</th>\n",
       "      <td>34</td>\n",
       "      <td>34</td>\n",
       "      <td>34</td>\n",
       "      <td>34</td>\n",
       "      <td>34</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_17Keller</th>\n",
       "      <th>Default</th>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>p_17Marinaccio</th>\n",
       "      <th>Purple</th>\n",
       "      <td>38</td>\n",
       "      <td>38</td>\n",
       "      <td>38</td>\n",
       "      <td>38</td>\n",
       "      <td>38</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">p_17Moscow</th>\n",
       "      <th>Default</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SuperBig</th>\n",
       "      <td>89</td>\n",
       "      <td>89</td>\n",
       "      <td>89</td>\n",
       "      <td>89</td>\n",
       "      <td>89</td>\n",
       "      <td>89</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               coord_x  coord_y  crop  exact  match  Image\n",
       "project         Group                                                     \n",
       "bbbc002         Default             72       72    72     72     72     72\n",
       "bbbc018         Default            144      144   144    144    144    144\n",
       "bbbc022         Default             43       43    43     43     43     43\n",
       "p_06Eggert      Default             49       49    49     49     49     49\n",
       "p_07mito        Default              5        5     5      5      5      5\n",
       "p_07mito_fibro  Default              1        1     1      1      1      1\n",
       "p_14ISBI        TissueBW            16       16    16     16     16     16\n",
       "p_15Stachler_bw Default              4        4     4      4      4      4\n",
       "p_15Stachler_c1 Purple              66       66    66     66     66     66\n",
       "p_15Stachler_c2 PinkAndPurple        4        4     4      4      4      4\n",
       "p_15jager       Default             13       13    13     13     13     13\n",
       "p_16ure_human   Default             84       84    84     84     84     84\n",
       "p_16ure_pig     Default             34       34    34     34     34     34\n",
       "p_17Keller      Default              6        6     6      6      6      6\n",
       "p_17Marinaccio  Purple              38       38    38     38     38     38\n",
       "p_17Moscow      Default              2        2     2      2      2      2\n",
       "                SuperBig            89       89    89     89     89     89"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training.groupby([\"project\", \"Group\"]).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "training.to_csv(\"training_metadata.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}