leffingwell.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Requirements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "required_desc = [\n",
    "'alcoholic', 'aldehydic', 'alliaceous', 'almond', 'amber', 'animal',\n",
    "'anisic', 'apple', 'apricot', 'aromatic', 'balsamic', 'banana', 'beefy',\n",
    "'bergamot', 'berry', 'bitter', 'black currant', 'brandy', 'burnt',\n",
    "'buttery', 'cabbage', 'camphoreous', 'caramellic', 'cedar', 'celery',\n",
    "'chamomile', 'cheesy', 'cherry', 'chocolate', 'cinnamon', 'citrus', 'clean',\n",
    "'clove', 'cocoa', 'coconut', 'coffee', 'cognac', 'cooked', 'cooling',\n",
    "'cortex', 'coumarinic', 'creamy', 'cucumber', 'dairy', 'dry', 'earthy',\n",
    "'ethereal', 'fatty', 'fermented', 'fishy', 'floral', 'fresh', 'fruit skin',\n",
    "'fruity', 'garlic', 'gassy', 'geranium', 'grape', 'grapefruit', 'grassy',\n",
    "'green', 'hawthorn', 'hay', 'hazelnut', 'herbal', 'honey', 'hyacinth',\n",
    "'jasmin', 'juicy', 'ketonic', 'lactonic', 'lavender', 'leafy', 'leathery',\n",
    "'lemon', 'lily', 'malty', 'meaty', 'medicinal', 'melon', 'metallic',\n",
    "'milky', 'mint', 'muguet', 'mushroom', 'musk', 'musty', 'natural', 'nutty',\n",
    "'odorless', 'oily', 'onion', 'orange', 'orangeflower', 'orris', 'ozone',\n",
    "'peach', 'pear', 'phenolic', 'pine', 'pineapple', 'plum', 'popcorn',\n",
    "'potato', 'powdery', 'pungent', 'radish', 'raspberry', 'ripe', 'roasted',\n",
    "'rose', 'rummy', 'sandalwood', 'savory', 'sharp', 'smoky', 'soapy',\n",
    "'solvent', 'sour', 'spicy', 'strawberry', 'sulfurous', 'sweaty', 'sweet',\n",
    "'tea', 'terpenic', 'tobacco', 'tomato', 'tropical', 'vanilla', 'vegetable',\n",
    "'vetiver', 'violet', 'warm', 'waxy', 'weedy', 'winey', 'woody'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "138"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(required_desc)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/aryanab/anaconda3/envs/deepchem/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import pyrfume\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[raw]\n",
    "\"leffingwell_data.csv\" = \"Odor labels for molecules as originally compiled by John Leffingwell and cleaned by Sanchez-Lengeling et al\"\n",
    "\"leffingwell_readme.pdf\" = \"Information about the dataset\"\n",
    "LICENSE = \"Licensing information and use restrictions according to the terms of John Leffingwell and Google\"\n",
    "\n",
    "[processed]\n",
    "\"molecules.csv\" = \"Information about odorant molecules used\"\n",
    "\"behavior.csv\" = \"Odor labels for each molecule (one column per label)\"\n",
    "\"behavior_sparse.csv\" = \"Odor labels for each molecule (single column)\"\n",
    "\"stimuli.csv\" = \"Maps stimulus to CID, negative numbers assigned to compounds without CIDs\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "molecules = pyrfume.load_data('leffingwell/molecules.csv', remote=True)\n",
    "behavior = pyrfume.load_data('leffingwell/behavior.csv', remote=True)\n",
    "behavior_sparse = pyrfume.load_data('leffingwell/behavior_sparse.csv', remote=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "possible tests: molecules\n",
    "- duplicate smiles?\n",
    "- convert all smiles to canonical smiles then check\n",
    "- check if all are valid smiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MolecularWeight</th>\n",
       "      <th>IsomericSMILES</th>\n",
       "      <th>IUPACName</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-955348933095</th>\n",
       "      <td>240.387</td>\n",
       "      <td>CCCCC=COC(=O)CCCCCCCC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hexenyl nonanoate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-923209957509</th>\n",
       "      <td>196.290</td>\n",
       "      <td>CC(=O)OCC1C=CC(C(C)C)CC1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tetrahydrocuminyl acetate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-874408321546</th>\n",
       "      <td>244.331</td>\n",
       "      <td>CCCCCCCCC(OC(C)=O)C(=O)OC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Methyl acetoxydecanoate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-873963935677</th>\n",
       "      <td>198.306</td>\n",
       "      <td>CCCCC=COC(=O)C(C)CCC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hexenyl methylvalerate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-862841422647</th>\n",
       "      <td>148.271</td>\n",
       "      <td>CCCC(S)COCC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Ethoxymethylbutanethiol</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               MolecularWeight             IsomericSMILES IUPACName   \n",
       "CID                                                                   \n",
       "-955348933095          240.387      CCCCC=COC(=O)CCCCCCCC       NaN  \\\n",
       "-923209957509          196.290   CC(=O)OCC1C=CC(C(C)C)CC1       NaN   \n",
       "-874408321546          244.331  CCCCCCCCC(OC(C)=O)C(=O)OC       NaN   \n",
       "-873963935677          198.306       CCCCC=COC(=O)C(C)CCC       NaN   \n",
       "-862841422647          148.271                CCCC(S)COCC       NaN   \n",
       "\n",
       "                                    name  \n",
       "CID                                       \n",
       "-955348933095          Hexenyl nonanoate  \n",
       "-923209957509  Tetrahydrocuminyl acetate  \n",
       "-874408321546    Methyl acetoxydecanoate  \n",
       "-873963935677     Hexenyl methylvalerate  \n",
       "-862841422647    Ethoxymethylbutanethiol  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "molecules.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "find '1-methyl-3-hexenyl acetate' aka => '4-hepten-2-ol, (e)-, acetate'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rdkit import Chem\n",
    "from rdkit.Chem import Draw\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def mol_img(smiles, name):\n",
    "    m = Chem.MolFromSmiles(smiles)\n",
    "    img = Draw.MolToImage(m)\n",
    "    # img.title(smiles)\n",
    "    # img.show(title = smiles)\n",
    "    with open(f\"imgs/'{name}'.png\", 'wb')as fp:\n",
    "        img.save(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "mol_img('CCC=CCC(C)OC(C)=O', '1-methyl-3-hexenyl acetate')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MolecularWeight</th>\n",
       "      <th>IsomericSMILES</th>\n",
       "      <th>IUPACName</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5352813</th>\n",
       "      <td>156.22</td>\n",
       "      <td>CC/C=C/CC(C)OC(=O)C</td>\n",
       "      <td>[(E)-hept-4-en-2-yl] acetate</td>\n",
       "      <td>4-hepten-2-ol, (e)-, acetate</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         MolecularWeight       IsomericSMILES                     IUPACName   \n",
       "CID                                                                           \n",
       "5352813           156.22  CC/C=C/CC(C)OC(=O)C  [(E)-hept-4-en-2-yl] acetate  \\\n",
       "\n",
       "                                 name  \n",
       "CID                                    \n",
       "5352813  4-hepten-2-ol, (e)-, acetate  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "molecules[molecules['IsomericSMILES']=='CC/C=C/CC(C)OC(=O)C']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MolecularWeight</th>\n",
       "      <th>IsomericSMILES</th>\n",
       "      <th>IUPACName</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8159</th>\n",
       "      <td>158.24</td>\n",
       "      <td>CCCCCCCOC(=O)C</td>\n",
       "      <td>heptyl acetate</td>\n",
       "      <td>heptyl acetate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82493</th>\n",
       "      <td>186.29</td>\n",
       "      <td>CC(C)CC(CC(C)C)OC(=O)C</td>\n",
       "      <td>2,6-dimethylheptan-4-yl acetate</td>\n",
       "      <td>2,6-dimethylheptan-4-yl acetate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87947</th>\n",
       "      <td>170.25</td>\n",
       "      <td>CC(CCC=C(C)C)OC(=O)C</td>\n",
       "      <td>6-methylhept-5-en-2-yl acetate</td>\n",
       "      <td>6-methylhept-5-en-2-yl acetate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97987</th>\n",
       "      <td>172.26</td>\n",
       "      <td>CCCCCCC(C)OC(=O)C</td>\n",
       "      <td>octan-2-yl acetate</td>\n",
       "      <td>1-methylheptyl acetate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118088</th>\n",
       "      <td>194.27</td>\n",
       "      <td>CC1=CC(C2CC1C2(C)C)OC(=O)C</td>\n",
       "      <td>(4,6,6-trimethyl-2-bicyclo[3.1.1]hept-3-enyl) ...</td>\n",
       "      <td>4,6,6-trimethylbicyclo[3.1.1]hept-3-en-2-yl ac...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>537615</th>\n",
       "      <td>158.24</td>\n",
       "      <td>CCCCC(CC)OC(=O)C</td>\n",
       "      <td>heptan-3-yl acetate</td>\n",
       "      <td>heptan-3-yl acetate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5352813</th>\n",
       "      <td>156.22</td>\n",
       "      <td>CC/C=C/CC(C)OC(=O)C</td>\n",
       "      <td>[(E)-hept-4-en-2-yl] acetate</td>\n",
       "      <td>4-hepten-2-ol, (e)-, acetate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5363206</th>\n",
       "      <td>156.22</td>\n",
       "      <td>CCC/C=C/CCOC(=O)C</td>\n",
       "      <td>[(E)-hept-3-enyl] acetate</td>\n",
       "      <td>(e)-hept-3-enyl acetate</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         MolecularWeight              IsomericSMILES   \n",
       "CID                                                    \n",
       "8159              158.24              CCCCCCCOC(=O)C  \\\n",
       "82493             186.29      CC(C)CC(CC(C)C)OC(=O)C   \n",
       "87947             170.25        CC(CCC=C(C)C)OC(=O)C   \n",
       "97987             172.26           CCCCCCC(C)OC(=O)C   \n",
       "118088            194.27  CC1=CC(C2CC1C2(C)C)OC(=O)C   \n",
       "537615            158.24            CCCCC(CC)OC(=O)C   \n",
       "5352813           156.22         CC/C=C/CC(C)OC(=O)C   \n",
       "5363206           156.22           CCC/C=C/CCOC(=O)C   \n",
       "\n",
       "                                                 IUPACName   \n",
       "CID                                                          \n",
       "8159                                        heptyl acetate  \\\n",
       "82493                      2,6-dimethylheptan-4-yl acetate   \n",
       "87947                       6-methylhept-5-en-2-yl acetate   \n",
       "97987                                   octan-2-yl acetate   \n",
       "118088   (4,6,6-trimethyl-2-bicyclo[3.1.1]hept-3-enyl) ...   \n",
       "537615                                 heptan-3-yl acetate   \n",
       "5352813                       [(E)-hept-4-en-2-yl] acetate   \n",
       "5363206                          [(E)-hept-3-enyl] acetate   \n",
       "\n",
       "                                                      name  \n",
       "CID                                                         \n",
       "8159                                        heptyl acetate  \n",
       "82493                      2,6-dimethylheptan-4-yl acetate  \n",
       "87947                       6-methylhept-5-en-2-yl acetate  \n",
       "97987                               1-methylheptyl acetate  \n",
       "118088   4,6,6-trimethylbicyclo[3.1.1]hept-3-en-2-yl ac...  \n",
       "537615                                 heptan-3-yl acetate  \n",
       "5352813                       4-hepten-2-ol, (e)-, acetate  \n",
       "5363206                            (e)-hept-3-enyl acetate  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k = molecules[molecules['name'].str.match('.*acetate.*')]\n",
    "l = k[k['name'].str.match('.*hep.*')]\n",
    "# 1-methyl-3-hexenyl acetate => CCC=CCC(C)OC(=O)C\n",
    "l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# l.apply(lambda x: mol_img(x['IsomericSMILES'], x['name']), axis=1)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "analysis continues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 3522 entries, -955348933095 to 162353069\n",
      "Data columns (total 4 columns):\n",
      " #   Column           Non-Null Count  Dtype  \n",
      "---  ------           --------------  -----  \n",
      " 0   MolecularWeight  3522 non-null   float64\n",
      " 1   IsomericSMILES   3522 non-null   object \n",
      " 2   IUPACName        3486 non-null   object \n",
      " 3   name             3522 non-null   object \n",
      "dtypes: float64(1), object(3)\n",
      "memory usage: 137.6+ KB\n"
     ]
    }
   ],
   "source": [
    "molecules.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3522"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(molecules.drop_duplicates())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3522"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_molecules = len(molecules['IsomericSMILES'].value_counts())\n",
    "total_molecules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rdkit import Chem\n",
    "\n",
    "def canonical_smiles(smiles):\n",
    "    return Chem.MolToSmiles(Chem.MolFromSmiles(smiles), isomericSmiles = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'CCC=CCC(C)OC(C)=O'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "canonical_smiles('CCC=CCC(C)OC(=O)C')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3522/3522 [00:00<00:00, 9229.19it/s]\n"
     ]
    }
   ],
   "source": [
    "molecules['canonicalSMILES'] = molecules['IsomericSMILES'].progress_apply(lambda x: canonical_smiles(x))\n",
    "assert len(molecules['canonicalSMILES'].value_counts()) == total_molecules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "molecules = molecules.drop(columns=['canonicalSMILES'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['MolecularWeight', 'IsomericSMILES', 'IUPACName', 'name'], dtype='object')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "molecules.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'CCCC(S)COCC'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "molecules['IsomericSMILES'][-862841422647]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "possible tests: behavior\n",
    "- basic\n",
    "- count of unique behaviours available, maybe pivot table\n",
    "- % count of each behaviour (sorted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>alcoholic</th>\n",
       "      <th>aldehydic</th>\n",
       "      <th>alliaceous</th>\n",
       "      <th>almond</th>\n",
       "      <th>animal</th>\n",
       "      <th>anisic</th>\n",
       "      <th>apple</th>\n",
       "      <th>apricot</th>\n",
       "      <th>aromatic</th>\n",
       "      <th>balsamic</th>\n",
       "      <th>...</th>\n",
       "      <th>tobacco</th>\n",
       "      <th>tomato</th>\n",
       "      <th>tropical</th>\n",
       "      <th>vanilla</th>\n",
       "      <th>vegetable</th>\n",
       "      <th>violet</th>\n",
       "      <th>warm</th>\n",
       "      <th>waxy</th>\n",
       "      <th>winey</th>\n",
       "      <th>woody</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Stimulus</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-955348933095</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-923209957509</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-874408321546</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-873963935677</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-862841422647</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 113 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               alcoholic  aldehydic  alliaceous  almond  animal  anisic   \n",
       "Stimulus                                                                  \n",
       "-955348933095          0          0           0       0       0       0  \\\n",
       "-923209957509          0          0           0       0       0       0   \n",
       "-874408321546          0          0           0       0       0       0   \n",
       "-873963935677          0          0           0       0       0       0   \n",
       "-862841422647          0          0           0       0       0       0   \n",
       "\n",
       "               apple  apricot  aromatic  balsamic  ...  tobacco  tomato   \n",
       "Stimulus                                           ...                    \n",
       "-955348933095      0        0         0         0  ...        0       0  \\\n",
       "-923209957509      0        0         0         0  ...        0       0   \n",
       "-874408321546      0        1         0         0  ...        0       0   \n",
       "-873963935677      1        0         0         0  ...        0       0   \n",
       "-862841422647      0        0         0         0  ...        0       0   \n",
       "\n",
       "               tropical  vanilla  vegetable  violet  warm  waxy  winey  woody  \n",
       "Stimulus                                                                       \n",
       "-955348933095         0        0          0       0     0     1      0      0  \n",
       "-923209957509         0        0          0       0     0     0      0      1  \n",
       "-874408321546         0        0          0       0     0     0      0      0  \n",
       "-873963935677         1        0          0       0     0     0      0      0  \n",
       "-862841422647         0        0          0       0     0     0      0      0  \n",
       "\n",
       "[5 rows x 113 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "behavior.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 3522 entries, -955348933095 to 162353069\n",
      "Columns: 113 entries, alcoholic to woody\n",
      "dtypes: int64(113)\n",
      "memory usage: 3.1 MB\n"
     ]
    }
   ],
   "source": [
    "behavior.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['alcoholic',\n",
       " 'aldehydic',\n",
       " 'alliaceous',\n",
       " 'almond',\n",
       " 'animal',\n",
       " 'anisic',\n",
       " 'apple',\n",
       " 'apricot',\n",
       " 'aromatic',\n",
       " 'balsamic',\n",
       " 'banana',\n",
       " 'beefy',\n",
       " 'berry',\n",
       " 'black currant',\n",
       " 'brandy',\n",
       " 'bread',\n",
       " 'brothy',\n",
       " 'burnt',\n",
       " 'buttery',\n",
       " 'cabbage',\n",
       " 'camphoreous',\n",
       " 'caramellic',\n",
       " 'catty',\n",
       " 'chamomile',\n",
       " 'cheesy',\n",
       " 'cherry',\n",
       " 'chicken',\n",
       " 'chocolate',\n",
       " 'cinnamon',\n",
       " 'citrus',\n",
       " 'cocoa',\n",
       " 'coconut',\n",
       " 'coffee',\n",
       " 'cognac',\n",
       " 'coumarinic',\n",
       " 'creamy',\n",
       " 'cucumber',\n",
       " 'dairy',\n",
       " 'dry',\n",
       " 'earthy',\n",
       " 'ethereal',\n",
       " 'fatty',\n",
       " 'fermented',\n",
       " 'fishy',\n",
       " 'floral',\n",
       " 'fresh',\n",
       " 'fruity',\n",
       " 'garlic',\n",
       " 'gasoline',\n",
       " 'grape',\n",
       " 'grapefruit',\n",
       " 'grassy',\n",
       " 'green',\n",
       " 'hay',\n",
       " 'hazelnut',\n",
       " 'herbal',\n",
       " 'honey',\n",
       " 'horseradish',\n",
       " 'jasmine',\n",
       " 'ketonic',\n",
       " 'leafy',\n",
       " 'leathery',\n",
       " 'lemon',\n",
       " 'malty',\n",
       " 'meaty',\n",
       " 'medicinal',\n",
       " 'melon',\n",
       " 'metallic',\n",
       " 'milky',\n",
       " 'mint',\n",
       " 'mushroom',\n",
       " 'musk',\n",
       " 'musty',\n",
       " 'nutty',\n",
       " 'odorless',\n",
       " 'oily',\n",
       " 'onion',\n",
       " 'orange',\n",
       " 'orris',\n",
       " 'peach',\n",
       " 'pear',\n",
       " 'phenolic',\n",
       " 'pine',\n",
       " 'pineapple',\n",
       " 'plum',\n",
       " 'popcorn',\n",
       " 'potato',\n",
       " 'pungent',\n",
       " 'radish',\n",
       " 'ripe',\n",
       " 'roasted',\n",
       " 'rose',\n",
       " 'rum',\n",
       " 'savory',\n",
       " 'sharp',\n",
       " 'smoky',\n",
       " 'solvent',\n",
       " 'sour',\n",
       " 'spicy',\n",
       " 'strawberry',\n",
       " 'sulfurous',\n",
       " 'sweet',\n",
       " 'tea',\n",
       " 'tobacco',\n",
       " 'tomato',\n",
       " 'tropical',\n",
       " 'vanilla',\n",
       " 'vegetable',\n",
       " 'violet',\n",
       " 'warm',\n",
       " 'waxy',\n",
       " 'winey',\n",
       " 'woody']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "odors = list(behavior.columns)\n",
    "odors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "113"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(odors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0\n",
       "1    301\n",
       "2    527\n",
       "3    860\n",
       "4    834\n",
       "5    568\n",
       "6    300\n",
       "7    103\n",
       "8     24\n",
       "9      5\n",
       "dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(behavior.sum(axis=1).sort_values()).groupby(by=0).size()\n",
    "# number of desc | number of molecules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Stimulus</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [count]\n",
       "Index: []"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# molecules with no descriptors\n",
    "pd.DataFrame(behavior.sum(axis=1).sort_values(), columns=['count']).query('count==0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[43263, 95594, 101363, 5463910, 605629]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# molecules with max (9) descriptors\n",
    "max_desc_molecules = list(dict(pd.DataFrame(behavior.sum(axis=1).sort_values(), columns=['count']).query('count==9')['count']).keys())\n",
    "max_desc_molecules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Stimulus\n",
       "43263      ['ethereal', 'green', 'fruity', 'banana', 'sha...\n",
       "95594      ['citrus', 'green', 'fruity', 'melon', 'almond...\n",
       "101363     ['citrus', 'ethereal', 'oily', 'fruity', 'fatt...\n",
       "5463910    ['rose', 'citrus', 'green', 'fruity', 'melon',...\n",
       "605629     ['creamy', 'green', 'fruity', 'milky', 'vanill...\n",
       "Name: Labels, dtype: object"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "behavior_sparse['Labels'][max_desc_molecules]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "behavior_wise = pd.melt(behavior)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 397986 entries, 0 to 397985\n",
      "Data columns (total 2 columns):\n",
      " #   Column    Non-Null Count   Dtype \n",
      "---  ------    --------------   ----- \n",
      " 0   variable  397986 non-null  object\n",
      " 1   value     397986 non-null  int64 \n",
      "dtypes: int64(1), object(1)\n",
      "memory usage: 6.1+ MB\n"
     ]
    }
   ],
   "source": [
    "behavior_wise.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "value\n",
       "0    385117\n",
       "1     12869\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "behavior_wise['value'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>variable</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>fruity</th>\n",
       "      <td>1391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>green</th>\n",
       "      <td>907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sweet</th>\n",
       "      <td>825</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>floral</th>\n",
       "      <td>553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fatty</th>\n",
       "      <td>407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>horseradish</th>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leathery</th>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>radish</th>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>catty</th>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>musk</th>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>113 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             value\n",
       "variable          \n",
       "fruity        1391\n",
       "green          907\n",
       "sweet          825\n",
       "floral         553\n",
       "fatty          407\n",
       "...            ...\n",
       "horseradish     20\n",
       "leathery        20\n",
       "radish          20\n",
       "catty           20\n",
       "musk            20\n",
       "\n",
       "[113 rows x 1 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# behavior.sum().sort_values(ascending=False) # same result\n",
    "behavior_wise.groupby(by='variable').agg(sum).sort_values(by=\"value\", ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Raw Labels</th>\n",
       "      <th>Labels</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Stimulus</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-955348933095</th>\n",
       "      <td>Herbal-green, waxy, oily, fruity</td>\n",
       "      <td>['green', 'oily', 'fruity', 'waxy', 'herbal']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-923209957509</th>\n",
       "      <td>Herbaceous, woody, slight spicy fruity odor</td>\n",
       "      <td>['woody', 'spicy', 'fruity', 'herbal']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-874408321546</th>\n",
       "      <td>delta-decalactone precursor; peach, apricot, b...</td>\n",
       "      <td>['peach', 'apricot', 'buttery']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-873963935677</th>\n",
       "      <td>Green, fruity, apple-like</td>\n",
       "      <td>['green', 'fruity', 'apple', 'tropical']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-862841422647</th>\n",
       "      <td>Catty urine; cassis in dilution</td>\n",
       "      <td>['catty']</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      Raw Labels   \n",
       "Stimulus                                                           \n",
       "-955348933095                   Herbal-green, waxy, oily, fruity  \\\n",
       "-923209957509        Herbaceous, woody, slight spicy fruity odor   \n",
       "-874408321546  delta-decalactone precursor; peach, apricot, b...   \n",
       "-873963935677                          Green, fruity, apple-like   \n",
       "-862841422647                    Catty urine; cassis in dilution   \n",
       "\n",
       "                                                      Labels  \n",
       "Stimulus                                                      \n",
       "-955348933095  ['green', 'oily', 'fruity', 'waxy', 'herbal']  \n",
       "-923209957509         ['woody', 'spicy', 'fruity', 'herbal']  \n",
       "-874408321546                ['peach', 'apricot', 'buttery']  \n",
       "-873963935677       ['green', 'fruity', 'apple', 'tropical']  \n",
       "-862841422647                                      ['catty']  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "behavior_sparse.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 3522 entries, -955348933095 to 162353069\n",
      "Data columns (total 2 columns):\n",
      " #   Column      Non-Null Count  Dtype \n",
      "---  ------      --------------  ----- \n",
      " 0   Raw Labels  3522 non-null   object\n",
      " 1   Labels      3522 non-null   object\n",
      "dtypes: object(2)\n",
      "memory usage: 211.6+ KB\n"
     ]
    }
   ],
   "source": [
    "behavior_sparse.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "behavior_sparse['descriptors'] = behavior_sparse['Labels'].apply(lambda x: ';'.join(eval(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Enhances creamy, cocoa aroma; also enhences meaty flavors'"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "behavior_sparse['Raw Labels'][41570]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Raw Labels</th>\n",
       "      <th>Labels</th>\n",
       "      <th>descriptors</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Stimulus</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-955348933095</th>\n",
       "      <td>Herbal-green, waxy, oily, fruity</td>\n",
       "      <td>['green', 'oily', 'fruity', 'waxy', 'herbal']</td>\n",
       "      <td>green;oily;fruity;waxy;herbal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-923209957509</th>\n",
       "      <td>Herbaceous, woody, slight spicy fruity odor</td>\n",
       "      <td>['woody', 'spicy', 'fruity', 'herbal']</td>\n",
       "      <td>woody;spicy;fruity;herbal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-874408321546</th>\n",
       "      <td>delta-decalactone precursor; peach, apricot, b...</td>\n",
       "      <td>['peach', 'apricot', 'buttery']</td>\n",
       "      <td>peach;apricot;buttery</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-873963935677</th>\n",
       "      <td>Green, fruity, apple-like</td>\n",
       "      <td>['green', 'fruity', 'apple', 'tropical']</td>\n",
       "      <td>green;fruity;apple;tropical</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-862841422647</th>\n",
       "      <td>Catty urine; cassis in dilution</td>\n",
       "      <td>['catty']</td>\n",
       "      <td>catty</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      Raw Labels   \n",
       "Stimulus                                                           \n",
       "-955348933095                   Herbal-green, waxy, oily, fruity  \\\n",
       "-923209957509        Herbaceous, woody, slight spicy fruity odor   \n",
       "-874408321546  delta-decalactone precursor; peach, apricot, b...   \n",
       "-873963935677                          Green, fruity, apple-like   \n",
       "-862841422647                    Catty urine; cassis in dilution   \n",
       "\n",
       "                                                      Labels   \n",
       "Stimulus                                                       \n",
       "-955348933095  ['green', 'oily', 'fruity', 'waxy', 'herbal']  \\\n",
       "-923209957509         ['woody', 'spicy', 'fruity', 'herbal']   \n",
       "-874408321546                ['peach', 'apricot', 'buttery']   \n",
       "-873963935677       ['green', 'fruity', 'apple', 'tropical']   \n",
       "-862841422647                                      ['catty']   \n",
       "\n",
       "                                 descriptors  \n",
       "Stimulus                                      \n",
       "-955348933095  green;oily;fruity;waxy;herbal  \n",
       "-923209957509      woody;spicy;fruity;herbal  \n",
       "-874408321546          peach;apricot;buttery  \n",
       "-873963935677    green;fruity;apple;tropical  \n",
       "-862841422647                          catty  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "behavior_sparse.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "spell correct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sulfurous; meaty on dilution; HVP, liver\n",
      "Nearly odorless; slight tingling taste; sweetness modiffier\n",
      "Pungent, breathtaking; in dilution a nutty alcoholic note; enhances citrus\n",
      "Odorless; sour taste; used in reaction flavors for meat\n",
      "Almost odorless; bland waxy taste; adds mouthfeel\n",
      "Burnt-rubber; meaty, poultry-like in dilution; modifier for fruit flavors\n",
      "Odorless; faint, bland taste; adds mouthfeel to low fat products\n",
      "Very weak ethereal-fruity odor;a mild sweet taste; normal use as solvent\n",
      "Essentially odorless; slightly bitter taste; used in reaction flavors\n",
      "Sweet, herbaceous-floral; orchid like odor; sweet, spice, fruity taste\n",
      "Strong, sweet floral-honey odor; sweet honey-like taste; slightly spicy\n",
      "Fatty-rancid; in dilution sweet, nutty, green, fatty-cognac like; citrus\n",
      "Faint, waxy, sweet odor; nearly tasteless; oily-creamy mouthfeel\n",
      "Almost odorless; oily-waxy; adds mouthfeel\n",
      "Animal-musty odor; burnt-oily, floral-hebaceous on dilution; green rooty\n",
      "Fatty-waxy; fatty mouthfeel; see comments\n",
      "Very faint, waxy-oily; nearly odorless; provides a waxy mouuthfeel\n",
      "Green; fatty; roasted at 8 ppm in sugar syrup\n",
      "Sulfurous, pungent; Fatty; winey at 8.5 ppm in coffee (See comments)\n",
      "Ethereal fruity-apple odor; apple-like taste; buttery-fermented notes\n",
      "Faint, waxy, sweet odor; nearly tasteless; creamy mouthfeel\n",
      "Emulsifier; Faint, oily-waxy odor; bitter taste above 1000 ppm\n",
      "Nearly odorless; faint fatty, bland taste; adds mouthfeel\n",
      "Astringent; fatty; earthy at 25 ppm in coffee; roasted\n",
      "Sulfurous; Catty, black currant note in dilution; also meaty\n",
      "Strong rubber-savory, brothy odor; nutty, earthy taste; fried, roasted\n",
      "Cooling; minty, herbaceous; odorless & cool & fresh if highly purified\n",
      "Coffee-like taste at 10 ppm; green earthy notes; cocoa, nutty\n",
      "Almost odorless; waxy; mouthfeel agent\n",
      "Cooked Chicken, meat aroma; onion-like; important to Durian flavor\n",
      "Earthy, bell pepper; vegetable; potato-like\n",
      "Green, nutty odor; slight coffee taste; caramel fruity at 40 ppm\n",
      "Cedary; impact compound of certain tobaccos; fish & berry  flavor improver\n",
      "Herbaceous-winey, musty-anisic odor; fruity; winey berry-currant taste\n",
      "Fruity green, pear-banana odor; nut-like; ripe fruity-pear pineapple taste\n",
      "Sweet, herbaceous, fatty odor; sweet green fruity taste; tea notes\n",
      "Fruity, caramelized pineapple odor; pineapple taste; milky, cheese notes\n",
      "Raw meat, blood-like in dilution; also catty, savory; enhances coffee\n",
      "Buttery, cooked caramel odor; caramel, nutty; buttery creamy taste\n",
      "1 ppm-cabbage vegetable; 0.1 ppm-popcorn-cereal taste; cheese\n",
      "Milky, oily-spicy odor; dairy milky-coconut taste; adds mouthfeel\n",
      "Sulfurous; green-fruity-tropical in dilution; also a onion-radish note\n",
      "Slight musty odor; slight bitter taste; used as presevative\n",
      "Sweet, sugary, caramel, bread like; intensely sweet caramel flavor; fruity\n",
      "Caramellic fruity; somewhat roasted; 2-pentanone precursor\n",
      "Sulfurous, vegetative, meaty; ripe cheese notes; tropical\n",
      "Fatty, sweet, citrus-aldehydic; cognac-rum notes; musky\n",
      "Blackcurrant; catty-sulfur note of Black currants; enhances coffee\n",
      "Roasted beef-like; typical note of heated meat; cereal, nutty\n",
      "Green, fatty, musty, waxy odor; green fruity taste; somewhat caramellic\n",
      "Tarry, tobacco-spicy odor; earthy; burnt, woody, whiskey, coumarin flavor\n",
      "Strongly green; privet-buttery like odor; green fruity flavor\n",
      "Sweet, orris-violet like; berry-like on dilution; a synergistic enhancer\n",
      "Pungent, spicy, fruity odor; fish, musty; shellfish in dilution\n",
      "Fruity, waxy-green, melon odor; waxy-green-tropical taste; also vegetative\n",
      "Sharp brandy-alcoholic note; imparts lift; improves juiciness in lemon\n",
      "Caramel-fruity; slight buttery odor; caramellic-fruity taste\n",
      "Floral, jasmine absolute character; coconut; See comments\n",
      "Woody, musty, green; nutty, cocoa, vegetative; useful in seafood flavors\n",
      "Cod liver oil note; Oily, Fatty, fishy; adds mandarin notes to citrus\n",
      "Fatty; savory, good in fatty flavors & fish flavors; mushroom in soup base\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Stimulus\n",
       "-955348933095    None\n",
       "-923209957509    None\n",
       "-874408321546    None\n",
       "-873963935677    None\n",
       "-862841422647    None\n",
       "                 ... \n",
       " 154241350       None\n",
       " 155323379       None\n",
       " 158699780       None\n",
       " 161504856       None\n",
       " 162353069       None\n",
       "Name: Raw Labels, Length: 3522, dtype: object"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find words\n",
    "def find(string):\n",
    "    if len(string.split(';')) > 2:\n",
    "        print(string)\n",
    "behavior_sparse['Raw Labels'].apply(lambda x: find(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "root_req_desc = required_desc.copy()\n",
    "for i in range(len(root_req_desc)):\n",
    "    root_req_desc[i] = root_req_desc[i][:-(int(0.4*len(root_req_desc[i])))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['alcoho',\n",
       " 'aldehy',\n",
       " 'alliac',\n",
       " 'almo',\n",
       " 'amb',\n",
       " 'anim',\n",
       " 'anis',\n",
       " 'app',\n",
       " 'apric',\n",
       " 'aroma',\n",
       " 'balsa',\n",
       " 'bana',\n",
       " 'bee',\n",
       " 'berga',\n",
       " 'ber',\n",
       " 'bitt',\n",
       " 'black cu',\n",
       " 'bran',\n",
       " 'bur',\n",
       " 'butte',\n",
       " 'cabba',\n",
       " 'camphor',\n",
       " 'carame',\n",
       " 'ced',\n",
       " 'cele',\n",
       " 'chamom',\n",
       " 'chee',\n",
       " 'cher',\n",
       " 'chocol',\n",
       " 'cinna',\n",
       " 'citr',\n",
       " 'cle',\n",
       " 'clo',\n",
       " 'coc',\n",
       " 'cocon',\n",
       " 'coff',\n",
       " 'cogn',\n",
       " 'cook',\n",
       " 'cooli',\n",
       " 'cort',\n",
       " 'coumar',\n",
       " 'crea',\n",
       " 'cucum',\n",
       " 'dai',\n",
       " 'dr',\n",
       " 'eart',\n",
       " 'ether',\n",
       " 'fat',\n",
       " 'fermen',\n",
       " 'fis',\n",
       " 'flor',\n",
       " 'fre',\n",
       " 'fruit ',\n",
       " 'frui',\n",
       " 'garl',\n",
       " 'gas',\n",
       " 'geran',\n",
       " 'gra',\n",
       " 'grapef',\n",
       " 'gras',\n",
       " 'gre',\n",
       " 'hawth',\n",
       " 'ha',\n",
       " 'hazel',\n",
       " 'herb',\n",
       " 'hon',\n",
       " 'hyaci',\n",
       " 'jasm',\n",
       " 'jui',\n",
       " 'keton',\n",
       " 'lacto',\n",
       " 'laven',\n",
       " 'lea',\n",
       " 'leath',\n",
       " 'lem',\n",
       " 'lil',\n",
       " 'mal',\n",
       " 'mea',\n",
       " 'medici',\n",
       " 'mel',\n",
       " 'metal',\n",
       " 'mil',\n",
       " 'min',\n",
       " 'mugu',\n",
       " 'mushr',\n",
       " 'mus',\n",
       " 'mus',\n",
       " 'natur',\n",
       " 'nut',\n",
       " 'odorl',\n",
       " 'oil',\n",
       " 'oni',\n",
       " 'oran',\n",
       " 'orangefl',\n",
       " 'orr',\n",
       " 'ozo',\n",
       " 'pea',\n",
       " 'pea',\n",
       " 'pheno',\n",
       " 'pin',\n",
       " 'pineap',\n",
       " 'plu',\n",
       " 'popco',\n",
       " 'pota',\n",
       " 'powde',\n",
       " 'punge',\n",
       " 'radi',\n",
       " 'raspbe',\n",
       " 'rip',\n",
       " 'roast',\n",
       " 'ros',\n",
       " 'rum',\n",
       " 'sandal',\n",
       " 'savo',\n",
       " 'sha',\n",
       " 'smo',\n",
       " 'soa',\n",
       " 'solve',\n",
       " 'sou',\n",
       " 'spi',\n",
       " 'strawb',\n",
       " 'sulfur',\n",
       " 'swea',\n",
       " 'swe',\n",
       " 'te',\n",
       " 'terpe',\n",
       " 'tobac',\n",
       " 'toma',\n",
       " 'tropi',\n",
       " 'vanil',\n",
       " 'vegeta',\n",
       " 'vetiv',\n",
       " 'viol',\n",
       " 'war',\n",
       " 'wax',\n",
       " 'wee',\n",
       " 'win',\n",
       " 'woo']"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "root_req_desc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "import enchant\n",
    "\n",
    "# Create an instance of the English dictionary\n",
    "dictionary = enchant.Dict(\"en_US\")\n",
    "\n",
    "# Check if a word is spelled correctly\n",
    "def spellcheck(word):\n",
    "    if dictionary.check(word):\n",
    "        # print(f\"{word} is spelled correctly.\")\n",
    "        pass\n",
    "    else:\n",
    "        if word in required_desc: # ignore spell error if word in required desc\n",
    "            return\n",
    "        for root in root_req_desc:\n",
    "            if root in word:    # only find spell error for words that have the certain root words\n",
    "                suggestions = dictionary.suggest(word)\n",
    "                if suggestions:\n",
    "                    print(f\"{word} is misspelled. Suggestions: {', '.join(suggestions)}\")\n",
    "                else:\n",
    "                    print(f\"{word} is misspelled, but no suggestions available.\")\n",
    "                break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "words_set = set()\n",
    "def get_spell_errors(string):\n",
    "    string = string.lower()\n",
    "    string = string.split(';')[0] #choose odor and not flavor\n",
    "    list_desc=re.sub(r'[!\"#$%&\\'()*\\-+,./:;<=>?@\\[\\]^_`{|}~]',\" \",string).split(\" \")\n",
    "\n",
    "    # # handle black currant\n",
    "    # if \"black\" in list_desc:\n",
    "    #     idx = list_desc.index('black')\n",
    "    #     if idx<len(list_desc)-1:\n",
    "    #         if list_desc[idx+1] == 'currant':\n",
    "    #             list_desc.pop(idx) #removes black\n",
    "    #             list_desc.pop(idx) # removes currant\n",
    "    #             list_desc.append(\"black currant\")\n",
    "    \n",
    "    for word in list_desc:\n",
    "        if len(word) == 0:\n",
    "            continue\n",
    "        elif word == ' ':\n",
    "            continue\n",
    "        else:\n",
    "            words_set.update([word])\n",
    "    # odor_set = set()\n",
    "    # for req_odor in required_desc:\n",
    "    #     if req_odor in list_desc:\n",
    "    #         odor_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Stimulus\n",
       "-955348933095    None\n",
       "-923209957509    None\n",
       "-874408321546    None\n",
       "-873963935677    None\n",
       "-862841422647    None\n",
       "                 ... \n",
       " 154241350       None\n",
       " 155323379       None\n",
       " 158699780       None\n",
       " 161504856       None\n",
       " 162353069       None\n",
       "Name: Raw Labels, Length: 3522, dtype: object"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "behavior_sparse['Raw Labels'].apply(lambda x: get_spell_errors(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "acetaldehyde is misspelled. Suggestions: asphalted, statehood, stilted, castellated, interluded, adulated, assaulted, astounded, satellited, escalated, osculated, interlarded, auscultated, stolidity, oscillated, postulated, adulterate, isolated\n",
      "acetataldehyde is misspelled, but no suggestions available.\n",
      "acetophenone is misspelled. Suggestions: acetone, Stephenson, acetylene, antiphon, antivenin, acetone's, Stephen, antiphons, Stephanie, Stephens, adenine, atoning, Stephen's, acetaminophen, antiphon's, acetylene's\n",
      "agrestic is misspelled. Suggestions: acrostic, agnostic, acrostics, aphoristic, acoustic, egoistic, juristic, eagerest, orgiastic, acrostic's\n",
      "alliaceious is misspelled. Suggestions: fallacious, alliances, malicious, salacious, alliance's, Alice's, aliases, siliceous, Delicious, audacious, delicious, luscious, alliance, fallacies, Alsace, Eliseo's, Alyce's, ulcerous, Allison's\n",
      "alliacious is misspelled. Suggestions: fallacious, malicious, salacious, Delicious, audacious, delicious, luscious, alliances, alliance's, Alice's, aliases, fallacies, Alicia's, Allison's\n",
      "ambery is misspelled. Suggestions: Amber, amber, ambler, Aubrey, Amber's, amber's, Emery, ember, emery, umber, umbra, Ampere, ampere, embers, amble, armory, embryo, imagery, abbr, ember's, umber's\n",
      "ambrette is misspelled. Suggestions: amaretto, ambient, ambulate, adumbrate, ambit, embitter, abrade, ambled, embroider, emirate, embrace, umbrage, ambrosia, umbrella\n",
      "ammoniacal is misspelled. Suggestions: ammonia cal, ammonia-cal, demoniacal, maniacal, demoniacally, maniacally, ironical, harmonically, demonically, ecumenical\n",
      "animalic is misspelled. Suggestions: animal, animals, animal's, anomalies, anomaly, anemic, angelic, animalcule, analog, anomaly's\n",
      "animalistic is misspelled. Suggestions: animistic, unrealistic, cannibalistic, inelastic, nihilistic, annalist, animals, animist, annalists, idealistic, analytic, animal's, animists, annalist's, minimalist, anapestic, animist's, minimalists, minimalist's\n",
      "animallic is misspelled. Suggestions: animal, animals, animal's, animalcule, anomalies, anomaly, anemic, anomalous, angelic, anomaly's, anemically, analog, inimically\n",
      "anisole is misspelled. Suggestions: insole, aisle, anise, insoles, Anatole, isle, insula, insole's, anise's\n",
      "applle is misspelled. Suggestions: Apple, apple, appall, Apollo, applier, apply, appalled, apples, applet, appeal, appalls, applied, applies, dapple, ample, ampule, Apple's, apple's\n",
      "artemisia is misspelled. Suggestions: Artemis, Artemis's, Amish, ATM's, atoms, items, Irtish, atom's, atomic, item's, airtime's, airtime, Atman, atishoo, atomize, itemize, ATM, edema, admission, armchair, atomizer, atom, item\n",
      "backnote is misspelled. Suggestions: banknote, back note, back-note, backbite, backbiter, backdate, backbit, beckoned, aconite, backdoor, backside, backwater\n",
      "backnotes is misspelled. Suggestions: banknotes, back notes, back-notes, backbites, banknote's, backbiters, backdates, aconites, backbiter's, backsides, backwaters, backwoods, aconite's, backside's, backwater's\n",
      "benzaldehyde is misspelled. Suggestions: Buxtehude, bobsledded, unsalted, insulted, insulated, consulted\n",
      "butterry is misspelled. Suggestions: buttery, battery, bitterer, but terry, but-terry, butterier, batterer, butter, buttered, Burberry, bitterly, butters, butter's, bitter, betray, Berry, Terry, berry, butte, buttery's, butty, terry, batter, better, buttress, battery's\n",
      "camomille is misspelled. Suggestions: chamomile, Camille, Camilla, commingle, compile, camisole, clammily\n",
      "camphaceous is misspelled. Suggestions: camphor's, curvaceous, campuses, emphases, canvases, coalfaces, compasses, composes, emphasis, coalface's, commerce's\n",
      "camphoraceous is misspelled. Suggestions: camphor's, amphora's, capricious, campfires, curvaceous, caprices, catchphrases, campfire's, comprises, caprice's, catchphrase's, compares, gracious, semaphores, Comoros, cameras, phrases, Cameroons, compare's, compressors, semaphore's, camera's, phrase's, compressor's, Cameroon's, Comoros's\n",
      "camphoraceousl is misspelled. Suggestions: capriciously, graciously, carnivorously, catchphrases, comprises, catchphrase's, avariciously, compresses, ferociously, compressors, compressor's\n",
      "camphorous is misspelled. Suggestions: camphor's, amphora's, Comoros, carnivorous, campfires, cumbrous, campfire's, cadaverous, Comoros's, semaphores, cameras, semaphore's, camera's\n",
      "cantelope is misspelled. Suggestions: antelope, cantaloupe, cant elope, cant-elope, cantaloupes, interlope, interloper, cantaloupe's, catalpa\n",
      "capronic is misspelled. Suggestions: cupric, capering, paranoiac, prong, Koranic, Capricorn, prang, prank\n",
      "carameellic is misspelled. Suggestions: caramel, caramelize, caramels, caramel's\n",
      "caramelic is misspelled. Suggestions: caramel, caramelize, caramels, caramel's\n",
      "cedary is misspelled. Suggestions: cedar, cedars, deary, cedar's, ceder, teary, ceders, celery, dray, dairy, diary, Sadr, dry, sectary, ceder's\n",
      "chamomille is misspelled. Suggestions: chamomile, chamomiles, chamomile's, Camille, chummily, Hamill, Camilla, chamois\n",
      "charcacter is misspelled. Suggestions: character, carjacked, charged, checkmate, chocolate, chickadee, cacti, checked, checkered, Choctaw, jacket, Chicago, chocked, chucked, shacked, sharked, chalked, changed, chuckled, hijacked, shackled, Cocteau, cockade, courgette, Chicago's, Chicagoan, chickened, chickweed, showcased, caged, caked, shellacked\n",
      "cheesey is misspelled. Suggestions: cheese, cheesy, cheesed, cheeses, cheesier, cheers, chews, Hersey, cheery, cheese's, Chase, Che's, chase, chess, chose, chesty, Cheney, cheeky, chaise, chaser, choose, choosy, Cheer's, cheer's, chew's, Chelsea, chess's\n",
      "cinnamate is misspelled. Suggestions: inmate, cinnamon, animate, cinema, cinematic, cinemas, unnamed, Minamoto, cellmate, cinema's, dynamite, reanimate, misnamed, Senate, senate, centime\n",
      "cinnamic is misspelled. Suggestions: cinnamon, cinematic, ceramic, dynamic, cinema, cynic, cinemas, gnomic, anemic, cinema's\n",
      "cinnamyl is misspelled. Suggestions: cinnamon, cinema, cinemas, minimal, namely, cinema's, enamel, animal, cynical, cynically, minimally, anomaly, zonal, zonally, singly, snarly, normal, snail, snarl\n",
      "citral is misspelled. Suggestions: mitral, Central, central, mistral, literal, astral, citric, citron, citrus, trail, trawl, trial, austral, cereal, mistrial, Tirol, littoral, citadel, Cyril, sidereal, steal, atrial, stroll, spiral, straw, stray\n",
      "citronellal is misspelled. Suggestions: citronella, citronella's\n",
      "citronellol is misspelled. Suggestions: citronella, citronella's, strongly, strangely, strangle\n",
      "citrusy is misspelled. Suggestions: citrus, citrus's, cirrus, citruses, Cyrus, ceteris, cites, cirrus's, cite's, citrons, strays, Cetus, Strauss, truss, cities, city's, estrus, Cyrus's, citron's, Cetus's, stray's\n",
      "contrbutes is misspelled. Suggestions: contributes, contributors, contributed, contribute, contributor's, contributor\n",
      "coumarin is misspelled. Suggestions: Comoran, Couperin, comparing, Qumran, Marin, cumin, Xamarin, Cameron, cowman, collaring, Carina, Corina, Corine, Marian, Marina, Marine, Marion, caring, coming, coring, curing, marina, marine, compering, cumbering, Corrine, cumming, Karin, Morin, murrain\n",
      "cuminic is misspelled. Suggestions: cumin, clinic, Dominic, cumin's, conic, cumming, comic, manic, Cummings, Dominica, Dominick, Gemini, coming, Germanic\n",
      "decalactone is misspelled. Suggestions: defalcating, deflecting, deselecting, dejecting, recollecting, delegating, neglecting\n",
      "descript is misspelled. Suggestions: descriptor, script, decrepit, descried, descriptors, disrupt, descriptive, described, typescript, discreet\n",
      "dodecalactone is misspelled. Suggestions: deescalating, defalcating, deselecting, dedicating, deflecting, detracting, deducting, deregulating, deadlocking, dejecting, detecting, delegating, collecting\n",
      "eartthy is misspelled. Suggestions: earthy, earth, earthly, earths, Arthur, arty, atty, earth's, earthed, earthen, earthier, oath, dearth, hearth\n",
      "estery is misspelled. Suggestions: eatery, Ester, ester, esters, estuary, easterly, Easter, astray, Eastern, Easters, Ester's, eastern, ester's, vestry, aster, story, Estes, artery, entry, mastery, mystery, austere, Easter's\n",
      "etheral is misspelled. Suggestions: ethereal, ethereally, Ethel, enthrall, ethical, thrall, ethyl, urethral\n",
      "fenugreek is misspelled. Suggestions: Greek, fingered, dungaree, filigree, dungarees, filigreed, filigrees, denigrate, enquirer, figure, gangrene, funereal, hungered, engorge, menagerie, engrave, Creek, creek, Negress, Negroes, enquirers, figured, figures, angered, foundered, Congreve, engraver, menageries, mongrel, dungaree's, Henrik, encore, engage, fanged, figure's, filigree's, hungrier, Ingres, engram, mongered, finagler, angrier, fanfare, finagle, encored, encores, engross, foundries, ingress, encore's, menagerie's, Ingres's\n",
      "fermenty is misspelled. Suggestions: ferment, ferments, ferment's, fermented, foment, fervent, Fremont, fitment, Fermat, fermenting, cement, deferment, figment, foments\n",
      "ferrmented is misspelled. Suggestions: fermented, fragmented, fomented, frequented, fronted, friended\n",
      "floralodor is misspelled. Suggestions: floral odor, floral-odor, floral, florid, Florida, floridly, fluoride, floodlit, florist, flailed, floored, floret, floured, freeloader, floodlight, freeload, plurality, frailty, frilled, flurried\n",
      "foenugreek is misspelled. Suggestions: corncrake, encourage, panegyric, phonograph, firecracker\n",
      "freen is misspelled. Suggestions: Green, green, free, Freon, freeing, freer, freed, frees, preen, Frauen, frown, Fran, firemen, foremen, freemen, Fern, Rene, fern, foreign, Freeman, fen, freeman, freshen, Irene, Fresno, Friend, friend, frozen, Efren, Frey, Wren, rein, wren, Freon's\n",
      "frui is misspelled. Suggestions: fruit, Fri, Frau, Fry, fro, fry, Fr, fr, fury, furor, Frey, fray, free, faro, fire, fur, fMRI, friar, frig, Fuji, RI, Ru, fare, fora, fore, fruity, Yuri, furl, furn, furry, furs, Freud, forum, frail, fraud, rue, Fr's, Frau's, Fri's, fur's\n",
      "fruiity is misspelled. Suggestions: fruity, fruit, fruitier, Frito, fruits, Friday, frailty, fortuity, fright, purity, fruit's, fruited, fruiting, Fritz, fritz, rutty\n",
      "fruiti is misspelled. Suggestions: fruit, fruity, Frito, fruits, fruitier, fruiting, fruit's, fruited, frat, fret, Fritz, fritz\n",
      "fruitti is misspelled. Suggestions: fruity, fruit, fruit ti, fruit-ti, Frito, fruitier, fritter, fruits, Pruitt, fruit's, fruited, fright, fruiting, frat, fret, Fritz, fritz, rutty\n",
      "fruiy is misspelled. Suggestions: fruit, fruity, fury, Fri, Fry, fry, furry, Frey, fray, Frau, fro, furor, Fr, fr, fairy, ferry, foray, friar, frig, Fri's\n",
      "grapefuit is misspelled. Suggestions: grapefruit, grapeshot, graffiti, graffito, graphite, grapevine, graphed, graved, gravity, griped, groped\n",
      "hawthorne is misspelled. Suggestions: Hawthorne, hawthorn, hawthorns, Hawthorne's, hawthorn's\n",
      "haylike is misspelled. Suggestions: hairlike, hay like, hay-like, alike, Hallie, haulage, halite, haulier, warlike, hulk, Hayek, Hale, hake, hale, hike, like, hankie, talkie, hayrick, Hoyle, haiku, halve, homelike, hornlike, Hollie, Hallie's\n",
      "heliotropin is misspelled. Suggestions: heliotrope, heliotropes, heliotrope's\n",
      "heliotropine is misspelled. Suggestions: heliotrope, heliotropes, heliotrope's\n",
      "herby is misspelled. Suggestions: hereby, herb, herbs, Derby, derby, Heb, Hebe, hobby, hubby, her by, her-by, Hebei, her, hey, herb's, herbal, HBO, hob, hub\n",
      "honeylike is misspelled. Suggestions: hornlike, honey like, honey-like, homelike, Honecker, unlike, hairlike, ringlike, winglike, Menelik, manlike, unalike, Henrik, honky\n",
      "hydrocabon is misspelled. Suggestions: hydrocarbon, hydrocarbons, hydrocarbon's, hydrogen, hydration, hydrophone, radiocarbon\n",
      "hydrocarbone is misspelled. Suggestions: hydrocarbon, hydrocarbons, hydrocarbon's\n",
      "iactonic is misspelled. Suggestions: acetonic, iconic, tectonic, Acton, Acton's\n",
      "isobutyrate is misspelled. Suggestions: obdurate, arbitrate, obtrude, Estrada, astride\n",
      "jasmone is misspelled. Suggestions: Jasmine, jasmine, jasmines, gasmen, Jason, gasman, Jasmine's, Jayson, Simone, jasmine's\n",
      "lactone is misspelled. Suggestions: lactose, lac tone, lac-tone, Acton, locating, Latiner, lactate, Latin, lectern, Laocoon, Latina, Latino, ketone, lacunae, acting, lactic, Clayton, lacking, lasting, Litton, carton, lacuna, lagoon, Landon, Linton, Lipton, Liston, lepton, octane, lacteal, lecture\n",
      "maltol is misspelled. Suggestions: malt, Maillol, maltose, Malta, malty, malts, Martel, Mattel, Melton, Milton, malt's, malted, mantel, maltier, multi, Mantle, mantle, melt, mildly, milt, molt, maillot, Malta's, metal, motel\n",
      "menthone is misspelled. Suggestions: methane, menthol, mention, Menuhin, monotone, Anthony\n",
      "mildl is misspelled. Suggestions: mildly, mild, Mill, mill, mold, middle, milady, milder, mildew, moldy, mild's, molds, wildly, meld, milt, melds, milts, mold's, meld's, milt's\n",
      "milkfat is misspelled. Suggestions: milk fat, milk-fat, milkiest, milkmaid, milked, Milford, mulct, milkweed\n",
      "mushoom is misspelled. Suggestions: mushroom, mush, mushy, mushier, mushing, musher, museum, mush's, mushed, mushes, shim, mushers, MASH, mash, mesh, mosh, much, sham, macho\n",
      "naphthalenic is misspelled. Suggestions: naphthalene, naphthalene's\n",
      "orangeblossom is misspelled. Suggestions: orange blossom, orange-blossom, evangelism, anabolism, Orientalism\n",
      "ozonic is misspelled. Suggestions: Ionic, ionic, iconic, ironic, oceanic, ozone, sonic, oozing, zinc, ozone's, cynic\n",
      "passionfruit is misspelled. Suggestions: passion fruit, passion-fruit, passionate, passionflower, infuriate, pianoforte, Pavarotti, personified, pioneered, Manfred, Winfred, pinafore, portioned, proofread, partnered, dishonored\n",
      "petitgrain is misspelled. Suggestions: pedicuring, multigrain, peregrine, pentagram, Pitcairn, pedestrian, pediatric, pattering, pottering, puttering, Praetorian, pestering, positron, praetorian, pediatrician, pettifogging, pedigree, picturing, tittering, tutoring, pituitaries, patron, pituitary, teetering, pediatrics, pedigreed, pedigrees, pedigree's\n",
      "phenolc is misspelled. Suggestions: phenol, phenol's, phonic\n",
      "phenylacetaldehyde is misspelled, but no suggestions available.\n",
      "pienapple is misspelled. Suggestions: pineapple, pineapples, pineapple's, Snapple, pinnacle, panoply, people, beanpole, pimple, pinhole, nipple, penile\n",
      "reminscent is misspelled. Suggestions: reminiscent, reminiscently, renascent, reminiscence, reminisced, reminiscing, luminescent, remnant, omniscient\n",
      "roasty is misspelled. Suggestions: roast, toasty, roasts, roaster, Rasta, Rusty, roost, roust, rusty, rosary, pasty, resat, tasty, toast, raster, roster, rosy, roast's, roasted, frosty, rest, rust, ratty, roister, rooster, boast, coast, hasty, nasty, raspy, roosts, rousts, roost's\n",
      "rosemay is misspelled. Suggestions: Rosemary, rosemary, rose may, rose-may, Roseau, roseate, seamy, Roseann, Rosetta, rosary, roses, Rosa, Rose, rose, rosy, Rose's, Rosella, dismay, rose's, Rosemarie, roomy, Zosma, Ramsay, bosomy, reseal, rosily, Rome, ream, resume, roes, seam, raceme, Rosie, rouse, Roseau's, raceway, reams, rosette, REM, ROM, Rom, rem, smear, Ramsey, roe's, Rosa's, romeos, ream's, Romeo's, romeo's\n",
      "rosey is misspelled. Suggestions: Rose, rose, rosy, Roseau, roes, Rosie, rise, rouse, roses, mosey, riser, rosier, Rosa, Ross, ruse, roues, rows, resew, res, Rios, rhos, rs, rues, Dorsey, horsey, Roy, roe, roe's, Rose's, rose's, R's, arose, prose, prosy, Ross's, roue's, Rio's, rho's, Roy's, row's, Rae's, Re's, re's, rue's\n",
      "salicylate is misspelled. Suggestions: saleslady, Sallust, solicited, solicit, sliced, sliest, solicitor, Szilard, salesladies, solicitude, cellulite, saleslady's, solubility\n",
      "savoury is misspelled. Suggestions: savory, Savoy, savor, savoy, savorier, savors, Savior, savior, savoys, savory's, savored, slavery, savor's, saver, sorry, Savoy's, savoy's\n",
      "solventy is misspelled. Suggestions: solvent, solvents, solvency, slovenly, solvent's, seventy\n",
      "somehat is misspelled. Suggestions: somewhat, some hat, some-hat, somewhats, someday, somehow, somerset, sunhat, smelt, sorehead, summat\n",
      "somehate is misspelled. Suggestions: somewhat, some hate, some-hate, somewhats, somerset, somewhere, someday, somehow, simulate, smeared, Semite, semester, smelter, sunhat, smite, smelt, sorehead, summat\n",
      "strawberr is misspelled. Suggestions: strawberry, strawberries, strawberry's, Strabo, strobe\n",
      "sulfuraceos is misspelled. Suggestions: sulfurous, pulverizes, sassafrases, slavery's, Alvarez's, solvency's\n",
      "sulfuraceous is misspelled. Suggestions: sulfurous, siliceous, sulfured, sulfurs, sulfur's, surfaces, sulfates, sulfuric, salacious, surface's, sulfate's, sufferance's, suffrage's, suffices, suffuses, sulfides, sluices, solaces, lubricious, salubrious, sufferance, sulfide's, sluice's, solace's, slavery's, ferocious, sufferers, sufferer's\n",
      "sulfureous is misspelled. Suggestions: sulfurous, sulfured, sulfurs, sulfur's, sulfates, sulfides, sulfuric, salubrious, sulfate's, sulfide's, sufferers, sufferer's\n",
      "sulfury is misspelled. Suggestions: sulfur, sulfurs, sulfured, sulfuric, sulfur's, silvery, sultry, sulfurous, slurry, soulfully, soulful, slavery, sulfa, sulfuring, salary\n",
      "tcocoa is misspelled. Suggestions: cocoa, coca, coco, Tosca, toccata, cacao, Cook, Togo, Tojo, cock, cook, taco, toga, took\n",
      "terpene is misspelled. Suggestions: deepen, tepee, Turpin, tarpon, terrine, tepees, deepened, steepen, teen, tern, tiepin, repine, tureen, Tyrone, reopen, teeny, topee, deepens, tuppence, depend, tenpin, toupee, tuppenny, Terran, Tirane, taping, typing, tarpons, temping, tepee's, Turpin's, tarpon's\n",
      "terpenes is misspelled. Suggestions: deepens, tepees, terseness, Serpens, tarpons, terrines, deepness, Turpin's, tarpon's, tepee's, Terence, steepens, teens, terns, tiepins, repines, tureens, Terrence, reopens, teen's, tern's, topees, tuppence, Serpens's, torpedoes, depends, tenpins, toupees, Torrens, tenpin's, tureen's, Terran's, Tyrone's, toupee's, terseness's, typing's, tenpins's\n",
      "terpentine is misspelled. Suggestions: turpentine, serpentine, repenting, turpentine's\n",
      "terpeny is misspelled. Suggestions: deepen, teeny, tuppenny, Turpin, tarpon, twopenny, steepen, teen, tern, tiepin, reopen, tepee, deepens, depend, tenpin, tureen, Terran, tepees, taping, typing, tarpons, temping, tepee's, Turpin's, tarpon's\n",
      "terpineol is misspelled. Suggestions: terminal, Turpin, terminally, tiepin, Turpin's, tarpon, tepidly, tiepins, panel, Tylenol, supinely, tarpons, torpidly, taping, tunnel, typing, spinal, tarpon's, teasingly, tellingly, torsional, Darnell, Parnell, Taiping, tapping, tipping, topping, topical, typical, typing's\n",
      "terpy is misspelled. Suggestions: Terry, terry, TARP, tarp, Twp, tepee, twp, Trey, trey, twerp, Troy, teary, troy, deep, try, tap, teeny, tip, top, weepy, temp, torpor, Teri, Terr, Tory, term, tern, terr, tray, Depp, Tupi, tape, topi, type, typo, Tempe, repay, tarps, tempo, turps, Derby, Harpy, Teddy, Terra, Terri, derby, harpy, peppy, tardy, tarry, tarty, teddy, telly, terse, turfy, Rep, dippy, rep, taupe, tear, tee, tier, toy, trap, trip, Earp, ropy, taper, Te, Ty, steep, RP, pert, tr, tree, Topsy, tepid, tipsy, tripe, trope, step, EDP, per, tar, tea, tor, tsp, tarp's, Euterpe\n",
      "tetrahydrolinalool is misspelled, but no suggestions available.\n",
      "thujonic is misspelled. Suggestions: iconic, conic, thong, thunk, thermionic, throng, hygienic, eugenic, laconic, picnic, Sejong, pathogenic\n",
      "topnote is misspelled. Suggestions: top note, top-note, topknot, topnotch, tinpot, tiptoe, Tonto, DuPont, topcoat, Toronto, topside, denote, donate, toned, tenet\n",
      "topnotes is misspelled. Suggestions: top notes, top-notes, topknots, topknot's, tiptoes, Tonto's, topcoats, tippets, tiptoe's, DuPont's, topcoat's, topsides, denotes, donates, Toronto's, tappets, tints, torpedoes, tenets, teapots, tint's, tornadoes, TNT's, tents, topside's, teapot's, tippet's, tent's, tenet's, tappet's\n",
      "trigeminal is misspelled. Suggestions: triennial, criminal, original, terminal, germinal, tribunal, tragedian, triggering, trimming\n",
      "tropicall is misspelled. Suggestions: tropically, tropical, tropic all, tropic-all, topically, topical, Tropicana, tropic, dropsical, typically, typical\n",
      "undernote is misspelled. Suggestions: under note, under-note, underwrote, underdone, undertone, undertow, underrate, Internet, indent, internet, underwrite, undercoat, underfoot, undershot, andante, entente, underneath, underwater, undercut\n",
      "undernotes is misspelled. Suggestions: under notes, under-notes, undertones, undertows, underrates, Internets, indents, indent's, underwrites, undercoats, undertone's, undertow's, andantes, ententes, undercoat's, underneaths, undersides, Internet's, undercuts, underpants, undercut's, andante's, entente's, underneath's, underside's, Underwood's, underpants's\n",
      "vanillic is misspelled. Suggestions: vanilla, vanillas, vanilla's\n",
      "vanillin is misspelled. Suggestions: vanilla, vanillas, Villon, villain, villein, aniline, lanolin, violin, vanilla's, vanishing, nailing, villainy, Ventolin, mainline, snailing, vainly, valuing, vanning, veiling, wailing, walling, willing\n",
      "vegetal is misspelled. Suggestions: vegetable, vegetate, vestal, varietal, victual, vital, wagtail, rectal\n",
      "vetivert is misspelled. Suggestions: divert, votive, velvet, Velveeta, velvety, vetoed, vetted, advert, vivider\n",
      "vetiveryl is misspelled. Suggestions: vertebral, votive, several, veteran, vitriol, severely, vertical, vertically, visceral, drivel, furtively, petrel, retrieval, ventral, antiviral, devilry, vividly, Tirol, viral, literal, venereal, deferral, votary, watery, medieval, petrol, Federal, federal, defiler, vivaria, Demerol, lateral, votary's\n"
     ]
    }
   ],
   "source": [
    "list_of_words = list(words_set)\n",
    "list_of_words.sort()\n",
    "for word in list_of_words:\n",
    "    spellcheck(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# manual spellcheck and filter\n",
    "spellcorrect = {}\n",
    "spellcorrect['acetaldehyde'] = ['acetataldehyde']\n",
    "spellcorrect['alliaceous'] = ['alliaceious', 'alliacious', 'alliacous']\n",
    "spellcorrect['amber'] = ['ambery']\n",
    "spellcorrect['animal'] = ['animalic', 'animalistic', 'animallic']\n",
    "spellcorrect['apple'] = ['applle']\n",
    "spellcorrect['balsamic'] = ['balsum']\n",
    "spellcorrect['black currant'] = ['blackurrant']\n",
    "spellcorrect['buttery'] = ['butterry']\n",
    "spellcorrect['camphoreous'] = ['camhorous', 'camphaceous', 'camphoraceous', 'camphoraceousl', 'camphorous']\n",
    "spellcorrect['caramellic'] = ['carameellic', 'caramelic', 'caramellized']\n",
    "spellcorrect['cedar'] = ['cedary']\n",
    "spellcorrect['chamomile'] = ['camomille', 'chamomille']\n",
    "spellcorrect['cheesy'] = ['cheesey']\n",
    "spellcorrect['cocoa'] = ['tcocoa']\n",
    "spellcorrect['cinnamon'] = ['cinnamate', 'cinnamic', 'cinnamyl']\n",
    "spellcorrect['citrus'] = ['cistus', 'citral', 'citronellal', 'citronellol', 'citrusy']\n",
    "spellcorrect['coumarinic'] = ['coumarin']\n",
    "spellcorrect['creamy'] = ['ceamy']\n",
    "spellcorrect['earthy'] = ['eartthy', 'eathy']\n",
    "spellcorrect['ethereal'] = ['etheral', 'etthereal']\n",
    "spellcorrect['fermented'] = ['fermenty', 'ferrmented']\n",
    "spellcorrect['fruity'] = ['frui', 'fruiity', 'fruiti', 'fruitti', 'fruiy', 'frutti', 'fruty', 'fuity', 'ruity']\n",
    "spellcorrect['floral'] = ['floralodor', 'flouve', 'foral']\n",
    "spellcorrect['green'] = ['freen', 'grreen', 'geen']\n",
    "spellcorrect['grapefruit'] = ['grapefuit']\n",
    "spellcorrect['hawthorn'] = ['hawthorne']\n",
    "spellcorrect['hay'] = ['haylike']\n",
    "spellcorrect['herbal'] = ['hebal', 'herby', 'hebaceous']\n",
    "spellcorrect['honey'] = ['honeylike']\n",
    "spellcorrect['jasmin'] = ['jasmone']\n",
    "spellcorrect['lemon'] = ['iime', 'limonene']\n",
    "spellcorrect['lactonic'] = ['decalactone', 'dodecalactone', 'iactonic', 'lactone', 'octalactone']\n",
    "spellcorrect['malty'] = ['maltol']\n",
    "spellcorrect['milky'] = ['mildl', 'milkfat']\n",
    "spellcorrect['mushroom'] = ['mushoom']\n",
    "spellcorrect['musk'] = ['ambrette']\n",
    "spellcorrect['onion'] = ['oinion']\n",
    "spellcorrect['odorless'] = ['odourless']\n",
    "spellcorrect['orange blossom'] = ['orangeblossom']\n",
    "spellcorrect['orange'] = ['ornge'] \n",
    "spellcorrect['ozone'] = ['ozonic']\n",
    "spellcorrect['passion fruit'] = ['passionfruit']\n",
    "spellcorrect['pineapple'] = ['pienapple', 'pieneapple']\n",
    "spellcorrect['popcorn'] = ['poocorn']\n",
    "spellcorrect['pungent'] = ['pumgent']\n",
    "spellcorrect['pine'] = ['pune', 'pne']\n",
    "spellcorrect['roasted'] = ['roasty']\n",
    "spellcorrect['rose'] = ['rosemay', 'rosey']\n",
    "spellcorrect['savory'] = ['savoury']\n",
    "spellcorrect['solvent'] = ['solventy']\n",
    "spellcorrect['strawberry'] = ['strawberr']\n",
    "spellcorrect['sulfurous'] = ['sufurous', 'sulfuraceos', 'sulfuraceous', 'sulfureous', 'sulfury', 'sulphur', 'sulphurol', 'sulphurous', 'sulphury']\n",
    "spellcorrect['spicy'] = ['sicy']\n",
    "spellcorrect['tobacco'] = ['tabac']\n",
    "spellcorrect['terpenic'] = ['terpene', 'terpenes', 'terpentine', 'terpeny', 'terpineol', 'terpy']\n",
    "spellcorrect['tropical'] = ['ltropical', 'tropicall']\n",
    "spellcorrect['vanilla'] = ['vanillic', 'vanillin', 'vanlla']\n",
    "spellcorrect['vegetable'] = ['vegetal']\n",
    "spellcorrect['vetiver'] = ['vertivert', 'vetivert', 'vetiveryl']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'acetaldehyde': ['acetataldehyde'],\n",
       " 'alliaceous': ['alliaceious', 'alliacious', 'alliacous'],\n",
       " 'amber': ['ambery'],\n",
       " 'animal': ['animalic', 'animalistic', 'animallic'],\n",
       " 'apple': ['applle'],\n",
       " 'balsamic': ['balsum'],\n",
       " 'black currant': ['blackurrant'],\n",
       " 'buttery': ['butterry'],\n",
       " 'camphoreous': ['camhorous',\n",
       "  'camphaceous',\n",
       "  'camphoraceous',\n",
       "  'camphoraceousl',\n",
       "  'camphorous'],\n",
       " 'caramellic': ['carameellic', 'caramelic', 'caramellized'],\n",
       " 'cedar': ['cedary'],\n",
       " 'chamomile': ['camomille', 'chamomille'],\n",
       " 'cheesy': ['cheesey'],\n",
       " 'cocoa': ['tcocoa'],\n",
       " 'cinnamon': ['cinnamate', 'cinnamic', 'cinnamyl'],\n",
       " 'citrus': ['cistus', 'citral', 'citronellal', 'citronellol', 'citrusy'],\n",
       " 'coumarinic': ['coumarin'],\n",
       " 'creamy': ['ceamy'],\n",
       " 'earthy': ['eartthy', 'eathy'],\n",
       " 'ethereal': ['etheral', 'etthereal'],\n",
       " 'fermented': ['fermenty', 'ferrmented'],\n",
       " 'fruity': ['frui',\n",
       "  'fruiity',\n",
       "  'fruiti',\n",
       "  'fruitti',\n",
       "  'fruiy',\n",
       "  'frutti',\n",
       "  'fruty',\n",
       "  'fuity',\n",
       "  'ruity'],\n",
       " 'floral': ['floralodor', 'flouve', 'foral'],\n",
       " 'green': ['freen', 'grreen', 'geen'],\n",
       " 'grapefruit': ['grapefuit'],\n",
       " 'hawthorn': ['hawthorne'],\n",
       " 'hay': ['haylike'],\n",
       " 'herbal': ['hebal', 'herby', 'hebaceous'],\n",
       " 'honey': ['honeylike'],\n",
       " 'jasmin': ['jasmone'],\n",
       " 'lemon': ['iime', 'limonene'],\n",
       " 'lactonic': ['decalactone',\n",
       "  'dodecalactone',\n",
       "  'iactonic',\n",
       "  'lactone',\n",
       "  'octalactone'],\n",
       " 'malty': ['maltol'],\n",
       " 'milky': ['mildl', 'milkfat'],\n",
       " 'mushroom': ['mushoom'],\n",
       " 'musk': ['ambrette'],\n",
       " 'onion': ['oinion'],\n",
       " 'odorless': ['odourless'],\n",
       " 'orange blossom': ['orangeblossom'],\n",
       " 'orange': ['ornge'],\n",
       " 'ozone': ['ozonic'],\n",
       " 'passion fruit': ['passionfruit'],\n",
       " 'pineapple': ['pienapple', 'pieneapple'],\n",
       " 'popcorn': ['poocorn'],\n",
       " 'pungent': ['pumgent'],\n",
       " 'pine': ['pune', 'pne'],\n",
       " 'roasted': ['roasty'],\n",
       " 'rose': ['rosemay', 'rosey'],\n",
       " 'savory': ['savoury'],\n",
       " 'solvent': ['solventy'],\n",
       " 'strawberry': ['strawberr'],\n",
       " 'sulfurous': ['sufurous',\n",
       "  'sulfuraceos',\n",
       "  'sulfuraceous',\n",
       "  'sulfureous',\n",
       "  'sulfury',\n",
       "  'sulphur',\n",
       "  'sulphurol',\n",
       "  'sulphurous',\n",
       "  'sulphury'],\n",
       " 'spicy': ['sicy'],\n",
       " 'tobacco': ['tabac'],\n",
       " 'terpenic': ['terpene',\n",
       "  'terpenes',\n",
       "  'terpentine',\n",
       "  'terpeny',\n",
       "  'terpineol',\n",
       "  'terpy'],\n",
       " 'tropical': ['ltropical', 'tropicall'],\n",
       " 'vanilla': ['vanillic', 'vanillin', 'vanlla'],\n",
       " 'vegetable': ['vegetal'],\n",
       " 'vetiver': ['vertivert', 'vetivert', 'vetiveryl']}"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spellcorrect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# correct spell errors trial 1\n",
    "from collections import defaultdict\n",
    "reverse_spellcorrect = defaultdict(list)\n",
    "for key, values in spellcorrect.items():\n",
    "    for value in values:\n",
    "        reverse_spellcorrect[value].append(key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'ornge' in reverse_spellcorrect.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['sulfurous']\n"
     ]
    }
   ],
   "source": [
    "if 'sulfuraceous' in reverse_spellcorrect.keys():\n",
    "    print(reverse_spellcorrect['sulfuraceous'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "def correct_spell_errors_v1(string):\n",
    "    string = string.lower()\n",
    "    string = string.split(';')[0] #choose odor and not flavor\n",
    "    list_desc=re.sub(r'[!\"#$%&\\'()*\\-+,./:;<=>?@\\[\\]^_`{|}~]',\" \",string).split(\" \")\n",
    "\n",
    "    # handle black currant\n",
    "    if \"black\" in list_desc:\n",
    "        idx = list_desc.index('black')\n",
    "        if idx<len(list_desc)-1:\n",
    "            if list_desc[idx+1] == 'currant':\n",
    "                list_desc.pop(idx) #removes black\n",
    "                list_desc.pop(idx) # removes currant\n",
    "                list_desc.append(\"black currant\")\n",
    "    \n",
    "    # handle orange flower\n",
    "    if \"orange\" in list_desc:\n",
    "        idx = list_desc.index('orange')\n",
    "        if idx<len(list_desc)-1:\n",
    "            if list_desc[idx+1] == 'flower':\n",
    "                list_desc.pop(idx) #removes orange\n",
    "                list_desc.pop(idx) # removes flower\n",
    "                list_desc.append(\"orangeflower\")\n",
    "    \n",
    "    # handle fruit skin\n",
    "    for idx in range(len(list_desc)):\n",
    "        if list_desc[idx] == 'skin' or list_desc[idx] == 'peel':\n",
    "            if idx != 0:\n",
    "                if list_desc[idx-1] in ['citrus', 'orange', 'apple', 'banana', 'lemon', 'pear', 'fruit', 'peach']:\n",
    "                    list_desc.append(\"fruit skin\")\n",
    "\n",
    "\n",
    "    words_set = set()\n",
    "    for word in list_desc:\n",
    "        flag = 0\n",
    "        if len(word) == 0:\n",
    "            continue\n",
    "        elif word == ' ':\n",
    "            continue\n",
    "        else:\n",
    "            for mispelled, corrected in reverse_spellcorrect.items():\n",
    "                if word == mispelled:\n",
    "                    words_set.update(corrected)\n",
    "                    flag = 1\n",
    "                    break\n",
    "            if flag == 0:\n",
    "                words_set.update([word])\n",
    "    new_string = \" \".join(words_set)\n",
    "    return new_string\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cream tropical fruity ripe sulfurous notes strawberry over cheese'"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "correct_spell_errors_v1('sulfuraceous, ltropical-over-ripe fruity, strawberry, cream & cheese notes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'roasted savory alliaceous sulfurous truffle pine'"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "correct_spell_errors_v1('sulfury, alliaceous, roasted, savoury, truffle, pne')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "behavior_sparse['cleaned raw labels'] = behavior_sparse['Raw Labels'].apply(lambda x: correct_spell_errors_v1(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Raw Labels</th>\n",
       "      <th>Labels</th>\n",
       "      <th>descriptors</th>\n",
       "      <th>cleaned raw labels</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Stimulus</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-955348933095</th>\n",
       "      <td>Herbal-green, waxy, oily, fruity</td>\n",
       "      <td>['green', 'oily', 'fruity', 'waxy', 'herbal']</td>\n",
       "      <td>green;oily;fruity;waxy;herbal</td>\n",
       "      <td>oily fruity waxy herbal green</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-923209957509</th>\n",
       "      <td>Herbaceous, woody, slight spicy fruity odor</td>\n",
       "      <td>['woody', 'spicy', 'fruity', 'herbal']</td>\n",
       "      <td>woody;spicy;fruity;herbal</td>\n",
       "      <td>fruity woody spicy slight odor herbaceous</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-874408321546</th>\n",
       "      <td>delta-decalactone precursor; peach, apricot, b...</td>\n",
       "      <td>['peach', 'apricot', 'buttery']</td>\n",
       "      <td>peach;apricot;buttery</td>\n",
       "      <td>precursor lactonic delta</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-873963935677</th>\n",
       "      <td>Green, fruity, apple-like</td>\n",
       "      <td>['green', 'fruity', 'apple', 'tropical']</td>\n",
       "      <td>green;fruity;apple;tropical</td>\n",
       "      <td>like green fruity apple</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-862841422647</th>\n",
       "      <td>Catty urine; cassis in dilution</td>\n",
       "      <td>['catty']</td>\n",
       "      <td>catty</td>\n",
       "      <td>urine catty</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      Raw Labels   \n",
       "Stimulus                                                           \n",
       "-955348933095                   Herbal-green, waxy, oily, fruity  \\\n",
       "-923209957509        Herbaceous, woody, slight spicy fruity odor   \n",
       "-874408321546  delta-decalactone precursor; peach, apricot, b...   \n",
       "-873963935677                          Green, fruity, apple-like   \n",
       "-862841422647                    Catty urine; cassis in dilution   \n",
       "\n",
       "                                                      Labels   \n",
       "Stimulus                                                       \n",
       "-955348933095  ['green', 'oily', 'fruity', 'waxy', 'herbal']  \\\n",
       "-923209957509         ['woody', 'spicy', 'fruity', 'herbal']   \n",
       "-874408321546                ['peach', 'apricot', 'buttery']   \n",
       "-873963935677       ['green', 'fruity', 'apple', 'tropical']   \n",
       "-862841422647                                      ['catty']   \n",
       "\n",
       "                                 descriptors   \n",
       "Stimulus                                       \n",
       "-955348933095  green;oily;fruity;waxy;herbal  \\\n",
       "-923209957509      woody;spicy;fruity;herbal   \n",
       "-874408321546          peach;apricot;buttery   \n",
       "-873963935677    green;fruity;apple;tropical   \n",
       "-862841422647                          catty   \n",
       "\n",
       "                                      cleaned raw labels  \n",
       "Stimulus                                                  \n",
       "-955348933095              oily fruity waxy herbal green  \n",
       "-923209957509  fruity woody spicy slight odor herbaceous  \n",
       "-874408321546                   precursor lactonic delta  \n",
       "-873963935677                    like green fruity apple  \n",
       "-862841422647                                urine catty  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "behavior_sparse.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "spell correction round 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "suff_root_req_desc = required_desc.copy()\n",
    "for i in range(len(suff_root_req_desc)):\n",
    "    suff_root_req_desc[i] = suff_root_req_desc[i][(int(0.4*len(suff_root_req_desc[i]))):]\n",
    "\n",
    "# tried with different values like: 0.4, 0.6, 0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['oholic',\n",
       " 'ehydic',\n",
       " 'aceous',\n",
       " 'mond',\n",
       " 'ber',\n",
       " 'imal',\n",
       " 'isic',\n",
       " 'ple',\n",
       " 'ricot',\n",
       " 'matic',\n",
       " 'samic',\n",
       " 'nana',\n",
       " 'efy',\n",
       " 'gamot',\n",
       " 'rry',\n",
       " 'tter',\n",
       " ' currant',\n",
       " 'andy',\n",
       " 'rnt',\n",
       " 'ttery',\n",
       " 'bbage',\n",
       " 'horeous',\n",
       " 'mellic',\n",
       " 'dar',\n",
       " 'lery',\n",
       " 'momile',\n",
       " 'eesy',\n",
       " 'erry',\n",
       " 'colate',\n",
       " 'namon',\n",
       " 'trus',\n",
       " 'ean',\n",
       " 'ove',\n",
       " 'coa',\n",
       " 'conut',\n",
       " 'ffee',\n",
       " 'gnac',\n",
       " 'oked',\n",
       " 'oling',\n",
       " 'rtex',\n",
       " 'arinic',\n",
       " 'eamy',\n",
       " 'umber',\n",
       " 'iry',\n",
       " 'ry',\n",
       " 'rthy',\n",
       " 'ereal',\n",
       " 'tty',\n",
       " 'mented',\n",
       " 'shy',\n",
       " 'oral',\n",
       " 'esh',\n",
       " 't skin',\n",
       " 'uity',\n",
       " 'rlic',\n",
       " 'ssy',\n",
       " 'anium',\n",
       " 'ape',\n",
       " 'efruit',\n",
       " 'assy',\n",
       " 'een',\n",
       " 'thorn',\n",
       " 'ay',\n",
       " 'elnut',\n",
       " 'rbal',\n",
       " 'ney',\n",
       " 'cinth',\n",
       " 'smin',\n",
       " 'icy',\n",
       " 'tonic',\n",
       " 'tonic',\n",
       " 'ender',\n",
       " 'afy',\n",
       " 'thery',\n",
       " 'mon',\n",
       " 'ily',\n",
       " 'lty',\n",
       " 'aty',\n",
       " 'icinal',\n",
       " 'lon',\n",
       " 'allic',\n",
       " 'lky',\n",
       " 'int',\n",
       " 'guet',\n",
       " 'hroom',\n",
       " 'usk',\n",
       " 'sty',\n",
       " 'tural',\n",
       " 'tty',\n",
       " 'rless',\n",
       " 'ily',\n",
       " 'ion',\n",
       " 'ange',\n",
       " 'geflower',\n",
       " 'ris',\n",
       " 'one',\n",
       " 'ach',\n",
       " 'ear',\n",
       " 'nolic',\n",
       " 'ine',\n",
       " 'eapple',\n",
       " 'lum',\n",
       " 'pcorn',\n",
       " 'tato',\n",
       " 'wdery',\n",
       " 'ngent',\n",
       " 'dish',\n",
       " 'pberry',\n",
       " 'ipe',\n",
       " 'asted',\n",
       " 'ose',\n",
       " 'mmy',\n",
       " 'alwood',\n",
       " 'vory',\n",
       " 'arp',\n",
       " 'oky',\n",
       " 'apy',\n",
       " 'lvent',\n",
       " 'our',\n",
       " 'icy',\n",
       " 'wberry',\n",
       " 'furous',\n",
       " 'eaty',\n",
       " 'eet',\n",
       " 'ea',\n",
       " 'penic',\n",
       " 'bacco',\n",
       " 'mato',\n",
       " 'pical',\n",
       " 'nilla',\n",
       " 'etable',\n",
       " 'tiver',\n",
       " 'olet',\n",
       " 'arm',\n",
       " 'axy',\n",
       " 'edy',\n",
       " 'ney',\n",
       " 'ody']"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "suff_root_req_desc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def spellcheck(word):\n",
    "    if dictionary.check(word):\n",
    "        # print(f\"{word} is spelled correctly.\")\n",
    "        pass\n",
    "    else:\n",
    "        if word in required_desc: # ignore spell error if word in required desc\n",
    "            return\n",
    "        for root in suff_root_req_desc:\n",
    "            if root in word:    # only find spell error for words that have the certain root words\n",
    "                suggestions = dictionary.suggest(word)\n",
    "                if suggestions:\n",
    "                    print(f\"{word} is misspelled. Suggestions: {', '.join(suggestions)}\")\n",
    "                else:\n",
    "                    print(f\"{word} is misspelled, but no suggestions available.\")\n",
    "                break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "words_set_v2 = set()\n",
    "def get_spell_errors_v2(string):\n",
    "    string = string.lower()\n",
    "    list_desc=string.split(\" \")\n",
    "\n",
    "    # handle black currant\n",
    "    if \"black\" in list_desc:\n",
    "        idx = list_desc.index('black')\n",
    "        if idx<len(list_desc)-1:\n",
    "            if list_desc[idx+1] == 'currant':\n",
    "                list_desc.pop(idx) #removes black\n",
    "                list_desc.pop(idx) # removes currant\n",
    "                list_desc.append(\"black currant\")\n",
    "\n",
    "    # handle orange flower\n",
    "    if \"orange\" in list_desc:\n",
    "        idx = list_desc.index('orange')\n",
    "        if idx<len(list_desc)-1:\n",
    "            if list_desc[idx+1] == 'flower':\n",
    "                list_desc.pop(idx) #removes orange\n",
    "                list_desc.pop(idx) # removes flower\n",
    "                list_desc.append(\"orangeflower\")\n",
    "    \n",
    "    # handle fruit skin\n",
    "    for idx in range(len(list_desc)):\n",
    "        if list_desc[idx] == 'skin' or list_desc[idx] == 'peel':\n",
    "            if idx != 0:\n",
    "                if list_desc[idx-1] in ['citrus', 'orange', 'apple', 'banana', 'lemon', 'pear', 'fruit', 'peach']:\n",
    "                    list_desc.append(\"fruit skin\")\n",
    "    \n",
    "    for word in list_desc:\n",
    "        if len(word) == 0:\n",
    "            continue\n",
    "        elif word == ' ':\n",
    "            continue\n",
    "        else:\n",
    "            words_set_v2.update([word])\n",
    "    # odor_set = set()\n",
    "    # for req_odor in required_desc:\n",
    "    #     if req_odor in list_desc:\n",
    "    #         odor_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Stimulus\n",
       "-955348933095    None\n",
       "-923209957509    None\n",
       "-874408321546    None\n",
       "-873963935677    None\n",
       "-862841422647    None\n",
       "                 ... \n",
       " 154241350       None\n",
       " 155323379       None\n",
       " 158699780       None\n",
       " 161504856       None\n",
       " 162353069       None\n",
       "Name: cleaned raw labels, Length: 3522, dtype: object"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "behavior_sparse['cleaned raw labels'].apply(lambda x: get_spell_errors_v2(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "acetophenone is misspelled. Suggestions: acetone, Stephenson, acetylene, antiphon, antivenin, acetone's, Stephen, antiphons, Stephanie, Stephens, adenine, atoning, Stephen's, acetaminophen, antiphon's, acetylene's\n",
      "ammoniacal is misspelled. Suggestions: ammonia cal, ammonia-cal, demoniacal, maniacal, demoniacally, maniacally, ironical, harmonically, demonically, ecumenical\n",
      "beany is misspelled. Suggestions: Bean, bean, Benny, beans, Meany, beady, meany, Ben, ban, beanie, Bern, bane, bani, been, bony, Barney, barney, barn, being, bonny, bunny, be any, be-any, brainy, brawny, Beau, Behan, Bran, bandy, beau, began, bendy, bent, bran, belay, Bethany, Ebony, bay, bey, ebony, any, Bean's, bean's, beaned, botany, bin, bun, Benz, band, bang, bank, bans, bear, bend, Ben's, ban's, Benny's\n",
      "breadcrust is misspelled. Suggestions: bread crust, bread-crust, broadcast, breadcrumb, breadfruit, breadfruits, readjust, breadfruit's\n",
      "bready is misspelled. Suggestions: Brady, bread, beady, ready, breads, Brad, brad, bred, breed, broad, broody, breadth, Bret, bared, bored, braid, brat, Bradly, Brandy, brandy, Bray, bead, bratty, bray, byroad, read, bread's, breaded, broadly, berate, brads, reedy, Brady's, Brad's, brad's\n",
      "caprylic is misspelled. Suggestions: acrylic, cupric, caprice, Capri, carbolic, garlic, caloric, calico, colic, Catholic, catholic, carpal, carpel, cryptic, Capri's, Capra, Carol, carol, relic, cleric\n",
      "carvone is misspelled. Suggestions: carving, carbine, carbon, carve, caving, crone, Carmine, carmine, coven, curving, Capone, Carson, Darvon, carton, craven, ravine, Carlene, canoe, cavern, Crane, crane, Carney, Carver, carver, craving, carved, carves, carvings, Gavin, cane, cave, cone, Avon, Carmen, careen, Calvin, Cavour, caravan, Lavonne, carrion, cartoon, coroner, crooner, carny, curve, Canon, Creon, canon, capon, clone, crony, croon, krone, forgone, carving's\n",
      "caryophyllene is misspelled. Suggestions: Caroline, Maryellen, carillon, carefuller, crystalline, Carolina, caroling, gramophone, grovelling, crestfallen\n",
      "chinese is misspelled. Suggestions: Chinese, chines, chine's, chins, Chin's, chin's, chinos, shines, chains, shiners, Ch'in's, China's, Chinese's, chain's, china's, chine, chino's, shine's, Hines, chicness, chinless, shins, cheese, chinks, Chan's, Chen's, shin's, shiner's, Chaney's, Cheney's, chink's\n",
      "clary is misspelled. Suggestions: Cary, Clay, clay, Clara, Clare, Clark, Claire, chary, glare, glory, Calgary, Calvary, claret, Cray, clearly, cray, Carey, Carly, Clair, Larry, Leary, carry, clear, clearer, Camry, clarify, clarity, cry, CARE, Cara, Carl, Carr, Cory, Gary, Lara, care, claw, cloy, Clare's, Clay's, clay's, Clara's\n",
      "collidine is misspelled. Suggestions: colliding, colluding, Claudine, collide, clouding, collating, collider, collude, collided, collides, Collin\n",
      "damascenone is misspelled. Suggestions: damson, damasking, dampening, damning, discerning, tumescence, timezone, tumescent\n",
      "estery is misspelled. Suggestions: eatery, Ester, ester, esters, estuary, easterly, Easter, astray, Eastern, Easters, Ester's, eastern, ester's, vestry, aster, story, Estes, artery, entry, mastery, mystery, austere, Easter's\n",
      "estry is misspelled. Suggestions: vestry, entry, estuary, astray, es try, es-try, eatery, estrus, Ester, ester, story, stray, ESR, EST, est, destroy, esters, pastry, extra, usury, EST's, Ester's, ester's\n",
      "flavour is misspelled. Suggestions: flavor, flavors, favor, flour, flair, flours, floury, flavor's, floor, flyover, flour's\n",
      "furfuryl is misspelled. Suggestions: fearful, fearfully, firefly, ferrule, freely, frilly, funeral, ferule, frail, frailly, frill, fifthly, Federal, federal, Farrell, Ferrell, fervidly, fibril, femoral, funereal, floral\n",
      "goaty is misspelled. Suggestions: goat, gouty, GATT, goats, got, goatee, Katy, coat, gate, ghat, goad, gout, gait, gotta, Getty, goody, gutty, gaiety, git, gloat, groat, gt, Gay, Goa, catty, gator, gaudy, gay, oat, goat's, grotty, GTE, God, cat, cot, gad, get, god, gut, jot, qty, GMAT, govt\n",
      "heliotropine is misspelled. Suggestions: heliotrope, heliotropes, heliotrope's\n",
      "hydrocarbone is misspelled. Suggestions: hydrocarbon, hydrocarbons, hydrocarbon's\n",
      "ionone is misspelled. Suggestions: Inonu, inane, Ionian, ion one, ion-one, Onion, onion, intone, inaner, none, anon, iPhone, iodine, ionize, inning, ozone, pinon, Danone, inline, intoner, online, undone, noon, inner, nine, Ono, inn, ion, non, one, Antone, anyone, insane, onions, Anne, Nona, Onion's, onion's, Inonu's\n",
      "menthone is misspelled. Suggestions: methane, menthol, mention, Menuhin, monotone, Anthony\n",
      "methional is misspelled. Suggestions: methanol, mythical, menthol, marginal, methane, ethanol, menial, thinly, maternal, methanol's, methane's\n",
      "nootkatone is misspelled. Suggestions: notating, nitrating, nocturne, nicotine, Stockton, contacting, negating, docketing\n",
      "odour is misspelled. Suggestions: odor, dour, Oder, odder, Oort, door, ode, out, odors, OD, order, our, odious, odium, ado, odd, Odom, doer, tour, ardor, Adar, OTOH, odor's\n",
      "pitty is misspelled. Suggestions: potty, putty, Pitt, pity, Patty, Petty, patty, petty, piety, pitta, putt, Pitts, Kitty, Mitty, bitty, ditty, kitty, pithy, titty, witty, pit, party, pita, Patti, peaty, Pitt's\n",
      "pyridine is misspelled. Suggestions: priding, pyrimidine, pristine, parading, prodding, purine, pride, Puritan, puritan, pyramiding, pyrite, riding, joyriding, praline, pricing, priming, prizing, prating, Paradise, deriding, paradise, periling, pirating\n",
      "salicylate is misspelled. Suggestions: saleslady, Sallust, solicited, solicit, sliced, sliest, solicitor, Szilard, salesladies, solicitude, cellulite, saleslady's, solubility\n",
      "styrene is misspelled. Suggestions: Styron, strewn, Sterne, Tyrone, serene, string, sterner, tureen, Stern, stern, staring, storing, strainer, strew, stringer, street, Sterno, steering, strain, screen, serener, strange, syringe, Stine, Stone, siren, stare, steno, stone, store, Styron's\n",
      "tangarine is misspelled. Suggestions: tangerine, tangerines, tangerine's, tangoing, angering, gangrene, tangling, ingrain, tinkering, Hungarian, danging, tanking, tonging, dungaree, tenuring, tonguing\n",
      "thujone is misspelled. Suggestions: throne, thine, thicken, thymine, thong, thunk, thane, thorn, Taejon, Theron, June, gone, Jon, think, thorny, Jane, Joni, cone, thug, thank\n",
      "urinaceous is misspelled. Suggestions: farinaceous, urinates, princes, urinals, Prince's, prince's, princess, urinal's, urinalysis, rinses, Orinoco's, Frances, prances, trances, rinse's, urinalyses, durance's, variances, erroneous, France's, prance's, prancers, trance's, Ignacio's, variance's, Frances's, evinces, prancer's, princess's\n"
     ]
    }
   ],
   "source": [
    "list_of_words = list(words_set_v2)\n",
    "list_of_words.sort()\n",
    "for word in list_of_words:\n",
    "    spellcheck(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # manual spellcheck and filter\n",
    "# spellcorrect_v2 = {}\n",
    "# spellcorrect_v2['creamy'] = ['ceamy']\n",
    "# spellcorrect_v2['green'] = ['geen']\n",
    "# spellcorrect_v2['herbal'] = ['hebaceous']\n",
    "# spellcorrect_v2['spicy'] = ['sicy']\n",
    "# spellcorrect_v2['pine'] = ['pne']\n",
    "\n",
    "# added to previous spell correction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Raw Labels</th>\n",
       "      <th>Labels</th>\n",
       "      <th>descriptors</th>\n",
       "      <th>cleaned raw labels</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Stimulus</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-955348933095</th>\n",
       "      <td>Herbal-green, waxy, oily, fruity</td>\n",
       "      <td>['green', 'oily', 'fruity', 'waxy', 'herbal']</td>\n",
       "      <td>green;oily;fruity;waxy;herbal</td>\n",
       "      <td>oily fruity waxy herbal green</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-923209957509</th>\n",
       "      <td>Herbaceous, woody, slight spicy fruity odor</td>\n",
       "      <td>['woody', 'spicy', 'fruity', 'herbal']</td>\n",
       "      <td>woody;spicy;fruity;herbal</td>\n",
       "      <td>fruity woody spicy slight odor herbaceous</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-874408321546</th>\n",
       "      <td>delta-decalactone precursor; peach, apricot, b...</td>\n",
       "      <td>['peach', 'apricot', 'buttery']</td>\n",
       "      <td>peach;apricot;buttery</td>\n",
       "      <td>precursor lactonic delta</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-873963935677</th>\n",
       "      <td>Green, fruity, apple-like</td>\n",
       "      <td>['green', 'fruity', 'apple', 'tropical']</td>\n",
       "      <td>green;fruity;apple;tropical</td>\n",
       "      <td>like green fruity apple</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-862841422647</th>\n",
       "      <td>Catty urine; cassis in dilution</td>\n",
       "      <td>['catty']</td>\n",
       "      <td>catty</td>\n",
       "      <td>urine catty</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      Raw Labels   \n",
       "Stimulus                                                           \n",
       "-955348933095                   Herbal-green, waxy, oily, fruity  \\\n",
       "-923209957509        Herbaceous, woody, slight spicy fruity odor   \n",
       "-874408321546  delta-decalactone precursor; peach, apricot, b...   \n",
       "-873963935677                          Green, fruity, apple-like   \n",
       "-862841422647                    Catty urine; cassis in dilution   \n",
       "\n",
       "                                                      Labels   \n",
       "Stimulus                                                       \n",
       "-955348933095  ['green', 'oily', 'fruity', 'waxy', 'herbal']  \\\n",
       "-923209957509         ['woody', 'spicy', 'fruity', 'herbal']   \n",
       "-874408321546                ['peach', 'apricot', 'buttery']   \n",
       "-873963935677       ['green', 'fruity', 'apple', 'tropical']   \n",
       "-862841422647                                      ['catty']   \n",
       "\n",
       "                                 descriptors   \n",
       "Stimulus                                       \n",
       "-955348933095  green;oily;fruity;waxy;herbal  \\\n",
       "-923209957509      woody;spicy;fruity;herbal   \n",
       "-874408321546          peach;apricot;buttery   \n",
       "-873963935677    green;fruity;apple;tropical   \n",
       "-862841422647                          catty   \n",
       "\n",
       "                                      cleaned raw labels  \n",
       "Stimulus                                                  \n",
       "-955348933095              oily fruity waxy herbal green  \n",
       "-923209957509  fruity woody spicy slight odor herbaceous  \n",
       "-874408321546                   precursor lactonic delta  \n",
       "-873963935677                    like green fruity apple  \n",
       "-862841422647                                urine catty  "
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "behavior_sparse.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "get labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "roots_dict = defaultdict(set)\n",
    "def connect_roots(cleaned_raw_labels):\n",
    "    raw_list = cleaned_raw_labels.split(' ')\n",
    "    for desc in required_desc:\n",
    "        root_set = set()\n",
    "        for word in raw_list:\n",
    "            if word == desc:\n",
    "                root_set.update([word])\n",
    "            elif re.match(f'{desc[:-(int(0.6*len(desc)))]}.*', word):\n",
    "                root_set.update([word])\n",
    "        roots_dict[desc].update(list(root_set))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Stimulus\n",
       "-955348933095    None\n",
       "-923209957509    None\n",
       "-874408321546    None\n",
       "-873963935677    None\n",
       "-862841422647    None\n",
       "                 ... \n",
       " 154241350       None\n",
       " 155323379       None\n",
       " 158699780       None\n",
       " 161504856       None\n",
       " 162353069       None\n",
       "Name: cleaned raw labels, Length: 3522, dtype: object"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "behavior_sparse['cleaned raw labels'].apply(lambda x: connect_roots(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(set,\n",
       "            {'alcoholic': {'alcohol', 'alcoholic'},\n",
       "             'aldehydic': {'aldehydic'},\n",
       "             'alliaceous': {'alliaceous', 'allium'},\n",
       "             'almond': {'almond', 'almonds', 'almost'},\n",
       "             'amber': {'amber',\n",
       "              'ambergris',\n",
       "              'amine',\n",
       "              'ammonia',\n",
       "              'ammoniacal',\n",
       "              'amyl'},\n",
       "             'animal': {'animal', 'anise', 'anisic', 'anisole'},\n",
       "             'anisic': {'animal', 'anise', 'anisic', 'anisole'},\n",
       "             'apple': {'apple', 'apricot'},\n",
       "             'apricot': {'apricot'},\n",
       "             'aromatic': {'aroma', 'aromas', 'aromatic'},\n",
       "             'balsamic': {'balsam', 'balsamic'},\n",
       "             'banana': {'banana'},\n",
       "             'beefy': {'bean',\n",
       "              'beans',\n",
       "              'beany',\n",
       "              'becomes',\n",
       "              'beef',\n",
       "              'beefsteak',\n",
       "              'beefy',\n",
       "              'beer',\n",
       "              'bell',\n",
       "              'benzaldehyde',\n",
       "              'benzene',\n",
       "              'benzoic',\n",
       "              'bergamot',\n",
       "              'berry',\n",
       "              'beta'},\n",
       "             'bergamot': {'bergamot'},\n",
       "             'berry': {'bean',\n",
       "              'beans',\n",
       "              'beany',\n",
       "              'becomes',\n",
       "              'beef',\n",
       "              'beefsteak',\n",
       "              'beefy',\n",
       "              'beer',\n",
       "              'bell',\n",
       "              'benzaldehyde',\n",
       "              'benzene',\n",
       "              'benzoic',\n",
       "              'bergamot',\n",
       "              'berry',\n",
       "              'beta'},\n",
       "             'bitter': {'bite', 'bitter', 'bitterness'},\n",
       "             'black currant': set(),\n",
       "             'brandy': {'braised', 'brandy'},\n",
       "             'burnt': {'buchu',\n",
       "              'bugs',\n",
       "              'burned',\n",
       "              'burning',\n",
       "              'burnt',\n",
       "              'but',\n",
       "              'butter',\n",
       "              'buttermilk',\n",
       "              'buttery'},\n",
       "             'buttery': {'but', 'butter', 'buttermilk', 'buttery'},\n",
       "             'cabbage': {'cabbage'},\n",
       "             'camphoreous': {'camphor', 'camphoreous'},\n",
       "             'caramellic': {'caramel', 'caramelized', 'caramellic', 'caraway'},\n",
       "             'cedar': {'cedar', 'celery', 'cereal'},\n",
       "             'celery': {'celery'},\n",
       "             'chamomile': {'chamomile'},\n",
       "             'cheesy': {'cheese', 'cheesy', 'chemical', 'cherries', 'cherry'},\n",
       "             'cherry': {'cheese', 'cheesy', 'chemical', 'cherries', 'cherry'},\n",
       "             'chocolate': {'chocolate'},\n",
       "             'cinnamon': {'cinnamon'},\n",
       "             'citrus': {'citrus'},\n",
       "             'clean': {'clam',\n",
       "              'clary',\n",
       "              'clean',\n",
       "              'clearing',\n",
       "              'clove',\n",
       "              'clover'},\n",
       "             'clove': {'clam',\n",
       "              'clary',\n",
       "              'clean',\n",
       "              'clearing',\n",
       "              'clove',\n",
       "              'clover'},\n",
       "             'cocoa': {'co',\n",
       "              'cocoa',\n",
       "              'coconut',\n",
       "              'cod',\n",
       "              'coffee',\n",
       "              'cognac',\n",
       "              'collidine',\n",
       "              'cologne',\n",
       "              'combination',\n",
       "              'comments',\n",
       "              'complex',\n",
       "              'component',\n",
       "              'compound',\n",
       "              'concentration',\n",
       "              'concentrations',\n",
       "              'concord',\n",
       "              'cone',\n",
       "              'confiferous',\n",
       "              'conifer',\n",
       "              'coniferous',\n",
       "              'connotation',\n",
       "              'connotations',\n",
       "              'contrbutes',\n",
       "              'contribute',\n",
       "              'contributes',\n",
       "              'contributor',\n",
       "              'cooked',\n",
       "              'cool',\n",
       "              'cooling',\n",
       "              'coriander',\n",
       "              'corn',\n",
       "              'costus',\n",
       "              'cotton',\n",
       "              'coumarinic'},\n",
       "             'coconut': {'cocoa', 'coconut'},\n",
       "             'coffee': {'coffee'},\n",
       "             'cognac': {'cognac'},\n",
       "             'cooked': {'cooked', 'cool', 'cooling'},\n",
       "             'cooling': {'cooked', 'cool', 'cooling'},\n",
       "             'cortex': {'coriander', 'corn'},\n",
       "             'coumarinic': {'coumarinic'},\n",
       "             'creamy': {'cream', 'creamy', 'cresolic', 'cress', 'cresylic'},\n",
       "             'cucumber': {'cucumber'},\n",
       "             'dairy': {'dairy', 'damascenone', 'dark'},\n",
       "             'dry': {'dried', 'dry'},\n",
       "             'earthy': {'earthy'},\n",
       "             'ethereal': {'ethereal'},\n",
       "             'fatty': {'faint',\n",
       "              'faintly',\n",
       "              'families',\n",
       "              'family',\n",
       "              'fat',\n",
       "              'fats',\n",
       "              'fatty'},\n",
       "             'fermented': {'fermented', 'fermenting'},\n",
       "             'fishy': {'fig', 'fir', 'fish', 'fishy', 'fixative', 'fixitive'},\n",
       "             'floral': {'flora', 'floral', 'flower', 'flowery'},\n",
       "             'fresh': {'fragrance',\n",
       "              'french',\n",
       "              'fresh',\n",
       "              'freshly',\n",
       "              'freshness',\n",
       "              'fried',\n",
       "              'fries',\n",
       "              'from',\n",
       "              'fruit',\n",
       "              'fruitiness',\n",
       "              'fruits',\n",
       "              'fruity'},\n",
       "             'fruit skin': {'fruit', 'fruitiness', 'fruits', 'fruity'},\n",
       "             'fruity': {'fruit', 'fruitiness', 'fruits', 'fruity'},\n",
       "             'garlic': {'garden', 'gardenia', 'garlic'},\n",
       "             'gassy': {'galbanum',\n",
       "              'garden',\n",
       "              'gardenia',\n",
       "              'garlic',\n",
       "              'gasoline',\n",
       "              'gassy'},\n",
       "             'geranium': {'geranium'},\n",
       "             'grape': {'grain',\n",
       "              'grape',\n",
       "              'grapefruit',\n",
       "              'grapes',\n",
       "              'grass',\n",
       "              'grassy',\n",
       "              'greasy',\n",
       "              'great',\n",
       "              'green',\n",
       "              'greenish',\n",
       "              'grilled',\n",
       "              'ground',\n",
       "              'grounds'},\n",
       "             'grapefruit': {'grape', 'grapefruit', 'grapes'},\n",
       "             'grassy': {'grain',\n",
       "              'grape',\n",
       "              'grapefruit',\n",
       "              'grapes',\n",
       "              'grass',\n",
       "              'grassy'},\n",
       "             'green': {'grain',\n",
       "              'grape',\n",
       "              'grapefruit',\n",
       "              'grapes',\n",
       "              'grass',\n",
       "              'grassy',\n",
       "              'greasy',\n",
       "              'great',\n",
       "              'green',\n",
       "              'greenish',\n",
       "              'grilled',\n",
       "              'ground',\n",
       "              'grounds'},\n",
       "             'hawthorn': {'hawthorn'},\n",
       "             'hay': {'ham',\n",
       "              'harsh',\n",
       "              'has',\n",
       "              'hawthorn',\n",
       "              'hay',\n",
       "              'hazel',\n",
       "              'hazelnut',\n",
       "              'hazelnuts'},\n",
       "             'hazelnut': {'hazel', 'hazelnut', 'hazelnuts'},\n",
       "             'herbal': {'herb', 'herbaceous', 'herbal'},\n",
       "             'honey': {'honey',\n",
       "              'honeydew',\n",
       "              'honeysuckle',\n",
       "              'horseradish',\n",
       "              'hot'},\n",
       "             'hyacinth': {'hyacinth'},\n",
       "             'jasmin': {'jasmin', 'jasmine'},\n",
       "             'juicy': {'juice', 'juicy', 'juniper'},\n",
       "             'ketonic': {'ketonic'},\n",
       "             'lactonic': {'lactonic'},\n",
       "             'lavender': {'lavender'},\n",
       "             'leafy': {'leaf',\n",
       "              'leafy',\n",
       "              'leather',\n",
       "              'leathery',\n",
       "              'leek',\n",
       "              'lemon',\n",
       "              'lemony',\n",
       "              'less',\n",
       "              'lettuce',\n",
       "              'levels'},\n",
       "             'leathery': {'leather', 'leathery'},\n",
       "             'lemon': {'leaf',\n",
       "              'leafy',\n",
       "              'leather',\n",
       "              'leathery',\n",
       "              'leek',\n",
       "              'lemon',\n",
       "              'lemony',\n",
       "              'less',\n",
       "              'lettuce',\n",
       "              'levels'},\n",
       "             'lily': {'licorice',\n",
       "              'lifting',\n",
       "              'light',\n",
       "              'like',\n",
       "              'lilac',\n",
       "              'lily',\n",
       "              'lime',\n",
       "              'linalool',\n",
       "              'linden',\n",
       "              'little',\n",
       "              'liver'},\n",
       "             'malty': {'malt',\n",
       "              'malty',\n",
       "              'mandarin',\n",
       "              'manderin',\n",
       "              'mango',\n",
       "              'maple',\n",
       "              'marine',\n",
       "              'masks',\n",
       "              'material',\n",
       "              'may'},\n",
       "             'meaty': {'meadow',\n",
       "              'meat',\n",
       "              'meats',\n",
       "              'meaty',\n",
       "              'mecaptan',\n",
       "              'medicinal',\n",
       "              'melon',\n",
       "              'menthol',\n",
       "              'menthone',\n",
       "              'mercaptan',\n",
       "              'metallic',\n",
       "              'methional'},\n",
       "             'medicinal': {'medicinal'},\n",
       "             'melon': {'meadow',\n",
       "              'meat',\n",
       "              'meats',\n",
       "              'meaty',\n",
       "              'mecaptan',\n",
       "              'medicinal',\n",
       "              'melon',\n",
       "              'menthol',\n",
       "              'menthone',\n",
       "              'mercaptan',\n",
       "              'metallic',\n",
       "              'methional'},\n",
       "             'metallic': {'metallic'},\n",
       "             'milky': {'mild',\n",
       "              'mildly',\n",
       "              'milk',\n",
       "              'milky',\n",
       "              'mimosa',\n",
       "              'mint',\n",
       "              'minty',\n",
       "              'mixed',\n",
       "              'mixture'},\n",
       "             'mint': {'mild',\n",
       "              'mildly',\n",
       "              'milk',\n",
       "              'milky',\n",
       "              'mimosa',\n",
       "              'mint',\n",
       "              'minty',\n",
       "              'mixed',\n",
       "              'mixture'},\n",
       "             'muguet': {'muguet'},\n",
       "             'mushroom': {'mushroom'},\n",
       "             'musk': {'much',\n",
       "              'muguet',\n",
       "              'muscat',\n",
       "              'mushroom',\n",
       "              'musk',\n",
       "              'muskmelon',\n",
       "              'musky',\n",
       "              'mustard',\n",
       "              'musty',\n",
       "              'mutton'},\n",
       "             'musty': {'much',\n",
       "              'muguet',\n",
       "              'muscat',\n",
       "              'mushroom',\n",
       "              'musk',\n",
       "              'muskmelon',\n",
       "              'musky',\n",
       "              'mustard',\n",
       "              'musty',\n",
       "              'mutton'},\n",
       "             'natural': {'natural'},\n",
       "             'nutty': {'nuance', 'nuances', 'nut', 'nutmeg', 'nuts', 'nutty'},\n",
       "             'odorless': {'odor', 'odorless'},\n",
       "             'oily': {'oil', 'oils', 'oily'},\n",
       "             'onion': {'on', 'onion', 'onions', 'only'},\n",
       "             'orange': {'orange', 'orangeflower'},\n",
       "             'orangeflower': {'orange', 'orangeflower'},\n",
       "             'orris': {'or',\n",
       "              'orange',\n",
       "              'orangeflower',\n",
       "              'orchid',\n",
       "              'origanum',\n",
       "              'orris'},\n",
       "             'ozone': {'ozone'},\n",
       "             'peach': {'pea',\n",
       "              'peach',\n",
       "              'peachy',\n",
       "              'peanut',\n",
       "              'peanuts',\n",
       "              'pear',\n",
       "              'pecan',\n",
       "              'peculiar',\n",
       "              'peculiiar',\n",
       "              'peel',\n",
       "              'peely',\n",
       "              'penetrating',\n",
       "              'pepper',\n",
       "              'peppermint',\n",
       "              'pepperoni',\n",
       "              'peppery',\n",
       "              'perilla',\n",
       "              'persistent',\n",
       "              'perspiration',\n",
       "              'petal',\n",
       "              'petitgrain',\n",
       "              'petrol'},\n",
       "             'pear': {'pea',\n",
       "              'peach',\n",
       "              'peachy',\n",
       "              'peanut',\n",
       "              'peanuts',\n",
       "              'pear',\n",
       "              'pecan',\n",
       "              'peculiar',\n",
       "              'peculiiar',\n",
       "              'peel',\n",
       "              'peely',\n",
       "              'penetrating',\n",
       "              'pepper',\n",
       "              'peppermint',\n",
       "              'pepperoni',\n",
       "              'peppery',\n",
       "              'perilla',\n",
       "              'persistent',\n",
       "              'perspiration',\n",
       "              'petal',\n",
       "              'petitgrain',\n",
       "              'petrol'},\n",
       "             'phenolic': {'phenol',\n",
       "              'phenolc',\n",
       "              'phenolic',\n",
       "              'phenylacetaldehyde',\n",
       "              'phenylacetic'},\n",
       "             'pine': {'pine',\n",
       "              'pineapple',\n",
       "              'piney',\n",
       "              'pistachio',\n",
       "              'pit',\n",
       "              'pith',\n",
       "              'pitty'},\n",
       "             'pineapple': {'pine', 'pineapple', 'piney'},\n",
       "             'plum': {'plant',\n",
       "              'plastic',\n",
       "              'pleasant',\n",
       "              'pleasantly',\n",
       "              'plum',\n",
       "              'plus'},\n",
       "             'popcorn': {'popcorn'},\n",
       "             'potato': {'pot', 'potato', 'potent'},\n",
       "             'powdery': {'powdery', 'power', 'powerful', 'powerfull'},\n",
       "             'pungent': {'pungent'},\n",
       "             'radish': {'radish'},\n",
       "             'raspberry': {'raspberry'},\n",
       "             'ripe': {'ribes', 'rich', 'rind', 'ripe', 'ripened'},\n",
       "             'roasted': {'roast', 'roasted'},\n",
       "             'rose': {'roast',\n",
       "              'roasted',\n",
       "              'roman',\n",
       "              'root',\n",
       "              'rooty',\n",
       "              'roquefort',\n",
       "              'rose',\n",
       "              'rosemary',\n",
       "              'rosy',\n",
       "              'rotten',\n",
       "              'rotting',\n",
       "              'rough'},\n",
       "             'rummy': {'rubber', 'rubbery', 'rubbing', 'rue', 'rum', 'rummy'},\n",
       "             'sandalwood': {'sandalwood'},\n",
       "             'savory': {'savory'},\n",
       "             'sharp': {'sharp',\n",
       "              'sheepy',\n",
       "              'shellfish',\n",
       "              'sherry',\n",
       "              'shiitake',\n",
       "              'shrimp'},\n",
       "             'smoky': {'smelling', 'smoke', 'smoked', 'smokey', 'smoky'},\n",
       "             'soapy': {'soapy',\n",
       "              'soft',\n",
       "              'solution',\n",
       "              'solvent',\n",
       "              'some',\n",
       "              'somehat',\n",
       "              'somehate',\n",
       "              'somewat',\n",
       "              'somewhat',\n",
       "              'soup',\n",
       "              'soupy',\n",
       "              'sour',\n",
       "              'soy'},\n",
       "             'solvent': {'solution', 'solvent'},\n",
       "             'sour': {'soapy',\n",
       "              'soft',\n",
       "              'solution',\n",
       "              'solvent',\n",
       "              'some',\n",
       "              'somehat',\n",
       "              'somehate',\n",
       "              'somewat',\n",
       "              'somewhat',\n",
       "              'soup',\n",
       "              'soupy',\n",
       "              'sour',\n",
       "              'soy'},\n",
       "             'spicy': {'sparkling',\n",
       "              'spearmint',\n",
       "              'spice',\n",
       "              'spicy',\n",
       "              'spike',\n",
       "              'sprouts'},\n",
       "             'strawberry': {'strange', 'straw', 'strawberry'},\n",
       "             'sulfurous': {'sulfide', 'sulfur', 'sulfuric', 'sulfurous'},\n",
       "             'sweaty': {'sweat', 'sweaty', 'sweet', 'sweetish', 'sweetness'},\n",
       "             'sweet': {'sweat', 'sweaty', 'sweet', 'sweetish', 'sweetness'},\n",
       "             'tea': {'tea',\n",
       "              'tenacious',\n",
       "              'terpenic',\n",
       "              'tetrahydrolinalool',\n",
       "              'texture'},\n",
       "             'terpenic': {'terpenic'},\n",
       "             'tobacco': {'tobacco'},\n",
       "             'tomato': {'tomato'},\n",
       "             'tropical': {'tropical'},\n",
       "             'vanilla': {'vanilla'},\n",
       "             'vegetable': {'vegetable', 'vegetables', 'vegetative'},\n",
       "             'vetiver': {'vetiver'},\n",
       "             'violet': {'violet'},\n",
       "             'warm': {'walnut',\n",
       "              'warm',\n",
       "              'warming',\n",
       "              'wasabi',\n",
       "              'watercress',\n",
       "              'watermelon',\n",
       "              'watery',\n",
       "              'waxy'},\n",
       "             'waxy': {'walnut',\n",
       "              'warm',\n",
       "              'warming',\n",
       "              'wasabi',\n",
       "              'watercress',\n",
       "              'watermelon',\n",
       "              'watery',\n",
       "              'waxy'},\n",
       "             'weedy': {'weak', 'weaker', 'weakly', 'weedy'},\n",
       "             'winey': {'william', 'wine', 'winey', 'wintergreen', 'with'},\n",
       "             'woody': {'wood', 'woody'}})"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roots_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# manually cleaned desc root combination\n",
    "merger_root_dict = {}\n",
    "merger_root_dict['alcoholic'] = ['alcohol', 'alcoholic']\n",
    "merger_root_dict['aldehydic'] = ['aldehydic']\n",
    "merger_root_dict['alliaceous'] = ['alliaceous', 'allium']\n",
    "merger_root_dict['almond'] = ['almond', 'almonds']\n",
    "merger_root_dict['amber'] = ['amber', 'ambergris']\n",
    "merger_root_dict['animal'] = ['animal']\n",
    "merger_root_dict['anisic'] = ['anise', 'anisic']\n",
    "merger_root_dict['apple'] = ['apple']\n",
    "merger_root_dict['apricot'] = ['apricot']\n",
    "merger_root_dict['aromatic'] = ['aromatic']\n",
    "merger_root_dict['balsamic'] = ['balsam', 'balsamic']\n",
    "merger_root_dict['banana'] = ['banana']\n",
    "merger_root_dict['beefy'] = ['beef', 'beefsteak', 'beefy']\n",
    "merger_root_dict['bergamot'] = ['bergamot']\n",
    "merger_root_dict['berry'] = ['berry']\n",
    "merger_root_dict['bitter'] = ['bitter', 'bitterness', 'bittersweet']\n",
    "merger_root_dict['black currant'] = ['black currant']\n",
    "merger_root_dict['brandy'] = ['brandy']\n",
    "merger_root_dict['burnt'] = ['burned', 'burning', 'burnt']\n",
    "merger_root_dict['buttery'] = ['butter', 'buttery']\n",
    "merger_root_dict['cabbage'] = ['cabbage']\n",
    "merger_root_dict['camphoreous'] = ['camphor', 'camphoreous']\n",
    "merger_root_dict['caramellic'] = ['caramel', 'caramelized', 'caramellic']\n",
    "merger_root_dict['cedar'] = ['cedar']\n",
    "merger_root_dict['celery'] = ['celery']\n",
    "merger_root_dict['chamomile'] = ['chamomile']\n",
    "merger_root_dict['cheesy'] = ['cheese', 'cheesy']\n",
    "merger_root_dict['cherry'] = ['cherries', 'cherry']\n",
    "merger_root_dict['chocolate'] = ['chocolate']\n",
    "merger_root_dict['cinnamon'] = ['cinnamon']\n",
    "merger_root_dict['citrus'] = ['citronella', 'citrus']\n",
    "merger_root_dict['clean'] = ['clean']\n",
    "merger_root_dict['clove'] = ['clove', 'clover']\n",
    "merger_root_dict['cocoa'] = ['cocoa']\n",
    "merger_root_dict['coconut'] = ['coconut']\n",
    "merger_root_dict['coffee'] = ['coffee']\n",
    "merger_root_dict['cognac'] = ['cognac']\n",
    "merger_root_dict['cooked'] = ['cooked']\n",
    "merger_root_dict['cooling'] = ['cool', 'cooling', 'coolness']\n",
    "merger_root_dict['cortex'] = ['cortex']\n",
    "merger_root_dict['coumarinic'] = ['coumarinic']\n",
    "merger_root_dict['creamy'] = ['cream', 'creamy']\n",
    "merger_root_dict['cucumber'] = ['cucumber']\n",
    "merger_root_dict['dairy'] = ['dairy']\n",
    "merger_root_dict['dry'] = ['dried', 'dry']\n",
    "merger_root_dict['earthy'] = ['earth', 'earthy']\n",
    "merger_root_dict['ethereal'] = ['ethereal']\n",
    "merger_root_dict['fatty'] = ['fat', 'fats', 'fatty']\n",
    "merger_root_dict['fermented'] = ['fermented', 'fermenting']\n",
    "merger_root_dict['fishy'] = ['fish', 'fishy']\n",
    "merger_root_dict['floral'] = ['flora', 'floral', 'flower', 'flowery']\n",
    "merger_root_dict['fresh'] = ['fresh', 'freshly', 'freshness']\n",
    "merger_root_dict['fruit skin'] = ['fruit skin']\n",
    "merger_root_dict['fruity'] = ['fruit', 'fruitier', 'fruitiness', 'fruits', 'fruity']\n",
    "merger_root_dict['garlic'] = ['garlic']\n",
    "merger_root_dict['gassy'] = ['gassy']\n",
    "merger_root_dict['geranium'] = ['geranium']\n",
    "merger_root_dict['grape'] = ['grape', 'grapes']\n",
    "merger_root_dict['grapefruit'] = ['grapefruit']\n",
    "merger_root_dict['grassy'] = ['grass', 'grassy']\n",
    "merger_root_dict['green'] = ['green', 'greenery', 'greenish']\n",
    "merger_root_dict['hawthorn'] = ['hawthorn']\n",
    "merger_root_dict['hay'] = ['hay']\n",
    "merger_root_dict['hazelnut'] = ['hazel', 'hazelnut', 'hazelnuts']\n",
    "merger_root_dict['herbal'] = ['herb', 'herbaceous', 'herbal']\n",
    "merger_root_dict['honey'] = ['honey', 'honeydew', 'honeysuckle']\n",
    "merger_root_dict['hyacinth'] = ['hyacinth']\n",
    "merger_root_dict['jasmin'] = ['jasmin', 'jasmine']\n",
    "merger_root_dict['juicy'] = ['juice', 'juiciness', 'juicy']\n",
    "merger_root_dict['ketonic'] = ['ketonic']\n",
    "merger_root_dict['lactonic'] = ['lactonic']\n",
    "merger_root_dict['lavender'] = ['lavender']\n",
    "merger_root_dict['leafy'] = ['leaf', 'leafy', 'lettuce']\n",
    "merger_root_dict['leathery'] = ['leather', 'leathery']\n",
    "merger_root_dict['lemon'] = ['lemon', 'lemony', 'lime']\n",
    "merger_root_dict['lily'] = ['lilac', 'lily']\n",
    "merger_root_dict['malty'] = ['malt', 'malty']\n",
    "merger_root_dict['meaty'] = ['meat', 'meats', 'meaty']\n",
    "merger_root_dict['medicinal'] = ['medicinal']\n",
    "merger_root_dict['melon'] = ['melon']\n",
    "merger_root_dict['metallic'] = ['metallic']\n",
    "merger_root_dict['milky'] = ['milk', 'milky']\n",
    "merger_root_dict['mint'] = ['mint', 'minty']\n",
    "merger_root_dict['muguet'] = ['muguet']\n",
    "merger_root_dict['mushroom'] = ['mushroom']\n",
    "merger_root_dict['musk'] = ['musk', 'musky']\n",
    "merger_root_dict['musty'] = ['musty']\n",
    "merger_root_dict['natural'] = ['natural']\n",
    "merger_root_dict['nutty'] = ['nut', 'nuts', 'nutty']\n",
    "merger_root_dict['odorless'] = ['odorless']\n",
    "merger_root_dict['oily'] = ['oil', 'oils', 'oily']\n",
    "merger_root_dict['onion'] = ['onion', 'onions']\n",
    "merger_root_dict['orange'] = ['orange']\n",
    "merger_root_dict['orangeflower'] = ['orangeflower']\n",
    "merger_root_dict['orris'] = ['orris']\n",
    "merger_root_dict['ozone'] = ['ozone']\n",
    "merger_root_dict['peach'] = ['peach', 'peachy']\n",
    "merger_root_dict['pear'] = ['pear']\n",
    "merger_root_dict['phenolic'] = ['phenol', 'phenolc', 'phenolic']\n",
    "merger_root_dict['pine'] = ['pine', 'piney']\n",
    "merger_root_dict['pineapple'] = ['pineapple']\n",
    "merger_root_dict['plum'] = ['plum']\n",
    "merger_root_dict['popcorn'] = ['popcorn']\n",
    "merger_root_dict['potato'] = ['potato']\n",
    "merger_root_dict['powdery'] = ['powdery']\n",
    "merger_root_dict['pungent'] = ['pungency', 'pungent']\n",
    "merger_root_dict['radish'] = ['radish', 'horseradish']\n",
    "merger_root_dict['raspberry'] = ['raspberry']\n",
    "merger_root_dict['ripe'] = ['ripe', 'ripened']\n",
    "merger_root_dict['roasted'] = ['roast', 'roasted']\n",
    "merger_root_dict['rose'] = ['rose', 'rosemary', 'rosy']\n",
    "merger_root_dict['rummy'] = ['rum', 'rummy']\n",
    "merger_root_dict['sandalwood'] = ['sandalwood']\n",
    "merger_root_dict['savory'] = ['savory']\n",
    "merger_root_dict['sharp'] = ['sharp']\n",
    "merger_root_dict['smoky'] = ['smoke', 'smoked', 'smokey', 'smoky']\n",
    "merger_root_dict['soapy'] = ['soapy']\n",
    "merger_root_dict['solvent'] = ['solvent']\n",
    "merger_root_dict['sour'] = ['sour']\n",
    "merger_root_dict['spicy'] = ['spice', 'spicy']\n",
    "merger_root_dict['strawberry'] = ['strawberry']\n",
    "merger_root_dict['sulfurous'] = ['sulfide', 'sulfur', 'sulfuric', 'sulfurous']\n",
    "merger_root_dict['sweaty'] = ['sweat','sweaty']\n",
    "merger_root_dict['sweet'] = ['sweet', 'sweeter', 'sweetish', 'sweetness']\n",
    "merger_root_dict['tea'] = ['tea']\n",
    "merger_root_dict['terpenic'] = ['terpenic']\n",
    "merger_root_dict['tobacco'] = ['tobacco', 'tobaccos']\n",
    "merger_root_dict['tomato'] = ['tomato']\n",
    "merger_root_dict['tropical'] = ['tropical']\n",
    "merger_root_dict['vanilla'] = ['vanilla']\n",
    "merger_root_dict['vegetable'] = ['vegetable', 'vegetables', 'vegetative']\n",
    "merger_root_dict['vetiver'] = ['vetiver']\n",
    "merger_root_dict['violet'] = ['violet']\n",
    "merger_root_dict['warm'] = ['warm', 'warming']\n",
    "merger_root_dict['waxy'] = ['waxy']\n",
    "merger_root_dict['weedy'] = ['weedy']\n",
    "merger_root_dict['winey'] = ['wine', 'wines', 'winey']\n",
    "merger_root_dict['woody'] = ['wood', 'woody']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "138"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(merger_root_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "reverse_merger_root_dict = defaultdict(list)\n",
    "for key, values in merger_root_dict.items():\n",
    "    for value in values:\n",
    "        reverse_merger_root_dict[value].append(key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'juiciness' in reverse_merger_root_dict.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_desc(cleaned_raw_labels):\n",
    "    desc_set = set()\n",
    "    raw_list = cleaned_raw_labels.split(' ')\n",
    "\n",
    "    # handle black currant\n",
    "    if \"black\" in raw_list:\n",
    "        idx = raw_list.index('black')\n",
    "        if idx<len(raw_list)-1:\n",
    "            if raw_list[idx+1] == 'currant':\n",
    "                raw_list.pop(idx) #removes black\n",
    "                raw_list.pop(idx) # removes currant\n",
    "                raw_list.append(\"black currant\")\n",
    "    \n",
    "    # handle orange flower\n",
    "    if \"orange\" in raw_list:\n",
    "        idx = raw_list.index('orange')\n",
    "        if idx<len(raw_list)-1:\n",
    "            if raw_list[idx+1] == 'flower':\n",
    "                raw_list.pop(idx) #removes orange\n",
    "                raw_list.pop(idx) # removes flower\n",
    "                raw_list.append(\"orangeflower\")\n",
    "    \n",
    "    # handle fruit skin\n",
    "    for idx in range(len(raw_list)):\n",
    "        if raw_list[idx] == 'skin' or raw_list[idx] == 'peel':\n",
    "            if idx != 0:\n",
    "                if raw_list[idx-1] in ['citrus', 'orange', 'apple', 'banana', 'lemon', 'pear', 'fruit', 'peach']:\n",
    "                    raw_list.append(\"fruit skin\")\n",
    "\n",
    "    for word in raw_list:\n",
    "        for root, desc in reverse_merger_root_dict.items():\n",
    "            if word == root:\n",
    "                desc_set.update(desc)\n",
    "                break\n",
    "    if len(desc_set) == 0:\n",
    "        return None\n",
    "    return ';'.join(desc_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'odorless;fruity;herbal;woody;spicy'"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_desc('fruity odorless herbaceous spicy odor slight woody')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "behavior_sparse['new_descriptors'] = behavior_sparse['cleaned raw labels'].apply(lambda x: clean_desc(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 3522 entries, -955348933095 to 162353069\n",
      "Data columns (total 5 columns):\n",
      " #   Column              Non-Null Count  Dtype \n",
      "---  ------              --------------  ----- \n",
      " 0   Raw Labels          3522 non-null   object\n",
      " 1   Labels              3522 non-null   object\n",
      " 2   descriptors         3522 non-null   object\n",
      " 3   cleaned raw labels  3522 non-null   object\n",
      " 4   new_descriptors     3488 non-null   object\n",
      "dtypes: object(5)\n",
      "memory usage: 294.1+ KB\n"
     ]
    }
   ],
   "source": [
    "behavior_sparse.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Raw Labels</th>\n",
       "      <th>Labels</th>\n",
       "      <th>descriptors</th>\n",
       "      <th>cleaned raw labels</th>\n",
       "      <th>new_descriptors</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Stimulus</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-955348933095</th>\n",
       "      <td>Herbal-green, waxy, oily, fruity</td>\n",
       "      <td>['green', 'oily', 'fruity', 'waxy', 'herbal']</td>\n",
       "      <td>green;oily;fruity;waxy;herbal</td>\n",
       "      <td>oily fruity waxy herbal green</td>\n",
       "      <td>oily;fruity;waxy;herbal;green</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-923209957509</th>\n",
       "      <td>Herbaceous, woody, slight spicy fruity odor</td>\n",
       "      <td>['woody', 'spicy', 'fruity', 'herbal']</td>\n",
       "      <td>woody;spicy;fruity;herbal</td>\n",
       "      <td>fruity woody spicy slight odor herbaceous</td>\n",
       "      <td>herbal;spicy;fruity;woody</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-874408321546</th>\n",
       "      <td>delta-decalactone precursor; peach, apricot, b...</td>\n",
       "      <td>['peach', 'apricot', 'buttery']</td>\n",
       "      <td>peach;apricot;buttery</td>\n",
       "      <td>precursor lactonic delta</td>\n",
       "      <td>lactonic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-873963935677</th>\n",
       "      <td>Green, fruity, apple-like</td>\n",
       "      <td>['green', 'fruity', 'apple', 'tropical']</td>\n",
       "      <td>green;fruity;apple;tropical</td>\n",
       "      <td>like green fruity apple</td>\n",
       "      <td>green;fruity;apple</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-862841422647</th>\n",
       "      <td>Catty urine; cassis in dilution</td>\n",
       "      <td>['catty']</td>\n",
       "      <td>catty</td>\n",
       "      <td>urine catty</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      Raw Labels   \n",
       "Stimulus                                                           \n",
       "-955348933095                   Herbal-green, waxy, oily, fruity  \\\n",
       "-923209957509        Herbaceous, woody, slight spicy fruity odor   \n",
       "-874408321546  delta-decalactone precursor; peach, apricot, b...   \n",
       "-873963935677                          Green, fruity, apple-like   \n",
       "-862841422647                    Catty urine; cassis in dilution   \n",
       "\n",
       "                                                      Labels   \n",
       "Stimulus                                                       \n",
       "-955348933095  ['green', 'oily', 'fruity', 'waxy', 'herbal']  \\\n",
       "-923209957509         ['woody', 'spicy', 'fruity', 'herbal']   \n",
       "-874408321546                ['peach', 'apricot', 'buttery']   \n",
       "-873963935677       ['green', 'fruity', 'apple', 'tropical']   \n",
       "-862841422647                                      ['catty']   \n",
       "\n",
       "                                 descriptors   \n",
       "Stimulus                                       \n",
       "-955348933095  green;oily;fruity;waxy;herbal  \\\n",
       "-923209957509      woody;spicy;fruity;herbal   \n",
       "-874408321546          peach;apricot;buttery   \n",
       "-873963935677    green;fruity;apple;tropical   \n",
       "-862841422647                          catty   \n",
       "\n",
       "                                      cleaned raw labels   \n",
       "Stimulus                                                   \n",
       "-955348933095              oily fruity waxy herbal green  \\\n",
       "-923209957509  fruity woody spicy slight odor herbaceous   \n",
       "-874408321546                   precursor lactonic delta   \n",
       "-873963935677                    like green fruity apple   \n",
       "-862841422647                                urine catty   \n",
       "\n",
       "                             new_descriptors  \n",
       "Stimulus                                      \n",
       "-955348933095  oily;fruity;waxy;herbal;green  \n",
       "-923209957509      herbal;spicy;fruity;woody  \n",
       "-874408321546                       lactonic  \n",
       "-873963935677             green;fruity;apple  \n",
       "-862841422647                           None  "
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "behavior_sparse.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Raw Labels</th>\n",
       "      <th>Labels</th>\n",
       "      <th>descriptors</th>\n",
       "      <th>cleaned raw labels</th>\n",
       "      <th>new_descriptors</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Stimulus</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-862841422647</th>\n",
       "      <td>Catty urine; cassis in dilution</td>\n",
       "      <td>['catty']</td>\n",
       "      <td>catty</td>\n",
       "      <td>urine catty</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-848964121442</th>\n",
       "      <td>See comments; components of Milk lactone</td>\n",
       "      <td>['milky']</td>\n",
       "      <td>milky</td>\n",
       "      <td>comments see</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-86422482515</th>\n",
       "      <td>Unstable; citrus, fatty, tropical</td>\n",
       "      <td>['fatty', 'citrus', 'tropical']</td>\n",
       "      <td>fatty;citrus;tropical</td>\n",
       "      <td>unstable</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>356</th>\n",
       "      <td>Hydrocarbon odor (gasoline-like)</td>\n",
       "      <td>['gasoline']</td>\n",
       "      <td>gasoline</td>\n",
       "      <td>hydrocarbon like odor gasoline</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4837</th>\n",
       "      <td>Mild salty taste; Somewhat fishy, ammonia odor</td>\n",
       "      <td>['fishy']</td>\n",
       "      <td>fishy</td>\n",
       "      <td>taste mild salty</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6736</th>\n",
       "      <td>Strong, offensive fecal odor; diluted - floral...</td>\n",
       "      <td>['fruity', 'animal', 'floral']</td>\n",
       "      <td>fruity;animal;floral</td>\n",
       "      <td>strong offensive odor fecal</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7363</th>\n",
       "      <td>Powerful; burnt, coffee-like, somewhat caramel...</td>\n",
       "      <td>['alliaceous', 'burnt', 'caramellic', 'coffee']</td>\n",
       "      <td>alliaceous;burnt;caramellic;coffee</td>\n",
       "      <td>powerful</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7438</th>\n",
       "      <td>Caraway-like; sweet spearmint-like odor</td>\n",
       "      <td>['sweet']</td>\n",
       "      <td>sweet</td>\n",
       "      <td>caraway like</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7894</th>\n",
       "      <td>Ammoniacal odor; fishy taste, on dilution a bl...</td>\n",
       "      <td>['cheesy']</td>\n",
       "      <td>cheesy</td>\n",
       "      <td>ammoniacal odor</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8141</th>\n",
       "      <td>Hydrocarbon odor (gasoline-like)</td>\n",
       "      <td>['gasoline']</td>\n",
       "      <td>gasoline</td>\n",
       "      <td>hydrocarbon like odor gasoline</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11507</th>\n",
       "      <td>Hydrocarbon odor</td>\n",
       "      <td>['gasoline']</td>\n",
       "      <td>gasoline</td>\n",
       "      <td>hydrocarbon odor</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11511</th>\n",
       "      <td>Hydrocarbon odor</td>\n",
       "      <td>['gasoline']</td>\n",
       "      <td>gasoline</td>\n",
       "      <td>hydrocarbon odor</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11551</th>\n",
       "      <td>Hydrocarbon odor, mild gasoline</td>\n",
       "      <td>['gasoline']</td>\n",
       "      <td>gasoline</td>\n",
       "      <td>hydrocarbon mild odor gasoline</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12388</th>\n",
       "      <td>Hydrocarbon odor</td>\n",
       "      <td>['gasoline']</td>\n",
       "      <td>gasoline</td>\n",
       "      <td>hydrocarbon odor</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12389</th>\n",
       "      <td>Mild hydrocarbon odor</td>\n",
       "      <td>['gasoline']</td>\n",
       "      <td>gasoline</td>\n",
       "      <td>hydrocarbon mild odor</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12507</th>\n",
       "      <td>A perspiration, delicatessen odor; contributes...</td>\n",
       "      <td>['rose']</td>\n",
       "      <td>rose</td>\n",
       "      <td>a odor perspiration delicatessen</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14928</th>\n",
       "      <td>Emulsifier; Faint, oily-waxy odor; bitter tast...</td>\n",
       "      <td>['oily', 'waxy']</td>\n",
       "      <td>oily;waxy</td>\n",
       "      <td>emulsifier</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15600</th>\n",
       "      <td>Hydrocarbon odor (gasoline-like)</td>\n",
       "      <td>['gasoline']</td>\n",
       "      <td>gasoline</td>\n",
       "      <td>hydrocarbon like odor gasoline</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16925</th>\n",
       "      <td>Astringent; fatty; earthy at 25 ppm in coffee;...</td>\n",
       "      <td>['fatty', 'roasted']</td>\n",
       "      <td>fatty;roasted</td>\n",
       "      <td>astringent</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20499</th>\n",
       "      <td>Powerful mercaptan-like; In dilution roasted, ...</td>\n",
       "      <td>['sulfurous', 'roasted', 'coffee', 'meaty']</td>\n",
       "      <td>sulfurous;roasted;coffee;meaty</td>\n",
       "      <td>like mercaptan powerful</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31285</th>\n",
       "      <td>Strong odor, hydrocarbon odor, gasoline odor</td>\n",
       "      <td>['gasoline']</td>\n",
       "      <td>gasoline</td>\n",
       "      <td>hydrocarbon strong odor gasoline</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61041</th>\n",
       "      <td>Powerful saffron aroma and taste; tobacco-camp...</td>\n",
       "      <td>['tobacco', 'camphoreous']</td>\n",
       "      <td>tobacco;camphoreous</td>\n",
       "      <td>taste saffron aroma powerful and</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61661</th>\n",
       "      <td>Strong mustard taste at 10 ppm in sugar syrup;...</td>\n",
       "      <td>['coffee']</td>\n",
       "      <td>coffee</td>\n",
       "      <td>taste sugar syrup in mustard 10 ppm at strong</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61982</th>\n",
       "      <td>\"Catty\" aroma; dull fruity black currant flavor</td>\n",
       "      <td>['catty']</td>\n",
       "      <td>catty</td>\n",
       "      <td>aroma catty</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79803</th>\n",
       "      <td>Strong amine odor; in dilution shrimp meat lik...</td>\n",
       "      <td>['meaty', 'fishy', 'cheesy', 'buttery']</td>\n",
       "      <td>meaty;fishy;cheesy;buttery</td>\n",
       "      <td>strong amine odor</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121677</th>\n",
       "      <td>Petrol hydrocarbon note; woody, somewhat licorice</td>\n",
       "      <td>['woody', 'gasoline']</td>\n",
       "      <td>woody;gasoline</td>\n",
       "      <td>hydrocarbon note petrol</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175606</th>\n",
       "      <td>Buchu like note; catty ribes note</td>\n",
       "      <td>['catty']</td>\n",
       "      <td>catty</td>\n",
       "      <td>buchu note like</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>520990</th>\n",
       "      <td>Diphenyloxide-like at 10 ppm in sugar syrup; s...</td>\n",
       "      <td>['green', 'sulfurous']</td>\n",
       "      <td>green;sulfurous</td>\n",
       "      <td>sugar like in syrup 10 ppm at diphenyloxide</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>526195</th>\n",
       "      <td>Blackcurrant; catty-sulfur note of Black curra...</td>\n",
       "      <td>['sulfurous', 'black currant', 'catty']</td>\n",
       "      <td>sulfurous;black currant;catty</td>\n",
       "      <td>blackcurrant</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5362572</th>\n",
       "      <td>\"Grilled\" note; roasted, animal, meat-like, sw...</td>\n",
       "      <td>['caramellic', 'meaty', 'roasted', 'nutty', 'a...</td>\n",
       "      <td>caramellic;meaty;roasted;nutty;animal</td>\n",
       "      <td>note grilled</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6997350</th>\n",
       "      <td>Powerful goaty odor; costus root like with dai...</td>\n",
       "      <td>['dairy', 'cheesy']</td>\n",
       "      <td>dairy;cheesy</td>\n",
       "      <td>odor powerful goaty</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11030188</th>\n",
       "      <td>Reminiscent of Spearmint; more ethereal than c...</td>\n",
       "      <td>['ethereal']</td>\n",
       "      <td>ethereal</td>\n",
       "      <td>spearmint reminiscent of</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15662052</th>\n",
       "      <td>Flavor modifier; may add caramellic notes to v...</td>\n",
       "      <td>['caramellic', 'vanilla']</td>\n",
       "      <td>caramellic;vanilla</td>\n",
       "      <td>modifier flavor</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71587434</th>\n",
       "      <td>Powerful shellfish seafood note on dilution; r...</td>\n",
       "      <td>['meaty', 'roasted']</td>\n",
       "      <td>meaty;roasted</td>\n",
       "      <td>seafood note on shellfish powerful dilution</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      Raw Labels   \n",
       "Stimulus                                                           \n",
       "-862841422647                    Catty urine; cassis in dilution  \\\n",
       "-848964121442           See comments; components of Milk lactone   \n",
       "-86422482515                   Unstable; citrus, fatty, tropical   \n",
       " 356                            Hydrocarbon odor (gasoline-like)   \n",
       " 4837             Mild salty taste; Somewhat fishy, ammonia odor   \n",
       " 6736          Strong, offensive fecal odor; diluted - floral...   \n",
       " 7363          Powerful; burnt, coffee-like, somewhat caramel...   \n",
       " 7438                    Caraway-like; sweet spearmint-like odor   \n",
       " 7894          Ammoniacal odor; fishy taste, on dilution a bl...   \n",
       " 8141                           Hydrocarbon odor (gasoline-like)   \n",
       " 11507                                          Hydrocarbon odor   \n",
       " 11511                                          Hydrocarbon odor   \n",
       " 11551                           Hydrocarbon odor, mild gasoline   \n",
       " 12388                                          Hydrocarbon odor   \n",
       " 12389                                     Mild hydrocarbon odor   \n",
       " 12507         A perspiration, delicatessen odor; contributes...   \n",
       " 14928         Emulsifier; Faint, oily-waxy odor; bitter tast...   \n",
       " 15600                          Hydrocarbon odor (gasoline-like)   \n",
       " 16925         Astringent; fatty; earthy at 25 ppm in coffee;...   \n",
       " 20499         Powerful mercaptan-like; In dilution roasted, ...   \n",
       " 31285              Strong odor, hydrocarbon odor, gasoline odor   \n",
       " 61041         Powerful saffron aroma and taste; tobacco-camp...   \n",
       " 61661         Strong mustard taste at 10 ppm in sugar syrup;...   \n",
       " 61982           \"Catty\" aroma; dull fruity black currant flavor   \n",
       " 79803         Strong amine odor; in dilution shrimp meat lik...   \n",
       " 121677        Petrol hydrocarbon note; woody, somewhat licorice   \n",
       " 175606                        Buchu like note; catty ribes note   \n",
       " 520990        Diphenyloxide-like at 10 ppm in sugar syrup; s...   \n",
       " 526195        Blackcurrant; catty-sulfur note of Black curra...   \n",
       " 5362572       \"Grilled\" note; roasted, animal, meat-like, sw...   \n",
       " 6997350       Powerful goaty odor; costus root like with dai...   \n",
       " 11030188      Reminiscent of Spearmint; more ethereal than c...   \n",
       " 15662052      Flavor modifier; may add caramellic notes to v...   \n",
       " 71587434      Powerful shellfish seafood note on dilution; r...   \n",
       "\n",
       "                                                          Labels   \n",
       "Stimulus                                                           \n",
       "-862841422647                                          ['catty']  \\\n",
       "-848964121442                                          ['milky']   \n",
       "-86422482515                     ['fatty', 'citrus', 'tropical']   \n",
       " 356                                                ['gasoline']   \n",
       " 4837                                                  ['fishy']   \n",
       " 6736                             ['fruity', 'animal', 'floral']   \n",
       " 7363            ['alliaceous', 'burnt', 'caramellic', 'coffee']   \n",
       " 7438                                                  ['sweet']   \n",
       " 7894                                                 ['cheesy']   \n",
       " 8141                                               ['gasoline']   \n",
       " 11507                                              ['gasoline']   \n",
       " 11511                                              ['gasoline']   \n",
       " 11551                                              ['gasoline']   \n",
       " 12388                                              ['gasoline']   \n",
       " 12389                                              ['gasoline']   \n",
       " 12507                                                  ['rose']   \n",
       " 14928                                          ['oily', 'waxy']   \n",
       " 15600                                              ['gasoline']   \n",
       " 16925                                      ['fatty', 'roasted']   \n",
       " 20499               ['sulfurous', 'roasted', 'coffee', 'meaty']   \n",
       " 31285                                              ['gasoline']   \n",
       " 61041                                ['tobacco', 'camphoreous']   \n",
       " 61661                                                ['coffee']   \n",
       " 61982                                                 ['catty']   \n",
       " 79803                   ['meaty', 'fishy', 'cheesy', 'buttery']   \n",
       " 121677                                    ['woody', 'gasoline']   \n",
       " 175606                                                ['catty']   \n",
       " 520990                                   ['green', 'sulfurous']   \n",
       " 526195                  ['sulfurous', 'black currant', 'catty']   \n",
       " 5362572       ['caramellic', 'meaty', 'roasted', 'nutty', 'a...   \n",
       " 6997350                                     ['dairy', 'cheesy']   \n",
       " 11030188                                           ['ethereal']   \n",
       " 15662052                              ['caramellic', 'vanilla']   \n",
       " 71587434                                   ['meaty', 'roasted']   \n",
       "\n",
       "                                         descriptors   \n",
       "Stimulus                                               \n",
       "-862841422647                                  catty  \\\n",
       "-848964121442                                  milky   \n",
       "-86422482515                   fatty;citrus;tropical   \n",
       " 356                                        gasoline   \n",
       " 4837                                          fishy   \n",
       " 6736                           fruity;animal;floral   \n",
       " 7363             alliaceous;burnt;caramellic;coffee   \n",
       " 7438                                          sweet   \n",
       " 7894                                         cheesy   \n",
       " 8141                                       gasoline   \n",
       " 11507                                      gasoline   \n",
       " 11511                                      gasoline   \n",
       " 11551                                      gasoline   \n",
       " 12388                                      gasoline   \n",
       " 12389                                      gasoline   \n",
       " 12507                                          rose   \n",
       " 14928                                     oily;waxy   \n",
       " 15600                                      gasoline   \n",
       " 16925                                 fatty;roasted   \n",
       " 20499                sulfurous;roasted;coffee;meaty   \n",
       " 31285                                      gasoline   \n",
       " 61041                           tobacco;camphoreous   \n",
       " 61661                                        coffee   \n",
       " 61982                                         catty   \n",
       " 79803                    meaty;fishy;cheesy;buttery   \n",
       " 121677                               woody;gasoline   \n",
       " 175606                                        catty   \n",
       " 520990                              green;sulfurous   \n",
       " 526195                sulfurous;black currant;catty   \n",
       " 5362572       caramellic;meaty;roasted;nutty;animal   \n",
       " 6997350                                dairy;cheesy   \n",
       " 11030188                                   ethereal   \n",
       " 15662052                         caramellic;vanilla   \n",
       " 71587434                              meaty;roasted   \n",
       "\n",
       "                                          cleaned raw labels new_descriptors  \n",
       "Stimulus                                                                      \n",
       "-862841422647                                    urine catty            None  \n",
       "-848964121442                                   comments see            None  \n",
       "-86422482515                                        unstable            None  \n",
       " 356                          hydrocarbon like odor gasoline            None  \n",
       " 4837                                       taste mild salty            None  \n",
       " 6736                            strong offensive odor fecal            None  \n",
       " 7363                                               powerful            None  \n",
       " 7438                                           caraway like            None  \n",
       " 7894                                        ammoniacal odor            None  \n",
       " 8141                         hydrocarbon like odor gasoline            None  \n",
       " 11507                                      hydrocarbon odor            None  \n",
       " 11511                                      hydrocarbon odor            None  \n",
       " 11551                        hydrocarbon mild odor gasoline            None  \n",
       " 12388                                      hydrocarbon odor            None  \n",
       " 12389                                 hydrocarbon mild odor            None  \n",
       " 12507                      a odor perspiration delicatessen            None  \n",
       " 14928                                            emulsifier            None  \n",
       " 15600                        hydrocarbon like odor gasoline            None  \n",
       " 16925                                            astringent            None  \n",
       " 20499                               like mercaptan powerful            None  \n",
       " 31285                      hydrocarbon strong odor gasoline            None  \n",
       " 61041                      taste saffron aroma powerful and            None  \n",
       " 61661         taste sugar syrup in mustard 10 ppm at strong            None  \n",
       " 61982                                           aroma catty            None  \n",
       " 79803                                     strong amine odor            None  \n",
       " 121677                              hydrocarbon note petrol            None  \n",
       " 175606                                      buchu note like            None  \n",
       " 520990          sugar like in syrup 10 ppm at diphenyloxide            None  \n",
       " 526195                                         blackcurrant            None  \n",
       " 5362572                                        note grilled            None  \n",
       " 6997350                                 odor powerful goaty            None  \n",
       " 11030188                           spearmint reminiscent of            None  \n",
       " 15662052                                    modifier flavor            None  \n",
       " 71587434        seafood note on shellfish powerful dilution            None  "
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "behavior_sparse[behavior_sparse['new_descriptors'].isna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_desc(descriptors, new_descriptors):\n",
    "    desc_set = set()\n",
    "    if new_descriptors is None:\n",
    "        return descriptors\n",
    "    else:\n",
    "        concat_desc = descriptors+';'+new_descriptors\n",
    "        desc_set.update(concat_desc.split(';'))\n",
    "    return ';'.join(desc_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'alliaceous;burnt;caramellic;coffee'"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "update_desc('alliaceous;burnt;caramellic;coffee', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'tropical;green;fruity;apple'"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "update_desc('green;fruity;apple;tropical', 'green;fruity;apple')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "behavior_sparse['new_descriptors'] = behavior_sparse.apply(lambda x: update_desc(x['descriptors'], x['new_descriptors']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Raw Labels</th>\n",
       "      <th>Labels</th>\n",
       "      <th>descriptors</th>\n",
       "      <th>cleaned raw labels</th>\n",
       "      <th>new_descriptors</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Stimulus</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-955348933095</th>\n",
       "      <td>Herbal-green, waxy, oily, fruity</td>\n",
       "      <td>['green', 'oily', 'fruity', 'waxy', 'herbal']</td>\n",
       "      <td>green;oily;fruity;waxy;herbal</td>\n",
       "      <td>oily fruity waxy herbal green</td>\n",
       "      <td>oily;fruity;waxy;herbal;green</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-923209957509</th>\n",
       "      <td>Herbaceous, woody, slight spicy fruity odor</td>\n",
       "      <td>['woody', 'spicy', 'fruity', 'herbal']</td>\n",
       "      <td>woody;spicy;fruity;herbal</td>\n",
       "      <td>fruity woody spicy slight odor herbaceous</td>\n",
       "      <td>herbal;spicy;fruity;woody</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-874408321546</th>\n",
       "      <td>delta-decalactone precursor; peach, apricot, b...</td>\n",
       "      <td>['peach', 'apricot', 'buttery']</td>\n",
       "      <td>peach;apricot;buttery</td>\n",
       "      <td>precursor lactonic delta</td>\n",
       "      <td>peach;lactonic;buttery;apricot</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-873963935677</th>\n",
       "      <td>Green, fruity, apple-like</td>\n",
       "      <td>['green', 'fruity', 'apple', 'tropical']</td>\n",
       "      <td>green;fruity;apple;tropical</td>\n",
       "      <td>like green fruity apple</td>\n",
       "      <td>tropical;green;fruity;apple</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-862841422647</th>\n",
       "      <td>Catty urine; cassis in dilution</td>\n",
       "      <td>['catty']</td>\n",
       "      <td>catty</td>\n",
       "      <td>urine catty</td>\n",
       "      <td>catty</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154241350</th>\n",
       "      <td>Green, allium (onion, garden cress), savory</td>\n",
       "      <td>['green', 'alliaceous', 'savory', 'onion']</td>\n",
       "      <td>green;alliaceous;savory;onion</td>\n",
       "      <td>allium cress savory onion garden green</td>\n",
       "      <td>savory;green;alliaceous;onion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155323379</th>\n",
       "      <td>Sulfury, alliaceous, roasted, savory, truffle</td>\n",
       "      <td>['roasted', 'alliaceous', 'savory', 'sulfurous']</td>\n",
       "      <td>roasted;alliaceous;savory;sulfurous</td>\n",
       "      <td>roasted savory alliaceous sulfurous truffle</td>\n",
       "      <td>savory;alliaceous;roasted;sulfurous</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158699780</th>\n",
       "      <td>Sulfuraceous, tropical-over-ripe fruity, straw...</td>\n",
       "      <td>['ripe', 'creamy', 'fruity', 'tropical', 'stra...</td>\n",
       "      <td>ripe;creamy;fruity;tropical;strawberry;sulfuro...</td>\n",
       "      <td>cream tropical fruity ripe sulfurous notes str...</td>\n",
       "      <td>creamy;tropical;cheesy;fruity;ripe;sulfurous;s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161504856</th>\n",
       "      <td>Green, tropical fruiy with waxy melon aspects</td>\n",
       "      <td>['green', 'fruity', 'melon', 'tropical', 'waxy']</td>\n",
       "      <td>green;fruity;melon;tropical;waxy</td>\n",
       "      <td>tropical with fruity waxy melon aspects green</td>\n",
       "      <td>tropical;fruity;waxy;melon;green</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162353069</th>\n",
       "      <td>Green fruity, allium (onion, garlic); see comm...</td>\n",
       "      <td>['green', 'fruity', 'alliaceous', 'garlic', 'o...</td>\n",
       "      <td>green;fruity;alliaceous;garlic;onion</td>\n",
       "      <td>allium fruity onion garlic green</td>\n",
       "      <td>alliaceous;fruity;onion;garlic;green</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3522 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      Raw Labels   \n",
       "Stimulus                                                           \n",
       "-955348933095                   Herbal-green, waxy, oily, fruity  \\\n",
       "-923209957509        Herbaceous, woody, slight spicy fruity odor   \n",
       "-874408321546  delta-decalactone precursor; peach, apricot, b...   \n",
       "-873963935677                          Green, fruity, apple-like   \n",
       "-862841422647                    Catty urine; cassis in dilution   \n",
       "...                                                          ...   \n",
       " 154241350           Green, allium (onion, garden cress), savory   \n",
       " 155323379         Sulfury, alliaceous, roasted, savory, truffle   \n",
       " 158699780     Sulfuraceous, tropical-over-ripe fruity, straw...   \n",
       " 161504856         Green, tropical fruiy with waxy melon aspects   \n",
       " 162353069     Green fruity, allium (onion, garlic); see comm...   \n",
       "\n",
       "                                                          Labels   \n",
       "Stimulus                                                           \n",
       "-955348933095      ['green', 'oily', 'fruity', 'waxy', 'herbal']  \\\n",
       "-923209957509             ['woody', 'spicy', 'fruity', 'herbal']   \n",
       "-874408321546                    ['peach', 'apricot', 'buttery']   \n",
       "-873963935677           ['green', 'fruity', 'apple', 'tropical']   \n",
       "-862841422647                                          ['catty']   \n",
       "...                                                          ...   \n",
       " 154241350            ['green', 'alliaceous', 'savory', 'onion']   \n",
       " 155323379      ['roasted', 'alliaceous', 'savory', 'sulfurous']   \n",
       " 158699780     ['ripe', 'creamy', 'fruity', 'tropical', 'stra...   \n",
       " 161504856      ['green', 'fruity', 'melon', 'tropical', 'waxy']   \n",
       " 162353069     ['green', 'fruity', 'alliaceous', 'garlic', 'o...   \n",
       "\n",
       "                                                     descriptors   \n",
       "Stimulus                                                           \n",
       "-955348933095                      green;oily;fruity;waxy;herbal  \\\n",
       "-923209957509                          woody;spicy;fruity;herbal   \n",
       "-874408321546                              peach;apricot;buttery   \n",
       "-873963935677                        green;fruity;apple;tropical   \n",
       "-862841422647                                              catty   \n",
       "...                                                          ...   \n",
       " 154241350                         green;alliaceous;savory;onion   \n",
       " 155323379                   roasted;alliaceous;savory;sulfurous   \n",
       " 158699780     ripe;creamy;fruity;tropical;strawberry;sulfuro...   \n",
       " 161504856                      green;fruity;melon;tropical;waxy   \n",
       " 162353069                  green;fruity;alliaceous;garlic;onion   \n",
       "\n",
       "                                              cleaned raw labels   \n",
       "Stimulus                                                           \n",
       "-955348933095                      oily fruity waxy herbal green  \\\n",
       "-923209957509          fruity woody spicy slight odor herbaceous   \n",
       "-874408321546                           precursor lactonic delta   \n",
       "-873963935677                            like green fruity apple   \n",
       "-862841422647                                        urine catty   \n",
       "...                                                          ...   \n",
       " 154241350                allium cress savory onion garden green   \n",
       " 155323379           roasted savory alliaceous sulfurous truffle   \n",
       " 158699780     cream tropical fruity ripe sulfurous notes str...   \n",
       " 161504856         tropical with fruity waxy melon aspects green   \n",
       " 162353069                      allium fruity onion garlic green   \n",
       "\n",
       "                                                 new_descriptors  \n",
       "Stimulus                                                          \n",
       "-955348933095                      oily;fruity;waxy;herbal;green  \n",
       "-923209957509                          herbal;spicy;fruity;woody  \n",
       "-874408321546                     peach;lactonic;buttery;apricot  \n",
       "-873963935677                        tropical;green;fruity;apple  \n",
       "-862841422647                                              catty  \n",
       "...                                                          ...  \n",
       " 154241350                         savory;green;alliaceous;onion  \n",
       " 155323379                   savory;alliaceous;roasted;sulfurous  \n",
       " 158699780     creamy;tropical;cheesy;fruity;ripe;sulfurous;s...  \n",
       " 161504856                      tropical;fruity;waxy;melon;green  \n",
       " 162353069                  alliaceous;fruity;onion;garlic;green  \n",
       "\n",
       "[3522 rows x 5 columns]"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "behavior_sparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Raw Labels</th>\n",
       "      <th>Labels</th>\n",
       "      <th>descriptors</th>\n",
       "      <th>cleaned raw labels</th>\n",
       "      <th>new_descriptors</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Stimulus</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>Nearly odorless; slight tingling taste; sweetn...</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless nearly</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>Pleasant sour taste; odorless</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>taste pleasant sour</td>\n",
       "      <td>odorless;sour</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>311</th>\n",
       "      <td>Odorless with a clean, acid, tart taste</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>taste odorless with clean a acid tart</td>\n",
       "      <td>odorless;clean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>453</th>\n",
       "      <td>Odorless with a pleasant, sweet taste</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>taste odorless pleasant with a sweet</td>\n",
       "      <td>odorless;sweet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>525</th>\n",
       "      <td>Almost odorless with a sharp tart acidic taste</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>taste almost odorless acidic with a tart sharp</td>\n",
       "      <td>odorless;sharp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54675810</th>\n",
       "      <td>Odorless with a clean acidulous taste</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>taste odorless with acidulous clean a</td>\n",
       "      <td>odorless;clean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54676860</th>\n",
       "      <td>Odorless with a sharp, pleasant acid taste</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>taste odorless pleasant with a acid sharp</td>\n",
       "      <td>odorless;sharp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57353225</th>\n",
       "      <td>Nearly odorless; Delicate floral waxy with ros...</td>\n",
       "      <td>['rose', 'odorless', 'waxy', 'floral']</td>\n",
       "      <td>rose;odorless;waxy;floral</td>\n",
       "      <td>odorless nearly</td>\n",
       "      <td>odorless;rose;waxy;floral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67120007</th>\n",
       "      <td>Virtually odorless, waxy</td>\n",
       "      <td>['waxy', 'odorless']</td>\n",
       "      <td>waxy;odorless</td>\n",
       "      <td>odorless waxy virtually</td>\n",
       "      <td>odorless;waxy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67253117</th>\n",
       "      <td>Nearly odorless, soft waxy, slight balsamic.</td>\n",
       "      <td>['balsamic', 'waxy', 'odorless']</td>\n",
       "      <td>balsamic;waxy;odorless</td>\n",
       "      <td>odorless balsamic waxy soft slight nearly</td>\n",
       "      <td>balsamic;waxy;odorless</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>71 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Raw Labels   \n",
       "Stimulus                                                      \n",
       "135       Nearly odorless; slight tingling taste; sweetn...  \\\n",
       "196                           Pleasant sour taste; odorless   \n",
       "311                 Odorless with a clean, acid, tart taste   \n",
       "453                   Odorless with a pleasant, sweet taste   \n",
       "525          Almost odorless with a sharp tart acidic taste   \n",
       "...                                                     ...   \n",
       "54675810              Odorless with a clean acidulous taste   \n",
       "54676860         Odorless with a sharp, pleasant acid taste   \n",
       "57353225  Nearly odorless; Delicate floral waxy with ros...   \n",
       "67120007                           Virtually odorless, waxy   \n",
       "67253117       Nearly odorless, soft waxy, slight balsamic.   \n",
       "\n",
       "                                          Labels                descriptors   \n",
       "Stimulus                                                                      \n",
       "135                                 ['odorless']                   odorless  \\\n",
       "196                                 ['odorless']                   odorless   \n",
       "311                                 ['odorless']                   odorless   \n",
       "453                                 ['odorless']                   odorless   \n",
       "525                                 ['odorless']                   odorless   \n",
       "...                                          ...                        ...   \n",
       "54675810                            ['odorless']                   odorless   \n",
       "54676860                            ['odorless']                   odorless   \n",
       "57353225  ['rose', 'odorless', 'waxy', 'floral']  rose;odorless;waxy;floral   \n",
       "67120007                    ['waxy', 'odorless']              waxy;odorless   \n",
       "67253117        ['balsamic', 'waxy', 'odorless']     balsamic;waxy;odorless   \n",
       "\n",
       "                                      cleaned raw labels   \n",
       "Stimulus                                                   \n",
       "135                                      odorless nearly  \\\n",
       "196                                  taste pleasant sour   \n",
       "311                taste odorless with clean a acid tart   \n",
       "453                 taste odorless pleasant with a sweet   \n",
       "525       taste almost odorless acidic with a tart sharp   \n",
       "...                                                  ...   \n",
       "54675810           taste odorless with acidulous clean a   \n",
       "54676860       taste odorless pleasant with a acid sharp   \n",
       "57353225                                 odorless nearly   \n",
       "67120007                         odorless waxy virtually   \n",
       "67253117       odorless balsamic waxy soft slight nearly   \n",
       "\n",
       "                    new_descriptors  \n",
       "Stimulus                             \n",
       "135                        odorless  \n",
       "196                   odorless;sour  \n",
       "311                  odorless;clean  \n",
       "453                  odorless;sweet  \n",
       "525                  odorless;sharp  \n",
       "...                             ...  \n",
       "54675810             odorless;clean  \n",
       "54676860             odorless;sharp  \n",
       "57353225  odorless;rose;waxy;floral  \n",
       "67120007              odorless;waxy  \n",
       "67253117     balsamic;waxy;odorless  \n",
       "\n",
       "[71 rows x 5 columns]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "behavior_sparse[behavior_sparse['new_descriptors'].str.match('.*odorless.*')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "# handle odorless\n",
    "def handle_odorless(desc_string):\n",
    "    desc_list = desc_string.split(';')\n",
    "    desc_set = set()\n",
    "    for desc in desc_list:\n",
    "        if desc == 'odorless':\n",
    "            continue\n",
    "        desc_set.update([desc])\n",
    "    if len(desc_set) == 0:\n",
    "        desc_set.update(['odorless'])\n",
    "    return ';'.join(desc_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'herbal;spicy;fruity;woody'"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "handle_odorless('fruity;spicy;odorless;herbal;woody')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'odorless'"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "handle_odorless('odorless')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "behavior_sparse['new_descriptors'] = behavior_sparse['new_descriptors'].apply(lambda x: handle_odorless(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Raw Labels</th>\n",
       "      <th>Labels</th>\n",
       "      <th>descriptors</th>\n",
       "      <th>cleaned raw labels</th>\n",
       "      <th>new_descriptors</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Stimulus</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>Nearly odorless; slight tingling taste; sweetn...</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless nearly</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>594</th>\n",
       "      <td>Odorless; sour taste; used in reaction flavors...</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>611</th>\n",
       "      <td>Odorless, flavor enhancer, umami character</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>umami odorless flavor character enhancer</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>757</th>\n",
       "      <td>Nearly odorless, acidic tart taste (see comments)</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>taste comments odorless see acidic nearly tart</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875</th>\n",
       "      <td>Odorless with a strong tart acid taste</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>taste odorless with a acid strong tart</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>985</th>\n",
       "      <td>Almost odorless; bland waxy taste; adds mouthfeel</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless almost</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1182</th>\n",
       "      <td>Almost odorless; sweet, bread-like taste</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless almost</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1491</th>\n",
       "      <td>Nearly odorless; sweetness modifier for sucros...</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless nearly</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2682</th>\n",
       "      <td>Odorless; faint, bland taste; adds mouthfeel t...</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4947</th>\n",
       "      <td>Almost odorless when pure</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>when odorless almost pure</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5054</th>\n",
       "      <td>Nearly odorless if pure; creamy, vanilla, swee...</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless pure nearly if</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5143</th>\n",
       "      <td>Nearly odorless; intense sweet taste, slight b...</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless nearly</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5810</th>\n",
       "      <td>Virtually odorless; possibly used in reaction ...</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless virtually</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6306</th>\n",
       "      <td>Essentially odorless; slightly bitter taste; u...</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless essentially</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6507</th>\n",
       "      <td>Virtually odorless; used as a solvent</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless virtually</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6919</th>\n",
       "      <td>Nearly odorless; clean sour taste with caramel...</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless nearly</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6970</th>\n",
       "      <td>Almost odorless; slighly medicinal, bitter-ast...</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless almost</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7175</th>\n",
       "      <td>Almost odorless and tasteless; antimicrobial p...</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless tasteless almost and</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7184</th>\n",
       "      <td>Almost odorless and tasteless; antimicrobial p...</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless tasteless almost and</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8201</th>\n",
       "      <td>Almost odorless; oily-waxy; adds mouthfeel</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless almost</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11370</th>\n",
       "      <td>Nearly odorless; weak, burning taste</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless nearly</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12588</th>\n",
       "      <td>Vitually odorless when pure; mild waxy-fatty f...</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>when odorless vitually pure</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14878</th>\n",
       "      <td>Almost odorless when pure; an emulsifier</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>when odorless almost pure</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15076</th>\n",
       "      <td>Nearly odorless; faint fatty, bland taste; add...</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless nearly</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66967</th>\n",
       "      <td>Essentially odorless; Mild fatty-waxy taste</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless essentially</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75232</th>\n",
       "      <td>Nearly odorless; adds mouthfeel</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless nearly</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92972</th>\n",
       "      <td>Odorless; bitter tonic taste at 100 ppm</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94206</th>\n",
       "      <td>Essentially odorless; Used in reaction flavors</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless essentially</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170623</th>\n",
       "      <td>Weak, nearly odorless; mild slightly fatty-wax...</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless nearly weak</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>638024</th>\n",
       "      <td>Odorless; with a delayed biting and burning taste</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643460</th>\n",
       "      <td>Odorless with a bland, acidulous taste</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>taste bland odorless with acidulous a</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10199004</th>\n",
       "      <td>Virtually odorless; Lingering coolness, good m...</td>\n",
       "      <td>['odorless']</td>\n",
       "      <td>odorless</td>\n",
       "      <td>odorless virtually</td>\n",
       "      <td>odorless</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Raw Labels        Labels   \n",
       "Stimulus                                                                    \n",
       "135       Nearly odorless; slight tingling taste; sweetn...  ['odorless']  \\\n",
       "594       Odorless; sour taste; used in reaction flavors...  ['odorless']   \n",
       "611              Odorless, flavor enhancer, umami character  ['odorless']   \n",
       "757       Nearly odorless, acidic tart taste (see comments)  ['odorless']   \n",
       "875                  Odorless with a strong tart acid taste  ['odorless']   \n",
       "985       Almost odorless; bland waxy taste; adds mouthfeel  ['odorless']   \n",
       "1182               Almost odorless; sweet, bread-like taste  ['odorless']   \n",
       "1491      Nearly odorless; sweetness modifier for sucros...  ['odorless']   \n",
       "2682      Odorless; faint, bland taste; adds mouthfeel t...  ['odorless']   \n",
       "4947                              Almost odorless when pure  ['odorless']   \n",
       "5054      Nearly odorless if pure; creamy, vanilla, swee...  ['odorless']   \n",
       "5143      Nearly odorless; intense sweet taste, slight b...  ['odorless']   \n",
       "5810      Virtually odorless; possibly used in reaction ...  ['odorless']   \n",
       "6306      Essentially odorless; slightly bitter taste; u...  ['odorless']   \n",
       "6507                  Virtually odorless; used as a solvent  ['odorless']   \n",
       "6919      Nearly odorless; clean sour taste with caramel...  ['odorless']   \n",
       "6970      Almost odorless; slighly medicinal, bitter-ast...  ['odorless']   \n",
       "7175      Almost odorless and tasteless; antimicrobial p...  ['odorless']   \n",
       "7184      Almost odorless and tasteless; antimicrobial p...  ['odorless']   \n",
       "8201             Almost odorless; oily-waxy; adds mouthfeel  ['odorless']   \n",
       "11370                  Nearly odorless; weak, burning taste  ['odorless']   \n",
       "12588     Vitually odorless when pure; mild waxy-fatty f...  ['odorless']   \n",
       "14878              Almost odorless when pure; an emulsifier  ['odorless']   \n",
       "15076     Nearly odorless; faint fatty, bland taste; add...  ['odorless']   \n",
       "66967           Essentially odorless; Mild fatty-waxy taste  ['odorless']   \n",
       "75232                       Nearly odorless; adds mouthfeel  ['odorless']   \n",
       "92972               Odorless; bitter tonic taste at 100 ppm  ['odorless']   \n",
       "94206        Essentially odorless; Used in reaction flavors  ['odorless']   \n",
       "170623    Weak, nearly odorless; mild slightly fatty-wax...  ['odorless']   \n",
       "638024    Odorless; with a delayed biting and burning taste  ['odorless']   \n",
       "643460               Odorless with a bland, acidulous taste  ['odorless']   \n",
       "10199004  Virtually odorless; Lingering coolness, good m...  ['odorless']   \n",
       "\n",
       "         descriptors                              cleaned raw labels   \n",
       "Stimulus                                                               \n",
       "135         odorless                                 odorless nearly  \\\n",
       "594         odorless                                        odorless   \n",
       "611         odorless        umami odorless flavor character enhancer   \n",
       "757         odorless  taste comments odorless see acidic nearly tart   \n",
       "875         odorless          taste odorless with a acid strong tart   \n",
       "985         odorless                                 odorless almost   \n",
       "1182        odorless                                 odorless almost   \n",
       "1491        odorless                                 odorless nearly   \n",
       "2682        odorless                                        odorless   \n",
       "4947        odorless                       when odorless almost pure   \n",
       "5054        odorless                         odorless pure nearly if   \n",
       "5143        odorless                                 odorless nearly   \n",
       "5810        odorless                              odorless virtually   \n",
       "6306        odorless                            odorless essentially   \n",
       "6507        odorless                              odorless virtually   \n",
       "6919        odorless                                 odorless nearly   \n",
       "6970        odorless                                 odorless almost   \n",
       "7175        odorless                   odorless tasteless almost and   \n",
       "7184        odorless                   odorless tasteless almost and   \n",
       "8201        odorless                                 odorless almost   \n",
       "11370       odorless                                 odorless nearly   \n",
       "12588       odorless                     when odorless vitually pure   \n",
       "14878       odorless                       when odorless almost pure   \n",
       "15076       odorless                                 odorless nearly   \n",
       "66967       odorless                            odorless essentially   \n",
       "75232       odorless                                 odorless nearly   \n",
       "92972       odorless                                        odorless   \n",
       "94206       odorless                            odorless essentially   \n",
       "170623      odorless                            odorless nearly weak   \n",
       "638024      odorless                                        odorless   \n",
       "643460      odorless           taste bland odorless with acidulous a   \n",
       "10199004    odorless                              odorless virtually   \n",
       "\n",
       "         new_descriptors  \n",
       "Stimulus                  \n",
       "135             odorless  \n",
       "594             odorless  \n",
       "611             odorless  \n",
       "757             odorless  \n",
       "875             odorless  \n",
       "985             odorless  \n",
       "1182            odorless  \n",
       "1491            odorless  \n",
       "2682            odorless  \n",
       "4947            odorless  \n",
       "5054            odorless  \n",
       "5143            odorless  \n",
       "5810            odorless  \n",
       "6306            odorless  \n",
       "6507            odorless  \n",
       "6919            odorless  \n",
       "6970            odorless  \n",
       "7175            odorless  \n",
       "7184            odorless  \n",
       "8201            odorless  \n",
       "11370           odorless  \n",
       "12588           odorless  \n",
       "14878           odorless  \n",
       "15076           odorless  \n",
       "66967           odorless  \n",
       "75232           odorless  \n",
       "92972           odorless  \n",
       "94206           odorless  \n",
       "170623          odorless  \n",
       "638024          odorless  \n",
       "643460          odorless  \n",
       "10199004        odorless  "
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "behavior_sparse[behavior_sparse['new_descriptors'].str.match('.*odorless.*')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Raw Labels</th>\n",
       "      <th>Labels</th>\n",
       "      <th>descriptors</th>\n",
       "      <th>cleaned raw labels</th>\n",
       "      <th>new_descriptors</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Stimulus</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-955348933095</th>\n",
       "      <td>Herbal-green, waxy, oily, fruity</td>\n",
       "      <td>['green', 'oily', 'fruity', 'waxy', 'herbal']</td>\n",
       "      <td>green;oily;fruity;waxy;herbal</td>\n",
       "      <td>oily fruity waxy herbal green</td>\n",
       "      <td>oily;fruity;waxy;herbal;green</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-923209957509</th>\n",
       "      <td>Herbaceous, woody, slight spicy fruity odor</td>\n",
       "      <td>['woody', 'spicy', 'fruity', 'herbal']</td>\n",
       "      <td>woody;spicy;fruity;herbal</td>\n",
       "      <td>fruity woody spicy slight odor herbaceous</td>\n",
       "      <td>spicy;fruity;herbal;woody</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-874408321546</th>\n",
       "      <td>delta-decalactone precursor; peach, apricot, b...</td>\n",
       "      <td>['peach', 'apricot', 'buttery']</td>\n",
       "      <td>peach;apricot;buttery</td>\n",
       "      <td>precursor lactonic delta</td>\n",
       "      <td>peach;lactonic;apricot;buttery</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-873963935677</th>\n",
       "      <td>Green, fruity, apple-like</td>\n",
       "      <td>['green', 'fruity', 'apple', 'tropical']</td>\n",
       "      <td>green;fruity;apple;tropical</td>\n",
       "      <td>like green fruity apple</td>\n",
       "      <td>tropical;green;fruity;apple</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-862841422647</th>\n",
       "      <td>Catty urine; cassis in dilution</td>\n",
       "      <td>['catty']</td>\n",
       "      <td>catty</td>\n",
       "      <td>urine catty</td>\n",
       "      <td>catty</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154241350</th>\n",
       "      <td>Green, allium (onion, garden cress), savory</td>\n",
       "      <td>['green', 'alliaceous', 'savory', 'onion']</td>\n",
       "      <td>green;alliaceous;savory;onion</td>\n",
       "      <td>allium cress savory onion garden green</td>\n",
       "      <td>savory;green;alliaceous;onion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155323379</th>\n",
       "      <td>Sulfury, alliaceous, roasted, savory, truffle</td>\n",
       "      <td>['roasted', 'alliaceous', 'savory', 'sulfurous']</td>\n",
       "      <td>roasted;alliaceous;savory;sulfurous</td>\n",
       "      <td>roasted savory alliaceous sulfurous truffle</td>\n",
       "      <td>savory;alliaceous;roasted;sulfurous</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158699780</th>\n",
       "      <td>Sulfuraceous, tropical-over-ripe fruity, straw...</td>\n",
       "      <td>['ripe', 'creamy', 'fruity', 'tropical', 'stra...</td>\n",
       "      <td>ripe;creamy;fruity;tropical;strawberry;sulfuro...</td>\n",
       "      <td>cream tropical fruity ripe sulfurous notes str...</td>\n",
       "      <td>creamy;tropical;cheesy;fruity;ripe;sulfurous;s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161504856</th>\n",
       "      <td>Green, tropical fruiy with waxy melon aspects</td>\n",
       "      <td>['green', 'fruity', 'melon', 'tropical', 'waxy']</td>\n",
       "      <td>green;fruity;melon;tropical;waxy</td>\n",
       "      <td>tropical with fruity waxy melon aspects green</td>\n",
       "      <td>tropical;fruity;waxy;melon;green</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162353069</th>\n",
       "      <td>Green fruity, allium (onion, garlic); see comm...</td>\n",
       "      <td>['green', 'fruity', 'alliaceous', 'garlic', 'o...</td>\n",
       "      <td>green;fruity;alliaceous;garlic;onion</td>\n",
       "      <td>allium fruity onion garlic green</td>\n",
       "      <td>fruity;alliaceous;onion;garlic;green</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3522 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      Raw Labels   \n",
       "Stimulus                                                           \n",
       "-955348933095                   Herbal-green, waxy, oily, fruity  \\\n",
       "-923209957509        Herbaceous, woody, slight spicy fruity odor   \n",
       "-874408321546  delta-decalactone precursor; peach, apricot, b...   \n",
       "-873963935677                          Green, fruity, apple-like   \n",
       "-862841422647                    Catty urine; cassis in dilution   \n",
       "...                                                          ...   \n",
       " 154241350           Green, allium (onion, garden cress), savory   \n",
       " 155323379         Sulfury, alliaceous, roasted, savory, truffle   \n",
       " 158699780     Sulfuraceous, tropical-over-ripe fruity, straw...   \n",
       " 161504856         Green, tropical fruiy with waxy melon aspects   \n",
       " 162353069     Green fruity, allium (onion, garlic); see comm...   \n",
       "\n",
       "                                                          Labels   \n",
       "Stimulus                                                           \n",
       "-955348933095      ['green', 'oily', 'fruity', 'waxy', 'herbal']  \\\n",
       "-923209957509             ['woody', 'spicy', 'fruity', 'herbal']   \n",
       "-874408321546                    ['peach', 'apricot', 'buttery']   \n",
       "-873963935677           ['green', 'fruity', 'apple', 'tropical']   \n",
       "-862841422647                                          ['catty']   \n",
       "...                                                          ...   \n",
       " 154241350            ['green', 'alliaceous', 'savory', 'onion']   \n",
       " 155323379      ['roasted', 'alliaceous', 'savory', 'sulfurous']   \n",
       " 158699780     ['ripe', 'creamy', 'fruity', 'tropical', 'stra...   \n",
       " 161504856      ['green', 'fruity', 'melon', 'tropical', 'waxy']   \n",
       " 162353069     ['green', 'fruity', 'alliaceous', 'garlic', 'o...   \n",
       "\n",
       "                                                     descriptors   \n",
       "Stimulus                                                           \n",
       "-955348933095                      green;oily;fruity;waxy;herbal  \\\n",
       "-923209957509                          woody;spicy;fruity;herbal   \n",
       "-874408321546                              peach;apricot;buttery   \n",
       "-873963935677                        green;fruity;apple;tropical   \n",
       "-862841422647                                              catty   \n",
       "...                                                          ...   \n",
       " 154241350                         green;alliaceous;savory;onion   \n",
       " 155323379                   roasted;alliaceous;savory;sulfurous   \n",
       " 158699780     ripe;creamy;fruity;tropical;strawberry;sulfuro...   \n",
       " 161504856                      green;fruity;melon;tropical;waxy   \n",
       " 162353069                  green;fruity;alliaceous;garlic;onion   \n",
       "\n",
       "                                              cleaned raw labels   \n",
       "Stimulus                                                           \n",
       "-955348933095                      oily fruity waxy herbal green  \\\n",
       "-923209957509          fruity woody spicy slight odor herbaceous   \n",
       "-874408321546                           precursor lactonic delta   \n",
       "-873963935677                            like green fruity apple   \n",
       "-862841422647                                        urine catty   \n",
       "...                                                          ...   \n",
       " 154241350                allium cress savory onion garden green   \n",
       " 155323379           roasted savory alliaceous sulfurous truffle   \n",
       " 158699780     cream tropical fruity ripe sulfurous notes str...   \n",
       " 161504856         tropical with fruity waxy melon aspects green   \n",
       " 162353069                      allium fruity onion garlic green   \n",
       "\n",
       "                                                 new_descriptors  \n",
       "Stimulus                                                          \n",
       "-955348933095                      oily;fruity;waxy;herbal;green  \n",
       "-923209957509                          spicy;fruity;herbal;woody  \n",
       "-874408321546                     peach;lactonic;apricot;buttery  \n",
       "-873963935677                        tropical;green;fruity;apple  \n",
       "-862841422647                                              catty  \n",
       "...                                                          ...  \n",
       " 154241350                         savory;green;alliaceous;onion  \n",
       " 155323379                   savory;alliaceous;roasted;sulfurous  \n",
       " 158699780     creamy;tropical;cheesy;fruity;ripe;sulfurous;s...  \n",
       " 161504856                      tropical;fruity;waxy;melon;green  \n",
       " 162353069                  fruity;alliaceous;onion;garlic;green  \n",
       "\n",
       "[3522 rows x 5 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "behavior_sparse"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>IsomericSMILES</th>\n",
       "      <th>descriptors</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-955348933095</th>\n",
       "      <td>CCCCC=COC(=O)CCCCCCCC</td>\n",
       "      <td>oily;fruity;waxy;herbal;green</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-923209957509</th>\n",
       "      <td>CC(=O)OCC1C=CC(C(C)C)CC1</td>\n",
       "      <td>spicy;fruity;herbal;woody</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-874408321546</th>\n",
       "      <td>CCCCCCCCC(OC(C)=O)C(=O)OC</td>\n",
       "      <td>peach;lactonic;apricot;buttery</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          IsomericSMILES                     descriptors\n",
       "CID                                                                     \n",
       "-955348933095      CCCCC=COC(=O)CCCCCCCC   oily;fruity;waxy;herbal;green\n",
       "-923209957509   CC(=O)OCC1C=CC(C(C)C)CC1       spicy;fruity;herbal;woody\n",
       "-874408321546  CCCCCCCCC(OC(C)=O)C(=O)OC  peach;lactonic;apricot;buttery"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mol_behaviors = pd.merge(molecules['IsomericSMILES'], behavior_sparse['new_descriptors'], how='inner', left_index=True, right_index=True)\n",
    "mol_behaviors = mol_behaviors.rename(columns={'new_descriptors':'descriptors'})\n",
    "mol_behaviors.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 3522 entries, -955348933095 to 162353069\n",
      "Data columns (total 2 columns):\n",
      " #   Column          Non-Null Count  Dtype \n",
      "---  ------          --------------  ----- \n",
      " 0   IsomericSMILES  3522 non-null   object\n",
      " 1   descriptors     3522 non-null   object\n",
      "dtypes: object(2)\n",
      "memory usage: 211.6+ KB\n"
     ]
    }
   ],
   "source": [
    "mol_behaviors.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_odors(desc, odor_list):\n",
    "    odors = desc.split(';')\n",
    "    for i in odors:\n",
    "        if i not in odor_list:\n",
    "            odor_list.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['alcoholic',\n",
       " 'aldehydic',\n",
       " 'alliaceous',\n",
       " 'almond',\n",
       " 'amber',\n",
       " 'animal',\n",
       " 'anisic',\n",
       " 'apple',\n",
       " 'apricot',\n",
       " 'aromatic',\n",
       " 'balsamic',\n",
       " 'banana',\n",
       " 'beefy',\n",
       " 'bergamot',\n",
       " 'berry',\n",
       " 'bitter',\n",
       " 'black currant',\n",
       " 'brandy',\n",
       " 'bread',\n",
       " 'brothy',\n",
       " 'burnt',\n",
       " 'buttery',\n",
       " 'cabbage',\n",
       " 'camphoreous',\n",
       " 'caramellic',\n",
       " 'catty',\n",
       " 'cedar',\n",
       " 'celery',\n",
       " 'chamomile',\n",
       " 'cheesy',\n",
       " 'cherry',\n",
       " 'chicken',\n",
       " 'chocolate',\n",
       " 'cinnamon',\n",
       " 'citrus',\n",
       " 'clean',\n",
       " 'clove',\n",
       " 'cocoa',\n",
       " 'coconut',\n",
       " 'coffee',\n",
       " 'cognac',\n",
       " 'cooked',\n",
       " 'cooling',\n",
       " 'coumarinic',\n",
       " 'creamy',\n",
       " 'cucumber',\n",
       " 'dairy',\n",
       " 'dry',\n",
       " 'earthy',\n",
       " 'ethereal',\n",
       " 'fatty',\n",
       " 'fermented',\n",
       " 'fishy',\n",
       " 'floral',\n",
       " 'fresh',\n",
       " 'fruit skin',\n",
       " 'fruity',\n",
       " 'garlic',\n",
       " 'gasoline',\n",
       " 'gassy',\n",
       " 'geranium',\n",
       " 'grape',\n",
       " 'grapefruit',\n",
       " 'grassy',\n",
       " 'green',\n",
       " 'hawthorn',\n",
       " 'hay',\n",
       " 'hazelnut',\n",
       " 'herbal',\n",
       " 'honey',\n",
       " 'horseradish',\n",
       " 'hyacinth',\n",
       " 'jasmin',\n",
       " 'jasmine',\n",
       " 'juicy',\n",
       " 'ketonic',\n",
       " 'lactonic',\n",
       " 'lavender',\n",
       " 'leafy',\n",
       " 'leathery',\n",
       " 'lemon',\n",
       " 'lily',\n",
       " 'malty',\n",
       " 'meaty',\n",
       " 'medicinal',\n",
       " 'melon',\n",
       " 'metallic',\n",
       " 'milky',\n",
       " 'mint',\n",
       " 'muguet',\n",
       " 'mushroom',\n",
       " 'musk',\n",
       " 'musty',\n",
       " 'natural',\n",
       " 'nutty',\n",
       " 'odorless',\n",
       " 'oily',\n",
       " 'onion',\n",
       " 'orange',\n",
       " 'orangeflower',\n",
       " 'orris',\n",
       " 'ozone',\n",
       " 'peach',\n",
       " 'pear',\n",
       " 'phenolic',\n",
       " 'pine',\n",
       " 'pineapple',\n",
       " 'plum',\n",
       " 'popcorn',\n",
       " 'potato',\n",
       " 'powdery',\n",
       " 'pungent',\n",
       " 'radish',\n",
       " 'raspberry',\n",
       " 'ripe',\n",
       " 'roasted',\n",
       " 'rose',\n",
       " 'rum',\n",
       " 'rummy',\n",
       " 'sandalwood',\n",
       " 'savory',\n",
       " 'sharp',\n",
       " 'smoky',\n",
       " 'soapy',\n",
       " 'solvent',\n",
       " 'sour',\n",
       " 'spicy',\n",
       " 'strawberry',\n",
       " 'sulfurous',\n",
       " 'sweaty',\n",
       " 'sweet',\n",
       " 'tea',\n",
       " 'terpenic',\n",
       " 'tobacco',\n",
       " 'tomato',\n",
       " 'tropical',\n",
       " 'vanilla',\n",
       " 'vegetable',\n",
       " 'vetiver',\n",
       " 'violet',\n",
       " 'warm',\n",
       " 'waxy',\n",
       " 'weedy',\n",
       " 'winey',\n",
       " 'woody']"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "odor_list = []\n",
    "mol_behaviors['descriptors'].apply(lambda x: get_odors(x, odor_list))\n",
    "odor_list.sort()\n",
    "odor_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "145"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(odor_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['cortex']"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "og_columns = odor_list\n",
    "# elements of required desc not in og_columns\n",
    "missing_desc = []\n",
    "for req in required_desc:\n",
    "    if req not in og_columns:\n",
    "        missing_desc.append(req)\n",
    "missing_desc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>desc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>alcoholic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aldehydic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>alliaceous</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>almond</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>amber</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>warm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>waxy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>weedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>winey</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144</th>\n",
       "      <td>woody</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>145 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           desc\n",
       "0     alcoholic\n",
       "1     aldehydic\n",
       "2    alliaceous\n",
       "3        almond\n",
       "4         amber\n",
       "..          ...\n",
       "140        warm\n",
       "141        waxy\n",
       "142       weedy\n",
       "143       winey\n",
       "144       woody\n",
       "\n",
       "[145 rows x 1 columns]"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "odors_df = pd.DataFrame(odor_list, columns=['desc'])\n",
    "odors_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>desc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>floral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>orangeflower</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            desc\n",
       "53        floral\n",
       "99  orangeflower"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "odors_df[odors_df['desc'].str.match('.*flo.*')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CID\n",
       "-955348933095    1\n",
       "-923209957509    1\n",
       "-874408321546    0\n",
       "-873963935677    1\n",
       "-862841422647    0\n",
       "Name: fruity, dtype: int64"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "odor_dummies = mol_behaviors['descriptors'].str.get_dummies(sep=';')\n",
    "odor_dummies['fruity'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "mol_behaviors_encoded = pd.concat([mol_behaviors, odor_dummies], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>IsomericSMILES</th>\n",
       "      <th>descriptors</th>\n",
       "      <th>alcoholic</th>\n",
       "      <th>aldehydic</th>\n",
       "      <th>alliaceous</th>\n",
       "      <th>almond</th>\n",
       "      <th>amber</th>\n",
       "      <th>animal</th>\n",
       "      <th>anisic</th>\n",
       "      <th>apple</th>\n",
       "      <th>...</th>\n",
       "      <th>tropical</th>\n",
       "      <th>vanilla</th>\n",
       "      <th>vegetable</th>\n",
       "      <th>vetiver</th>\n",
       "      <th>violet</th>\n",
       "      <th>warm</th>\n",
       "      <th>waxy</th>\n",
       "      <th>weedy</th>\n",
       "      <th>winey</th>\n",
       "      <th>woody</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-955348933095</th>\n",
       "      <td>CCCCC=COC(=O)CCCCCCCC</td>\n",
       "      <td>oily;fruity;waxy;herbal;green</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-923209957509</th>\n",
       "      <td>CC(=O)OCC1C=CC(C(C)C)CC1</td>\n",
       "      <td>spicy;fruity;herbal;woody</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-874408321546</th>\n",
       "      <td>CCCCCCCCC(OC(C)=O)C(=O)OC</td>\n",
       "      <td>peach;lactonic;apricot;buttery</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 147 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          IsomericSMILES                     descriptors   \n",
       "CID                                                                        \n",
       "-955348933095      CCCCC=COC(=O)CCCCCCCC   oily;fruity;waxy;herbal;green  \\\n",
       "-923209957509   CC(=O)OCC1C=CC(C(C)C)CC1       spicy;fruity;herbal;woody   \n",
       "-874408321546  CCCCCCCCC(OC(C)=O)C(=O)OC  peach;lactonic;apricot;buttery   \n",
       "\n",
       "               alcoholic  aldehydic  alliaceous  almond  amber  animal   \n",
       "CID                                                                      \n",
       "-955348933095          0          0           0       0      0       0  \\\n",
       "-923209957509          0          0           0       0      0       0   \n",
       "-874408321546          0          0           0       0      0       0   \n",
       "\n",
       "               anisic  apple  ...  tropical  vanilla  vegetable  vetiver   \n",
       "CID                           ...                                          \n",
       "-955348933095       0      0  ...         0        0          0        0  \\\n",
       "-923209957509       0      0  ...         0        0          0        0   \n",
       "-874408321546       0      0  ...         0        0          0        0   \n",
       "\n",
       "               violet  warm  waxy  weedy  winey  woody  \n",
       "CID                                                     \n",
       "-955348933095       0     0     1      0      0      0  \n",
       "-923209957509       0     0     0      0      0      1  \n",
       "-874408321546       0     0     0      0      0      0  \n",
       "\n",
       "[3 rows x 147 columns]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mol_behaviors_encoded.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 3522 entries, -955348933095 to 162353069\n",
      "Columns: 147 entries, IsomericSMILES to woody\n",
      "dtypes: int64(145), object(2)\n",
      "memory usage: 4.1+ MB\n"
     ]
    }
   ],
   "source": [
    "mol_behaviors_encoded.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded_behavior = mol_behaviors_encoded.drop(columns=['IsomericSMILES', 'descriptors'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0\n",
       "1     232\n",
       "2     454\n",
       "3     779\n",
       "4     895\n",
       "5     622\n",
       "6     368\n",
       "7     126\n",
       "8      36\n",
       "9       9\n",
       "10      1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(encoded_behavior.sum(axis=1).sort_values()).groupby(by=0).size()\n",
    "# number of desc | number of molecules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CID</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [count]\n",
       "Index: []"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# molecules with no descriptors\n",
    "pd.DataFrame(encoded_behavior.sum(axis=1).sort_values(), columns=['count']).query('count==0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fruity': 1413,\n",
       " 'green': 913,\n",
       " 'sweet': 842,\n",
       " 'floral': 558,\n",
       " 'fatty': 412,\n",
       " 'herbal': 408,\n",
       " 'sulfurous': 253,\n",
       " 'waxy': 248,\n",
       " 'apple': 240,\n",
       " 'fresh': 240,\n",
       " 'nutty': 230,\n",
       " 'meaty': 226,\n",
       " 'spicy': 214,\n",
       " 'woody': 213,\n",
       " 'citrus': 212,\n",
       " 'oily': 209,\n",
       " 'tropical': 204,\n",
       " 'roasted': 195,\n",
       " 'ethereal': 194,\n",
       " 'winey': 177,\n",
       " 'earthy': 174,\n",
       " 'vegetable': 166,\n",
       " 'balsamic': 166,\n",
       " 'caramellic': 165,\n",
       " 'rose': 165,\n",
       " 'cheesy': 146,\n",
       " 'pineapple': 142,\n",
       " 'burnt': 138,\n",
       " 'onion': 127,\n",
       " 'mint': 123,\n",
       " 'pungent': 119,\n",
       " 'fermented': 110,\n",
       " 'buttery': 98,\n",
       " 'creamy': 93,\n",
       " 'musty': 92,\n",
       " 'honey': 91,\n",
       " 'pear': 91,\n",
       " 'melon': 86,\n",
       " 'garlic': 85,\n",
       " 'savory': 84,\n",
       " 'alcoholic': 84,\n",
       " 'phenolic': 81,\n",
       " 'cocoa': 80,\n",
       " 'cognac': 78,\n",
       " 'berry': 73,\n",
       " 'dairy': 73,\n",
       " 'camphoreous': 70,\n",
       " 'vanilla': 70,\n",
       " 'banana': 70,\n",
       " 'alliaceous': 69,\n",
       " 'coffee': 63,\n",
       " 'orange': 60,\n",
       " 'rum': 59,\n",
       " 'mushroom': 59,\n",
       " 'tobacco': 58,\n",
       " 'fishy': 58,\n",
       " 'leafy': 56,\n",
       " 'sharp': 56,\n",
       " 'rummy': 56,\n",
       " 'aromatic': 55,\n",
       " 'almond': 53,\n",
       " 'sour': 51,\n",
       " 'peach': 51,\n",
       " 'grape': 50,\n",
       " 'smoky': 49,\n",
       " 'animal': 48,\n",
       " 'gasoline': 48,\n",
       " 'anisic': 45,\n",
       " 'cooked': 44,\n",
       " 'solvent': 43,\n",
       " 'coconut': 43,\n",
       " 'warm': 43,\n",
       " 'jasmine': 42,\n",
       " 'milky': 42,\n",
       " 'hay': 41,\n",
       " 'cherry': 41,\n",
       " 'lemon': 39,\n",
       " 'jasmin': 39,\n",
       " 'coumarinic': 38,\n",
       " 'pine': 37,\n",
       " 'apricot': 36,\n",
       " 'bitter': 36,\n",
       " 'medicinal': 36,\n",
       " 'radish': 35,\n",
       " 'tea': 35,\n",
       " 'dry': 35,\n",
       " 'hazelnut': 34,\n",
       " 'ripe': 32,\n",
       " 'odorless': 32,\n",
       " 'brandy': 31,\n",
       " 'ketonic': 30,\n",
       " 'plum': 29,\n",
       " 'potato': 29,\n",
       " 'grassy': 29,\n",
       " 'chicken': 29,\n",
       " 'grapefruit': 29,\n",
       " 'chocolate': 28,\n",
       " 'strawberry': 27,\n",
       " 'bread': 27,\n",
       " 'beefy': 27,\n",
       " 'cabbage': 27,\n",
       " 'malty': 26,\n",
       " 'cinnamon': 26,\n",
       " 'orris': 26,\n",
       " 'violet': 26,\n",
       " 'metallic': 25,\n",
       " 'cucumber': 25,\n",
       " 'gassy': 25,\n",
       " 'aldehydic': 25,\n",
       " 'tomato': 24,\n",
       " 'chamomile': 24,\n",
       " 'popcorn': 24,\n",
       " 'lactonic': 24,\n",
       " 'black currant': 23,\n",
       " 'brothy': 23,\n",
       " 'musk': 22,\n",
       " 'juicy': 21,\n",
       " 'lily': 21,\n",
       " 'leathery': 20,\n",
       " 'catty': 20,\n",
       " 'horseradish': 20,\n",
       " 'clove': 19,\n",
       " 'cooling': 17,\n",
       " 'fruit skin': 16,\n",
       " 'geranium': 16,\n",
       " 'hyacinth': 15,\n",
       " 'terpenic': 15,\n",
       " 'sweaty': 14,\n",
       " 'raspberry': 14,\n",
       " 'celery': 13,\n",
       " 'lavender': 12,\n",
       " 'amber': 11,\n",
       " 'natural': 10,\n",
       " 'bergamot': 9,\n",
       " 'clean': 8,\n",
       " 'ozone': 8,\n",
       " 'muguet': 8,\n",
       " 'cedar': 7,\n",
       " 'hawthorn': 5,\n",
       " 'vetiver': 5,\n",
       " 'sandalwood': 4,\n",
       " 'soapy': 3,\n",
       " 'powdery': 3,\n",
       " 'orangeflower': 3,\n",
       " 'weedy': 1}"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dict(encoded_behavior.sum().sort_values(ascending=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['cortex']"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "og_columns = list(encoded_behavior.columns)\n",
    "# elements of required desc not in og_columns\n",
    "missing_desc = []\n",
    "for req in required_desc:\n",
    "    if req not in og_columns:\n",
    "        missing_desc.append(req)\n",
    "missing_desc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_required_desc = required_desc.copy()\n",
    "for i in missing_desc:\n",
    "    new_required_desc.remove(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_req_desc(desc_string):\n",
    "    desc_list = desc_string.split(';')\n",
    "    desc_set = set()\n",
    "    for desc in desc_list:\n",
    "        if desc in required_desc:\n",
    "            desc_set.update([desc])\n",
    "    return ';'.join(desc_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'caramellic;creamy;sweet'"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_req_desc('sweet;creamy;brown;caramellic;testing')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "mol_behaviors_encoded['Updated_Desc_v2'] = mol_behaviors_encoded['descriptors'].apply(lambda x: get_req_desc(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>IsomericSMILES</th>\n",
       "      <th>descriptors</th>\n",
       "      <th>alcoholic</th>\n",
       "      <th>aldehydic</th>\n",
       "      <th>alliaceous</th>\n",
       "      <th>almond</th>\n",
       "      <th>amber</th>\n",
       "      <th>animal</th>\n",
       "      <th>anisic</th>\n",
       "      <th>apple</th>\n",
       "      <th>...</th>\n",
       "      <th>vanilla</th>\n",
       "      <th>vegetable</th>\n",
       "      <th>vetiver</th>\n",
       "      <th>violet</th>\n",
       "      <th>warm</th>\n",
       "      <th>waxy</th>\n",
       "      <th>weedy</th>\n",
       "      <th>winey</th>\n",
       "      <th>woody</th>\n",
       "      <th>Updated_Desc_v2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-955348933095</th>\n",
       "      <td>CCCCC=COC(=O)CCCCCCCC</td>\n",
       "      <td>oily;fruity;waxy;herbal;green</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>oily;fruity;waxy;herbal;green</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-923209957509</th>\n",
       "      <td>CC(=O)OCC1C=CC(C(C)C)CC1</td>\n",
       "      <td>spicy;fruity;herbal;woody</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>herbal;spicy;fruity;woody</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-874408321546</th>\n",
       "      <td>CCCCCCCCC(OC(C)=O)C(=O)OC</td>\n",
       "      <td>peach;lactonic;apricot;buttery</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>peach;lactonic;apricot;buttery</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-873963935677</th>\n",
       "      <td>CCCCC=COC(=O)C(C)CCC</td>\n",
       "      <td>tropical;green;fruity;apple</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>tropical;green;fruity;apple</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-862841422647</th>\n",
       "      <td>CCCC(S)COCC</td>\n",
       "      <td>catty</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 148 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          IsomericSMILES                     descriptors   \n",
       "CID                                                                        \n",
       "-955348933095      CCCCC=COC(=O)CCCCCCCC   oily;fruity;waxy;herbal;green  \\\n",
       "-923209957509   CC(=O)OCC1C=CC(C(C)C)CC1       spicy;fruity;herbal;woody   \n",
       "-874408321546  CCCCCCCCC(OC(C)=O)C(=O)OC  peach;lactonic;apricot;buttery   \n",
       "-873963935677       CCCCC=COC(=O)C(C)CCC     tropical;green;fruity;apple   \n",
       "-862841422647                CCCC(S)COCC                           catty   \n",
       "\n",
       "               alcoholic  aldehydic  alliaceous  almond  amber  animal   \n",
       "CID                                                                      \n",
       "-955348933095          0          0           0       0      0       0  \\\n",
       "-923209957509          0          0           0       0      0       0   \n",
       "-874408321546          0          0           0       0      0       0   \n",
       "-873963935677          0          0           0       0      0       0   \n",
       "-862841422647          0          0           0       0      0       0   \n",
       "\n",
       "               anisic  apple  ...  vanilla  vegetable  vetiver  violet  warm   \n",
       "CID                           ...                                              \n",
       "-955348933095       0      0  ...        0          0        0       0     0  \\\n",
       "-923209957509       0      0  ...        0          0        0       0     0   \n",
       "-874408321546       0      0  ...        0          0        0       0     0   \n",
       "-873963935677       0      1  ...        0          0        0       0     0   \n",
       "-862841422647       0      0  ...        0          0        0       0     0   \n",
       "\n",
       "               waxy  weedy  winey  woody                 Updated_Desc_v2  \n",
       "CID                                                                       \n",
       "-955348933095     1      0      0      0   oily;fruity;waxy;herbal;green  \n",
       "-923209957509     0      0      0      1       herbal;spicy;fruity;woody  \n",
       "-874408321546     0      0      0      0  peach;lactonic;apricot;buttery  \n",
       "-873963935677     0      0      0      0     tropical;green;fruity;apple  \n",
       "-862841422647     0      0      0      0                                  \n",
       "\n",
       "[5 rows x 148 columns]"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mol_behaviors_encoded.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "required_leffingwell_dataset = mol_behaviors_encoded[['IsomericSMILES', 'Updated_Desc_v2']+new_required_desc]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>IsomericSMILES</th>\n",
       "      <th>Updated_Desc_v2</th>\n",
       "      <th>alcoholic</th>\n",
       "      <th>aldehydic</th>\n",
       "      <th>alliaceous</th>\n",
       "      <th>almond</th>\n",
       "      <th>amber</th>\n",
       "      <th>animal</th>\n",
       "      <th>anisic</th>\n",
       "      <th>apple</th>\n",
       "      <th>...</th>\n",
       "      <th>tropical</th>\n",
       "      <th>vanilla</th>\n",
       "      <th>vegetable</th>\n",
       "      <th>vetiver</th>\n",
       "      <th>violet</th>\n",
       "      <th>warm</th>\n",
       "      <th>waxy</th>\n",
       "      <th>weedy</th>\n",
       "      <th>winey</th>\n",
       "      <th>woody</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-955348933095</th>\n",
       "      <td>CCCCC=COC(=O)CCCCCCCC</td>\n",
       "      <td>oily;fruity;waxy;herbal;green</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-923209957509</th>\n",
       "      <td>CC(=O)OCC1C=CC(C(C)C)CC1</td>\n",
       "      <td>herbal;spicy;fruity;woody</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-874408321546</th>\n",
       "      <td>CCCCCCCCC(OC(C)=O)C(=O)OC</td>\n",
       "      <td>peach;lactonic;apricot;buttery</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-873963935677</th>\n",
       "      <td>CCCCC=COC(=O)C(C)CCC</td>\n",
       "      <td>tropical;green;fruity;apple</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-862841422647</th>\n",
       "      <td>CCCC(S)COCC</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 139 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          IsomericSMILES                 Updated_Desc_v2   \n",
       "CID                                                                        \n",
       "-955348933095      CCCCC=COC(=O)CCCCCCCC   oily;fruity;waxy;herbal;green  \\\n",
       "-923209957509   CC(=O)OCC1C=CC(C(C)C)CC1       herbal;spicy;fruity;woody   \n",
       "-874408321546  CCCCCCCCC(OC(C)=O)C(=O)OC  peach;lactonic;apricot;buttery   \n",
       "-873963935677       CCCCC=COC(=O)C(C)CCC     tropical;green;fruity;apple   \n",
       "-862841422647                CCCC(S)COCC                                   \n",
       "\n",
       "               alcoholic  aldehydic  alliaceous  almond  amber  animal   \n",
       "CID                                                                      \n",
       "-955348933095          0          0           0       0      0       0  \\\n",
       "-923209957509          0          0           0       0      0       0   \n",
       "-874408321546          0          0           0       0      0       0   \n",
       "-873963935677          0          0           0       0      0       0   \n",
       "-862841422647          0          0           0       0      0       0   \n",
       "\n",
       "               anisic  apple  ...  tropical  vanilla  vegetable  vetiver   \n",
       "CID                           ...                                          \n",
       "-955348933095       0      0  ...         0        0          0        0  \\\n",
       "-923209957509       0      0  ...         0        0          0        0   \n",
       "-874408321546       0      0  ...         0        0          0        0   \n",
       "-873963935677       0      1  ...         1        0          0        0   \n",
       "-862841422647       0      0  ...         0        0          0        0   \n",
       "\n",
       "               violet  warm  waxy  weedy  winey  woody  \n",
       "CID                                                     \n",
       "-955348933095       0     0     1      0      0      0  \n",
       "-923209957509       0     0     0      0      0      1  \n",
       "-874408321546       0     0     0      0      0      0  \n",
       "-873963935677       0     0     0      0      0      0  \n",
       "-862841422647       0     0     0      0      0      0  \n",
       "\n",
       "[5 rows x 139 columns]"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "required_leffingwell_dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 3522 entries, -955348933095 to 162353069\n",
      "Columns: 139 entries, IsomericSMILES to woody\n",
      "dtypes: int64(137), object(2)\n",
      "memory usage: 3.9+ MB\n"
     ]
    }
   ],
   "source": [
    "required_leffingwell_dataset.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>alcoholic</th>\n",
       "      <th>aldehydic</th>\n",
       "      <th>alliaceous</th>\n",
       "      <th>almond</th>\n",
       "      <th>amber</th>\n",
       "      <th>animal</th>\n",
       "      <th>anisic</th>\n",
       "      <th>apple</th>\n",
       "      <th>apricot</th>\n",
       "      <th>aromatic</th>\n",
       "      <th>...</th>\n",
       "      <th>tropical</th>\n",
       "      <th>vanilla</th>\n",
       "      <th>vegetable</th>\n",
       "      <th>vetiver</th>\n",
       "      <th>violet</th>\n",
       "      <th>warm</th>\n",
       "      <th>waxy</th>\n",
       "      <th>weedy</th>\n",
       "      <th>winey</th>\n",
       "      <th>woody</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-955348933095</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-923209957509</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-874408321546</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-873963935677</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-862841422647</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154241350</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155323379</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158699780</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161504856</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162353069</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3522 rows × 137 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               alcoholic  aldehydic  alliaceous  almond  amber  animal   \n",
       "CID                                                                      \n",
       "-955348933095          0          0           0       0      0       0  \\\n",
       "-923209957509          0          0           0       0      0       0   \n",
       "-874408321546          0          0           0       0      0       0   \n",
       "-873963935677          0          0           0       0      0       0   \n",
       "-862841422647          0          0           0       0      0       0   \n",
       "...                  ...        ...         ...     ...    ...     ...   \n",
       " 154241350             0          0           1       0      0       0   \n",
       " 155323379             0          0           1       0      0       0   \n",
       " 158699780             0          0           0       0      0       0   \n",
       " 161504856             0          0           0       0      0       0   \n",
       " 162353069             0          0           1       0      0       0   \n",
       "\n",
       "               anisic  apple  apricot  aromatic  ...  tropical  vanilla   \n",
       "CID                                              ...                      \n",
       "-955348933095       0      0        0         0  ...         0        0  \\\n",
       "-923209957509       0      0        0         0  ...         0        0   \n",
       "-874408321546       0      0        1         0  ...         0        0   \n",
       "-873963935677       0      1        0         0  ...         1        0   \n",
       "-862841422647       0      0        0         0  ...         0        0   \n",
       "...               ...    ...      ...       ...  ...       ...      ...   \n",
       " 154241350          0      0        0         0  ...         0        0   \n",
       " 155323379          0      0        0         0  ...         0        0   \n",
       " 158699780          0      0        0         0  ...         1        0   \n",
       " 161504856          0      0        0         0  ...         1        0   \n",
       " 162353069          0      0        0         0  ...         0        0   \n",
       "\n",
       "               vegetable  vetiver  violet  warm  waxy  weedy  winey  woody  \n",
       "CID                                                                         \n",
       "-955348933095          0        0       0     0     1      0      0      0  \n",
       "-923209957509          0        0       0     0     0      0      0      1  \n",
       "-874408321546          0        0       0     0     0      0      0      0  \n",
       "-873963935677          0        0       0     0     0      0      0      0  \n",
       "-862841422647          0        0       0     0     0      0      0      0  \n",
       "...                  ...      ...     ...   ...   ...    ...    ...    ...  \n",
       " 154241350             0        0       0     0     0      0      0      0  \n",
       " 155323379             0        0       0     0     0      0      0      0  \n",
       " 158699780             0        0       0     0     0      0      0      0  \n",
       " 161504856             0        0       0     0     1      0      0      0  \n",
       " 162353069             0        0       0     0     0      0      0      0  \n",
       "\n",
       "[3522 rows x 137 columns]"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "required_encoded = required_leffingwell_dataset.drop(columns=['IsomericSMILES', 'Updated_Desc_v2'])\n",
    "required_encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CID</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>175606</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12388</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8141</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-862841422647</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11511</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31285</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11507</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15600</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12389</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>356</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11551</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61982</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               count\n",
       "CID                 \n",
       " 175606            0\n",
       " 12388             0\n",
       " 8141              0\n",
       "-862841422647      0\n",
       " 11511             0\n",
       " 31285             0\n",
       " 11507             0\n",
       " 15600             0\n",
       " 12389             0\n",
       " 356               0\n",
       " 11551             0\n",
       " 61982             0"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "no_desc_df = pd.DataFrame(required_encoded.sum(axis=1).sort_values(), columns=['count']).query('count==0')\n",
    "no_desc_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>IsomericSMILES</th>\n",
       "      <th>Updated_Desc_v2</th>\n",
       "      <th>alcoholic</th>\n",
       "      <th>aldehydic</th>\n",
       "      <th>alliaceous</th>\n",
       "      <th>almond</th>\n",
       "      <th>amber</th>\n",
       "      <th>animal</th>\n",
       "      <th>anisic</th>\n",
       "      <th>apple</th>\n",
       "      <th>...</th>\n",
       "      <th>tropical</th>\n",
       "      <th>vanilla</th>\n",
       "      <th>vegetable</th>\n",
       "      <th>vetiver</th>\n",
       "      <th>violet</th>\n",
       "      <th>warm</th>\n",
       "      <th>waxy</th>\n",
       "      <th>weedy</th>\n",
       "      <th>winey</th>\n",
       "      <th>woody</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CCCCC=COC(=O)CCCCCCCC</td>\n",
       "      <td>oily;fruity;waxy;herbal;green</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CC(=O)OCC1C=CC(C(C)C)CC1</td>\n",
       "      <td>herbal;spicy;fruity;woody</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CCCCCCCCC(OC(C)=O)C(=O)OC</td>\n",
       "      <td>peach;lactonic;apricot;buttery</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CCCCC=COC(=O)C(C)CCC</td>\n",
       "      <td>tropical;green;fruity;apple</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CCCCCCCC=CC(=O)OC(CCCCCCCC)C(=O)O</td>\n",
       "      <td>milky</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 139 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                      IsomericSMILES                 Updated_Desc_v2   \n",
       "0              CCCCC=COC(=O)CCCCCCCC   oily;fruity;waxy;herbal;green  \\\n",
       "1           CC(=O)OCC1C=CC(C(C)C)CC1       herbal;spicy;fruity;woody   \n",
       "2          CCCCCCCCC(OC(C)=O)C(=O)OC  peach;lactonic;apricot;buttery   \n",
       "3               CCCCC=COC(=O)C(C)CCC     tropical;green;fruity;apple   \n",
       "4  CCCCCCCC=CC(=O)OC(CCCCCCCC)C(=O)O                           milky   \n",
       "\n",
       "   alcoholic  aldehydic  alliaceous  almond  amber  animal  anisic  apple   \n",
       "0          0          0           0       0      0       0       0      0  \\\n",
       "1          0          0           0       0      0       0       0      0   \n",
       "2          0          0           0       0      0       0       0      0   \n",
       "3          0          0           0       0      0       0       0      1   \n",
       "4          0          0           0       0      0       0       0      0   \n",
       "\n",
       "   ...  tropical  vanilla  vegetable  vetiver  violet  warm  waxy  weedy   \n",
       "0  ...         0        0          0        0       0     0     1      0  \\\n",
       "1  ...         0        0          0        0       0     0     0      0   \n",
       "2  ...         0        0          0        0       0     0     0      0   \n",
       "3  ...         1        0          0        0       0     0     0      0   \n",
       "4  ...         0        0          0        0       0     0     0      0   \n",
       "\n",
       "   winey  woody  \n",
       "0      0      0  \n",
       "1      0      1  \n",
       "2      0      0  \n",
       "3      0      0  \n",
       "4      0      0  \n",
       "\n",
       "[5 rows x 139 columns]"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "required_leffingwell_dataset_v2 = required_leffingwell_dataset.drop(no_desc_df.index).reset_index(drop=True)\n",
    "required_leffingwell_dataset_v2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 3510 entries, 0 to 3509\n",
      "Columns: 139 entries, IsomericSMILES to woody\n",
      "dtypes: int64(137), object(2)\n",
      "memory usage: 3.7+ MB\n"
     ]
    }
   ],
   "source": [
    "required_leffingwell_dataset_v2.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "required_encoded_v2 = required_leffingwell_dataset_v2.drop(columns=['IsomericSMILES', 'Updated_Desc_v2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [count]\n",
       "Index: []"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# molecules with no descriptors\n",
    "pd.DataFrame(required_encoded_v2.sum(axis=1).sort_values(), columns=['count']).query('count==0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['alcoholic',\n",
       " 'aldehydic',\n",
       " 'alliaceous',\n",
       " 'almond',\n",
       " 'amber',\n",
       " 'animal',\n",
       " 'anisic',\n",
       " 'apple',\n",
       " 'apricot',\n",
       " 'aromatic',\n",
       " 'balsamic',\n",
       " 'banana',\n",
       " 'beefy',\n",
       " 'bergamot',\n",
       " 'berry',\n",
       " 'bitter',\n",
       " 'black currant',\n",
       " 'brandy',\n",
       " 'burnt',\n",
       " 'buttery',\n",
       " 'cabbage',\n",
       " 'camphoreous',\n",
       " 'caramellic',\n",
       " 'cedar',\n",
       " 'celery',\n",
       " 'chamomile',\n",
       " 'cheesy',\n",
       " 'cherry',\n",
       " 'chocolate',\n",
       " 'cinnamon',\n",
       " 'citrus',\n",
       " 'clean',\n",
       " 'clove',\n",
       " 'cocoa',\n",
       " 'coconut',\n",
       " 'coffee',\n",
       " 'cognac',\n",
       " 'cooked',\n",
       " 'cooling',\n",
       " 'coumarinic',\n",
       " 'creamy',\n",
       " 'cucumber',\n",
       " 'dairy',\n",
       " 'dry',\n",
       " 'earthy',\n",
       " 'ethereal',\n",
       " 'fatty',\n",
       " 'fermented',\n",
       " 'fishy',\n",
       " 'floral',\n",
       " 'fresh',\n",
       " 'fruit skin',\n",
       " 'fruity',\n",
       " 'garlic',\n",
       " 'gassy',\n",
       " 'geranium',\n",
       " 'grape',\n",
       " 'grapefruit',\n",
       " 'grassy',\n",
       " 'green',\n",
       " 'hawthorn',\n",
       " 'hay',\n",
       " 'hazelnut',\n",
       " 'herbal',\n",
       " 'honey',\n",
       " 'hyacinth',\n",
       " 'jasmin',\n",
       " 'juicy',\n",
       " 'ketonic',\n",
       " 'lactonic',\n",
       " 'lavender',\n",
       " 'leafy',\n",
       " 'leathery',\n",
       " 'lemon',\n",
       " 'lily',\n",
       " 'malty',\n",
       " 'meaty',\n",
       " 'medicinal',\n",
       " 'melon',\n",
       " 'metallic',\n",
       " 'milky',\n",
       " 'mint',\n",
       " 'muguet',\n",
       " 'mushroom',\n",
       " 'musk',\n",
       " 'musty',\n",
       " 'natural',\n",
       " 'nutty',\n",
       " 'odorless',\n",
       " 'oily',\n",
       " 'onion',\n",
       " 'orange',\n",
       " 'orangeflower',\n",
       " 'orris',\n",
       " 'ozone',\n",
       " 'peach',\n",
       " 'pear',\n",
       " 'phenolic',\n",
       " 'pine',\n",
       " 'pineapple',\n",
       " 'plum',\n",
       " 'popcorn',\n",
       " 'potato',\n",
       " 'powdery',\n",
       " 'pungent',\n",
       " 'radish',\n",
       " 'raspberry',\n",
       " 'ripe',\n",
       " 'roasted',\n",
       " 'rose',\n",
       " 'rummy',\n",
       " 'sandalwood',\n",
       " 'savory',\n",
       " 'sharp',\n",
       " 'smoky',\n",
       " 'soapy',\n",
       " 'solvent',\n",
       " 'sour',\n",
       " 'spicy',\n",
       " 'strawberry',\n",
       " 'sulfurous',\n",
       " 'sweaty',\n",
       " 'sweet',\n",
       " 'tea',\n",
       " 'terpenic',\n",
       " 'tobacco',\n",
       " 'tomato',\n",
       " 'tropical',\n",
       " 'vanilla',\n",
       " 'vegetable',\n",
       " 'vetiver',\n",
       " 'violet',\n",
       " 'warm',\n",
       " 'waxy',\n",
       " 'weedy',\n",
       " 'winey',\n",
       " 'woody']"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(required_encoded_v2.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "fruity          1413\n",
       "green            913\n",
       "sweet            842\n",
       "floral           558\n",
       "fatty            412\n",
       "                ... \n",
       "sandalwood         4\n",
       "powdery            3\n",
       "orangeflower       3\n",
       "soapy              3\n",
       "weedy              1\n",
       "Length: 137, dtype: int64"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "required_encoded_v2.sum().sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>IsomericSMILES</th>\n",
       "      <th>Updated_Desc_v2</th>\n",
       "      <th>alcoholic</th>\n",
       "      <th>aldehydic</th>\n",
       "      <th>alliaceous</th>\n",
       "      <th>almond</th>\n",
       "      <th>amber</th>\n",
       "      <th>animal</th>\n",
       "      <th>anisic</th>\n",
       "      <th>apple</th>\n",
       "      <th>...</th>\n",
       "      <th>tropical</th>\n",
       "      <th>vanilla</th>\n",
       "      <th>vegetable</th>\n",
       "      <th>vetiver</th>\n",
       "      <th>violet</th>\n",
       "      <th>warm</th>\n",
       "      <th>waxy</th>\n",
       "      <th>weedy</th>\n",
       "      <th>winey</th>\n",
       "      <th>woody</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>0 rows × 139 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [IsomericSMILES, Updated_Desc_v2, alcoholic, aldehydic, alliaceous, almond, amber, animal, anisic, apple, apricot, aromatic, balsamic, banana, beefy, bergamot, berry, bitter, black currant, brandy, burnt, buttery, cabbage, camphoreous, caramellic, cedar, celery, chamomile, cheesy, cherry, chocolate, cinnamon, citrus, clean, clove, cocoa, coconut, coffee, cognac, cooked, cooling, coumarinic, creamy, cucumber, dairy, dry, earthy, ethereal, fatty, fermented, fishy, floral, fresh, fruit skin, fruity, garlic, gassy, geranium, grape, grapefruit, grassy, green, hawthorn, hay, hazelnut, herbal, honey, hyacinth, jasmin, juicy, ketonic, lactonic, lavender, leafy, leathery, lemon, lily, malty, meaty, medicinal, melon, metallic, milky, mint, muguet, mushroom, musk, musty, natural, nutty, odorless, oily, onion, orange, orangeflower, orris, ozone, peach, pear, phenolic, ...]\n",
       "Index: []\n",
       "\n",
       "[0 rows x 139 columns]"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "required_leffingwell_dataset_v2[required_leffingwell_dataset_v2['IsomericSMILES'].duplicated(keep=False)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>IsomericSMILES</th>\n",
       "      <th>Updated_Desc_v2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2731</th>\n",
       "      <td>CC/C=C/CC(C)OC(=O)C</td>\n",
       "      <td>green;fruity</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           IsomericSMILES Updated_Desc_v2\n",
       "2731  CC/C=C/CC(C)OC(=O)C    green;fruity"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "required_leffingwell_dataset_v2[['IsomericSMILES', 'Updated_Desc_v2']][required_leffingwell_dataset_v2['IsomericSMILES']=='CC/C=C/CC(C)OC(=O)C']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "required_leffingwell_dataset_v2.to_csv('curated_leffingwell.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "odor",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}