Evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>(500, 501)</th>\n",
       "      <th>(501, 502)</th>\n",
       "      <th>(502, 503)</th>\n",
       "      <th>(503, 504)</th>\n",
       "      <th>(504, 505)</th>\n",
       "      <th>(505, 506)</th>\n",
       "      <th>(506, 507)</th>\n",
       "      <th>(507, 508)</th>\n",
       "      <th>(508, 509)</th>\n",
       "      <th>...</th>\n",
       "      <th>(3990, 3991)</th>\n",
       "      <th>(3991, 3992)</th>\n",
       "      <th>(3992, 3993)</th>\n",
       "      <th>(3993, 3994)</th>\n",
       "      <th>(3994, 3995)</th>\n",
       "      <th>(3995, 3996)</th>\n",
       "      <th>(3996, 3997)</th>\n",
       "      <th>(3997, 3998)</th>\n",
       "      <th>(3998, 3999)</th>\n",
       "      <th>(3999, 4000)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>./20131112_P136sols/monopeaklists/20131112_P13...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>./20131112_P136sols/monopeaklists/20131112_P13...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>./20131112_P136sols/monopeaklists/20131112_P13...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>./20131112_P136sols/monopeaklists/20131112_P13...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>./20131112_P136sols/monopeaklists/20131112_P13...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 3501 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            filename  (500, 501)  (501, 502)  \\\n",
       "0  ./20131112_P136sols/monopeaklists/20131112_P13...           0           0   \n",
       "1  ./20131112_P136sols/monopeaklists/20131112_P13...           0           0   \n",
       "2  ./20131112_P136sols/monopeaklists/20131112_P13...           0           0   \n",
       "3  ./20131112_P136sols/monopeaklists/20131112_P13...           0           0   \n",
       "4  ./20131112_P136sols/monopeaklists/20131112_P13...           0           0   \n",
       "\n",
       "   (502, 503)  (503, 504)  (504, 505)  (505, 506)  (506, 507)  (507, 508)  \\\n",
       "0           0           0           0           0           0           0   \n",
       "1           0           0           0           0           0           0   \n",
       "2           0           0           0           0           0           0   \n",
       "3           0           0           0           0           0           0   \n",
       "4           0           0           0           0           0           0   \n",
       "\n",
       "   (508, 509)      ...       (3990, 3991)  (3991, 3992)  (3992, 3993)  \\\n",
       "0           0      ...                  0             0             0   \n",
       "1           0      ...                  0             0             0   \n",
       "2           0      ...                  0             0             0   \n",
       "3           0      ...                  0             0             0   \n",
       "4           0      ...                  0             0             0   \n",
       "\n",
       "   (3993, 3994)  (3994, 3995)  (3995, 3996)  (3996, 3997)  (3997, 3998)  \\\n",
       "0             0             0             0             0             0   \n",
       "1             0             0             0             0             0   \n",
       "2             0             0             0             0             0   \n",
       "3             0             0             0             0             0   \n",
       "4             0             0             0             0             0   \n",
       "\n",
       "   (3998, 3999)  (3999, 4000)  \n",
       "0             0             0  \n",
       "1             0             0  \n",
       "2             0             0  \n",
       "3             0             0  \n",
       "4             0             0  \n",
       "\n",
       "[5 rows x 3501 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import csv\n",
    "import os\n",
    "import glob\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "#data import/show\n",
    "\n",
    "data = pd.read_csv('datalabel1.csv', index_col = [0])\n",
    "data.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduced1 = data.loc[:, (data != 0).any(axis=0)] #with labels, no zero rows\n",
    "filelabels = reduced1.iloc[:,0] # labels only \n",
    "reduced = reduced1.iloc[:,1:] # no labels\n",
    "\n",
    "count = 0\n",
    "for index, row in reduced1.iterrows():\n",
    "    if row['(2705, 2706)'] == 1:\n",
    "        count += 1\n",
    "labels = reduced1.iloc[:,0]\n",
    "str = labels.tolist()\n",
    "#print(count)\n",
    "#print(reduced1['(1706, 1707)'])\n",
    "#reduced.shape\n",
    "df1 = reduced1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Search data for biomarkers with Gu's "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = reduced1\n",
    "## Apodemus\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(1293, 1294)'] == 1 and row['(1443, 1444)'] == 1 and row ['(1451, 1452)'] == 1 and row ['(1592, 1593)'] == 1 and row['(2595, 2596)'] == 1 and row['(2680, 2681)'] == 1 and row['(2695, 2696)'] == 1 and row['(2941, 2942)'] == 1:\n",
    "        print(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Mammuthus\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(1706, 1707)'] == 1 and row['(2705, 2706)'] == 1 and row['(1251, 1252)'] == 1 and row ['(1518, 1519)'] == 1 and row['(1568, 1569)'] == 1 and row['(1756, 1757)'] == 1 and row['(1772, 1773)'] == 1:\n",
    "        print(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Equus\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(1706, 1707)'] == 1 and row['(2705, 2706)'] == 1 and row ['(2106, 2107)'] == 1 and row['(1604, 1605)'] == 1 and row['(1783, 1784)'] == 1 and row['(1182, 1183)'] == 1 and row['(1235, 1236)'] == 1 and row['(1311, 1312)'] == 1 and row['(1427, 1428)'] == 1 and row['(1501, 1502)'] == 1  and row['(1530, 1531)'] == 1 and row['(2583, 2584)'] == 1:\n",
    "        print(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Coelodonta\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(1706, 1707)'] == 1 and row['(2705, 2706)'] == 1 and row ['(2106, 2107)'] == 1 and row['(1604, 1605)'] == 1 and row['(1783, 1784)'] == 1 and row['(1261, 1262)'] == 1 and row['(1453, 1454)'] == 1 and row['(1473, 1474)'] == 1 and row['(1560, 1561)'] == 1 and row['(2571, 2572)'] == 1:\n",
    "        print(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Bos/Bison\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(1706, 1707)'] == 1 and row['(2705, 2706)'] == 1 and row ['(2106, 2107)'] == 1 and row['(1832, 1833)'] == 1 and row['(1848, 1849)'] == 1 and row['(1427, 1428)'] == 1 and row['(1435, 1436)'] == 1 and row['(3018, 3019)'] == 1 and row['(1208, 1209)'] == 1 and row['(2853, 2854)'] == 1:\n",
    "        print(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Cervine\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(1706, 1707)'] == 1 and row['(2705, 2706)'] == 1 and row ['(2106, 2107)'] == 1 and row['(1832, 1833)'] == 1 and row['(1848, 1849)'] == 1 and row['(1427, 1428)'] == 1 and row['(1435, 1436)'] == 1 and row['(3018, 3019)'] == 1 and row['(1311, 1312)'] == 1 and row['(2899, 2900)'] == 1 and row['(1550, 1551)'] == 1 and row['(2583, 2584)'] == 1:\n",
    "        print(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Rangifer\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(1706, 1707)'] == 1 and row['(2705, 2706)'] == 1 and row ['(2106, 2107)'] == 1 and row['(1832, 1833)'] == 1 and row['(1848, 1849)'] == 1 and row['(1427, 1428)'] == 1 and row['(1435, 1436)'] == 1 and row['(3018, 3019)'] == 1 and row['(1311, 1312)'] == 1 and row['(2899, 2900)'] == 1 and row['(1580, 1581)'] == 1 and row['(2557, 2558)'] == 1:\n",
    "        print(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Lepus\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(1706, 1707)'] == 1 and row['(2705, 2706)'] == 1 and row ['(2106, 2107)'] == 1 and row['(1832, 1833)'] == 1 and row['(1848, 1849)'] == 1 and row['(1453, 1454)'] == 1 and row['(1311, 1312)'] == 1 and row['(1501, 1502)'] == 1 and row['(1532, 1533)'] == 1 and row['(1608, 1609)'] == 1 and row['(2525, 2526)'] == 1:\n",
    "        print(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Crocuta/Panthera\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(1706, 1707)'] == 1 and row['(2705, 2706)'] == 1 and row ['(2106, 2107)'] == 1 and row['(1832, 1833)'] == 1 and row['(1848, 1849)'] == 1 and row['(1453, 1454)'] == 1 and row['(1261, 1262)'] == 1 and row['(1459, 1460)'] == 1 and row['(1560, 1561)'] == 1 and row['(2976, 2977)'] == 1 and row['(1588, 1589)'] == 1  and row['(2247, 2248)'] == 1:\n",
    "        print(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Mustela\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(1706, 1707)'] == 1 and row['(2705, 2706)'] == 1 and row ['(2106, 2107)'] == 1 and row['(1832, 1833)'] == 1 and row['(1848, 1849)'] == 1 and row['(1453, 1454)'] == 1 and row['(1261, 1262)'] == 1 and row['(1459, 1460)'] == 1 and row['(1560, 1561)'] == 1 and row['(2976, 2977)'] == 1 and row['(2216, 2217)'] == 1  and row['(866, 867)'] == 1 and row['(1235, 1236)'] == 1 and row['(1548, 1549)'] == 1 and row['(1680, 1681)'] == 1 and row['(2147, 2148)'] == 1:\n",
    "        print(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Ursus\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(1706, 1707)'] == 1 and row['(2705, 2706)'] == 1 and row ['(2106, 2107)'] == 1 and row['(1832, 1833)'] == 1 and row['(1848, 1849)'] == 1 and row['(1453, 1454)'] == 1 and row['(1261, 1262)'] == 1 and row['(1459, 1460)'] == 1 and row['(1560, 1561)'] == 1 and row['(2976, 2977)'] == 1 and row['(2216, 2217)'] == 1  and row['(836, 837)'] == 1 and row['(1690, 1691)'] == 1 and row['(1706, 1707)'] == 1 and row['(1233, 1234)'] == 1 and row['(2135, 2136)'] == 1 and row['(2163, 2164)'] == 1 and row['(2599, 2600)'] == 1:\n",
    "        print(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Canid\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(1706, 1707)'] == 1 and row['(2705, 2706)'] == 1 and row ['(2106, 2107)'] == 1 and row['(1832, 1833)'] == 1 and row['(1848, 1849)'] == 1 and row['(1453, 1454)'] == 1 and row['(1261, 1262)'] == 1 and row['(1459, 1460)'] == 1 and row['(1560, 1561)'] == 1 and row['(2976, 2977)'] == 1 and row['(2216, 2217)'] == 1  and row['(836, 837)'] == 1 and row['(1690, 1691)'] == 1 and row['(1706, 1707)'] == 1 and row['(2131, 2132)'] == 1 and row['(2613, 2614)'] == 1:\n",
    "        print(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##df.loc[(df['column_name'] == some_value) & df['other_column'].isin(some_values)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#check for \"mammuthus\" biomarkers in closely clustered samples\n",
    "\n",
    "for index, row in reduced1.iterrows():\n",
    "    if row['filename'] == './20131112_P136sols/monopeaklists/20131112_P136sols_B2.csv': #& row['(2705, 2706)'] == 1 :\n",
    "        print('/20131112_P136sols/monopeaklists/20131112_P136sols_B2.csv -> sample found')\n",
    "        if row['(1706, 1707)'] == 1:\n",
    "            print(\"yes: 1706, 1707\")\n",
    "        if row['(2705, 2706)'] == 1:\n",
    "            print(\"yes: 2705, 2706\")\n",
    "        if row['(1518, 1519)'] == 1:\n",
    "            print(\"yes: 1518, 1519\")\n",
    "        if row['(1251, 1252)'] == 1:\n",
    "            print(\"yes: 1251, 1252\")\n",
    "        if row['(1568, 1569)'] == 1:\n",
    "            print(\"yes: 1568, 1569\")\n",
    "        if row['(1756, 1757)'] == 1:\n",
    "            print(\"yes: 1756, 1757\")\n",
    "        if row['(1772, 1773)'] == 1:\n",
    "            print(\"yes: 1772, 1773\")\n",
    "            \n",
    "            \n",
    "for index, row in reduced1.iterrows():\n",
    "    if row['filename'] == './20131112_P136sols/monopeaklists/20131112_P136sols_D10.csv':\n",
    "        print('./20131112_P136sols/monopeaklists/20131112_P136sols_D10.csv -> sample found')\n",
    "        if row['(1706, 1707)'] == 1:\n",
    "            print(\"yes: 1706, 1707\")\n",
    "        if row['(2705, 2706)'] == 1:\n",
    "            print(\"yes: 2705, 2706\")\n",
    "        if row['(1518, 1519)'] == 1:\n",
    "            print(\"yes: 1518, 1519\")\n",
    "        if row['(1251, 1252)'] == 1:\n",
    "            print(\"yes: 1251, 1252\")\n",
    "        if row['(1568, 1569)'] == 1:\n",
    "            print(\"yes: 1568, 1569\")\n",
    "        if row['(1756, 1757)'] == 1:\n",
    "            print(\"yes: 1756, 1757\")\n",
    "        if row['(1772, 1773)'] == 1:\n",
    "            print(\"yes: 1772, 1773\")\n",
    "            \n",
    "                        \n",
    "for index, row in reduced1.iterrows():\n",
    "    if row['filename'] == './20131112_P140sols/monopeaklists/20131112_P140sols_I19.csv':\n",
    "        print('./20131112_P140sols/monopeaklists/20131112_P140sols_I19.csv -> sample found')\n",
    "        if row['(1706, 1707)'] == 1:\n",
    "            print(\"yes: 1706, 1707\")\n",
    "        if row['(2705, 2706)'] == 1:\n",
    "            print(\"yes: 2705, 2706\")\n",
    "        if row['(1518, 1519)'] == 1:\n",
    "            print(\"yes: 1518, 1519\")\n",
    "        if row['(1251, 1252)'] == 1:\n",
    "            print(\"yes: 1251, 1252\")\n",
    "        if row['(1568, 1569)'] == 1:\n",
    "            print(\"yes: 1568, 1569\")\n",
    "        if row['(1756, 1757)'] == 1:\n",
    "            print(\"yes: 1756, 1757\")\n",
    "        if row['(1772, 1773)'] == 1:\n",
    "            print(\"yes: 1772, 1773\")\n",
    "            \n",
    "for index, row in reduced1.iterrows():\n",
    "    if row['filename'] == './20131209_P96solsRun/monopeaklists/20131209_P96solsRun_L5.csv':\n",
    "        print('./20131209_P96solsRun/monopeaklists/20131209_P96solsRun_L5.csv -> sample found')\n",
    "        if row['(1706, 1707)'] == 1:\n",
    "            print(\"yes: 1706, 1707\")\n",
    "        if row['(2705, 2706)'] == 1:\n",
    "            print(\"yes: 2705, 2706\")\n",
    "        if row['(1518, 1519)'] == 1:\n",
    "            print(\"yes: 1518, 1519\")\n",
    "        if row['(1251, 1252)'] == 1:\n",
    "            print(\"yes: 1251, 1252\")\n",
    "        if row['(1568, 1569)'] == 1:\n",
    "            print(\"yes: 1568, 1569\")\n",
    "        if row['(1756, 1757)'] == 1:\n",
    "            print(\"yes: 1756, 1757\")\n",
    "        if row['(1772, 1773)'] == 1:\n",
    "            print(\"yes: 1772, 1773\")\n",
    "            \n",
    "            \n",
    "for index, row in reduced1.iterrows():\n",
    "    if row['filename'] == './20131217_PH100solRun/monopeaklists/cals_L24.csv':\n",
    "        print('./20131217_PH100solRun/monopeaklists/cals_L24.csv -> sample found')\n",
    "        if row['(1706, 1707)'] == 1:\n",
    "            print(\"yes: 1706, 1707\")\n",
    "        if row['(2705, 2706)'] == 1:\n",
    "            print(\"yes: 2705, 2706\")\n",
    "        if row['(1518, 1519)'] == 1:\n",
    "            print(\"yes: 1518, 1519\")\n",
    "        if row['(1251, 1252)'] == 1:\n",
    "            print(\"yes: 1251, 1252\")\n",
    "        if row['(1568, 1569)'] == 1:\n",
    "            print(\"yes: 1568, 1569\")\n",
    "        if row['(1756, 1757)'] == 1:\n",
    "            print(\"yes: 1756, 1757\")\n",
    "        if row['(1772, 1773)'] == 1:\n",
    "            print(\"yes: 1772, 1773\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Search data for biomarkers with old"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = reduced1\n",
    "## Apodemus\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(1293, 1294)'] == 1 and row['(1443, 1444)'] == 1 and row ['(1451, 1452)'] == 1 and row ['(1592, 1593)'] == 1 and row['(2695, 2696)'] == 1 and row['(2941, 2942)'] == 1:\n",
    "        print(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./20140116_PH56SOLrun/monopeaklists/20140116_PH56SOLrun_N4.csv\n",
      "./20140116_PH56SOLrun/monopeaklists/20140116_PH56SOLrun_F12.csv\n",
      "./20140113_PH88SOL/monopeaklists/20140113_PH88SOL_O1.csv\n",
      "./20140113_PH88SOL/monopeaklists/20140113_PH88SOL_O21.csv\n",
      "./20140113_PH88SOL/monopeaklists/20140113_PH88SOL_I24.csv\n",
      "./20140113_PH88SOL/monopeaklists/20140113_PH88SOL_K21.csv\n",
      "./20140113_PH88SOL/monopeaklists/20140113_PH88SOL_C18.csv\n",
      "./20140113_PH88SOL/monopeaklists/20140113_PH88SOL_A24.csv\n",
      "./20140113_PH88SOL/monopeaklists/20140113_PH88SOL_I9.csv\n",
      "./20140113_PH88SOL/monopeaklists/20140113_PH88SOL_F6.csv\n",
      "./20131112_P132sols/monopeaklists/20131112_P132sols_F20.csv\n",
      "./20131112_P132sols/monopeaklists/20131112_P132sols_J21.csv\n",
      "./20131112_P132sols/monopeaklists/20131112_P132sols_J23.csv\n",
      "./20131112_P132sols/monopeaklists/20131112_P132sols_D15.csv\n",
      "./20140116_PH52SOLrun/monopeaklists/20140116_PH52SOLrun_E7.csv\n",
      "./20140115_PH80SOLrun/monopeaklists/20140115_PH80SOLrun_J6.csv\n",
      "./20140115_PH80SOLrun/monopeaklists/20140115_PH80SOLrun_O11.csv\n",
      "./20131209_P96solsRun/monopeaklists/20131209_P96solsRun_O17.csv\n",
      "./20140116_PH60SOLrun/monopeaklists/20140116_PH60SOLrun_G10.csv\n",
      "./20140116_PH60SOLrun/monopeaklists/20140116_PH60SOLrun_J11.csv\n",
      "./20140114_PH84SOLrun/monopeaklists/20140114_PH84SOLrun_E12.csv\n",
      "./20131218_PH92solRun/monopeaklists/20131218_PH92solRun_H22.csv\n",
      "./20140115_PH72SOLrun/monopeaklists/20140115_PH72SOLrun_I1.csv\n",
      "./20140115_PH72SOLrun/monopeaklists/20140115_PH72SOLrun_C17.csv\n"
     ]
    }
   ],
   "source": [
    "## Mammuthus\n",
    "\n",
    "listmam = []\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(2705, 2706)'] == 1 and row['(1251, 1252)'] == 1 and row['(1568, 1569)'] == 1:\n",
    "        print(row[0])\n",
    "        listmam.append(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                               filename  (699, 700)  \\\n",
      "542   ./20140116_PH56SOLrun/monopeaklists/20140116_P...           0   \n",
      "593   ./20140116_PH56SOLrun/monopeaklists/20140116_P...           0   \n",
      "710   ./20140113_PH88SOL/monopeaklists/20140113_PH88...           0   \n",
      "740   ./20140113_PH88SOL/monopeaklists/20140113_PH88...           0   \n",
      "758   ./20140113_PH88SOL/monopeaklists/20140113_PH88...           0   \n",
      "833   ./20140113_PH88SOL/monopeaklists/20140113_PH88...           0   \n",
      "854   ./20140113_PH88SOL/monopeaklists/20140113_PH88...           0   \n",
      "878   ./20140113_PH88SOL/monopeaklists/20140113_PH88...           0   \n",
      "908   ./20140113_PH88SOL/monopeaklists/20140113_PH88...           0   \n",
      "914   ./20140113_PH88SOL/monopeaklists/20140113_PH88...           0   \n",
      "1105  ./20131112_P132sols/monopeaklists/20131112_P13...           0   \n",
      "1338  ./20131112_P132sols/monopeaklists/20131112_P13...           0   \n",
      "1356  ./20131112_P132sols/monopeaklists/20131112_P13...           0   \n",
      "1377  ./20131112_P132sols/monopeaklists/20131112_P13...           0   \n",
      "1450  ./20140116_PH52SOLrun/monopeaklists/20140116_P...           0   \n",
      "1871  ./20140115_PH80SOLrun/monopeaklists/20140115_P...           0   \n",
      "2110  ./20140115_PH80SOLrun/monopeaklists/20140115_P...           0   \n",
      "2773  ./20131209_P96solsRun/monopeaklists/20131209_P...           0   \n",
      "2953  ./20140116_PH60SOLrun/monopeaklists/20140116_P...           0   \n",
      "3175  ./20140116_PH60SOLrun/monopeaklists/20140116_P...           0   \n",
      "4068  ./20140114_PH84SOLrun/monopeaklists/20140114_P...           0   \n",
      "4407  ./20131218_PH92solRun/monopeaklists/20131218_P...           0   \n",
      "5077  ./20140115_PH72SOLrun/monopeaklists/20140115_P...           0   \n",
      "5148  ./20140115_PH72SOLrun/monopeaklists/20140115_P...           0   \n",
      "\n",
      "      (700, 701)  (701, 702)  (702, 703)  (703, 704)  (704, 705)  (705, 706)  \\\n",
      "542            0           0           0           0           0           0   \n",
      "593            0           0           0           0           0           0   \n",
      "710            0           0           0           0           0           0   \n",
      "740            0           0           0           0           0           0   \n",
      "758            0           0           0           0           0           0   \n",
      "833            0           0           0           0           0           0   \n",
      "854            0           0           0           0           0           0   \n",
      "878            0           0           0           0           0           0   \n",
      "908            0           0           0           0           0           0   \n",
      "914            0           0           0           0           0           0   \n",
      "1105           0           0           0           0           0           0   \n",
      "1338           0           0           0           0           0           0   \n",
      "1356           0           0           0           0           0           0   \n",
      "1377           0           0           1           0           0           0   \n",
      "1450           0           0           0           0           0           0   \n",
      "1871           0           0           0           0           0           0   \n",
      "2110           0           0           0           0           0           0   \n",
      "2773           0           0           0           0           0           0   \n",
      "2953           0           0           0           0           0           0   \n",
      "3175           0           0           0           0           0           0   \n",
      "4068           0           0           0           0           0           0   \n",
      "4407           0           0           0           0           0           0   \n",
      "5077           0           0           0           0           0           0   \n",
      "5148           0           0           0           1           0           0   \n",
      "\n",
      "      (706, 707)  (707, 708)      ...       (3690, 3691)  (3691, 3692)  \\\n",
      "542            0           0      ...                  0             0   \n",
      "593            0           1      ...                  0             0   \n",
      "710            0           0      ...                  0             0   \n",
      "740            0           1      ...                  0             0   \n",
      "758            0           0      ...                  1             0   \n",
      "833            0           0      ...                  0             0   \n",
      "854            0           0      ...                  0             0   \n",
      "878            0           0      ...                  0             0   \n",
      "908            0           0      ...                  0             0   \n",
      "914            0           0      ...                  0             0   \n",
      "1105           0           0      ...                  0             0   \n",
      "1338           0           0      ...                  0             0   \n",
      "1356           0           0      ...                  0             0   \n",
      "1377           0           0      ...                  0             0   \n",
      "1450           0           0      ...                  0             0   \n",
      "1871           0           0      ...                  0             0   \n",
      "2110           0           0      ...                  0             0   \n",
      "2773           0           0      ...                  0             0   \n",
      "2953           0           0      ...                  0             0   \n",
      "3175           0           0      ...                  0             0   \n",
      "4068           0           0      ...                  0             0   \n",
      "4407           0           0      ...                  0             0   \n",
      "5077           0           0      ...                  0             0   \n",
      "5148           0           0      ...                  0             0   \n",
      "\n",
      "      (3692, 3693)  (3693, 3694)  (3694, 3695)  (3695, 3696)  (3696, 3697)  \\\n",
      "542              0             0             0             0             0   \n",
      "593              0             0             0             0             0   \n",
      "710              0             0             0             0             0   \n",
      "740              0             0             0             0             0   \n",
      "758              0             0             0             0             0   \n",
      "833              0             0             0             0             0   \n",
      "854              0             0             0             0             0   \n",
      "878              0             0             0             0             0   \n",
      "908              0             0             0             0             0   \n",
      "914              0             0             0             0             0   \n",
      "1105             0             0             0             0             0   \n",
      "1338             0             0             0             0             0   \n",
      "1356             0             0             0             0             0   \n",
      "1377             0             0             0             0             0   \n",
      "1450             0             0             0             0             0   \n",
      "1871             0             0             0             0             0   \n",
      "2110             0             0             0             0             0   \n",
      "2773             0             0             0             0             0   \n",
      "2953             0             0             0             0             0   \n",
      "3175             0             0             0             0             0   \n",
      "4068             0             0             0             0             0   \n",
      "4407             0             0             0             0             0   \n",
      "5077             0             0             0             0             0   \n",
      "5148             0             0             0             0             0   \n",
      "\n",
      "      (3697, 3698)  (3698, 3699)  (3699, 3700)  \n",
      "542              0             0             0  \n",
      "593              0             0             0  \n",
      "710              0             0             0  \n",
      "740              0             0             0  \n",
      "758              0             0             0  \n",
      "833              0             0             0  \n",
      "854              0             0             0  \n",
      "878              0             0             0  \n",
      "908              0             0             0  \n",
      "914              0             0             0  \n",
      "1105             0             0             0  \n",
      "1338             1             0             0  \n",
      "1356             0             0             0  \n",
      "1377             0             0             0  \n",
      "1450             0             0             0  \n",
      "1871             0             0             0  \n",
      "2110             0             0             0  \n",
      "2773             0             0             0  \n",
      "2953             0             0             0  \n",
      "3175             0             0             0  \n",
      "4068             0             0             0  \n",
      "4407             0             0             0  \n",
      "5077             0             0             0  \n",
      "5148             0             0             0  \n",
      "\n",
      "[24 rows x 3002 columns]\n",
      "[542, 593, 710, 740, 758, 833, 854, 878, 908, 914, 1105, 1338, 1356, 1377, 1450, 1871, 2110, 2773, 2953, 3175, 4068, 4407, 5077, 5148]\n"
     ]
    }
   ],
   "source": [
    "#mamlist\n",
    "lismam = []\n",
    "mampd = df1.loc[df1['filename'].isin(listmam)]\n",
    "print(mampd)\n",
    "for i in mampd.index[:]:\n",
    "    lismam.append(i) \n",
    "print(lismam)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1 0 ... 1 1 1]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "\n",
    "\n",
    "#compute distances, fit data\n",
    "kmeans = KMeans(n_clusters=3)\n",
    "kmeans.fit(reduced)\n",
    "y_kmeans = kmeans.predict(reduced)\n",
    "print(y_kmeans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-2.70365657 -0.42521802 -1.03048798]\n",
      " [ 0.96876586 -0.67075299 -0.91845675]\n",
      " [-3.27233784  0.21626324 -1.56281267]\n",
      " ...\n",
      " [ 0.49912673 -2.0679248   1.69560446]\n",
      " [ 4.00550603  0.09933486 -0.31804291]\n",
      " [ 3.31135306  0.0454949  -0.37167566]]\n",
      "[[-2.70365657 -0.42521802 -1.03048798]\n",
      " [ 0.96876586 -0.67075299 -0.91845675]\n",
      " [-3.27233784  0.21626324 -1.56281267]\n",
      " ...\n",
      " [ 0.49912673 -2.0679248   1.69560446]\n",
      " [ 4.00550603  0.09933486 -0.31804291]\n",
      " [ 3.31135306  0.0454949  -0.37167566]]\n",
      "<class 'numpy.ndarray'>\n"
     ]
    }
   ],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "#reduce data to 2,3 dimensions\n",
    "data_reduced3 = PCA(n_components=3).fit_transform(reduced)\n",
    "slice2 = data_reduced3[2953]\n",
    "np.asarray(slice2)\n",
    "print(data_reduced3)\n",
    "data_reduced2 = PCA(n_components=2).fit_transform(reduced)\n",
    "print(data_reduced3)\n",
    "print(type(slice2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1.62741663 -0.90412839  2.11352524]\n",
      " [-2.97975837 -1.13605267 -0.10106213]\n",
      " [-1.0020822   2.03008988  0.20958323]\n",
      " [-1.456646    0.27135042  1.27854738]\n",
      " [ 1.24004806 -0.67588991 -1.00660315]\n",
      " [ 3.21008239 -0.22683291 -0.46732944]\n",
      " [-0.05242667  2.04248896 -2.84399973]\n",
      " [ 2.00732988  1.68640513 -2.40733409]\n",
      " [ 2.87564146  1.2797911  -2.15123968]\n",
      " [-1.57302066  1.39889682 -1.04351068]\n",
      " [ 2.38913768  1.81988306  0.68679852]\n",
      " [ 1.37738335  1.33237435 -1.81922562]\n",
      " [-0.52325962  3.00863939  0.91524173]\n",
      " [ 4.39296095  0.60739777 -1.03276176]\n",
      " [ 1.88762188 -1.9715168   1.29598705]\n",
      " [ 4.11875993 -0.48110957  0.27923227]\n",
      " [-2.86681771 -0.77058128 -0.49537388]\n",
      " [-0.56377713  3.24707896  0.65537743]\n",
      " [-2.21407769 -0.20184476 -0.86284492]\n",
      " [ 2.28482177 -0.88966864  2.44235903]\n",
      " [ 2.30487428 -2.20947136  1.60575116]\n",
      " [-2.0724143   0.50588391  0.12222239]\n",
      " [ 1.25965195  1.07227408  2.83545002]\n",
      " [ 1.41481019 -2.49948106  1.69092353]]\n"
     ]
    }
   ],
   "source": [
    "lismam = np.array(lismam)\n",
    "comam = data_reduced3[lismam]\n",
    "    \n",
    "print(comam)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'comam' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-44-64c874b5f47f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m trace2 = go.Scatter3d(\n\u001b[0;32m---> 26\u001b[0;31m     \u001b[0mx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcomam\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m     \u001b[0my\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcomam\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m     \u001b[0mz\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcomam\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'comam' is not defined"
     ]
    }
   ],
   "source": [
    "#visualise\n",
    "\n",
    "import plotly.plotly as py\n",
    "import plotly.graph_objs as go\n",
    "\n",
    "import colorlover as cl\n",
    "\n",
    "\n",
    "trace = go.Scatter3d(\n",
    "    x=data_reduced3[:, 0],\n",
    "    y=data_reduced3[:, 1],\n",
    "    z=data_reduced3[:, 2],\n",
    "    mode='markers',\n",
    "    text = str,\n",
    "    name = 'Data',\n",
    "\n",
    "    marker=dict(color=y_kmeans,size=2.7,\n",
    "        opacity=0.7,\n",
    "        colorscale='Rainbow',\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "trace2 = go.Scatter3d(\n",
    "    x=comam[:,0],\n",
    "    y=comam[:,1],\n",
    "    z=comam[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Mammuthus\",\n",
    "    name = \"Known taxa\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(10, 0, 0, .8)',\n",
    "     size=13,\n",
    "                symbol='circle',\n",
    "        opacity=0.6,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "dataMeans = [trace, trace2]\n",
    "\n",
    "layout = go.Layout(\n",
    "    showlegend=False,\n",
    "    title='K-means, 2 clusters, samples found with old markers'\n",
    "   \n",
    "    \n",
    ")\n",
    "\n",
    "fig = go.Figure(data=dataMeans, layout=layout)\n",
    "py.iplot(fig, filename='simple-3d-scatter')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Equus\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(2705, 2706)'] == 1 and row['(1604, 1605)'] == 1 and row['(1182, 1183)'] == 1 and row['(1235, 1236)'] == 1 and row['(1311, 1312)'] == 1 and row['(1427, 1428)'] == 1 and row['(1530, 1531)'] == 1 and row['(2583, 2584)'] == 1:\n",
    "        print(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./20131112_P136sols/monopeaklists/20131112_P136sols_B2.csv\n",
      "./20140113_PH88SOL/monopeaklists/20140113_PH88SOL_D11.csv\n",
      "./20140113_PH88SOL/monopeaklists/20140113_PH88SOL_E11.csv\n",
      "./20140113_PH88SOL/monopeaklists/20140113_PH88SOL_G13.csv\n",
      "./20140113_PH88SOL/monopeaklists/20140113_PH88SOL_O12.csv\n",
      "./20131112_P132sols/monopeaklists/20131112_P132sols_K23.csv\n",
      "./20131112_P132sols/monopeaklists/20131112_P132sols_O24.csv\n",
      "./20131112_P132sols/monopeaklists/20131112_P132sols_K3.csv\n",
      "./20131112_P132sols/monopeaklists/20131112_P132sols_N19.csv\n",
      "./20131209_P96solsRun/monopeaklists/20131209_P96solsRun_A11.csv\n",
      "./20131209_P96solsRun/monopeaklists/20131209_P96solsRun_H5.csv\n",
      "./20131209_P96solsRun/monopeaklists/20131209_P96solsRun_D19.csv\n",
      "./20140114_PH84SOLrun/monopeaklists/20140114_PH84SOLrun_M23.csv\n",
      "./20131218_PH92solRun/monopeaklists/20131218_PH92solRun_I17.csv\n",
      "./20131218_PH92solRun/monopeaklists/20131218_PH92solRun_E18.csv\n"
     ]
    }
   ],
   "source": [
    "## Coelodonta\n",
    "listcoe =[]\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(2705, 2706)'] == 1 and row['(1604, 1605)'] == 1 and row['(1261, 1262)'] == 1 and row['(1453, 1454)'] == 1 and row['(1473, 1474)'] == 1:\n",
    "        print(row[0])\n",
    "        listcoe.append(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-2.20351719  0.30294737 -0.09820481]\n",
      " [-1.28707761  3.22449799 -0.33008863]\n",
      " [-1.10963972  2.08329071 -0.59611273]\n",
      " [-1.17497165  3.57713007  1.43609384]\n",
      " [-1.22234991  2.71781245  1.8008343 ]\n",
      " [-0.99361736  3.18761513  0.57265599]\n",
      " [-0.51282112  2.97977358  1.64857897]\n",
      " [-1.24987223  3.58514843  0.19340574]\n",
      " [-0.83659801  3.89690113  0.519311  ]\n",
      " [-1.23680398  2.42833639  1.67714897]\n",
      " [-1.51657158  1.79152133  1.74950578]\n",
      " [-1.46957401  2.84350324  1.38281618]\n",
      " [-1.55210458  2.60386392 -0.33144822]\n",
      " [-0.7731999   3.45163814  2.31629919]\n",
      " [-1.45760547  3.11826156  0.66059137]]\n"
     ]
    }
   ],
   "source": [
    "#coelist\n",
    "liscoe = []\n",
    "coepd = df1.loc[df1['filename'].isin(listcoe)]\n",
    "#print(coepd)\n",
    "for i in coepd.index[:]:\n",
    "    liscoe.append(i) \n",
    "#print(liscoe)\n",
    "liscoe = np.array(liscoe)\n",
    "cocoe = data_reduced3[liscoe]\n",
    "print(cocoe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe id=\"igraph\" scrolling=\"no\" style=\"border:none;\" seamless=\"seamless\" src=\"https://plot.ly/~NataliaKudr/2.embed\" height=\"525px\" width=\"100%\"></iframe>"
      ],
      "text/plain": [
       "<plotly.tools.PlotlyDisplay object>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#visualise\n",
    "\n",
    "import plotly.plotly as py\n",
    "import plotly.graph_objs as go\n",
    "\n",
    "import colorlover as cl\n",
    "\n",
    "\n",
    "trace = go.Scatter3d(\n",
    "    x=data_reduced3[:, 0],\n",
    "    y=data_reduced3[:, 1],\n",
    "    z=data_reduced3[:, 2],\n",
    "    mode='markers',\n",
    "    text = str,\n",
    "    name = 'Data',\n",
    "\n",
    "    marker=dict(color=y_kmeans,size=2.7,\n",
    "        opacity=0.7,\n",
    "        colorscale='Rainbow',\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "trace2 = go.Scatter3d(\n",
    "    x=cocoe[:,0],\n",
    "    y=cocoe[:,1],\n",
    "    z=cocoe[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Coelodonta\",\n",
    "    name = \"Known taxa\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(10, 0, 0, .8)',\n",
    "     size=13,\n",
    "                symbol='circle',\n",
    "        opacity=0.6,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "dataMeans = [trace, trace2]\n",
    "\n",
    "layout = go.Layout(\n",
    "    showlegend=False,\n",
    "    title='K-means, 2 clusters, samples found with old markers'\n",
    "   \n",
    "    \n",
    ")\n",
    "\n",
    "fig = go.Figure(data=dataMeans, layout=layout)\n",
    "py.iplot(fig, filename='simple-3d-scatter')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Bos/Bison\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(2705, 2706)'] == 1 and row['(1832, 1833)'] == 1 and row['(1848, 1849)'] == 1 and row['(1427, 1428)'] == 1 and row['(1435, 1436)'] == 1 and row['(3018, 3019)'] == 1 and row['(1208, 1209)'] == 1 and row['(2853, 2854)'] == 1:\n",
    "        print(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./20131218_PH92solRun/monopeaklists/20131218_PH92solRun_C10.csv\n"
     ]
    }
   ],
   "source": [
    "## Cervine\n",
    "listcer =[]\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(2705, 2706)'] == 1 and row['(1832, 1833)'] == 1 and row['(1848, 1849)'] == 1 and row['(1427, 1428)'] == 1 and row['(1435, 1436)'] == 1 and row['(3018, 3019)'] == 1 and row['(1311, 1312)'] == 1 and row['(2899, 2900)'] == 1 and row['(1550, 1551)'] == 1:\n",
    "        print(row[0])\n",
    "        listcer.append(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.95316775  3.42458515  2.16038418]]\n"
     ]
    }
   ],
   "source": [
    "#ranlist\n",
    "liscer = []\n",
    "cerpd = df1.loc[df1['filename'].isin(listcer)]\n",
    "#print(coepd)\n",
    "for i in cerpd.index[:]:\n",
    "    liscer.append(i) \n",
    "#print(liscoe)\n",
    "liscer = np.array(liscer)\n",
    "cocer = data_reduced3[liscer]\n",
    "print(cocer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./20131112_P132sols/monopeaklists/20131112_P132sols_O7.csv\n",
      "./20131112_P132sols/monopeaklists/20131112_P132sols_O2.csv\n",
      "./20131112_P132sols/monopeaklists/20131112_P132sols_F20.csv\n",
      "./20131112_P132sols/monopeaklists/20131112_P132sols_M16.csv\n",
      "./20131112_P132sols/monopeaklists/20131112_P132sols_G9.csv\n",
      "./20131112_P132sols/monopeaklists/20131112_P132sols_G15.csv\n",
      "./20131112_P132sols/monopeaklists/20131112_P132sols_G1.csv\n",
      "./20131209_P96solsRun/monopeaklists/20131209_P96solsRun_K23.csv\n",
      "./20140115_PH76SOLrun/monopeaklists/20140115_PH76SOLrun_K14.csv\n",
      "./20131218_PH92solRun/monopeaklists/20131218_PH92solRun_H15.csv\n",
      "./20131218_PH92solRun/monopeaklists/20131218_PH92solRun_A21.csv\n",
      "./20131218_PH92solRun/monopeaklists/20131218_PH92solRun_A16.csv\n",
      "./20131218_PH92solRun/monopeaklists/20131218_PH92solRun_M23.csv\n",
      "./20140115_PH72SOLrun/monopeaklists/20140115_PH72SOLrun_N23.csv\n"
     ]
    }
   ],
   "source": [
    "## Rangifer\n",
    "listran =[]\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(2705, 2706)'] == 1 and row ['(2106, 2107)'] == 1 and row['(1832, 1833)'] == 1 and row['(1848, 1849)'] == 1 and row['(1427, 1428)'] == 1 and row['(1435, 1436)'] == 1 and row['(3018, 3019)'] == 1 and row['(1311, 1312)'] == 1 and row['(2899, 2900)'] == 1 and row['(1580, 1581)'] == 1:\n",
    "        print(row[0])\n",
    "        listran.append(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.27523853  4.47049934  2.88147234]\n",
      " [-1.22244699  3.89165223  1.24572638]\n",
      " [ 2.38913762  1.81998865  0.68634383]\n",
      " [-0.51318745  5.00021171  1.99281836]\n",
      " [ 1.7352403   3.75838168  2.03007129]\n",
      " [-0.88937163  4.09855087  1.46698561]\n",
      " [ 0.3827827   3.27782128  2.8706344 ]\n",
      " [-0.66946238  3.09353748  3.15808675]\n",
      " [-0.25917503  4.64188732  2.66711598]\n",
      " [-0.74250694  4.38981949  2.80103256]\n",
      " [-0.74200118  4.49905303  2.45756686]\n",
      " [-0.59834922  4.12317412  2.32884614]\n",
      " [-0.29450267  4.45541116  3.08997339]\n",
      " [ 0.82693344 -0.63329926  1.46203536]]\n"
     ]
    }
   ],
   "source": [
    "#ranlist\n",
    "lisran = []\n",
    "ranpd = df1.loc[df1['filename'].isin(listran)]\n",
    "#print(coepd)\n",
    "for i in ranpd.index[:]:\n",
    "    lisran.append(i) \n",
    "#print(liscoe)\n",
    "lisran = np.array(lisran)\n",
    "coran = data_reduced3[lisran]\n",
    "print(coran)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe id=\"igraph\" scrolling=\"no\" style=\"border:none;\" seamless=\"seamless\" src=\"https://plot.ly/~NataliaKudr/2.embed\" height=\"525px\" width=\"100%\"></iframe>"
      ],
      "text/plain": [
       "<plotly.tools.PlotlyDisplay object>"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#visualise\n",
    "\n",
    "import plotly.plotly as py\n",
    "import plotly.graph_objs as go\n",
    "\n",
    "import colorlover as cl\n",
    "\n",
    "\n",
    "trace = go.Scatter3d(\n",
    "    x=data_reduced3[:, 0],\n",
    "    y=data_reduced3[:, 1],\n",
    "    z=data_reduced3[:, 2],\n",
    "    mode='markers',\n",
    "    text = str,\n",
    "    name = 'Data',\n",
    "\n",
    "    marker=dict(color=y_kmeans,size=2.7,\n",
    "        opacity=0.7,\n",
    "        colorscale='Rainbow',\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "trace2 = go.Scatter3d(\n",
    "    x=coran[:,0],\n",
    "    y=coran[:,1],\n",
    "    z=coran[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Rangifer\",\n",
    "    name = \"Known taxa\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(10, 0, 0, .8)',\n",
    "     size=13,\n",
    "                symbol='circle',\n",
    "        opacity=0.6,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "dataMeans = [trace, trace2]\n",
    "\n",
    "layout = go.Layout(\n",
    "    showlegend=False,\n",
    "    title='K-means, 2 clusters, samples found with old markers'\n",
    "   \n",
    "    \n",
    ")\n",
    "\n",
    "fig = go.Figure(data=dataMeans, layout=layout)\n",
    "py.iplot(fig, filename='simple-3d-scatter')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./20140113_PH88SOL/monopeaklists/20140113_PH88SOL_I9.csv\n",
      "./20140113_PH88SOL/monopeaklists/20140113_PH88SOL_K15.csv\n",
      "./20131112_P132sols/monopeaklists/20131112_P132sols_F20.csv\n",
      "./20131112_P132sols/monopeaklists/20131112_P132sols_K16.csv\n",
      "./20131209_P96solsRun/monopeaklists/20131209_P96solsRun_B2.csv\n",
      "./20131209_P96solsRun/monopeaklists/20131209_P96solsRun_I17.csv\n",
      "./20131218_PH92solRun/monopeaklists/20131218_PH92solRun_C22.csv\n"
     ]
    }
   ],
   "source": [
    "## Lepus\n",
    "listlep =[]\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(2705, 2706)'] == 1 and row['(1832, 1833)'] == 1 and row['(1848, 1849)'] == 1 and row['(1453, 1454)'] == 1 and row['(1311, 1312)'] == 1 and row['(1501, 1502)'] == 1 and row['(1532, 1533)'] == 1 and row['(1608, 1609)'] == 1:\n",
    "        print(row[0])\n",
    "        listlep.append(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 2.87564149  1.27973722 -2.1510189 ]\n",
      " [ 2.30848492  0.05416137  0.01547437]\n",
      " [ 2.38913762  1.81998865  0.68634383]\n",
      " [ 3.30283045  0.87102842 -1.43463242]\n",
      " [ 3.75362994  1.69351822 -1.99587727]\n",
      " [ 3.39868957  1.01737832 -1.53400486]\n",
      " [-0.16803062  4.78530918  2.89791861]]\n"
     ]
    }
   ],
   "source": [
    "#leplist\n",
    "lislep = []\n",
    "leppd = df1.loc[df1['filename'].isin(listlep)]\n",
    "#print(coepd)\n",
    "for i in leppd.index[:]:\n",
    "    lislep.append(i) \n",
    "#print(liscoe)\n",
    "lislep = np.array(lislep)\n",
    "colep = data_reduced3[lislep]\n",
    "print(colep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe id=\"igraph\" scrolling=\"no\" style=\"border:none;\" seamless=\"seamless\" src=\"https://plot.ly/~NataliaKudr/2.embed\" height=\"525px\" width=\"100%\"></iframe>"
      ],
      "text/plain": [
       "<plotly.tools.PlotlyDisplay object>"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#visualise\n",
    "\n",
    "import plotly.plotly as py\n",
    "import plotly.graph_objs as go\n",
    "\n",
    "import colorlover as cl\n",
    "\n",
    "\n",
    "trace = go.Scatter3d(\n",
    "    x=data_reduced3[:, 0],\n",
    "    y=data_reduced3[:, 1],\n",
    "    z=data_reduced3[:, 2],\n",
    "    mode='markers',\n",
    "    text = str,\n",
    "    name = 'Data',\n",
    "\n",
    "    marker=dict(color=y_kmeans,size=2.7,\n",
    "        opacity=0.7,\n",
    "        colorscale='Rainbow',\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "trace2 = go.Scatter3d(\n",
    "    x=colep[:,0],\n",
    "    y=colep[:,1],\n",
    "    z=colep[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Lepus\",\n",
    "    name = \"Known taxa\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(10, 0, 0, .8)',\n",
    "     size=13,\n",
    "                symbol='circle',\n",
    "        opacity=0.6,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "dataMeans = [trace, trace2]\n",
    "\n",
    "layout = go.Layout(\n",
    "    showlegend=False,\n",
    "    title='K-means, 2 clusters, samples found with old markers'\n",
    "   \n",
    "    \n",
    ")\n",
    "\n",
    "fig = go.Figure(data=dataMeans, layout=layout)\n",
    "py.iplot(fig, filename='simple-3d-scatter')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./20140115_PH76SOLrun/monopeaklists/20140115_PH76SOLrun_E6.csv\n",
      "./20140115_PH76SOLrun/monopeaklists/20140115_PH76SOLrun_F5.csv\n"
     ]
    }
   ],
   "source": [
    "## Crocuta/Panthera\n",
    "listcro =[]\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(2705, 2706)'] == 1 and row['(1832, 1833)'] == 1 and row['(1848, 1849)'] == 1 and row['(1453, 1454)'] == 1 and row['(1261, 1262)'] == 1 and row['(1459, 1460)'] == 1 and row['(2247, 2248)'] == 1:\n",
    "        print(row[0])\n",
    "        listcro.append(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[2.91101202 0.08203382 2.26456222]\n",
      " [3.02099566 1.28725073 0.31054773]]\n"
     ]
    }
   ],
   "source": [
    "#crolist\n",
    "liscro = []\n",
    "cropd = df1.loc[df1['filename'].isin(listcro)]\n",
    "#print(coepd)\n",
    "for i in cropd.index[:]:\n",
    "    liscro.append(i) \n",
    "#print(liscoe)\n",
    "liscro = np.array(liscro)\n",
    "cocro = data_reduced3[liscro]\n",
    "print(cocro)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe id=\"igraph\" scrolling=\"no\" style=\"border:none;\" seamless=\"seamless\" src=\"https://plot.ly/~NataliaKudr/2.embed\" height=\"525px\" width=\"100%\"></iframe>"
      ],
      "text/plain": [
       "<plotly.tools.PlotlyDisplay object>"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#visualise\n",
    "\n",
    "import plotly.plotly as py\n",
    "import plotly.graph_objs as go\n",
    "\n",
    "import colorlover as cl\n",
    "\n",
    "\n",
    "trace = go.Scatter3d(\n",
    "    x=data_reduced3[:, 0],\n",
    "    y=data_reduced3[:, 1],\n",
    "    z=data_reduced3[:, 2],\n",
    "    mode='markers',\n",
    "    text = str,\n",
    "    name = 'Data',\n",
    "\n",
    "    marker=dict(color=y_kmeans,size=2.7,\n",
    "        opacity=0.7,\n",
    "        colorscale='Rainbow',\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "trace2 = go.Scatter3d(\n",
    "    x=cocro[:,0],\n",
    "    y=cocro[:,1],\n",
    "    z=cocro[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Crocuta/Panthera\",\n",
    "    name = \"Known taxa\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(10, 0, 0, .8)',\n",
    "     size=13,\n",
    "                symbol='circle',\n",
    "        opacity=0.6,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "dataMeans = [trace, trace2]\n",
    "\n",
    "layout = go.Layout(\n",
    "    showlegend=False,\n",
    "    title='K-means, 2 clusters, samples found with old markers'\n",
    "   \n",
    "    \n",
    ")\n",
    "\n",
    "fig = go.Figure(data=dataMeans, layout=layout)\n",
    "py.iplot(fig, filename='simple-3d-scatter')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Mustela\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(2705, 2706)'] == 1 and row['(1832, 1833)'] == 1 and row['(1848, 1849)'] == 1 and row['(1453, 1454)'] == 1 and row['(1261, 1262)'] == 1 and row['(1459, 1460)'] == 1 and row['(2216, 2217)'] == 1 and row['(1235, 1236)'] == 1 and row['(1548, 1549)'] == 1 and row['(1680, 1681)'] == 1 and row['(2147, 2148)'] == 1:\n",
    "        print(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Ursus\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(2705, 2706)'] == 1 and row['(1832, 1833)'] == 1 and row['(1848, 1849)'] == 1 and row['(1453, 1454)'] == 1 and row['(1261, 1262)'] == 1 and row['(1459, 1460)'] == 1 and row['(2216, 2217)'] == 1 and row['(1690, 1691)'] == 1 and row['(2135, 2136)'] == 1 and row['(2163, 2164)'] == 1 and row['(2599, 2600)'] == 1:\n",
    "        print(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./20140113_PH88SOL/monopeaklists/20140113_PH88SOL_D11.csv\n",
      "./20140115_PH80SOLrun/monopeaklists/20140115_PH80SOLrun_J15.csv\n",
      "['./20140113_PH88SOL/monopeaklists/20140113_PH88SOL_D11.csv', './20140115_PH80SOLrun/monopeaklists/20140115_PH80SOLrun_J15.csv']\n"
     ]
    }
   ],
   "source": [
    "## Canid\n",
    "listcan =[]\n",
    "for index, row in df1.iterrows():\n",
    "    if row['(2705, 2706)'] == 1 and row['(1832, 1833)'] == 1 and row['(1848, 1849)'] == 1 and row['(1453, 1454)'] == 1 and row['(1261, 1262)'] == 1 and row['(1459, 1460)'] == 1 and row['(2216, 2217)'] == 1 and row['(1690, 1691)'] == 1 and row['(2131, 2132)'] == 1:\n",
    "        print(row[0])\n",
    "        listcan.append(row[0])\n",
    "print(listcan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-1.28707761  3.22449799 -0.33008863]\n",
      " [-1.25598299  2.08200587  0.87085083]]\n"
     ]
    }
   ],
   "source": [
    "#canlist\n",
    "liscan = []\n",
    "canpd = df1.loc[df1['filename'].isin(listcan)]\n",
    "#print(coepd)\n",
    "for i in canpd.index[:]:\n",
    "    liscan.append(i) \n",
    "#print(liscoe)\n",
    "liscan = np.array(liscan)\n",
    "cocan = data_reduced3[liscan]\n",
    "print(cocan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe id=\"igraph\" scrolling=\"no\" style=\"border:none;\" seamless=\"seamless\" src=\"https://plot.ly/~NataliaKudr/2.embed\" height=\"525px\" width=\"100%\"></iframe>"
      ],
      "text/plain": [
       "<plotly.tools.PlotlyDisplay object>"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#visualise\n",
    "\n",
    "import plotly.plotly as py\n",
    "import plotly.graph_objs as go\n",
    "\n",
    "import colorlover as cl\n",
    "\n",
    "\n",
    "trace = go.Scatter3d(\n",
    "    x=data_reduced3[:, 0],\n",
    "    y=data_reduced3[:, 1],\n",
    "    z=data_reduced3[:, 2],\n",
    "    mode='markers',\n",
    "    text = str,\n",
    "    name = 'Data',\n",
    "\n",
    "    marker=dict(color=y_kmeans,size=2.7,\n",
    "        opacity=0.7,\n",
    "        colorscale='Rainbow',\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "trace2 = go.Scatter3d(\n",
    "    x=cocan[:,0],\n",
    "    y=cocan[:,1],\n",
    "    z=cocan[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Canid\",\n",
    "    name = \"Known taxa\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(10, 0, 0, .8)',\n",
    "     size=13,\n",
    "                symbol='circle',\n",
    "        opacity=0.6,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "dataMeans = [trace, trace2]\n",
    "\n",
    "layout = go.Layout(\n",
    "    showlegend=False,\n",
    "    title='K-means, 2 clusters, samples found without novel markers'\n",
    "   \n",
    "    \n",
    ")\n",
    "\n",
    "fig = go.Figure(data=dataMeans, layout=layout)\n",
    "py.iplot(fig, filename='simple-3d-scatter')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe id=\"igraph\" scrolling=\"no\" style=\"border:none;\" seamless=\"seamless\" src=\"https://plot.ly/~NataliaKudr/2.embed\" height=\"525px\" width=\"100%\"></iframe>"
      ],
      "text/plain": [
       "<plotly.tools.PlotlyDisplay object>"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#visualise\n",
    "\n",
    "import plotly.plotly as py\n",
    "import plotly.graph_objs as go\n",
    "\n",
    "import colorlover as cl\n",
    "\n",
    "\n",
    "trace = go.Scatter3d(\n",
    "    x=data_reduced3[:, 0],\n",
    "    y=data_reduced3[:, 1],\n",
    "    z=data_reduced3[:, 2],\n",
    "    mode='markers',\n",
    "    text = str,\n",
    "    name = 'Full Dataset',\n",
    "\n",
    "    marker=dict(color= y_kmeans, size=2,\n",
    "        opacity=0.9,\n",
    "       colorscale='Blues',\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "trace2 = go.Scatter3d(\n",
    "    x=cocan[:,0],\n",
    "    y=cocan[:,1],\n",
    "    z=cocan[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Canid\",\n",
    "    name = \"Canid\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(10, 0, 110, .8)',\n",
    "     size=8,\n",
    "                symbol='circle',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "trace3 = go.Scatter3d(\n",
    "    x=comam[:,0],\n",
    "    y=comam[:,1],\n",
    "    z=comam[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Mammuthus\",\n",
    "    name = \"Mammuthus\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(252.0, 141.0, 89.0, 0.8)',\n",
    "     size=8,\n",
    "                symbol='cross',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "trace4 = go.Scatter3d(\n",
    "    x=cocoe[:,0],\n",
    "    y=cocoe[:,1],\n",
    "    z=cocoe[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Coelodonta\",\n",
    "    name = \"Coelodonta\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(255.0, 200.0, 150.0, .8)',\n",
    "     size=8,\n",
    "                symbol='square',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "trace5 = go.Scatter3d(\n",
    "    x=coran[:,0],\n",
    "    y=coran[:,1],\n",
    "    z=coran[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Rangifer\",\n",
    "    name = \"Rangifer\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(145.0, 191.0, 219.0, .8)',\n",
    "     size=8,\n",
    "                symbol='diamond',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "trace6 = go.Scatter3d(\n",
    "    x=colep[:,0],\n",
    "    y=colep[:,1],\n",
    "    z=colep[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Lepus\",\n",
    "    name = \"Lepus\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(145.0, 191.0, 0.0, .8)',\n",
    "     size=8,\n",
    "                symbol='square-open',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "trace7 = go.Scatter3d(\n",
    "    x=cocro[:,0],\n",
    "    y=cocro[:,1],\n",
    "    z=cocro[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Crocuta/Panthera\",\n",
    "    name = \"Crocuta/Panthera\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(145.0, 0.0, 0.0, .8)',\n",
    "     size=8,\n",
    "                symbol='circle-open',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "trace8 = go.Scatter3d(\n",
    "    x=cocer[:,0],\n",
    "    y=cocer[:,1],\n",
    "    z=cocer[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Cervine\",\n",
    "    name = \"Cervine\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(145.0, 102.0, 0.0, .8)',\n",
    "     size=8,\n",
    "                symbol='diamond-open',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "dataMeans = [trace, trace2,trace3,trace4,trace5,trace6,trace7,trace8]\n",
    "\n",
    "layout = go.Layout(\n",
    "    showlegend=True,\n",
    "    title='K-means Clustering: 3 clusters; Taxa Identified without novel Biomarkermarkers'\n",
    "   \n",
    "    \n",
    ")\n",
    "\n",
    "fig = go.Figure(data=dataMeans, layout=layout)\n",
    "py.iplot(fig, filename='simple-3d-scatter')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Init: initializing centroids\n",
      "Init: initializing clusters\n",
      "Starting iterations...\n",
      "Run 1, iteration: 1/100, moves: 2244, cost: 662568.0\n",
      "Run 1, iteration: 2/100, moves: 598, cost: 662056.0\n",
      "Run 1, iteration: 3/100, moves: 98, cost: 662018.0\n",
      "Run 1, iteration: 4/100, moves: 10, cost: 662018.0\n",
      "Init: initializing centroids\n",
      "Init: initializing clusters\n",
      "Starting iterations...\n",
      "Run 2, iteration: 1/100, moves: 2066, cost: 660384.0\n",
      "Run 2, iteration: 2/100, moves: 376, cost: 660148.0\n",
      "Run 2, iteration: 3/100, moves: 72, cost: 660123.0\n",
      "Run 2, iteration: 4/100, moves: 8, cost: 660123.0\n",
      "Init: initializing centroids\n",
      "Init: initializing clusters\n",
      "Starting iterations...\n",
      "Run 3, iteration: 1/100, moves: 3806, cost: 662295.0\n",
      "Run 3, iteration: 2/100, moves: 448, cost: 661648.0\n",
      "Run 3, iteration: 3/100, moves: 68, cost: 661648.0\n",
      "Best run was number 2\n",
      "[[0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]]\n",
      "[0 2 0 ... 2 1 1]\n"
     ]
    }
   ],
   "source": [
    "from kmodes.kmodes import KModes\n",
    "\n",
    "km = KModes(n_clusters=3, init='Huang', n_init=3, verbose=1)\n",
    "clusters = km.fit_predict(reduced)\n",
    "labelsmodes = km.labels_\n",
    "centroids = km.cluster_centroids_\n",
    "\n",
    "#Print the cluster centroids\n",
    "print(km.cluster_centroids_)\n",
    "print(km.labels_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe id=\"igraph\" scrolling=\"no\" style=\"border:none;\" seamless=\"seamless\" src=\"https://plot.ly/~NataliaKudr/2.embed\" height=\"525px\" width=\"100%\"></iframe>"
      ],
      "text/plain": [
       "<plotly.tools.PlotlyDisplay object>"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#visualise\n",
    "\n",
    "import plotly.plotly as py\n",
    "import plotly.graph_objs as go\n",
    "\n",
    "import colorlover as cl\n",
    "\n",
    "\n",
    "trace = go.Scatter3d(\n",
    "    x=data_reduced3[:, 0],\n",
    "    y=data_reduced3[:, 1],\n",
    "    z=data_reduced3[:, 2],\n",
    "    mode='markers',\n",
    "    text = str,\n",
    "    name = 'Full Dataset',\n",
    "\n",
    "    marker=dict(color= y_kmeans, size=2,\n",
    "        opacity=0.9,\n",
    "       colorscale='Blues',\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "trace2 = go.Scatter3d(\n",
    "    x=cocan[:,0],\n",
    "    y=cocan[:,1],\n",
    "    z=cocan[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Canid\",\n",
    "    name = \"Canid\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(10, 0, 110, .8)',\n",
    "     size=8,\n",
    "                symbol='circle',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "trace3 = go.Scatter3d(\n",
    "    x=comam[:,0],\n",
    "    y=comam[:,1],\n",
    "    z=comam[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Mammuthus\",\n",
    "    name = \"Mammuthus\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(252.0, 141.0, 89.0, 0.8)',\n",
    "     size=8,\n",
    "                symbol='cross',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "trace4 = go.Scatter3d(\n",
    "    x=cocoe[:,0],\n",
    "    y=cocoe[:,1],\n",
    "    z=cocoe[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Coelodonta\",\n",
    "    name = \"Coelodonta\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(255.0, 200.0, 150.0, .8)',\n",
    "     size=8,\n",
    "                symbol='square',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "trace5 = go.Scatter3d(\n",
    "    x=coran[:,0],\n",
    "    y=coran[:,1],\n",
    "    z=coran[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Rangifer\",\n",
    "    name = \"Rangifer\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(145.0, 191.0, 219.0, .8)',\n",
    "     size=8,\n",
    "                symbol='diamond',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "trace6 = go.Scatter3d(\n",
    "    x=colep[:,0],\n",
    "    y=colep[:,1],\n",
    "    z=colep[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Lepus\",\n",
    "    name = \"Lepus\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(145.0, 191.0, 0.0, .8)',\n",
    "     size=8,\n",
    "                symbol='square-open',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "trace7 = go.Scatter3d(\n",
    "    x=cocro[:,0],\n",
    "    y=cocro[:,1],\n",
    "    z=cocro[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Crocuta/Panthera\",\n",
    "    name = \"Crocuta/Panthera\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(145.0, 0.0, 0.0, .8)',\n",
    "     size=8,\n",
    "                symbol='circle-open',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "trace8 = go.Scatter3d(\n",
    "    x=cocer[:,0],\n",
    "    y=cocer[:,1],\n",
    "    z=cocer[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Cervine\",\n",
    "    name = \"Cervine\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(145.0, 102.0, 0.0, .8)',\n",
    "     size=8,\n",
    "                symbol='diamond-open',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "dataMeans = [trace, trace2,trace3,trace4,trace5,trace6,trace7,trace8]\n",
    "\n",
    "layout = go.Layout(\n",
    "    showlegend=True,\n",
    "    title='K-modes Clustering: 3 clusters; Taxa Identified without novel Biomarkermarkers'\n",
    "   \n",
    "    \n",
    ")\n",
    "\n",
    "fig = go.Figure(data=dataMeans, layout=layout)\n",
    "py.iplot(fig, filename='simple-3d-scatter')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As it can be seen from the figure, many taxa are dispersed around and do not localise. This figure is based on biomarkers identified previously to Gu's paper as with the new biomarkers only 1 sample was identfiable. coordinates the same but not the same file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## duplicate coordinates check\n",
    "\n",
    "full = listlep\n",
    "full.extend(listcoe)\n",
    "full.extend(listmam)\n",
    "full.extend(listcan)\n",
    "full.extend(listcro)\n",
    "full.extend(listran)\n",
    "print(len(full))\n",
    "print(full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fulllist\n",
    "lisfull = []\n",
    "fullpd = df1.loc[df1['filename'].isin(full)]\n",
    "print(fullpd)\n",
    "\n",
    "for i in fullpd.index[:]:\n",
    "    lisfull.append(i) \n",
    "#print(liscoe)\n",
    "lisfull = np.array(lisfull)\n",
    "cofull = data_reduced3[lisfull]\n",
    "print(cofull)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "no_dupes = [x for n, x in enumerate(cofull) if x not in cofull[:n]]\n",
    "print (no_dupes) # [[1], [2], [3], [5]]\n",
    "\n",
    "dupes = [x for n, x in enumerate(cofull) if x in cofull[:n]]\n",
    "print (dupes) # [[1], [3]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2.38913763  1.81997382  0.68673084 mam/lep/ran\n",
    "./20131112_P132sols/monopeaklists/20131112_P132sols_F20.csv\n",
    "./20131112_P132sols/monopeaklists/20131112_P132sols_F20.csv\n",
    "./20131112_P132sols/monopeaklists/20131112_P132sols_F20.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-1.28707758  3.22445118 -0.3305387 can/coe\n",
    "./20140113_PH88SOL/monopeaklists/20140113_PH88SOL_D11.csv\n",
    "./20140113_PH88SOL/monopeaklists/20140113_PH88SOL_D11.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2.87564149  1.27970445 -2.15111653 mam/lep\n",
    "./20140113_PH88SOL/monopeaklists/20140113_PH88SOL_I9.csv\n",
    "./20140113_PH88SOL/monopeaklists/20140113_PH88SOL_I9.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['./20131209_P96solsRun/monopeaklists/20131209_P96solsRun_N7.csv', './20131218_PH92solRun/monopeaklists/20131218_PH92solRun_G17.csv', './20140116_PH60SOLrun/monopeaklists/20140116_PH60SOLrun_C21.csv', './20131112_P140sols/monopeaklists/20131112_P140sols_H10.csv', './20131112_P140sols/monopeaklists/20131112_P140sols_I24.csv', './20131112_P140sols/monopeaklists/20131112_P140sols_L9.csv']\n"
     ]
    }
   ],
   "source": [
    "#ML mammoth\n",
    "mlmamlist = []\n",
    "\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131209_P96solsRun_N7.csv'):\n",
    "        mlmamlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131218_PH92solRun_G17.csv'):\n",
    "        mlmamlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH60SOLrun_C21.csv'):\n",
    "        mlmamlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131112_P140sols_H10.csv'):\n",
    "        mlmamlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131112_P140sols_I24.csv'):\n",
    "        mlmamlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131112_P140sols_L9.csv'):\n",
    "        mlmamlist.append(row[0])\n",
    "        \n",
    "print(mlmamlist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-1.32866718  2.44563248 -0.18796156]\n",
      " [-1.78668642  1.43925438 -0.16079045]\n",
      " [-1.87190208  0.39793946  0.39187263]\n",
      " [-0.84708478  2.1430092   1.30897789]\n",
      " [-1.11334134  0.89351299  1.43099102]\n",
      " [-0.93631285  2.5061577   0.8021261 ]]\n"
     ]
    }
   ],
   "source": [
    "#mlmamlist\n",
    "lismlmam = []\n",
    "mlmampd = df1.loc[df1['filename'].isin(mlmamlist)]\n",
    "#print(coepd)\n",
    "for i in mlmampd.index[:]:\n",
    "    lismlmam.append(i) \n",
    "#print(liscoe)\n",
    "lismlmam = np.array(lismlmam)\n",
    "comlmam = data_reduced3[lismlmam]\n",
    "print(comlmam)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['./20131112_P140sols/monopeaklists/20131112_P140sols_P6.csv', './20140114_PH84SOLrun/monopeaklists/20140114_PH84SOLrun_I18.csv', './20140116_PH52SOLrun/monopeaklists/20140116_PH52SOLrun_G17.csv']\n"
     ]
    }
   ],
   "source": [
    "#ML mustelid\n",
    "mlmuslist = []\n",
    "\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131112_P140sols_P6.csv'):\n",
    "        mlmuslist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140114_PH84SOLrun_I18.csv'):\n",
    "        mlmuslist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH52SOLrun_G17.csv'):\n",
    "        mlmuslist.append(row[0])\n",
    "\n",
    "        \n",
    "print(mlmuslist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.07918732  2.16333666  2.57952495]\n",
      " [-1.72783472  1.53062732 -1.00637202]\n",
      " [-1.0004468   2.76120966 -1.02518206]]\n"
     ]
    }
   ],
   "source": [
    "#mlmuslist\n",
    "lismlmus = []\n",
    "mlmuspd = df1.loc[df1['filename'].isin(mlmuslist)]\n",
    "#print(coepd)\n",
    "for i in mlmuspd.index[:]:\n",
    "    lismlmus.append(i) \n",
    "#print(liscoe)\n",
    "lismlmus = np.array(lismlmus)\n",
    "comlmus = data_reduced3[lismlmus]\n",
    "print(comlmus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['./20131112_P132sols/monopeaklists/20131112_P132sols_E23.csv', './20131112_P136sols/monopeaklists/20131112_P136sols_C18.csv', './20131217_PH100solRun/monopeaklists/20131217_PH100solRun_K5.csv', './20131217_PH100solRun/monopeaklists/20131217_PH100solRun_L8.csv', './20131218_PH92solRun/monopeaklists/20131218_PH92solRun_B24.csv', './20131218_PH92solRun/monopeaklists/20131218_PH92solRun_G5.csv', './20140113_PH88SOL/monopeaklists/20140113_PH88SOL_B5.csv', './20140113_PH88SOL/monopeaklists/20140113_PH88SOL_G19.csv', './20140113_PH88SOL/monopeaklists/20140113_PH88SOL_P6.csv', './20140114_PH84SOLrun/monopeaklists/20140114_PH84SOLrun_K20.csv', './20140115_PH76SOLrun/monopeaklists/20140115_PH76SOLrun_F23.csv', './20140115_PH80SOLrun/monopeaklists/20140115_PH80SOLrun_K6.csv']\n"
     ]
    }
   ],
   "source": [
    "#ML lagomorph\n",
    "mllaglist = []\n",
    "\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131112_P132sols_E23.csv'):\n",
    "        mllaglist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131112_P136sols_C18.csv'):\n",
    "        mllaglist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131217_PH100solRun_K5.csv'):\n",
    "        mllaglist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131217_PH100solRun_L8.csv'):\n",
    "        mllaglist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131218_PH92solRun_B24.csv'):\n",
    "        mllaglist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131218_PH92solRun_G5.csv'):\n",
    "        mllaglist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140113_PH88SOL_B5.csv'):\n",
    "        mllaglist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140113_PH88SOL_G19.csv'):\n",
    "        mllaglist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140113_PH88SOL_P6.csv'):\n",
    "        mllaglist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140114_PH84SOLrun_K20.csv'):\n",
    "        mllaglist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140115_PH76SOLrun_F23.csv'):\n",
    "        mllaglist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140115_PH80SOLrun_K6.csv'):\n",
    "        mllaglist.append(row[0])\n",
    "\n",
    "        \n",
    "print(mllaglist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-1.18272281  1.89810655 -1.41451459]\n",
      " [-1.20481081  2.36996002 -1.31444973]\n",
      " [ 0.18039588  3.1424201  -0.11473219]\n",
      " [-0.75687435  2.2597665  -1.06975221]\n",
      " [ 0.78600367  2.61530158  0.83959236]\n",
      " [ 0.05750132 -0.40310743  3.01210288]\n",
      " [ 0.91699846  1.44987234  2.63060183]\n",
      " [-0.27859218  2.17111282 -1.43904589]\n",
      " [ 1.11054649  3.37281854  0.84195082]\n",
      " [ 0.91825176  2.0216935   1.19473007]\n",
      " [-1.55199888  0.77346517  0.32403766]\n",
      " [-1.30698557  0.92262337  0.28028077]]\n"
     ]
    }
   ],
   "source": [
    "#mllaglist\n",
    "lismllag = []\n",
    "mllagpd = df1.loc[df1['filename'].isin(mllaglist)]\n",
    "#print(coepd)\n",
    "for i in mllagpd.index[:]:\n",
    "    lismllag.append(i) \n",
    "#print(liscoe)\n",
    "lismllag = np.array(lismllag)\n",
    "comllag = data_reduced3[lismllag]\n",
    "print(comllag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['./20140113_PH88SOL/monopeaklists/20140113_PH88SOL_B19.csv', './20140113_PH88SOL/monopeaklists/20140113_PH88SOL_B8.csv', './20140113_PH88SOL/monopeaklists/20140113_PH88SOL_C13.csv']\n"
     ]
    }
   ],
   "source": [
    "#ML hyaena\n",
    "mlhyalist = []\n",
    "\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140113_PH88SOL_B19.csv'):\n",
    "        mlhyalist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140113_PH88SOL_B8.csv'):\n",
    "        mlhyalist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140113_PH88SOL_C13.csv'):\n",
    "        mlhyalist.append(row[0])\n",
    "\n",
    "\n",
    "        \n",
    "print(mlhyalist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-1.25183356  2.79071666  0.68186569]\n",
      " [-0.87563297  2.0456752   1.94788703]\n",
      " [-0.84296327  3.36234913  0.03782778]]\n"
     ]
    }
   ],
   "source": [
    "#mlhyalist\n",
    "lismlhya = []\n",
    "mlhyapd = df1.loc[df1['filename'].isin(mlhyalist)]\n",
    "#print(coepd)\n",
    "for i in mlhyapd.index[:]:\n",
    "    lismlhya.append(i) \n",
    "#print(liscoe)\n",
    "lismlhya = np.array(lismlhya)\n",
    "comlhya = data_reduced3[lismlhya]\n",
    "print(comlhya)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['./20131112_P132sols/monopeaklists/20131112_P132sols_G23.csv', './20131112_P136sols/monopeaklists/20131112_P136sols_E20.csv', './20131209_P96solsRun/monopeaklists/20131209_P96solsRun_B5.csv', './20131209_P96solsRun/monopeaklists/20131209_P96solsRun_D10.csv', './20131217_PH100solRun/monopeaklists/20131217_PH100solRun_A13.csv', './20131217_PH100solRun/monopeaklists/20131217_PH100solRun_D13.csv', './20131217_PH100solRun/monopeaklists/20131217_PH100solRun_I22.csv', './20131217_PH100solRun/monopeaklists/20131217_PH100solRun_N14.csv', './20140114_PH84SOLrun/monopeaklists/20140114_PH84SOLrun_A10.csv', './20140114_PH84SOLrun/monopeaklists/20140114_PH84SOLrun_E9.csv', './20140115_PH80SOLrun/monopeaklists/20140115_PH80SOLrun_I24.csv', './20140115_PH80SOLrun/monopeaklists/20140115_PH80SOLrun_P7.csv', './20140116_PH52SOLrun/monopeaklists/20140116_PH52SOLrun_K4.csv']\n"
     ]
    }
   ],
   "source": [
    "#ML horse\n",
    "mlhorlist = []\n",
    "\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131112_P132sols_G23.csv'):\n",
    "        mlhorlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131112_P136sols_E20.csv'):\n",
    "        mlhorlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131209_P96solsRun_B5.csv'):\n",
    "        mlhorlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131209_P96solsRun_D10.csv'):\n",
    "        mlhorlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131217_PH100solRun_A13.csv'):\n",
    "        mlhorlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131217_PH100solRun_D13.csv'):\n",
    "        mlhorlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131217_PH100solRun_I22.csv'):\n",
    "        mlhorlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131217_PH100solRun_N14.csv'):\n",
    "        mlhorlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140114_PH84SOLrun_A10.csv'):\n",
    "        mlhorlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140114_PH84SOLrun_E9.csv'):\n",
    "        mlhorlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140115_PH80SOLrun_I24.csv'):\n",
    "        mlhorlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140115_PH80SOLrun_P7.csv'):\n",
    "        mlhorlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH52SOLrun_K4.csv'):\n",
    "        mlhorlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140303_PH100solRun_A13.csv'):\n",
    "        mlhorlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140303_PH100solRun_D13.csv'):\n",
    "        mlhorlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140303_PH100solRun_I22.csv'):\n",
    "        mlhorlist.append(row[0])\n",
    "\n",
    "print(mlhorlist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-2.31482363  0.20052974  0.32618126]\n",
      " [-0.61815278  2.22009158  2.4437809 ]\n",
      " [-0.35562086  2.051928    2.24817015]\n",
      " [-0.4352371   1.67541477  1.8024358 ]\n",
      " [-0.07259796  1.20377252  3.03955209]\n",
      " [-0.94355788  1.03963681  2.85332377]\n",
      " [-0.97617224  1.30717198  2.36953403]\n",
      " [ 0.05682925  1.65706512  3.53596702]\n",
      " [-0.65097147  0.66634592  3.19815691]\n",
      " [-0.82063309  2.99878409  1.16195423]\n",
      " [-0.25009762  3.02677233  1.37475307]\n",
      " [-0.15272038  2.73842493  2.06865324]\n",
      " [-1.7263481   1.86465342 -0.31393473]]\n"
     ]
    }
   ],
   "source": [
    "#mlhorlist\n",
    "lismlhor = []\n",
    "mlhorpd = df1.loc[df1['filename'].isin(mlhorlist)]\n",
    "#print(coepd)\n",
    "for i in mlhorpd.index[:]:\n",
    "    lismlhor.append(i) \n",
    "#print(liscoe)\n",
    "lismlhor = np.array(lismlhor)\n",
    "comlhor = data_reduced3[lismlhor]\n",
    "print(comlhor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['./20131112_P132sols/monopeaklists/20131112_P132sols_D20.csv', './20131112_P132sols/monopeaklists/20131112_P132sols_E4.csv', './20131112_P132sols/monopeaklists/20131112_P132sols_E6.csv', './20131112_P132sols/monopeaklists/20131112_P132sols_K14.csv', './20131112_P132sols/monopeaklists/20131112_P132sols_K6.csv', './20131209_P96solsRun/monopeaklists/20131209_P96solsRun_E9.csv', './20131218_PH92solRun/monopeaklists/20131218_PH92solRun_C23.csv', './20131218_PH92solRun/monopeaklists/20131218_PH92solRun_F3.csv', './20140113_PH88SOL/monopeaklists/20140113_PH88SOL_D11.csv', './20140113_PH88SOL/monopeaklists/20140113_PH88SOL_E4.csv', './20140114_PH84SOLrun/monopeaklists/20140114_PH84SOLrun_D1.csv', './20140114_PH84SOLrun/monopeaklists/20140114_PH84SOLrun_D1.csv', './20140116_PH52SOLrun/monopeaklists/20140116_PH52SOLrun_O8.csv']\n"
     ]
    }
   ],
   "source": [
    "#ML fox\n",
    "mlfoxlist = []\n",
    "\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131112_P132sols_D20.csv'):\n",
    "        mlfoxlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131112_P132sols_E4.csv'):\n",
    "        mlfoxlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131112_P132sols_E6.csv'):\n",
    "        mlfoxlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131112_P132sols_K14.csv'):\n",
    "        mlfoxlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131112_P132sols_K6.csv'):\n",
    "        mlfoxlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131209_P96solsRun_E9.csv'):\n",
    "        mlfoxlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131218_PH92solRun_C23.csv'):\n",
    "        mlfoxlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131218_PH92solRun_F3.csv'):\n",
    "        mlfoxlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140113_PH88SOL_D11.csv'):\n",
    "        mlfoxlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140113_PH88SOL_E4.csv'):\n",
    "        mlfoxlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140114_PH84SOLrun_D1.csv'):\n",
    "        mlfoxlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140114_PH84SOLrun_D1.csv'):\n",
    "        mlfoxlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH52SOLrun_O8.csv'):\n",
    "        mlfoxlist.append(row[0])\n",
    "        \n",
    "print(mlfoxlist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-1.28707759e+00  3.22455030e+00 -3.30072877e-01]\n",
      " [-4.38715074e-01  3.01402760e+00 -2.57368928e-01]\n",
      " [-5.99206140e-01  1.44948703e+00  3.56656974e+00]\n",
      " [-2.86694023e-01  1.47743538e+00  3.09543372e+00]\n",
      " [-1.60990012e-03  2.82761266e+00  2.46113319e+00]\n",
      " [-1.24217351e+00  2.65985274e+00  9.95511002e-01]\n",
      " [-1.80366864e-02  4.13513065e+00  1.77755759e+00]\n",
      " [-6.35041982e-01  1.57510754e+00  2.79333916e+00]\n",
      " [ 2.11507300e-01  1.19281666e+00  3.37222915e+00]\n",
      " [-1.82525952e-01  8.21102010e-01  4.05437221e+00]\n",
      " [-9.82886608e-01  2.71128707e+00  2.36593767e+00]\n",
      " [ 2.85016865e-02  3.26276423e+00  2.48701213e+00]]\n"
     ]
    }
   ],
   "source": [
    "#mlfoxlist\n",
    "lismlfox = []\n",
    "mlfoxpd = df1.loc[df1['filename'].isin(mlfoxlist)]\n",
    "#print(coepd)\n",
    "for i in mlfoxpd.index[:]:\n",
    "    lismlfox.append(i) \n",
    "#print(liscoe)\n",
    "lismlfox = np.array(lismlfox)\n",
    "comlfox = data_reduced3[lismlfox]\n",
    "print(comlfox)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['./20131112_P132sols/monopeaklists/20131112_P132sols_A1.csv', './20131112_P132sols/monopeaklists/20131112_P132sols_E1.csv', './20131112_P132sols/monopeaklists/20131112_P132sols_I1.csv', './20131112_P132sols/monopeaklists/20131112_P132sols_K10.csv', './20131112_P132sols/monopeaklists/20131112_P132sols_L23.csv', './20131112_P132sols/monopeaklists/20131112_P132sols_P4.csv', './20131209_P96solsRun/monopeaklists/20131209_P96solsRun_A1.csv', './20131209_P96solsRun/monopeaklists/20131209_P96solsRun_E1.csv', './20131209_P96solsRun/monopeaklists/20131209_P96solsRun_H19.csv', './20131209_P96solsRun/monopeaklists/20131209_P96solsRun_I1.csv', './20131209_P96solsRun/monopeaklists/20131209_P96solsRun_L6.csv', './20131209_P96solsRun/monopeaklists/20131209_P96solsRun_M1.csv', './20131217_PH100solRun/monopeaklists/20131217_PH100solRun_A1.csv', './20131217_PH100solRun/monopeaklists/20131217_PH100solRun_E1.csv', './20131217_PH100solRun/monopeaklists/20131217_PH100solRun_E11.csv', './20131217_PH100solRun/monopeaklists/20131217_PH100solRun_G18.csv', './20131218_PH92solRun/monopeaklists/20131218_PH92solRun_C3.csv', './20131218_PH92solRun/monopeaklists/20131218_PH92solRun_E1.csv', './20131218_PH92solRun/monopeaklists/20131218_PH92solRun_I23.csv', './20131218_PH92solRun/monopeaklists/20131218_PH92solRun_J13.csv', './20131218_PH92solRun/monopeaklists/20131218_PH92solRun_L17.csv', './20140113_PH88SOL/monopeaklists/20140113_PH88SOL_H15.csv', './20140113_PH88SOL/monopeaklists/20140113_PH88SOL_J2.csv', './20140113_PH88SOL/monopeaklists/20140113_PH88SOL_M1.csv', './20140114_PH84SOLrun/monopeaklists/20140114_PH84SOLrun_J20.csv', './20140114_PH84SOLrun/monopeaklists/20140114_PH84SOLrun_L13.csv', './20140114_PH84SOLrun/monopeaklists/20140114_PH84SOLrun_L24.csv', './20140114_PH84SOLrun/monopeaklists/20140114_PH84SOLrun_P24.csv', './20140115_PH72SOLrun/monopeaklists/20140115_PH72SOLrun_E1.csv', './20140115_PH72SOLrun/monopeaklists/20140115_PH72SOLrun_H11.csv', './20140115_PH72SOLrun/monopeaklists/20140115_PH72SOLrun_M1.csv', './20140115_PH76SOLrun/monopeaklists/20140115_PH76SOLrun_E1.csv', './20140115_PH76SOLrun/monopeaklists/20140115_PH76SOLrun_I1.csv', './20140115_PH76SOLrun/monopeaklists/20140115_PH76SOLrun_M1.csv', './20140115_PH80SOLrun/monopeaklists/20140115_PH80SOLrun_A1.csv', './20140115_PH80SOLrun/monopeaklists/20140115_PH80SOLrun_E1.csv', './20140115_PH80SOLrun/monopeaklists/20140115_PH80SOLrun_J5.csv', './20140115_PH80SOLrun/monopeaklists/20140115_PH80SOLrun_L24.csv', './20140115_PH80SOLrun/monopeaklists/20140115_PH80SOLrun_N13.csv', './20140116_PH52SOLrun/monopeaklists/20140116_PH52SOLrun_A1.csv', './20140116_PH52SOLrun/monopeaklists/20140116_PH52SOLrun_E1.csv', './20140116_PH52SOLrun/monopeaklists/20140116_PH52SOLrun_F18.csv', './20140116_PH52SOLrun/monopeaklists/20140116_PH52SOLrun_G20.csv', './20140116_PH52SOLrun/monopeaklists/20140116_PH52SOLrun_H1.csv', './20140116_PH52SOLrun/monopeaklists/20140116_PH52SOLrun_M1.csv', './20140116_PH56SOLrun/monopeaklists/20140116_PH56SOLrun_A1.csv', './20140116_PH56SOLrun/monopeaklists/20140116_PH56SOLrun_E1.csv', './20140116_PH56SOLrun/monopeaklists/20140116_PH56SOLrun_I1.csv', './20140116_PH56SOLrun/monopeaklists/20140116_PH56SOLrun_M1.csv', './20140116_PH60SOLrun/monopeaklists/20140116_PH60SOLrun_A1.csv', './20140116_PH60SOLrun/monopeaklists/20140116_PH60SOLrun_E1.csv', './20140116_PH56SOLrun/monopeaklists/20140116_PH56SOLrun_A1.csv', './20140116_PH56SOLrun/monopeaklists/20140116_PH56SOLrun_E1.csv', './20140116_PH60SOLrun/monopeaklists/20140116_PH60SOLrun_I1.csv', './20140116_PH60SOLrun/monopeaklists/20140116_PH60SOLrun_J2.csv', './20140116_PH60SOLrun/monopeaklists/20140116_PH60SOLrun_M1.csv']\n"
     ]
    }
   ],
   "source": [
    "#ML bovine\n",
    "mlbovlist = []\n",
    "\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131112_P132sols_A1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131112_P132sols_E1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131112_P132sols_I1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131112_P132sols_K10.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131112_P132sols_L23.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131112_P132sols_P4.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131209_P96solsRun_A1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131209_P96solsRun_E1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131209_P96solsRun_H19.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131209_P96solsRun_I1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131209_P96solsRun_L6.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131209_P96solsRun_M1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131217_PH100solRun_A1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131217_PH100solRun_E1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131217_PH100solRun_E11.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131217_PH100solRun_G18.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131218_PH92solRun_C3.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131218_PH92solRun_E1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131218_PH92solRun_I23.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131218_PH92solRun_J13.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131218_PH92solRun_L17.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140113_PH88SOL_H15.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140113_PH88SOL_J2.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140113_PH88SOL_M1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140114_PH84SOLrun_J20.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140114_PH84SOLrun_L13.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140114_PH84SOLrun_L24.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140114_PH84SOLrun_P24.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140115_PH72SOLrun_E1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140115_PH72SOLrun_H11.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140115_PH72SOLrun_M1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140115_PH76SOLrun_E1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140115_PH76SOLrun_I1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140115_PH76SOLrun_M1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140115_PH80SOLrun_A1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140115_PH80SOLrun_E1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140115_PH80SOLrun_J5.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140115_PH80SOLrun_L24.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140115_PH80SOLrun_N13.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH52SOLrun_A1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH52SOLrun_E1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH52SOLrun_F18.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH52SOLrun_G20.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH52SOLrun_H1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH52SOLrun_M1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH56SOLrun_A1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH56SOLrun_E1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH56SOLrun_I1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH56SOLrun_M1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH60SOLrun_A1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH60SOLrun_E1.csv'):\n",
    "        mlbovlist.append(row[0])       \n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH56SOLrun_A1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH56SOLrun_E1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH60SOLrun_I1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH60SOLrun_J2.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH60SOLrun_M1.csv'):\n",
    "        mlbovlist.append(row[0])\n",
    " \n",
    "        \n",
    "print(mlbovlist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-2.16560526  2.59233187 -0.67574335]\n",
      " [-2.27968043  2.65821836 -0.46219206]\n",
      " [-2.16736431  2.45352361 -0.17846114]\n",
      " [-2.0742367   2.26388242 -0.03245643]\n",
      " [-2.05755981  1.10612489 -0.50565577]\n",
      " [-1.80906881  1.05082532 -0.92191719]\n",
      " [-1.13791005  3.74345326  1.5977396 ]\n",
      " [-1.83100675  1.93685251 -1.05052999]\n",
      " [-0.9308022   0.38368568  0.65609005]\n",
      " [-2.27570352  2.49759066 -0.81063275]\n",
      " [-1.16655481  3.54979111  1.92789882]\n",
      " [-1.13463801  3.91987338  1.81189663]\n",
      " [-0.99855241  0.59329575  0.73448605]\n",
      " [ 0.17861635  2.18474914  3.14401334]\n",
      " [-0.92542904  0.63953968  4.0414386 ]\n",
      " [-0.54344334  4.12822368  2.21206104]\n",
      " [-1.69553999  0.33250487 -0.35974015]\n",
      " [-0.88229501  4.32373538  2.45213006]\n",
      " [-0.77925445  4.30559095  2.75783188]\n",
      " [-1.93628597 -0.96042972  1.90171697]\n",
      " [-3.12806777  0.99549806 -0.45840383]\n",
      " [-2.5257436   2.60590013 -0.1576212 ]\n",
      " [-0.26989483  3.27445824  4.47704029]\n",
      " [-2.54991798  2.14390256  0.22069776]\n",
      " [-1.78499226 -0.26390932  0.50251002]\n",
      " [-1.20171267  3.86639472  0.41262869]\n",
      " [-1.49421316  3.00688532  2.06025648]\n",
      " [-1.36988064  3.10482907  1.43964334]\n",
      " [-2.33256419  2.77675546 -0.2449427 ]\n",
      " [-1.9937234   1.27571197 -1.23825329]\n",
      " [-0.8643344   0.32773787  0.25463625]\n",
      " [-2.28891432  2.41375354 -0.28415983]\n",
      " [-2.71584161  2.29818482 -0.49618866]\n",
      " [-2.1612233   2.31660148 -0.52978562]\n",
      " [-2.39499846  2.61019456 -0.22464779]\n",
      " [-2.14081146  2.54712416  0.10994262]\n",
      " [-2.07722061  2.66036753 -0.15723148]\n",
      " [-2.25315505  2.80967771  0.25704651]\n",
      " [-2.94340705  0.75810238 -1.50198209]\n",
      " [-1.96403402  1.41837297 -1.0649372 ]\n",
      " [-3.07639686  0.86022585 -0.24483057]\n",
      " [-3.19094552  0.6217132  -0.12092193]\n",
      " [-1.41566981  0.04184287  0.62624579]\n",
      " [-2.03615107 -0.22616243  0.49372998]\n",
      " [-1.54043155  1.55137898  3.2985362 ]\n",
      " [-1.44685681  3.81337654  0.29549707]\n",
      " [-1.60212978  0.5444526   0.99300422]\n",
      " [-2.07076075  0.53258743 -0.11947039]\n",
      " [-0.97003243  3.51533321  1.24070653]\n",
      " [-2.12065665  0.07969898  0.13052131]\n",
      " [-1.76880817  3.29435218  1.21053598]\n",
      " [-2.47126375  1.77321782  0.35025756]\n",
      " [-1.50461235  0.02507461  0.70163536]\n",
      " [-2.29424421  2.83387751 -0.51970518]]\n"
     ]
    }
   ],
   "source": [
    "#mlbovlist\n",
    "lismlbov = []\n",
    "mlbovpd = df1.loc[df1['filename'].isin(mlbovlist)]\n",
    "#print(coepd)\n",
    "for i in mlbovpd.index[:]:\n",
    "    lismlbov.append(i) \n",
    "#print(liscoe)\n",
    "lismlbov = np.array(lismlbov)\n",
    "comlbov = data_reduced3[lismlbov]\n",
    "print(comlbov)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['./20131112_P136sols/monopeaklists/20131112_P136sols_C11.csv', './20131209_P96solsRun/monopeaklists/20131209_P96solsRun_D20.csv', './20131217_PH100solRun/monopeaklists/20131217_PH100solRun_M6.csv', './20131217_PH100solRun/monopeaklists/20131217_PH100solRun_N2.csv', './20140113_PH88SOL/monopeaklists/20140113_PH88SOL_A14.csv', './20140113_PH88SOL/monopeaklists/20140113_PH88SOL_H4.csv', './20140113_PH88SOL/monopeaklists/20140113_PH88SOL_P14.csv', './20140114_PH84SOLrun/monopeaklists/20140114_PH84SOLrun_M20.csv', './20140116_PH52SOLrun/monopeaklists/20140116_PH52SOLrun_J17.csv', './20140116_PH56SOLrun/monopeaklists/20140116_PH56SOLrun_L16.csv', './20140116_PH60SOLrun/monopeaklists/20140116_PH60SOLrun_G11.csv', './20140116_PH60SOLrun/monopeaklists/20140116_PH60SOLrun_M2.csv', './20140116_PH60SOLrun/monopeaklists/20140116_PH60SOLrun_M22.csv']\n"
     ]
    }
   ],
   "source": [
    "#ML bear\n",
    "mlbealist = []\n",
    "\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131112_P136sols_C11.csv'):\n",
    "        mlbealist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131209_P96solsRun_D20.csv'):\n",
    "        mlbealist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131217_PH100solRun_M6.csv'):\n",
    "        mlbealist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20131217_PH100solRun_N2.csv'):\n",
    "        mlbealist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140113_PH88SOL_A14.csv'):\n",
    "        mlbealist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140113_PH88SOL_H4.csv'):\n",
    "        mlbealist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140113_PH88SOL_P14.csv'):\n",
    "        mlbealist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140114_PH84SOLrun_M20.csv'):\n",
    "        mlbealist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH52SOLrun_J17.csv'):\n",
    "        mlbealist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH56SOLrun_L16.csv'):\n",
    "        mlbealist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH60SOLrun_G11.csv'):\n",
    "        mlbealist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH60SOLrun_M2.csv'):\n",
    "        mlbealist.append(row[0])\n",
    "for index, row in df1.iterrows():\n",
    "    if row['filename'].endswith('20140116_PH60SOLrun_M22.csv'):\n",
    "        mlbealist.append(row[0])\n",
    "\n",
    "print(mlbealist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-2.22364051  1.21298014 -0.45596212]\n",
      " [-0.36424171  1.98468822  2.64016159]\n",
      " [-0.43612845  3.44965974  1.45869453]\n",
      " [-0.92600595  3.42991371  1.3136208 ]\n",
      " [ 0.41683438  3.55299118  1.06839065]\n",
      " [-0.73195096  1.29764722  2.581004  ]\n",
      " [-0.4917287   1.51852333  3.22973722]\n",
      " [ 0.00793639  1.46370182  3.18192043]\n",
      " [ 0.04653859  1.98810693  2.98040812]\n",
      " [-0.71289406  2.17647053  2.83691427]\n",
      " [-1.0467064   2.96992446 -0.1848055 ]\n",
      " [-2.01542392  1.84189129 -0.11923792]\n",
      " [-1.08665222  2.45337185  0.40481992]]\n"
     ]
    }
   ],
   "source": [
    "#mlbealist\n",
    "lismlbea = []\n",
    "mlbeapd = df1.loc[df1['filename'].isin(mlbealist)]\n",
    "#print(coepd)\n",
    "for i in mlbeapd.index[:]:\n",
    "    lismlbea.append(i) \n",
    "#print(liscoe)\n",
    "lismlbea = np.array(lismlbea)\n",
    "comlbea = data_reduced3[lismlbea]\n",
    "print(comlbea)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe id=\"igraph\" scrolling=\"no\" style=\"border:none;\" seamless=\"seamless\" src=\"https://plot.ly/~NataliaKudr/2.embed\" height=\"525px\" width=\"100%\"></iframe>"
      ],
      "text/plain": [
       "<plotly.tools.PlotlyDisplay object>"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#visualise\n",
    "\n",
    "import plotly.plotly as py\n",
    "import plotly.graph_objs as go\n",
    "\n",
    "import colorlover as cl\n",
    "\n",
    "\n",
    "trace = go.Scatter3d(\n",
    "    x=data_reduced3[:, 0],\n",
    "    y=data_reduced3[:, 1],\n",
    "    z=data_reduced3[:, 2],\n",
    "    mode='markers',\n",
    "    text = str,\n",
    "    name = 'Full Dataset',\n",
    "\n",
    "    marker=dict(color= clusters, size=2,\n",
    "        opacity=0.9,\n",
    "       colorscale='Blues',\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "trace2 = go.Scatter3d(\n",
    "    x=comlhya[:,0],\n",
    "    y=comlhya[:,1],\n",
    "    z=comlhya[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Hyaena\",\n",
    "    name = \"Hyaena\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(10, 0, 110, .8)',\n",
    "     size=8,\n",
    "                symbol='circle',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "trace3 = go.Scatter3d(\n",
    "    x=comlmam[:,0],\n",
    "    y=comlmam[:,1],\n",
    "    z=comlmam[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Mammuth\",\n",
    "    name = \"Mammoth\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(252.0, 141.0, 89.0, 0.8)',\n",
    "     size=8,\n",
    "                symbol='cross',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "trace4 = go.Scatter3d(\n",
    "    x=comlbea[:,0],\n",
    "    y=comlbea[:,1],\n",
    "    z=comlbea[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Bear\",\n",
    "    name = \"Bear\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(255.0, 200.0, 150.0, .8)',\n",
    "     size=8,\n",
    "                symbol='square',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "trace5 = go.Scatter3d(\n",
    "    x=comlbov[:,0],\n",
    "    y=comlbov[:,1],\n",
    "    z=comlbov[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Bovine\",\n",
    "    name = \"Bovine\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(145.0, 191.0, 219.0, .8)',\n",
    "     size=8,\n",
    "                symbol='diamond',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "trace6 = go.Scatter3d(\n",
    "    x=comlmus[:,0],\n",
    "    y=comlmus[:,1],\n",
    "    z=comlmus[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Mustelid\",\n",
    "    name = \"Mustelid\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(145.0, 191.0, 0.0, .8)',\n",
    "     size=8,\n",
    "                symbol='square-open',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "trace7 = go.Scatter3d(\n",
    "    x=comlhor[:,0],\n",
    "    y=comlhor[:,1],\n",
    "    z=comlhor[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Horse\",\n",
    "    name = \"Horse\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(145.0, 0.0, 0.0, .8)',\n",
    "     size=8,\n",
    "                symbol='circle-open',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "trace8 = go.Scatter3d(\n",
    "    x=comlfox[:,0],\n",
    "    y=comlfox[:,1],\n",
    "    z=comlfox[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Fox\",\n",
    "    name = \"Fox\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(145.0, 102.0, 0.0, .8)',\n",
    "     size=8,\n",
    "                symbol='diamond-open',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "trace9 = go.Scatter3d(\n",
    "    x=comllag[:,0],\n",
    "    y=comllag[:,1],\n",
    "    z=comllag[:,2],\n",
    "    mode='markers',\n",
    "    text = \"Lagomorph\",\n",
    "    name = \"Lagomorph\",\n",
    "    \n",
    "    marker=dict(color = 'rgba(0.0, 102.0, 120.0, .8)',\n",
    "     size=5,\n",
    "                symbol='x',\n",
    "        opacity=0.9,\n",
    "                \n",
    "\n",
    "    )\n",
    ")\n",
    "\n",
    "dataMeans = [trace, trace2,trace3,trace4,trace5,trace6,trace7,trace8, trace9]\n",
    "\n",
    "layout = go.Layout(\n",
    "    showlegend=True,\n",
    "    title='K-means Clustering: 3 clusters; ML identifies taxa'\n",
    "   \n",
    "    \n",
    ")\n",
    "\n",
    "fig = go.Figure(data=dataMeans, layout=layout)\n",
    "py.iplot(fig, filename='simple-3d-scatter')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}