2_prepare_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process the raw data from pyapm and bpnsdata\n",
    "This notebook is the code to process the output given after processing the data with pypam and bpnsdata\n",
    "For more information about this process please contact clea.parcerisas@vliz.be or check the documentation of both packages\n",
    "https://lifewatch-pypam.readthedocs.io/en/latest/\n",
    "https://github.com/lifewatch/bpnsdata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "ExecuteTime": {
     "end_time": "2023-07-24T17:24:56.618684100Z",
     "start_time": "2023-07-24T17:24:55.935247800Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\cleap\\Envs\\wadden_soundscape\\lib\\site-packages\\geopandas\\_compat.py:124: UserWarning: The Shapely GEOS version (3.10.2-CAPI-1.16.0) is incompatible with the GEOS version PyGEOS was compiled with (3.10.4-CAPI-1.16.2). Conversions between both will be slow.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import pathlib\n",
    "\n",
    "import geopandas\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import xarray\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "ExecuteTime": {
     "end_time": "2023-07-24T17:26:15.450271100Z",
     "start_time": "2023-07-24T17:26:15.431117400Z"
    }
   },
   "outputs": [],
   "source": [
    "# Don't forget to add the ones you added before\n",
    "ENV_LABELS = [\n",
    "    'route_density',\n",
    "    'season',\n",
    "    'moon_phase',\n",
    "    'day_moment',\n",
    "    'sediment_type',\n",
    "    'bathymetry',\n",
    "    'tide'\n",
    "]\n",
    "\n",
    "CHUNK_LENGTH = 10.0 # This has to be the SAME one you chose in 0_create_dataset (binsize)\n",
    "\n",
    "# Set the min and max frequencies to use (has to be a range smaller or equal to the one you selected in 0_create_dataset)\n",
    "MAX_FREQ = 20000\n",
    "MIN_FREQ = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "ExecuteTime": {
     "end_time": "2023-07-24T17:26:20.194978900Z",
     "start_time": "2023-07-24T17:26:20.169962200Z"
    }
   },
   "outputs": [],
   "source": [
    "# Write down which of the variables are CATEGORIES and not NUMERICAL\n",
    "CATEGORICAL_VARS = ['day_moment', 'sediment_type', 'tide','seabed_habitat', 'deployment_name']\n",
    "CYCLIC_VARS = ['season', 'moon_phase']\n",
    "\n",
    "vars_dtypes = {\n",
    "    'route_density': int,\n",
    "    'season': int,\n",
    "    'moon_phase': np.float16,\n",
    "    'day_moment': 'category',\n",
    "    'sediment_type': 'category',\n",
    "    'bathymetry': np.float16,\n",
    "    'tide': 'category',\n",
    "    'deployment_name': 'category'\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "ExecuteTime": {
     "end_time": "2023-07-24T17:26:30.199932800Z",
     "start_time": "2023-07-24T17:26:30.162910600Z"
    }
   },
   "outputs": [],
   "source": [
    "# Define the folders\n",
    "data_path = pathlib.Path('./data/raw_data/')\n",
    "processed_data_path = pathlib.Path('./data/processed/')\n",
    "raw_data_path = pathlib.Path('./data/raw_data/deployments/deployments')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "ExecuteTime": {
     "end_time": "2023-07-24T17:26:38.801500700Z",
     "start_time": "2023-07-24T17:26:38.766100Z"
    }
   },
   "outputs": [],
   "source": [
    "# Read the metadata csv\n",
    "metadata = pd.read_csv(data_path.joinpath('data_summary_mda.csv'))\n",
    "\n",
    "# Create the empty output vars\n",
    "df_features = pd.DataFrame()\n",
    "df_sample = pd.DataFrame()\n",
    "df_env = pd.DataFrame()\n",
    "df_geo = geopandas.GeoDataFrame()\n",
    "df_labels = pd.DataFrame()\n",
    "\n",
    "# Define the names of the vars that will be used\n",
    "# HERE IMPORTANT TO DECIDE IF YOU PROCESS 'oct3' or 'millidecade_bands'\n",
    "features_var = 'oct3'\n",
    "deployment_columns = ['deployment_name', 'datetime']\n",
    "freqticks = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "ExecuteTime": {
     "end_time": "2023-07-24T17:36:53.771153500Z",
     "start_time": "2023-07-24T17:36:53.592776900Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 15/15 [00:00<00:00, 24.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total amount of time recorded 1041.0233333333333 h\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Join all the deployments in one DataFrame\n",
    "df = pd.DataFrame()\n",
    "total_acoustic_time = 0\n",
    "for idx in tqdm(metadata.index, total=len(metadata)):\n",
    "    deployment_row = metadata.loc[idx]\n",
    "    env_name = '%s_%s_env.nc' % (idx, deployment_row.deployment_name)\n",
    "    env_path = processed_data_path.joinpath(env_name)\n",
    "    deployment_file_name = '%s_%s.nc' % (idx, deployment_row.deployment_name)\n",
    "    name = deployment_row['deployment_name']\n",
    "    deployment = xarray.open_dataset(env_path)\n",
    "\n",
    "    # Get the geometry\n",
    "    geo_series = geopandas.GeoSeries(data=geopandas.points_from_xy(x=deployment['longitude'],\n",
    "                                                                   y=deployment['latitude']),\n",
    "                                     crs='EPSG:4326')\n",
    "\n",
    "    # Eliminate the frequencies below MIN_FREQ and above MAX_FREQ\n",
    "    deployment = deployment.sel(frequency=deployment.frequency[deployment.frequency < MAX_FREQ])\n",
    "    deployment = deployment.sel(frequency=deployment.frequency[deployment.frequency > MIN_FREQ])\n",
    "    deployment_duration = deployment.datetime.max() - deployment.datetime.min()\n",
    "    total_acoustic_time += deployment_duration\n",
    "    deployment = deployment[ENV_LABELS + [features_var] + deployment_columns].dropna('id', 'any')\n",
    "    clean_freqticks = deployment.frequency.values\n",
    "\n",
    "    # Create a pandas df with all the wanted values\n",
    "    values_arr = deployment[features_var].values\n",
    "    df_deployment = pd.DataFrame(values_arr)\n",
    "    df_deployment = df_deployment.astype(np.float16)\n",
    "\n",
    "    for env in ENV_LABELS + deployment_columns:\n",
    "        df_deployment[env] = deployment[env]\n",
    "\n",
    "    df_deployment = geopandas.GeoDataFrame(df_deployment, geometry=geo_series)\n",
    "\n",
    "    df = pd.concat([df, df_deployment], ignore_index=True)\n",
    "    deployment.close()\n",
    "\n",
    "# print the total acoustic time\n",
    "print('Total amount of time recorded %s h' % (total_acoustic_time.values / np.timedelta64(1, 'h')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Some data clean up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "ExecuteTime": {
     "end_time": "2023-07-24T16:12:05.492126500Z",
     "start_time": "2023-07-24T16:12:05.362458300Z"
    }
   },
   "outputs": [],
   "source": [
    "# Change the data types to save some computational power and memory\n",
    "# Some operations\n",
    "df = df.replace(['Civil twilight', 'Astronomical twilight', 'Nautical twilight'], ['Twilight', 'Twilight', 'Twilight'])\n",
    "df['bathymetry'] = -1 * df['bathymetry']\n",
    "\n",
    "# Categorical vars to category for efficient storage and processing\n",
    "for env, env_type in vars_dtypes.items():\n",
    "    df[env] = df[env].astype(env_type)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save the outputs to work on with the next script"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "ExecuteTime": {
     "end_time": "2023-07-24T16:12:11.145855400Z",
     "start_time": "2023-07-24T16:12:11.115028300Z"
    }
   },
   "outputs": [],
   "source": [
    "# Filter the deployments to skip if there were any\n",
    "np.save(processed_data_path.joinpath('used_freqticks.npy'), clean_freqticks)\n",
    "df.to_pickle(processed_data_path.joinpath('df_complete.pkl'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "data": {
      "text/plain": "             0         1         2         3         4         5         6   \n0     80.18750  81.25000  79.62500  73.62500  74.75000  74.25000  73.37500  \\\n1     82.62500  82.25000  80.31250  74.50000  76.37500  75.18750  72.00000   \n2     80.50000  80.75000  79.06250  73.25000  74.62500  73.93750  72.25000   \n3     81.87500  81.31250  77.62500  73.18750  76.25000  74.37500  70.75000   \n4     80.93750  81.25000  79.87500  73.87500  75.06250  73.81250  71.18750   \n...        ...       ...       ...       ...       ...       ...       ...   \n7111  60.09375  57.78125  57.46875  56.40625  53.18750  54.53125  56.87500   \n7112  58.68750  57.03125  57.12500  55.65625  51.78125  53.37500  56.75000   \n7113  60.75000  58.28125  58.96875  55.50000  52.65625  53.71875  55.68750   \n7114  62.12500  58.25000  57.21875  54.71875  53.81250  55.37500  59.87500   \n7115  48.31250  46.71875  43.53125  43.62500  42.62500  44.65625  47.40625   \n\n             7         8         9  ...  moon_phase  day_moment   \n0     72.81250  71.50000  71.25000  ...    6.175781    Twilight  \\\n1     70.75000  70.56250  70.06250  ...    6.175781    Twilight   \n2     70.75000  70.31250  69.50000  ...    6.175781    Twilight   \n3     70.00000  69.75000  70.25000  ...    6.175781    Twilight   \n4     70.31250  70.37500  70.00000  ...    6.175781    Twilight   \n...        ...       ...       ...  ...         ...         ...   \n7111  58.68750  60.93750  62.71875  ...    3.501953    Twilight   \n7112  59.37500  61.53125  62.40625  ...    3.501953    Twilight   \n7113  58.75000  61.50000  62.87500  ...    3.501953    Twilight   \n7114  68.56250  71.56250  68.93750  ...    3.501953    Twilight   \n7115  51.28125  52.81250  53.81250  ...    3.501953    Twilight   \n\n      sediment_type  bathymetry  tide  deployment_name            datetime   \n0              reef    2.482422   low        NLOyster1 2022-05-29 22:00:09  \\\n1              reef    2.482422   low        NLOyster1 2022-05-29 22:00:19   \n2              reef    2.482422   low        NLOyster1 2022-05-29 22:00:29   \n3              reef    2.482422   low        NLOyster1 2022-05-29 22:00:39   \n4              reef    2.482422   low        NLOyster1 2022-05-29 22:00:49   \n...             ...         ...   ...              ...                 ...   \n7111     coarsesand    1.554688   low    CoarseSandLow 2022-06-15 22:40:19   \n7112     coarsesand    1.554688   low    CoarseSandLow 2022-06-15 22:40:29   \n7113     coarsesand    1.554688   low    CoarseSandLow 2022-06-15 22:40:39   \n7114     coarsesand    1.554688   low    CoarseSandLow 2022-06-15 22:40:49   \n7115     coarsesand    1.554688   low    CoarseSandLow 2022-06-15 22:40:59   \n\n                      geometry        20       21  \n0     POINT (4.88076 53.06843)       NaN      NaN  \n1     POINT (4.88076 53.06843)       NaN      NaN  \n2     POINT (4.88076 53.06843)       NaN      NaN  \n3     POINT (4.88076 53.06843)       NaN      NaN  \n4     POINT (4.88076 53.06843)       NaN      NaN  \n...                        ...       ...      ...  \n7111  POINT (5.20231 53.06843)  66.81250  67.1875  \n7112  POINT (5.20231 53.06843)  67.37500  68.0625  \n7113  POINT (5.20231 53.06843)  66.93750  67.3125  \n7114  POINT (5.20231 53.06843)  68.25000  69.3750  \n7115  POINT (5.20231 53.06843)  55.46875  55.5625  \n\n[7116 rows x 32 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n      <th>7</th>\n      <th>8</th>\n      <th>9</th>\n      <th>...</th>\n      <th>moon_phase</th>\n      <th>day_moment</th>\n      <th>sediment_type</th>\n      <th>bathymetry</th>\n      <th>tide</th>\n      <th>deployment_name</th>\n      <th>datetime</th>\n      <th>geometry</th>\n      <th>20</th>\n      <th>21</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>80.18750</td>\n      <td>81.25000</td>\n      <td>79.62500</td>\n      <td>73.62500</td>\n      <td>74.75000</td>\n      <td>74.25000</td>\n      <td>73.37500</td>\n      <td>72.81250</td>\n      <td>71.50000</td>\n      <td>71.25000</td>\n      <td>...</td>\n      <td>6.175781</td>\n      <td>Twilight</td>\n      <td>reef</td>\n      <td>2.482422</td>\n      <td>low</td>\n      <td>NLOyster1</td>\n      <td>2022-05-29 22:00:09</td>\n      <td>POINT (4.88076 53.06843)</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>82.62500</td>\n      <td>82.25000</td>\n      <td>80.31250</td>\n      <td>74.50000</td>\n      <td>76.37500</td>\n      <td>75.18750</td>\n      <td>72.00000</td>\n      <td>70.75000</td>\n      <td>70.56250</td>\n      <td>70.06250</td>\n      <td>...</td>\n      <td>6.175781</td>\n      <td>Twilight</td>\n      <td>reef</td>\n      <td>2.482422</td>\n      <td>low</td>\n      <td>NLOyster1</td>\n      <td>2022-05-29 22:00:19</td>\n      <td>POINT (4.88076 53.06843)</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>80.50000</td>\n      <td>80.75000</td>\n      <td>79.06250</td>\n      <td>73.25000</td>\n      <td>74.62500</td>\n      <td>73.93750</td>\n      <td>72.25000</td>\n      <td>70.75000</td>\n      <td>70.31250</td>\n      <td>69.50000</td>\n      <td>...</td>\n      <td>6.175781</td>\n      <td>Twilight</td>\n      <td>reef</td>\n      <td>2.482422</td>\n      <td>low</td>\n      <td>NLOyster1</td>\n      <td>2022-05-29 22:00:29</td>\n      <td>POINT (4.88076 53.06843)</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>81.87500</td>\n      <td>81.31250</td>\n      <td>77.62500</td>\n      <td>73.18750</td>\n      <td>76.25000</td>\n      <td>74.37500</td>\n      <td>70.75000</td>\n      <td>70.00000</td>\n      <td>69.75000</td>\n      <td>70.25000</td>\n      <td>...</td>\n      <td>6.175781</td>\n      <td>Twilight</td>\n      <td>reef</td>\n      <td>2.482422</td>\n      <td>low</td>\n      <td>NLOyster1</td>\n      <td>2022-05-29 22:00:39</td>\n      <td>POINT (4.88076 53.06843)</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>80.93750</td>\n      <td>81.25000</td>\n      <td>79.87500</td>\n      <td>73.87500</td>\n      <td>75.06250</td>\n      <td>73.81250</td>\n      <td>71.18750</td>\n      <td>70.31250</td>\n      <td>70.37500</td>\n      <td>70.00000</td>\n      <td>...</td>\n      <td>6.175781</td>\n      <td>Twilight</td>\n      <td>reef</td>\n      <td>2.482422</td>\n      <td>low</td>\n      <td>NLOyster1</td>\n      <td>2022-05-29 22:00:49</td>\n      <td>POINT (4.88076 53.06843)</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>7111</th>\n      <td>60.09375</td>\n      <td>57.78125</td>\n      <td>57.46875</td>\n      <td>56.40625</td>\n      <td>53.18750</td>\n      <td>54.53125</td>\n      <td>56.87500</td>\n      <td>58.68750</td>\n      <td>60.93750</td>\n      <td>62.71875</td>\n      <td>...</td>\n      <td>3.501953</td>\n      <td>Twilight</td>\n      <td>coarsesand</td>\n      <td>1.554688</td>\n      <td>low</td>\n      <td>CoarseSandLow</td>\n      <td>2022-06-15 22:40:19</td>\n      <td>POINT (5.20231 53.06843)</td>\n      <td>66.81250</td>\n      <td>67.1875</td>\n    </tr>\n    <tr>\n      <th>7112</th>\n      <td>58.68750</td>\n      <td>57.03125</td>\n      <td>57.12500</td>\n      <td>55.65625</td>\n      <td>51.78125</td>\n      <td>53.37500</td>\n      <td>56.75000</td>\n      <td>59.37500</td>\n      <td>61.53125</td>\n      <td>62.40625</td>\n      <td>...</td>\n      <td>3.501953</td>\n      <td>Twilight</td>\n      <td>coarsesand</td>\n      <td>1.554688</td>\n      <td>low</td>\n      <td>CoarseSandLow</td>\n      <td>2022-06-15 22:40:29</td>\n      <td>POINT (5.20231 53.06843)</td>\n      <td>67.37500</td>\n      <td>68.0625</td>\n    </tr>\n    <tr>\n      <th>7113</th>\n      <td>60.75000</td>\n      <td>58.28125</td>\n      <td>58.96875</td>\n      <td>55.50000</td>\n      <td>52.65625</td>\n      <td>53.71875</td>\n      <td>55.68750</td>\n      <td>58.75000</td>\n      <td>61.50000</td>\n      <td>62.87500</td>\n      <td>...</td>\n      <td>3.501953</td>\n      <td>Twilight</td>\n      <td>coarsesand</td>\n      <td>1.554688</td>\n      <td>low</td>\n      <td>CoarseSandLow</td>\n      <td>2022-06-15 22:40:39</td>\n      <td>POINT (5.20231 53.06843)</td>\n      <td>66.93750</td>\n      <td>67.3125</td>\n    </tr>\n    <tr>\n      <th>7114</th>\n      <td>62.12500</td>\n      <td>58.25000</td>\n      <td>57.21875</td>\n      <td>54.71875</td>\n      <td>53.81250</td>\n      <td>55.37500</td>\n      <td>59.87500</td>\n      <td>68.56250</td>\n      <td>71.56250</td>\n      <td>68.93750</td>\n      <td>...</td>\n      <td>3.501953</td>\n      <td>Twilight</td>\n      <td>coarsesand</td>\n      <td>1.554688</td>\n      <td>low</td>\n      <td>CoarseSandLow</td>\n      <td>2022-06-15 22:40:49</td>\n      <td>POINT (5.20231 53.06843)</td>\n      <td>68.25000</td>\n      <td>69.3750</td>\n    </tr>\n    <tr>\n      <th>7115</th>\n      <td>48.31250</td>\n      <td>46.71875</td>\n      <td>43.53125</td>\n      <td>43.62500</td>\n      <td>42.62500</td>\n      <td>44.65625</td>\n      <td>47.40625</td>\n      <td>51.28125</td>\n      <td>52.81250</td>\n      <td>53.81250</td>\n      <td>...</td>\n      <td>3.501953</td>\n      <td>Twilight</td>\n      <td>coarsesand</td>\n      <td>1.554688</td>\n      <td>low</td>\n      <td>CoarseSandLow</td>\n      <td>2022-06-15 22:40:59</td>\n      <td>POINT (5.20231 53.06843)</td>\n      <td>55.46875</td>\n      <td>55.5625</td>\n    </tr>\n  </tbody>\n</table>\n<p>7116 rows × 32 columns</p>\n</div>"
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-24T16:12:13.053085500Z",
     "start_time": "2023-07-24T16:12:12.994777100Z"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}