ConstruçãoDataset_MapeamentoIA_JEH.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "32391584",
   "metadata": {},
   "source": [
    "# Construção dos Datasets "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "59c13936",
   "metadata": {},
   "source": [
    "## Projeto de Mapeamento Marinho Apresentado Nas Jornadas de Engenharia Hidrográfica"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3d3f7f7",
   "metadata": {},
   "source": [
    "## Diogo Ceddia Porto Silva"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ceb467d5",
   "metadata": {},
   "source": [
    "#### Importação de bibliotecas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6d9d3250",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pyproj import Proj\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "from pykdtree.kdtree import KDTree\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97d260a6",
   "metadata": {},
   "source": [
    "#### Carregando dados Brutos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "14220898",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Carregando todos os arquivos BATIMÉTRICOS de todas as resoluções.\n",
    "df_bat_2m = pd.read_csv(\"C:\\DCPS\\GitHub\\Dados_MapeamentoIA_JEH\\Madeira_13e14_WGS84_UTM28N_2m.txt\",\n",
    "                      skiprows=1,delim_whitespace=True,names=['utmx','utmy','z'])\n",
    "df_bat_2m['z_res'] = '2m'\n",
    "\n",
    "df_bat_4m = pd.read_csv(\"C:\\DCPS\\GitHub\\Dados_MapeamentoIA_JEH\\Madeira_13e14_WGS84_UTM28N_4m.txt\",\n",
    "                      skiprows=1,delim_whitespace=True,names=['utmx','utmy','z'])\n",
    "df_bat_4m['z_res'] = '4m'\n",
    "\n",
    "# Criando arquivo único da batimetria, com coluna associada à resolução.\n",
    "df_bat = pd.concat([df_bat_2m,df_bat_4m],axis=0)\n",
    "del df_bat_2m,df_bat_4m\n",
    "\n",
    "# Carregando dado SÍSMICO\n",
    "df_sbp = pd.read_csv(r\"C:\\DCPS\\GitHub\\Dados_MapeamentoIA_JEH\\Thickness_MadeiraNW.txt\",sep=';')\n",
    "\n",
    "# Convertendo de projeção geográfica WGS84 para UTM WGS84\n",
    "df_sbp['LATITUDE'] = df_sbp['LATITUDE'].str.replace(',','.')\n",
    "df_sbp['LATITUDE'] = df_sbp['LATITUDE'].astype(\"float\")\n",
    "df_sbp['LONGITUDE'] = df_sbp['LONGITUDE'].str.replace(',','.')\n",
    "df_sbp['LONGITUDE'] = df_sbp['LONGITUDE'].astype(\"float\")\n",
    "df_sbp['THICKNESS_'] = df_sbp['THICKNESS_'].str.replace(',','.')\n",
    "df_sbp['THICKNESS_'] = df_sbp['THICKNESS_'].astype(\"float\")\n",
    "df_sbp['classe'] = 1\n",
    "df_sbp['classe'][df_sbp[df_sbp['THICKNESS_']!=0].index] = 0\n",
    "\n",
    "myProj = Proj(\"+proj=utm +zone=28 +north +ellps=WGS84 +datum=WGS84 +units=m\")\n",
    "utmx, utmy = myProj(df_sbp.LONGITUDE, df_sbp.LATITUDE) \n",
    "df_sbp['utmx'] = utmx\n",
    "df_sbp['utmy'] = utmy\n",
    "\n",
    "# Pegando somente colunas de interesse da sísmica\n",
    "df_sbp = df_sbp[['utmx','utmy','classe']]\n",
    "del utmx,utmy\n",
    "\n",
    "# Carregando dados de BACKSCATTER\n",
    "df_bs = pd.read_csv(r\"C:\\DCPS\\GitHub\\Dados_MapeamentoIA_JEH\\backscatter.txt\",names = ['utmx','utmy','bs'])\n",
    "\n",
    "# Carregando dados de DECLIVIDADE DO FUNDO\n",
    "df_sl = pd.read_csv(r\"C:\\DCPS\\GitHub\\Dados_MapeamentoIA_JEH\\slope.xyz\",names = ['utmx','utmy','sl'])\n",
    "\n",
    "# Carregando dados de ORIENTAÇÃO DA DECLIVIDADE DO FUNDO\n",
    "df_as = pd.read_csv(r\"C:\\DCPS\\GitHub\\Dados_MapeamentoIA_JEH\\aspect.xyz\",names = ['utmx','utmy','as'])\n",
    "\n",
    "# Carregando dados da DECLIVIDADE SUBMARINA\n",
    "df_ds = pd.read_csv(r\"C:\\DCPS\\GitHub\\Dados_MapeamentoIA_JEH\\declive submarino.txt\",usecols=[0,1],names = ['utmx','utmy'])\n",
    "\n",
    "# Carregando dados de LINHA DE COSTA\n",
    "df_lc = pd.read_csv(r\"C:\\DCPS\\GitHub\\Dados_MapeamentoIA_JEH\\linha de costa.txt\",usecols=[0,1],names = ['utmx','utmy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f63cae91",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.003000497817993164\n"
     ]
    }
   ],
   "source": [
    "# Pegando distância do declive submarino e a linha de costa. \n",
    "kd_tree = KDTree(df_lc[['utmx','utmy']].values)\n",
    "start = time.time()\n",
    "dist, idx = kd_tree.query(df_ds[['utmx','utmy']].values, k=1)\n",
    "end = time.time()\n",
    "print(end - start)\n",
    "df_ds['dist_lc'] = dist"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab02638a",
   "metadata": {},
   "source": [
    "#### Construido Dataset de Treino/Validação"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "74dfb019",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1356348991394043\n",
      "0.13068413734436035\n",
      "0.16422748565673828\n",
      "0.18649911880493164\n",
      "0.10423040390014648\n",
      "0.11671328544616699\n"
     ]
    }
   ],
   "source": [
    "# PYKDTREE\n",
    "\n",
    "data = df_sbp\n",
    "\n",
    "# Correlacionando sísmica com batimetria\n",
    "kd_tree = KDTree(df_bat[['utmx','utmy']].values)\n",
    "start = time.time()\n",
    "dist, idx = kd_tree.query(df_sbp[['utmx','utmy']].values, k=1)\n",
    "end = time.time()\n",
    "print(end - start)\n",
    "# Salvando correlação de batimetria no dataset final\n",
    "data['z'] = df_bat.z.iloc[idx].values\n",
    "data['z_res'] = df_bat.z_res.iloc[idx].values\n",
    "data['z_dist'] = dist\n",
    "\n",
    "# Correlacionando sísmica com backscatter\n",
    "kd_tree = KDTree(df_bs[['utmx','utmy']].values)\n",
    "start = time.time()\n",
    "dist, idx = kd_tree.query(df_sbp[['utmx','utmy']].values, k=1)\n",
    "end = time.time()\n",
    "print(end - start)\n",
    "# Salvando correlação de batimetria no dataset final\n",
    "data['bs'] = df_bs.bs.iloc[idx].values\n",
    "data['bs_dist'] = dist\n",
    "\n",
    "# Correlacionando sísmica com declividade do fundo\n",
    "kd_tree = KDTree(df_sl[['utmx','utmy']].values)\n",
    "start = time.time()\n",
    "dist, idx = kd_tree.query(df_sbp[['utmx','utmy']].values, k=1)\n",
    "end = time.time()\n",
    "print(end - start)\n",
    "# Salvando correlação de declividade do fundo no dataset final\n",
    "data['sl'] = df_sl.sl.iloc[idx].values\n",
    "data['sl_dist'] = dist\n",
    "\n",
    "# Correlacionando sísmica com orientação da declividade do fundo\n",
    "kd_tree = KDTree(df_as[['utmx','utmy']].values)\n",
    "start = time.time()\n",
    "dist, idx = kd_tree.query(df_sbp[['utmx','utmy']].values, k=1)\n",
    "end = time.time()\n",
    "print(end - start)\n",
    "# Salvando correlação de declividade do fundo no dataset final\n",
    "data['as'] = df_as['as'].iloc[idx].values\n",
    "data['as_dist'] = dist\n",
    "\n",
    "# DISTANCIA EUCLIDIANA DA LINHA DE COSTA\n",
    "kd_tree = KDTree(df_lc[['utmx','utmy']].values)\n",
    "start = time.time()\n",
    "dist, idx = kd_tree.query(df_sbp[['utmx','utmy']].values, k=1)\n",
    "end = time.time()\n",
    "print(end - start)\n",
    "data['dist_lc'] = dist\n",
    "\n",
    "# DISTANCIA EUCLIDIANA DA DECLIVE SUBMARINO\n",
    "kd_tree = KDTree(df_ds[['utmx','utmy']].values)\n",
    "start = time.time()\n",
    "dist, idx = kd_tree.query(df_sbp[['utmx','utmy']].values, k=1)\n",
    "end = time.time()\n",
    "print(end - start)\n",
    "data = data.reset_index(drop=True)\n",
    "data['dist_ds'] = dist\n",
    "#data['dist_ds'] = data['dist_ds']*(np.sign(data['dist_ds'] - df_ds['dist_lc'].iloc[idx].reset_index(drop=True)))\n",
    "#data['dist_ds_lc'] = df_ds['dist_lc'].iloc[idx].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c583170",
   "metadata": {},
   "source": [
    "#### Construindo Dataset de Extrapolação"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9759368e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.2078871726989746\n",
      "2.489129066467285\n",
      "2.500565767288208\n",
      "2.0352015495300293\n",
      "2.1158087253570557\n"
     ]
    }
   ],
   "source": [
    "# PYKDTREE\n",
    "\n",
    "data_all = df_bat\n",
    "\n",
    "# Correlacionando BATIMETRIA COM BACKSCATTER\n",
    "kd_tree = KDTree(df_bs[['utmx','utmy']].values)\n",
    "start = time.time()\n",
    "dist, idx = kd_tree.query(df_bat[['utmx','utmy']].values, k=1)\n",
    "end = time.time()\n",
    "print(end - start)\n",
    "data_all['bs'] = df_bs.bs.iloc[idx].values\n",
    "data_all['bs_dist'] = dist\n",
    "\n",
    "# Correlacionando BATIMETRIA COM DECLIVIDADE DO FUNDO\n",
    "kd_tree = KDTree(df_sl[['utmx','utmy']].values)\n",
    "start = time.time()\n",
    "dist, idx = kd_tree.query(df_bat[['utmx','utmy']].values, k=1)\n",
    "end = time.time()\n",
    "print(end - start)\n",
    "data_all['sl'] = df_sl.sl.iloc[idx].values\n",
    "data_all['sl_dist'] = dist\n",
    "\n",
    "# Correlacionando BATIMETRIA COM ORIENTAÇÃO DA DECLIVIDADE DO FUNDO\n",
    "kd_tree = KDTree(df_as[['utmx','utmy']].values)\n",
    "start = time.time()\n",
    "dist, idx = kd_tree.query(df_bat[['utmx','utmy']].values, k=1)\n",
    "end = time.time()\n",
    "print(end - start)\n",
    "data_all['as'] = df_as['as'].iloc[idx].values\n",
    "data_all['as_dist'] = dist\n",
    "\n",
    "# DISTANCIA EUCLIDIANA DA LINHA DE COSTA\n",
    "kd_tree = KDTree(df_lc[['utmx','utmy']].values)\n",
    "start = time.time()\n",
    "dist, idx = kd_tree.query(df_bat[['utmx','utmy']].values, k=1)\n",
    "end = time.time()\n",
    "print(end - start)\n",
    "data_all['dist_lc'] = dist\n",
    "\n",
    "# DISTANCIA EUCLIDIANA DA DECLIVE SUBMARINO\n",
    "kd_tree = KDTree(df_ds[['utmx','utmy']].values)\n",
    "start = time.time()\n",
    "dist, idx = kd_tree.query(df_bat[['utmx','utmy']].values, k=1)\n",
    "end = time.time()\n",
    "print(end - start)\n",
    "data_all['dist_ds'] = dist\n",
    "data_all = data_all.reset_index(drop=True)\n",
    "#data_all['dist_ds'] = data_all['dist_ds']*(np.sign(data_all['dist_ds'] - df_ds['dist_lc'].iloc[idx].reset_index(drop=True)))\n",
    "#data_all['dist_ds_lc'] = df_ds['dist_lc'].iloc[idx].reset_index(drop=True)\n",
    "\n",
    "del df_bat, df_bs, df_ds, df_lc, df_sbp, df_sl, dist, start, end, idx, kd_tree, df_as"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7555d074",
   "metadata": {},
   "source": [
    "#### Filtrando todo atributo que dista 10m+ do grid pivot."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7ce35649",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.reset_index(drop=True)\n",
    "data = data.drop(np.where(data.z_dist>10)[0]).reset_index(drop=True)\n",
    "data = data.drop(np.where(data.bs_dist>10)[0]).reset_index(drop=True)\n",
    "data = data.drop(np.where(data.sl_dist>10)[0]).reset_index(drop=True)\n",
    "data = data.drop(np.where(data.as_dist>10)[0]).reset_index(drop=True)\n",
    "data_all = data_all.reset_index(drop=True)\n",
    "data_all = data_all.drop(np.where(data_all.bs_dist>10)[0]).reset_index(drop=True)\n",
    "data_all = data_all.drop(np.where(data_all.sl_dist>10)[0]).reset_index(drop=True)\n",
    "data_all = data_all.drop(np.where(data_all.as_dist>10)[0]).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c7712d1",
   "metadata": {},
   "source": [
    "#### Filtrando dados duplicados"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e5f9fa3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#print('\\nQuantidade de linhas no dataset original:',len(data))\n",
    "#duplicateDFRow = data[data.duplicated(subset=['z','bs','sl','as','classe'])]\n",
    "#print('Quantidade de linhas duplicadas:', len(duplicateDFRow))\n",
    "#print('% filtrado:', np.round(len(duplicateDFRow)/len(data)*100,3))\n",
    "#data.drop(axis=0, index=duplicateDFRow.index, inplace=True)\n",
    "#data = data.reset_index(drop=True)\n",
    "#print('Quantidade de linhas do novo dataset:',len(data),'\\n')\n",
    "#del duplicateDFRow\n",
    "#print('% de sedimento:',np.round((len(np.where(data.classe==0)[0])/len(data))*100,3))\n",
    "#print('% de rocha:',np.round((len(np.where(data.classe==1)[0])/len(data))*100,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4f2c10cb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Quantidade de linhas no dataset original: 5925070\n",
      "Quantidade de linhas duplicadas: 5950\n",
      "% filtrado: 0.1\n",
      "Quantidade de linhas do novo dataset: 5919120 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('\\nQuantidade de linhas no dataset original:',len(data_all))\n",
    "duplicateDFRow = data_all[data_all.duplicated(subset=['z','bs','sl','as'])]\n",
    "print('Quantidade de linhas duplicadas:', len(duplicateDFRow))\n",
    "print('% filtrado:', np.round(len(duplicateDFRow)/len(data_all)*100,3))\n",
    "data_all.drop(axis=0, index=duplicateDFRow.index, inplace=True)\n",
    "data_all = data_all.reset_index(drop=True)\n",
    "print('Quantidade de linhas do novo dataset:',len(data_all),'\\n')\n",
    "del duplicateDFRow"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb81f47c",
   "metadata": {},
   "source": [
    "#### Filtrando dados que estão no declive submarino"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1ca71911",
   "metadata": {},
   "outputs": [],
   "source": [
    "#data = data.drop(np.where(data['dist_ds']<50)[0]).reset_index(drop=True)\n",
    "#data_all = data_all.drop(np.where(data_all['dist_ds']<50)[0]).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aae2d8e5",
   "metadata": {},
   "source": [
    "#### Descrição e detalhamento dos dados"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "181f20d9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 191598 entries, 0 to 191597\n",
      "Data columns (total 14 columns):\n",
      " #   Column   Non-Null Count   Dtype  \n",
      "---  ------   --------------   -----  \n",
      " 0   utmx     191598 non-null  float64\n",
      " 1   utmy     191598 non-null  float64\n",
      " 2   classe   191598 non-null  int64  \n",
      " 3   z        191598 non-null  float64\n",
      " 4   z_res    191598 non-null  object \n",
      " 5   z_dist   191598 non-null  float64\n",
      " 6   bs       191598 non-null  float64\n",
      " 7   bs_dist  191598 non-null  float64\n",
      " 8   sl       191598 non-null  float64\n",
      " 9   sl_dist  191598 non-null  float64\n",
      " 10  as       191598 non-null  float64\n",
      " 11  as_dist  191598 non-null  float64\n",
      " 12  dist_lc  191598 non-null  float64\n",
      " 13  dist_ds  191598 non-null  float64\n",
      "dtypes: float64(12), int64(1), object(1)\n",
      "memory usage: 20.5+ MB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f35e6834",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>utmx</th>\n",
       "      <th>utmy</th>\n",
       "      <th>classe</th>\n",
       "      <th>z</th>\n",
       "      <th>z_dist</th>\n",
       "      <th>bs</th>\n",
       "      <th>bs_dist</th>\n",
       "      <th>sl</th>\n",
       "      <th>sl_dist</th>\n",
       "      <th>as</th>\n",
       "      <th>as_dist</th>\n",
       "      <th>dist_lc</th>\n",
       "      <th>dist_ds</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>191598.000000</td>\n",
       "      <td>1.915980e+05</td>\n",
       "      <td>191598.000000</td>\n",
       "      <td>191598.000000</td>\n",
       "      <td>191598.000000</td>\n",
       "      <td>191598.000000</td>\n",
       "      <td>191598.000000</td>\n",
       "      <td>191598.000000</td>\n",
       "      <td>191598.000000</td>\n",
       "      <td>191598.000000</td>\n",
       "      <td>191598.000000</td>\n",
       "      <td>191598.000000</td>\n",
       "      <td>191598.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>285717.867186</td>\n",
       "      <td>3.629459e+06</td>\n",
       "      <td>0.400114</td>\n",
       "      <td>70.553797</td>\n",
       "      <td>1.362341</td>\n",
       "      <td>-19.806302</td>\n",
       "      <td>2.167814</td>\n",
       "      <td>2.427746</td>\n",
       "      <td>0.770287</td>\n",
       "      <td>112.430045</td>\n",
       "      <td>0.770287</td>\n",
       "      <td>3523.354167</td>\n",
       "      <td>1998.196859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2644.643443</td>\n",
       "      <td>2.303986e+03</td>\n",
       "      <td>0.489922</td>\n",
       "      <td>14.035168</td>\n",
       "      <td>0.662312</td>\n",
       "      <td>6.531755</td>\n",
       "      <td>0.809359</td>\n",
       "      <td>5.550080</td>\n",
       "      <td>0.323013</td>\n",
       "      <td>83.926307</td>\n",
       "      <td>0.323013</td>\n",
       "      <td>2140.623294</td>\n",
       "      <td>1336.726379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>280127.962030</td>\n",
       "      <td>3.623894e+06</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>39.346000</td>\n",
       "      <td>0.005770</td>\n",
       "      <td>-41.341000</td>\n",
       "      <td>0.005050</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.005770</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.005770</td>\n",
       "      <td>715.801462</td>\n",
       "      <td>2.491801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>282641.572615</td>\n",
       "      <td>3.627810e+06</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>60.659000</td>\n",
       "      <td>0.867293</td>\n",
       "      <td>-24.971000</td>\n",
       "      <td>1.595921</td>\n",
       "      <td>0.650000</td>\n",
       "      <td>0.565119</td>\n",
       "      <td>57.000000</td>\n",
       "      <td>0.565119</td>\n",
       "      <td>1850.328125</td>\n",
       "      <td>911.224655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>286673.440596</td>\n",
       "      <td>3.629665e+06</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>70.911000</td>\n",
       "      <td>1.335847</td>\n",
       "      <td>-20.870000</td>\n",
       "      <td>2.258685</td>\n",
       "      <td>1.140000</td>\n",
       "      <td>0.800351</td>\n",
       "      <td>82.000000</td>\n",
       "      <td>0.800351</td>\n",
       "      <td>2632.794801</td>\n",
       "      <td>1618.190985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>287695.587580</td>\n",
       "      <td>3.631278e+06</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>78.198000</td>\n",
       "      <td>1.855233</td>\n",
       "      <td>-13.310000</td>\n",
       "      <td>2.769188</td>\n",
       "      <td>1.860000</td>\n",
       "      <td>0.978651</td>\n",
       "      <td>145.000000</td>\n",
       "      <td>0.978651</td>\n",
       "      <td>5472.919231</td>\n",
       "      <td>3140.065377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>290013.835834</td>\n",
       "      <td>3.633793e+06</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>149.974000</td>\n",
       "      <td>9.988894</td>\n",
       "      <td>2.119000</td>\n",
       "      <td>6.576014</td>\n",
       "      <td>89.000000</td>\n",
       "      <td>9.856238</td>\n",
       "      <td>359.000000</td>\n",
       "      <td>9.856238</td>\n",
       "      <td>8561.918000</td>\n",
       "      <td>4824.692303</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                utmx          utmy         classe              z  \\\n",
       "count  191598.000000  1.915980e+05  191598.000000  191598.000000   \n",
       "mean   285717.867186  3.629459e+06       0.400114      70.553797   \n",
       "std      2644.643443  2.303986e+03       0.489922      14.035168   \n",
       "min    280127.962030  3.623894e+06       0.000000      39.346000   \n",
       "25%    282641.572615  3.627810e+06       0.000000      60.659000   \n",
       "50%    286673.440596  3.629665e+06       0.000000      70.911000   \n",
       "75%    287695.587580  3.631278e+06       1.000000      78.198000   \n",
       "max    290013.835834  3.633793e+06       1.000000     149.974000   \n",
       "\n",
       "              z_dist             bs        bs_dist             sl  \\\n",
       "count  191598.000000  191598.000000  191598.000000  191598.000000   \n",
       "mean        1.362341     -19.806302       2.167814       2.427746   \n",
       "std         0.662312       6.531755       0.809359       5.550080   \n",
       "min         0.005770     -41.341000       0.005050       0.000000   \n",
       "25%         0.867293     -24.971000       1.595921       0.650000   \n",
       "50%         1.335847     -20.870000       2.258685       1.140000   \n",
       "75%         1.855233     -13.310000       2.769188       1.860000   \n",
       "max         9.988894       2.119000       6.576014      89.000000   \n",
       "\n",
       "             sl_dist             as        as_dist        dist_lc  \\\n",
       "count  191598.000000  191598.000000  191598.000000  191598.000000   \n",
       "mean        0.770287     112.430045       0.770287    3523.354167   \n",
       "std         0.323013      83.926307       0.323013    2140.623294   \n",
       "min         0.005770       0.000000       0.005770     715.801462   \n",
       "25%         0.565119      57.000000       0.565119    1850.328125   \n",
       "50%         0.800351      82.000000       0.800351    2632.794801   \n",
       "75%         0.978651     145.000000       0.978651    5472.919231   \n",
       "max         9.856238     359.000000       9.856238    8561.918000   \n",
       "\n",
       "             dist_ds  \n",
       "count  191598.000000  \n",
       "mean     1998.196859  \n",
       "std      1336.726379  \n",
       "min         2.491801  \n",
       "25%       911.224655  \n",
       "50%      1618.190985  \n",
       "75%      3140.065377  \n",
       "max      4824.692303  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1e46dd85",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 5919120 entries, 0 to 5919119\n",
      "Data columns (total 12 columns):\n",
      " #   Column   Dtype  \n",
      "---  ------   -----  \n",
      " 0   utmx     float64\n",
      " 1   utmy     float64\n",
      " 2   z        float64\n",
      " 3   z_res    object \n",
      " 4   bs       float64\n",
      " 5   bs_dist  float64\n",
      " 6   sl       float64\n",
      " 7   sl_dist  float64\n",
      " 8   as       float64\n",
      " 9   as_dist  float64\n",
      " 10  dist_lc  float64\n",
      " 11  dist_ds  float64\n",
      "dtypes: float64(11), object(1)\n",
      "memory usage: 541.9+ MB\n"
     ]
    }
   ],
   "source": [
    "data_all.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b9b833aa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>utmx</th>\n",
       "      <th>utmy</th>\n",
       "      <th>z</th>\n",
       "      <th>bs</th>\n",
       "      <th>bs_dist</th>\n",
       "      <th>sl</th>\n",
       "      <th>sl_dist</th>\n",
       "      <th>as</th>\n",
       "      <th>as_dist</th>\n",
       "      <th>dist_lc</th>\n",
       "      <th>dist_ds</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>5.919120e+06</td>\n",
       "      <td>5.919120e+06</td>\n",
       "      <td>5.919120e+06</td>\n",
       "      <td>5.919120e+06</td>\n",
       "      <td>5.919120e+06</td>\n",
       "      <td>5.919120e+06</td>\n",
       "      <td>5919120.0</td>\n",
       "      <td>5.919120e+06</td>\n",
       "      <td>5919120.0</td>\n",
       "      <td>5.919120e+06</td>\n",
       "      <td>5.919120e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.859595e+05</td>\n",
       "      <td>3.630915e+06</td>\n",
       "      <td>7.058757e+01</td>\n",
       "      <td>-1.916879e+01</td>\n",
       "      <td>2.163977e+00</td>\n",
       "      <td>2.974167e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.275024e+02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.289966e+03</td>\n",
       "      <td>1.733538e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2.457484e+03</td>\n",
       "      <td>3.240108e+03</td>\n",
       "      <td>1.715595e+01</td>\n",
       "      <td>6.409923e+00</td>\n",
       "      <td>8.056143e-01</td>\n",
       "      <td>5.509527e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.354982e+01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.938720e+03</td>\n",
       "      <td>1.209329e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>2.801360e+05</td>\n",
       "      <td>3.623904e+06</td>\n",
       "      <td>3.003900e+01</td>\n",
       "      <td>-4.859100e+01</td>\n",
       "      <td>5.656854e-03</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.133646e+02</td>\n",
       "      <td>1.221843e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.841280e+05</td>\n",
       "      <td>3.628392e+06</td>\n",
       "      <td>5.670000e+01</td>\n",
       "      <td>-2.434100e+01</td>\n",
       "      <td>1.595724e+00</td>\n",
       "      <td>7.700000e-01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.500000e+01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.729975e+03</td>\n",
       "      <td>8.452185e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.864400e+05</td>\n",
       "      <td>3.630768e+06</td>\n",
       "      <td>7.089100e+01</td>\n",
       "      <td>-1.961000e+01</td>\n",
       "      <td>2.256461e+00</td>\n",
       "      <td>1.370000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.110000e+02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.662756e+03</td>\n",
       "      <td>1.458822e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.878880e+05</td>\n",
       "      <td>3.633524e+06</td>\n",
       "      <td>7.965200e+01</td>\n",
       "      <td>-1.331000e+01</td>\n",
       "      <td>2.763794e+00</td>\n",
       "      <td>2.410000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.650000e+02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.482468e+03</td>\n",
       "      <td>2.518825e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.902520e+05</td>\n",
       "      <td>3.637124e+06</td>\n",
       "      <td>1.499990e+02</td>\n",
       "      <td>7.479000e+00</td>\n",
       "      <td>8.148684e+00</td>\n",
       "      <td>8.935000e+01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.590000e+02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.662566e+03</td>\n",
       "      <td>4.853595e+03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               utmx          utmy             z            bs       bs_dist  \\\n",
       "count  5.919120e+06  5.919120e+06  5.919120e+06  5.919120e+06  5.919120e+06   \n",
       "mean   2.859595e+05  3.630915e+06  7.058757e+01 -1.916879e+01  2.163977e+00   \n",
       "std    2.457484e+03  3.240108e+03  1.715595e+01  6.409923e+00  8.056143e-01   \n",
       "min    2.801360e+05  3.623904e+06  3.003900e+01 -4.859100e+01  5.656854e-03   \n",
       "25%    2.841280e+05  3.628392e+06  5.670000e+01 -2.434100e+01  1.595724e+00   \n",
       "50%    2.864400e+05  3.630768e+06  7.089100e+01 -1.961000e+01  2.256461e+00   \n",
       "75%    2.878880e+05  3.633524e+06  7.965200e+01 -1.331000e+01  2.763794e+00   \n",
       "max    2.902520e+05  3.637124e+06  1.499990e+02  7.479000e+00  8.148684e+00   \n",
       "\n",
       "                 sl    sl_dist            as    as_dist       dist_lc  \\\n",
       "count  5.919120e+06  5919120.0  5.919120e+06  5919120.0  5.919120e+06   \n",
       "mean   2.974167e+00        0.0  1.275024e+02        0.0  3.289966e+03   \n",
       "std    5.509527e+00        0.0  8.354982e+01        0.0  1.938720e+03   \n",
       "min    0.000000e+00        0.0  0.000000e+00        0.0  7.133646e+02   \n",
       "25%    7.700000e-01        0.0  6.500000e+01        0.0  1.729975e+03   \n",
       "50%    1.370000e+00        0.0  1.110000e+02        0.0  2.662756e+03   \n",
       "75%    2.410000e+00        0.0  1.650000e+02        0.0  4.482468e+03   \n",
       "max    8.935000e+01        0.0  3.590000e+02        0.0  8.662566e+03   \n",
       "\n",
       "            dist_ds  \n",
       "count  5.919120e+06  \n",
       "mean   1.733538e+03  \n",
       "std    1.209329e+03  \n",
       "min    1.221843e-01  \n",
       "25%    8.452185e+02  \n",
       "50%    1.458822e+03  \n",
       "75%    2.518825e+03  \n",
       "max    4.853595e+03  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_all.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0c232bb1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>utmx</th>\n",
       "      <th>utmy</th>\n",
       "      <th>z</th>\n",
       "      <th>z_res</th>\n",
       "      <th>bs</th>\n",
       "      <th>bs_dist</th>\n",
       "      <th>sl</th>\n",
       "      <th>sl_dist</th>\n",
       "      <th>as</th>\n",
       "      <th>as_dist</th>\n",
       "      <th>dist_lc</th>\n",
       "      <th>dist_ds</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>289984.0</td>\n",
       "      <td>3624500.0</td>\n",
       "      <td>59.993</td>\n",
       "      <td>2m</td>\n",
       "      <td>-23.081</td>\n",
       "      <td>2.618635</td>\n",
       "      <td>1.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2152.248438</td>\n",
       "      <td>1709.201926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>289986.0</td>\n",
       "      <td>3624500.0</td>\n",
       "      <td>59.987</td>\n",
       "      <td>2m</td>\n",
       "      <td>-23.081</td>\n",
       "      <td>3.678213</td>\n",
       "      <td>88.09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2150.919495</td>\n",
       "      <td>1711.109044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>289984.0</td>\n",
       "      <td>3624502.0</td>\n",
       "      <td>59.961</td>\n",
       "      <td>2m</td>\n",
       "      <td>-23.081</td>\n",
       "      <td>0.853960</td>\n",
       "      <td>1.27</td>\n",
       "      <td>0.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2150.754683</td>\n",
       "      <td>1708.600241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>289986.0</td>\n",
       "      <td>3624502.0</td>\n",
       "      <td>59.930</td>\n",
       "      <td>2m</td>\n",
       "      <td>-23.081</td>\n",
       "      <td>2.720523</td>\n",
       "      <td>88.09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2149.424817</td>\n",
       "      <td>1710.508029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>289982.0</td>\n",
       "      <td>3624504.0</td>\n",
       "      <td>59.984</td>\n",
       "      <td>2m</td>\n",
       "      <td>-23.081</td>\n",
       "      <td>1.982233</td>\n",
       "      <td>1.36</td>\n",
       "      <td>0.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2150.593579</td>\n",
       "      <td>1706.092438</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       utmx       utmy       z z_res      bs   bs_dist     sl  sl_dist    as  \\\n",
       "0  289984.0  3624500.0  59.993    2m -23.081  2.618635   1.18      0.0  31.0   \n",
       "1  289986.0  3624500.0  59.987    2m -23.081  3.678213  88.09      0.0   4.0   \n",
       "2  289984.0  3624502.0  59.961    2m -23.081  0.853960   1.27      0.0  56.0   \n",
       "3  289986.0  3624502.0  59.930    2m -23.081  2.720523  88.09      0.0  20.0   \n",
       "4  289982.0  3624504.0  59.984    2m -23.081  1.982233   1.36      0.0  62.0   \n",
       "\n",
       "   as_dist      dist_lc      dist_ds  \n",
       "0      0.0  2152.248438  1709.201926  \n",
       "1      0.0  2150.919495  1711.109044  \n",
       "2      0.0  2150.754683  1708.600241  \n",
       "3      0.0  2149.424817  1710.508029  \n",
       "4      0.0  2150.593579  1706.092438  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_all.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af9f3a5a",
   "metadata": {},
   "source": [
    "#### Salvando"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "cbef7c03",
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv(r\"C:\\DCPS\\GitHub\\Dados_MapeamentoIA_JEH\\dado_linhas.txt\",index=False,header=True)\n",
    "data_all.to_csv(r\"C:\\DCPS\\GitHub\\Dados_MapeamentoIA_JEH\\dado_tudo.txt\",index=False,header=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}