data_loading.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting the data into Torch\n",
    "Torch is really bad at handling external data. It's native data format is t7. This is a collection of notes on getting data loaded in for use in the ML contest.\n",
    "\n",
    "I've included Scott Locklin's <a href=\"https://github.com/locklin/torch-things/blob/master/csv2t7.sh\">script</a> in this directory for converting csv to t7. The raw CSV files must be stripped of text (t7 only accepts numerals) including headers before feeding them into the script. I also had to condition the NaNs in the PE logs in the 'facies_vectors' data. They are now 9999s."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "******FACIES_VECTORS FILE STATISTICS*******\t\n"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "counting lines\n"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "lines = 4149\n"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "columns = 9\n",
       "creating header\n",
       "dumping data; this may take a moment...\n"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "******NOFACIES_DATA FILE STATISTICS*******\t\n"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "counting lines\n"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "lines = 830\n"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "columns = 8\n",
       "creating header\n"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "dumping data; this may take a moment...\n"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "******TRAINING_DATA FILE STATISTICS*******\t\n"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "counting lines\n"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "lines = 3232\n"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "columns = 9\n",
       "creating header\n",
       "dumping data; this may take a moment...\n"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "******VALIDATION_DATA_NO_FACIES FILE STATISTICS*******\t\n"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "counting lines\n"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "lines = 830\n"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "columns = 8\n",
       "creating header\n",
       "dumping data; this may take a moment...\n"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "******WELL_DATA_WITH_FACIES FILE STATISTICS*******\t\n"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "counting lines\n"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "lines = 830\n"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "columns = 10\n",
       "creating header\n",
       "dumping data; this may take a moment...\n"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "-- facies_vectors\n",
    "    -- remove CSV header and replace NaNs with 99999\n",
    "os.execute(\"sed '1d' ~/2016-ml-contest/facies_vectors.csv | \\\n",
    "            awk '{gsub(/,,/,\\\",99999,\\\")}; 1' > dat/facies_vectors.csv\")\n",
    "\n",
    "    -- delete formation names and well names (all textual data)\n",
    "os.execute(\"cut --complement -f 3 -d, dat/facies_vectors.csv > tmpfile; \\\n",
    "            cut --complement -f 2 -d, tmpfile > dat/facies_vectors.csv\")\n",
    "\n",
    "    -- convert CSV to t7\n",
    "print(\"******FACIES_VECTORS FILE STATISTICS*******\")\n",
    "os.execute(\"bash csv2t7.sh dat/facies_vectors.csv dat/facies_vectors.t7\");\n",
    "\n",
    "-- repeat for all files\n",
    "    -- nofacies_data\n",
    "os.execute(\"sed '1d' ~/2016-ml-contest/nofacies_data.csv | \\\n",
    "            awk '{gsub(/,,/,\\\",99999,\\\")}; 1' > dat/nofacies_data.csv\")\n",
    "os.execute(\"cut --complement -f 2 -d, dat/nofacies_data.csv > tmpfile; \\\n",
    "            cut --complement -f 1 -d, tmpfile > dat/nofacies_data.csv\")\n",
    "print(\"******NOFACIES_DATA FILE STATISTICS*******\")\n",
    "os.execute(\"bash csv2t7.sh dat/nofacies_data.csv dat/nofacies_data.t7\");\n",
    "    -- training_data\n",
    "os.execute(\"sed '1d' ~/2016-ml-contest/training_data.csv | \\\n",
    "            awk '{gsub(/,,/,\\\",99999,\\\")}; 1' > dat/training_data.csv\")\n",
    "os.execute(\"cut --complement -f 3 -d, dat/training_data.csv > tmpfile; \\\n",
    "            cut --complement -f 2 -d, tmpfile > dat/training_data.csv\")\n",
    "print(\"******TRAINING_DATA FILE STATISTICS*******\")\n",
    "os.execute(\"bash csv2t7.sh dat/training_data.csv dat/training_data.t7\");\n",
    "    -- validation_data_nofacies\n",
    "os.execute(\"sed '1d' ~/2016-ml-contest/validation_data_nofacies.csv | \\\n",
    "            awk '{gsub(/,,/,\\\",99999,\\\")}; 1' > dat/validation_data_nofacies.csv\")\n",
    "os.execute(\"cut --complement -f 2 -d, dat/validation_data_nofacies.csv > tmpfile; \\\n",
    "            cut --complement -f 1 -d, tmpfile > dat/validation_data_nofacies.csv\")\n",
    "print(\"******VALIDATION_DATA_NO_FACIES FILE STATISTICS*******\")\n",
    "os.execute(\"bash csv2t7.sh dat/validation_data_nofacies.csv dat/validation_data_nofacies.t7\");\n",
    "    -- well_data_with_facies\n",
    "os.execute(\"sed '1d' ~/2016-ml-contest/well_data_with_facies.csv | \\\n",
    "            awk '{gsub(/,,/,\\\",99999,\\\")}; 1' > dat/well_data_with_facies.csv\")\n",
    "os.execute(\"cut --complement -f 3 -d, dat/well_data_with_facies.csv > tmpfile; \\\n",
    "            cut --complement -f 2 -d, tmpfile > dat/well_data_with_facies.csv\")\n",
    "print(\"******WELL_DATA_WITH_FACIES FILE STATISTICS*******\")\n",
    "os.execute(\"bash csv2t7.sh dat/well_data_with_facies.csv dat/well_data_with_facies.t7\");\n",
    "\n",
    "-- clean\n",
    "os.execute(\"rm tmpfile dat/*.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "file = torch.DiskFile('dat/facies_vectors.t7', 'r')\n",
    "facies = file:readObject()\n",
    "file:close()\n",
    "file = torch.DiskFile('dat/nofacies_data.t7', 'r')\n",
    "nofacies = file:readObject()\n",
    "file:close()\n",
    "file = torch.DiskFile('dat/training_data.t7', 'r')\n",
    "training = file:readObject()\n",
    "file:close()\n",
    "file = torch.DiskFile('dat/validation_data_nofacies.t7', 'r')\n",
    "validate = file:readObject()\n",
    "file:close()\n",
    "file = torch.DiskFile('dat/well_data_with_facies.t7', 'r')\n",
    "well = file:readObject()\n",
    "file:close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "facies size: \t4149\tx\t9\t\n",
       "nofacies size: \t830\tx\t8\t\n",
       "training size: \t3232\tx\t9\t\n",
       "validate size: \t830\tx\t8\t\n",
       "well size: \t830\tx\t10\t\n"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"facies size: \", facies:size()[1], \"x\", facies:size()[2])\n",
    "print(\"nofacies size: \", nofacies:size()[1], \"x\", nofacies:size()[2])\n",
    "print(\"training size: \", training:size()[1], \"x\", training:size()[2])\n",
    "print(\"validate size: \", validate:size()[1], \"x\", validate:size()[2])\n",
    "print(\"well size: \", well:size()[1], \"x\", well:size()[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "iTorch",
   "language": "lua",
   "name": "itorch"
  },
  "language_info": {
   "name": "lua",
   "version": "5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}