Sediment Concentration from USGS Station.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Objective\n",
    "\n",
    "Download the data for gaging station based on the USGS site number and plot the result."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_station_data(site_no=\"0702200\"):\n",
    "    # importing the requests library \n",
    "    import requests \n",
    "    import xml.etree.ElementTree as ET\n",
    "    from zipfile import ZipFile\n",
    "    import io\n",
    "    from io import BytesIO\n",
    "    # defining the api-endpoint  \n",
    "    URL = \"https://cida.usgs.gov/\"\n",
    "    API_ENDPOINT = \"https://cida.usgs.gov/sediment/data\"\n",
    "    SITE_NO = \"07022000\"\n",
    "    ogcfilter = ET.Element(\"ogc:Filter\",attrib={\"xmlns:ogc\":\"http://www.opengis.net/ogc\"})\n",
    "    property_is = ET.SubElement(ogcfilter, \"ogc:PropertyIsEqualTo\", attrib={\"matchCase\":\"true\"})\n",
    "    ET.SubElement(property_is, \"ogc:PropertyName\").text = \"SITE_NO\"\n",
    "    ET.SubElement(property_is, \"ogc:Literal\").text = SITE_NO\n",
    "    ogcfilter_string = ET.tostring(ogcfilter).decode()  \n",
    "    data = {'email' : 'user@usace.army.mil',\n",
    "            'format': 'tsv',\n",
    "            'directDownload':'true',\n",
    "            'dataTypes': 'sites_data_daily_discrete_',\n",
    "            'dailyFilter' : ogcfilter_string,\n",
    "            'discreteFilter' : ogcfilter_string\n",
    "           }\n",
    "\n",
    "    # sending post request and saving response as response object \n",
    "    search = requests.post(url = API_ENDPOINT, data = data, allow_redirects=True)  \n",
    "    download = requests.get(url=URL+search.text)\n",
    "    zipfile = ZipFile(BytesIO(download._content))\n",
    "    return zipfile\n",
    "zipfile = get_station_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_comments(filename):\n",
    "    ncomments=0\n",
    "    comments=''\n",
    "    with zipfile.open(filename) as f:\n",
    "        commentline = f.readline().decode()\n",
    "        while commentline:\n",
    "            if commentline[0] == '#' or commentline[:2] == r'\"#':\n",
    "                ncomments +=1\n",
    "                comments += commentline.strip(r'\"').strip('#')\n",
    "                commentline=f.readline().decode()\n",
    "            else:\n",
    "                commentline=None\n",
    "    return (ncomments, comments)\n",
    "(ncomments, comments) = get_comments('discrete_data.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DATETIME</th>\n",
       "      <th>SSC</th>\n",
       "      <th>DAILYFLOW</th>\n",
       "      <th>P63</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1973-01-30 10:30:00</td>\n",
       "      <td>306</td>\n",
       "      <td>369000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1973-02-21 10:50:00</td>\n",
       "      <td>436</td>\n",
       "      <td>237000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1973-06-21 11:00:00</td>\n",
       "      <td>468</td>\n",
       "      <td>408000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1973-07-18 09:30:00</td>\n",
       "      <td>227</td>\n",
       "      <td>179000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1973-08-14 10:00:00</td>\n",
       "      <td>211</td>\n",
       "      <td>208000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              DATETIME   SSC  DAILYFLOW  P63\n",
       "0  1973-01-30 10:30:00   306     369000  NaN\n",
       "1  1973-02-21 10:50:00   436     237000  NaN\n",
       "2  1973-06-21 11:00:00   468     408000  NaN\n",
       "3  1973-07-18 09:30:00   227     179000  NaN\n",
       "4  1973-08-14 10:00:00   211     208000  NaN"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas\n",
    "discrete_data = pandas.read_table(zipfile.open('discrete_data.tsv'),header=ncomments,sep=\"\\t\").dropna(axis=1,how='all')\n",
    "drop_columns=[]\n",
    "for k in discrete_data.columns:\n",
    "    if k not in ['DATETIME','SSC','DAILYFLOW','P63']:\n",
    "        drop_columns.append(k)\n",
    "discrete_data.drop(drop_columns,axis=1).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3f7472ddbfa144159d4f549f6f6f41a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib ipympl\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "ssc = np.zeros(discrete_data.SSC.shape,'d')\n",
    "for i,ssci in enumerate(discrete_data.SSC):\n",
    "    if 'A' in ssci:\n",
    "        ssc[i] = float(ssci[2:])\n",
    "    else:\n",
    "        ssc[i] = float(ssci)\n",
    "p63 = np.zeros(discrete_data.P63.shape,'d')\n",
    "for i,p63i in enumerate(discrete_data.P63):\n",
    "    if type(p63i) is not float:\n",
    "        if 'A' in p63i:\n",
    "            p63[i] = float(p63i[2:])\n",
    "        else:\n",
    "            p63[i] = float(p63i)\n",
    "    else:\n",
    "        p63[i] = p63i\n",
    "plt.figure()\n",
    "plt.scatter(discrete_data.DAILYFLOW,ssc)\n",
    "sc = plt.scatter(discrete_data.DAILYFLOW,p63)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9553a5ffe1e742b78b74ab21ac37f499",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "date = np.array(discrete_data.DATETIME, dtype='datetime64')\n",
    "p = plt.plot(date,discrete_data.DAILYFLOW)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = zipfile.open('discrete_data.tsv')\n",
    "newfile = open('test.tsv','w')\n",
    "for line in file.readlines():\n",
    "    newfile.writelines(line.decode())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>agency_cd</th>\n",
       "      <th>site_no</th>\n",
       "      <th>datetime</th>\n",
       "      <th>DAILY_FLOW</th>\n",
       "      <th>DAILY_FLOW_QUAL</th>\n",
       "      <th>DAILY_SSC</th>\n",
       "      <th>DAILY_SSC_QUAL</th>\n",
       "      <th>DAILY_SSL</th>\n",
       "      <th>DAILY_SSL_QUAL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>USGS</td>\n",
       "      <td>7022000</td>\n",
       "      <td>1933-04-01</td>\n",
       "      <td>191000</td>\n",
       "      <td>A</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>USGS</td>\n",
       "      <td>7022000</td>\n",
       "      <td>1933-04-02</td>\n",
       "      <td>224000</td>\n",
       "      <td>A</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>USGS</td>\n",
       "      <td>7022000</td>\n",
       "      <td>1933-04-03</td>\n",
       "      <td>257000</td>\n",
       "      <td>A</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>USGS</td>\n",
       "      <td>7022000</td>\n",
       "      <td>1933-04-04</td>\n",
       "      <td>279000</td>\n",
       "      <td>A</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>USGS</td>\n",
       "      <td>7022000</td>\n",
       "      <td>1933-04-05</td>\n",
       "      <td>298000</td>\n",
       "      <td>A</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  agency_cd  site_no    datetime  DAILY_FLOW DAILY_FLOW_QUAL  DAILY_SSC  \\\n",
       "0      USGS  7022000  1933-04-01      191000               A        NaN   \n",
       "1      USGS  7022000  1933-04-02      224000               A        NaN   \n",
       "2      USGS  7022000  1933-04-03      257000               A        NaN   \n",
       "3      USGS  7022000  1933-04-04      279000               A        NaN   \n",
       "4      USGS  7022000  1933-04-05      298000               A        NaN   \n",
       "\n",
       "  DAILY_SSC_QUAL  DAILY_SSL DAILY_SSL_QUAL  \n",
       "0            NaN        NaN            NaN  \n",
       "1            NaN        NaN            NaN  \n",
       "2            NaN        NaN            NaN  \n",
       "3            NaN        NaN            NaN  \n",
       "4            NaN        NaN            NaN  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(ncomments, comments) = get_comments('daily_data.tsv')\n",
    "daily_data = pandas.read_table(zipfile.open('daily_data.tsv'),header=ncomments).dropna(axis=1,how='all')\n",
    "daily_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d159f2d86fa8469588390124edf4eedb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "sc = plt.scatter(daily_data.DAILY_FLOW,daily_data.DAILY_SSC)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": false,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": false,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}