water_level_pipeline.md

Water Level Pipeline

• A series of functions to be added to the filter-stations module in pypi to evalute which TAHMO stations to use that corroborates with the water level
• All begins with the coordinates of the gauging station(location of the monitoring sensor)

import os
from pathlib import Path
import haversine as hs
import pandas as pd
import numpy as np
import datetime
import statsmodels.api as sm
from matplotlib.dates import DateFormatter
import matplotlib.pyplot as plt
import warnings
import dateutil.parser
warnings.filterwarnings('ignore')

# config_path
config_path = os.path.join(Path(os.getcwd()).parent.parent.absolute(), 'config.json')

from test import retreive_data, Interactive_maps, Filter, pipeline, Water_level
import json
# Authentication
with open(config_path) as f:
    conf = json.load(f)

apiKey = conf['apiKey']
apiSecret = conf['apiSecret']
map_api_key = conf['map_api_key']
ret = retreive_data(apiKey, apiSecret, map_api_key)
pipe = pipeline(apiKey, apiSecret, map_api_key)
maps = Interactive_maps(apiKey, apiSecret, map_api_key)
filt = Filter(apiKey, apiSecret, map_api_key)
wl = Water_level()

Loading data

# muringato lat,lon
muringato_loc = wl.coordinates('muringato')
# ewaso lat,lon
ewaso_loc = wl.coordinates('ewaso')

# weather data from 2020 to 2022
weather_data = filt.filter_pr('2020-01-01', '2022-12-31', country='Kenya')

# water level data from muringato and ewaso
muringato_data = wl.water_level_data('muringato')
ewaso_data = wl.water_level_data('ewaso')

Retrieving precipitation data from BigQuery...

start_date = '2021-02-11'
end_date = '2021-12-01'
muringato_data_s6_2021 = muringato_data[['muringato_sensor6']].loc[start_date:end_date]
muringato_data_s6_2021.columns = ['water_level']

list(pd.date_range(start=start_date, end=end_date, freq='D').difference(muringato_data[['muringato_sensor6']].index))

[]

# drop weather stations with missing data
weather_stations_data = weather_data.dropna(axis=1)
weather_stations_data.isna().sum().sum()
weather_stations_data = weather_stations_data.loc[start_date:end_date]

above, below = pipe.shed_stations(weather_stations_data,
                   muringato_data_s6_2021,
                   muringato_loc,
                   100,
                   lag=50
                   )

below_stations = [i.split('_')[0] for i in below.keys()]
print(below_stations)
below_stations_metadata = ret.get_stations_info(multipleStations=below_stations)[['code', 'location.latitude', 'location.longitude']]

['TA00028', 'TA00108', 'TA00062', 'TA00074', 'TA00184', 'TA00288', 'TA00441']

below_stations_metadata['distance']= below_stations_metadata.apply(lambda row: hs.haversine((muringato_loc[0], 
                                                                                             muringato_loc[1]), (row['location.latitude'], 
                                                                                                             row['location.longitude'])), axis=1)
below_stations_metadata.sort_values(by='distance')

<style scoped> .dataframe tbody tr th:only-of-type { vertical-align: middle; }

.dataframe tbody tr th {
    vertical-align: top;
}

.dataframe thead th {
    text-align: right;
}

</style>

	code	location.latitude	location.longitude	distance
68	TA00074	-0.566080	37.074412	21.623104
26	TA00028	0.055219	37.136747	54.874887
96	TA00108	-0.991916	37.507288	88.864489
167	TA00184	-1.464180	35.287930	220.249152
398	TA00441	-0.599323	34.744223	247.636699
266	TA00288	0.697736	34.865137	263.620540
56	TA00062	1.273419	35.085363	280.155770

# Interactive visuals
import plotly.express as px
import plotly.graph_objects as go

fig = px.scatter_mapbox(below_stations_metadata, 
                        lat="location.latitude", 
                        lon="location.longitude", 
                        hover_name="code", 
                        hover_data=["distance"],
                        color_discrete_sequence=["fuchsia"],
                        zoom=8,
                        height=800,
                        )
# update marker size
fig.update_traces(marker=dict(size=10))
# add a point for the central station
fig.add_trace(go.Scattermapbox(
        lat=[muringato_loc[0]],
        lon=[muringato_loc[1]],
        mode='markers',
        marker=go.scattermapbox.Marker(
            size=14
        ),
        text=['Muringato gauging station'],
    ))

fig.update_layout(
    mapbox_style="carto-positron",
    margin={"r":0,"t":0,"l":0,"b":0},
    showlegend=False
)
fig.show()

pipe.plot_figs(
    weather_stations_data,
    list(muringato_data_s6_2021['water_level']),
    list(below.keys()),
    date=dateutil.parser.parse(str(muringato_data_s6_2021.index[0])).strftime('%d-%m-%Y'), 
    save=False   
)

Begin plotting!