Skip to content

Commit

Permalink
0.4.1
Browse files Browse the repository at this point in the history 
- Fixed errors in count_time(), adjust_temporal_counts() and sensing_depth()
- Improved documentation
  • Loading branch information
@joaquinperaza
joaquinperaza committed Jul 3, 2023
1 parent 1a17f36 commit 387a201
Show file tree
Hide file tree
Showing 15 changed files with 294 additions and 341 deletions.
23 changes: 10 additions & 13 deletions build/lib/crnpy/crnpy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,8 @@ def fill_missing_timestamps(df, col='timestamp', format='%Y-%m-%d %H:%M:%S', fre

def count_time(counts=None, timestamp_col=None):
"""Approximate counting time.
The function will calculate the approximate counting time for each observation by taking the difference between
consecutive timestamps. If counts has a DateTimeIndex, timestamp_col is not needed.
Args:
counts (pandas.DataFrame): DataFrame with neutron counts, might have DateTimeIndex.
Expand Down Expand Up @@ -119,7 +121,7 @@ def fill_counts(counts, count_times=None, timestamp_col=None, expected_time=Fals
Args:
counts (pandas.DataFrame): DataFrame with neutron counts, might have DateTimeIndex.
count_time (pandas.Series or pandas.DataFrame): Counting time in seconds. If a DataFrame is provided, it must have the same number of columns as `counts`.
count_times (pandas.Series or pandas.DataFrame): Counting time in seconds. If a DataFrame is provided, it must have the same number of columns as `counts`.
timestamp_col (pandas.Series): Series with timestamps. If counts has a DateTimeIndex, timestamp_col is not needed.
expected_time (int): Expected counting time in seconds. If not provided, it is calculated as the median of the counting times.
threshold (float): Minimum fraction of the neutron integration time. Default is 0.25.
Expand Down Expand Up @@ -185,7 +187,7 @@ def fill_counts(counts, count_times=None, timestamp_col=None, expected_time=Fals
counts = counts.interpolate(method='linear', limit=limit, limit_direction='both').round()
return counts

def adjust_temporal_counts(counts, count_time=3600, count_times=None, timestamp_col=None):
def adjust_temporal_counts(counts, count_time=None, count_times=None, timestamp_col=None):
"""Normalize neutron counts to the desired counting time.
Args:
Expand Down Expand Up @@ -734,7 +736,7 @@ def counts_to_vwc(counts, N0, Wlat, Wsoc ,bulk_density, a0=0.0808,a1=0.372,a2=0.



def sensing_depth(vwc, pressure, p_ref, bulk_density, Wlat, dist, method='Schron_2017'):
def sensing_depth(vwc, pressure, p_ref, bulk_density, Wlat, dist=None, method='Schron_2017'):
# Convert docstring to google format
"""Function that computes the estimated sensing depth of the cosmic-ray neutron probe.
The function offers several methods to compute the depth at which 86 % of the neutrons
Expand Down Expand Up @@ -764,7 +766,6 @@ def sensing_depth(vwc, pressure, p_ref, bulk_density, Wlat, dist, method='Schron

# Determine sensing depth (D86)
if method == 'Schron_2017':

# See Appendix A of Schrön et al. (2017)
Fp = 0.4922 / (0.86 - np.exp(-1 * pressure / p_ref));
Fveg = 0
Expand All @@ -779,6 +780,8 @@ def sensing_depth(vwc, pressure, p_ref, bulk_density, Wlat, dist, method='Schron

elif method == 'Franz_2012':
results = 5.8/(bulk_density*Wlat+vwc+0.0829)
else:
raise ValueError('Method not recognized. Please select either "Schron_2017" or "Franz_2012".')

return results

Expand Down Expand Up @@ -943,8 +946,8 @@ def storage(sm,T=1,Z_surface=150,Z_subsurface=1000):

def cutoff_rigidity(lat,lon):
"""Function to estimate the approximate cutoff rigidity for any point on Earth according to the
tabulated data of Smart and Shea, 2019. Values are approximations so that users have an idea of
what neutron detectors from the Neutron Monitor Database (NMD).
tabulated data of Smart and Shea, 2019. The returned value can be used to select the appropriate
neutron monitor station to estimate the cosmic-ray neutron intensity at the location of interest.
Args:
lat (float): Geographic latitude in decimal degrees. Value in range -90 to 90
Expand Down Expand Up @@ -1058,7 +1061,7 @@ def estimate_lattice_water(clay_content, total_carbon=None):
![img1](img/lattice_water_simple.png) | ![img2](img/lattice_water_multiple.png)
:-------------------------:|:-------------------------:
$\omega_{lat} = 0.097 * clay(\%)$ | $\omega_{lat} = -0.028 + 0.077 * clay(\%) + 0.459 * carbon(\%)$
Linear regression [lattice water (%) as a function of clay (%)] done with data from Soil Water Processes Lab and Dong and Ochsner (2018) | Multiple linear regression [lattice water (%) as a function of clay (%) and soil carbon (%)] done with data from Soil Water Processes Lab.
Linear regression [lattice water (%) as a function of clay (%)] done with data from Kansas Sate University - Soil Water Processes Lab. | Multiple linear regression [lattice water (%) as a function of clay (%) and soil carbon (%)] done with data from Soil Water Processes Lab.
Args:
clay_content (float): Clay content in the soil in percent.
Expand All @@ -1067,12 +1070,6 @@ def estimate_lattice_water(clay_content, total_carbon=None):
Returns:
(float): Amount of water in the lattice of clay minerals in percent
References:
Dong, J., & Ochsner, T. E. (2018). Soil texture often exerts a stronger influence than precipitation
on mesoscale soil moisture patterns. Water Resources Research, 54, 2199– 2211.
https://doi.org/10.1002/2017WR021692
"""
if total_carbon is None:
lattice_water = 0.097 * clay_content
Expand Down
12 changes: 6 additions & 6 deletions docs/correction_routines.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ Suggested steps for processing raw neutron counts from a roving CRNP. Dashed lin
Suggested steps for processing raw neutron counts from a stationary CRNP. Dashed lines indicate optional steps. See the complete [example notebook](../examples/rover/Hydroinnova_rover_example/).

### Incoming neutron flux
The CRNPy package includes a complete set of methods for correcting the raw observed neutron counts for natural variation in the incoming neutron flux, including a set of tools for searching and downloading data from a reference neutron monitoring station from the NMDB database (www.nmdb.eu) by proposing the most similar stations after analyzing the cut-off rigidity of the reference station and the estimated cut-off rigidity value for the studied location as a form of finding stations under a similar earth electromagnetic field (Klein et al., 2009; Shea & Smart, 2019; Smart & Shea, 2001).
The CRNPy package includes a complete set of methods for correcting the raw observed neutron counts for natural variation in the incoming neutron flux, including a set of tools for searching and downloading data from a reference neutron monitoring station from the NMDB database (www.nmdb.eu) by proposing the most similar stations after analyzing the cut-off rigidity of the reference station and the estimated cut-off rigidity value for the studied location as a form of finding stations under a similar earth electromagnetic field (Klein et al., 2009; Shea & Smart., 2019).

| Incoming neutron flux correction factor|
|---------------------------------|
Expand All @@ -25,10 +25,10 @@ The CRNPy package includes a complete set of methods for correcting the raw obse

!!! info "Implementation"

See [crnpy.crnpy.cutoff_rigidity][], [crnpy.crnpy.find_neutron_detectors][], [crnpy.crnpy.get_incoming_neutron_flux][], [crnpy.crnpy.interpolate_incoming_flux][] and [crnpy.crnpy.atm_correction][] documentation for the implementation details.
See [crnpy.crnpy.cutoff_rigidity][], [crnpy.crnpy.find_neutron_monitor][], [crnpy.crnpy.get_incoming_neutron_flux][], [crnpy.crnpy.interpolate_incoming_flux][] and [crnpy.crnpy.incoming_flux_correction][] documentation for the implementation details.

### Atmospheric corrections
The package also provides functions for correcting raw neutron counts for atmospheric pressure, humidity, and temperature variations, (Andreasen et al., 2017; Rosolem et al., 2013; Zreda et al., 2012).
The package also provides functions for correcting raw neutron counts for atmospheric pressure, humidity, and temperature variations, (Andreasen et al., 2017; Rosolem et al., 2013).

| Pressure correction | Atmospheric water correction |
|---------------------|------------------------------|
Expand All @@ -40,10 +40,10 @@ The package also provides functions for correcting raw neutron counts for atmosp

!!! info "Implementation"

See [crnpy.crnpy.atm_correction][] documentation for the implementation details.
See [crnpy.crnpy.humidity_correction][] and [crnpy.crnpy.pressure_correction][] documentation for the implementation details.

### Biomass correction
The library provides a function for correcting neutron counts for the effects of above-ground biomass by combining an approach for estimating biomass water equivalent (BWE) from in-situ biomass samples (Wahbi et al., 2018) and the BWE correction factor (Baatz et al., 2015).
The library provides a function for correcting neutron counts for the effects of above-ground biomass by combining an approach for estimating biomass water equivalent (BWE) from in-situ biomass samples and the BWE correction factor (Baatz et al., 2015).

| Biomass correction |
|--------------------|
Expand All @@ -57,7 +57,7 @@ The library provides a function for correcting neutron counts for the effects of
See [crnpy.crnpy.bwe_correction][] and [crnpy.crnpy.biomass_to_bwe][] documentation for the implementation details.

### Road correction
Due to the known high sensitivity closer to the detector, use cases like rover surveys could require the use of a correction factor accounting for the differences between the field soil water content and the road water content, CRNPy implements the methodology proposed by Schrön et al. (2018).
Due to the known high sensitivity closer to the detector, use cases like rover surveys could require the use of a correction factor accounting for the differences between the field soil water content and the road water content (Schrön et al., (2018)).

| Road correction |
|-----------------|
Expand Down
55 changes: 21 additions & 34 deletions docs/examples/calibration/calibration.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,9 @@
"\n",
"# Load the soil samples data and the CRNP dataset using pandas\n",
"df_soil = pd.read_csv(\"https://raw.githubusercontent.com/soilwater/crnpy/main/docs/examples/calibration/soil_data.csv\")\n",
"df_station = pd.read_csv(\"https://raw.githubusercontent.com/soilwater/crnpy/main/docs/examples/calibration/station_data.csv\", skiprows=[0,2,3])\n",
"\n",
"# Load the station data"
"# Load the station data\n",
"df_station = pd.read_csv(\"https://raw.githubusercontent.com/soilwater/crnpy/main/docs/examples/calibration/station_data.csv\", skiprows=[0,2,3])\n"
]
},
{
Expand Down Expand Up @@ -125,9 +125,7 @@
"source": [
"## Neutron count processing\n",
"\n",
"Following the example for processing CRNP data from [RDT detectors](../../stationary/example_RDT_station/) neutron counts recorded while the field sampling was done need to be corrected.\n",
"\n",
"[This dataset](https://github.com/soilwater/crnpy/blob/main/docs/examples/calibration/soil_data.xlsx) contains the raw counts and atmospheric data recorded by the CRNP while the field sampling was done.\n"
"Following a similar approach as the example for [Stationary CRNP](../../stationary/example_RDT_station/) neutron counts recorded while the field sampling was done will be corrected."
]
},
{
Expand Down Expand Up @@ -451,17 +449,26 @@
"40 NEWK Newark 2.40 50 True\n",
"28 KIEL2 KielRT 2.36 54 True\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\jperaza\\AppData\\Local\\anaconda3\\envs\\crnpy\\lib\\site-packages\\crnpy\\crnpy.py:473: UserWarning: Reference incoming neutron flux not provided. Using first value of incoming neutron flux.\n",
" warnings.warn('Reference incoming neutron flux not provided. Using first value of incoming neutron flux.')\n"
]
}
],
"source": [
"# Find stations with cutoff rigidity similar to estimated by lat,lon, \n",
"# filtering the time window from experiment setup to the end of the calibration\n",
"# Find stations with cutoff rigidity similar to the one estimated by lat,lon, \n",
"# Filtering the time window from experiment setup to the end of the calibration\n",
"crnpy.find_neutron_monitor(crnpy.cutoff_rigidity(39.1, -96.6), start_date = deployment_date, end_date = calibration_end)\n",
"\n",
"#Download data for one of the similar stations and add to df\n",
"incoming_neutrons = crnpy.get_incoming_neutron_flux(deployment_date, calibration_end, station=\"DRBS\", utc_offset=-5)\n",
"df['incoming_flux']=crnpy.interpolate_incoming_flux(incoming_neutrons, timestamps=df['TIMESTAMP'])\n",
"ref_incoming_flux = incoming_neutrons.iloc[0]\n"
"ref_incoming_flux = incoming_neutrons.iloc[0]\n",
"df['corrected'] = crnpy.incoming_flux_correction(df['total_counts'], incoming_neutrons=df['incoming_flux'])\n"
]
},
{
Expand All @@ -479,34 +486,14 @@
"execution_count": 6,
"id": "6a146046-a47d-4018-91e9-213075a70da9",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"No count time columns provided. Using timestamp index to compute count time.\n",
"Using median count time as expected count time: 3600.0\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\jperaza\\AppData\\Local\\anaconda3\\envs\\crnpy\\lib\\site-packages\\crnpy\\crnpy.py:471: UserWarning: Reference incoming neutron flux not provided. Using first value of incoming neutron flux.\n",
" warnings.warn('Reference incoming neutron flux not provided. Using first value of incoming neutron flux.')\n"
]
}
],
"outputs": [],
"source": [
"# Fill NaN values in atmospheric data\n",
"df[['pressure', 'RH', 'T']] = crnpy.fill_missing_atm(df[['barometric_pressure_Avg', 'relative_humidity_Avg', 'air_temperature_Avg']])\n",
"# Correct count by atmospheric variables and incoming flux\n",
"df['total_counts'] = crnpy.fill_counts(df['total_counts'])\n",
"\n",
"# Correct count by atmospheric variables and incoming flux\n",
"df['corrected'] = crnpy.humidity_correction(df['total_counts'],humidity=df['RH'], temp=df['T'], Aref=0)\n",
"df['corrected'] = crnpy.pressure_correction(df['corrected'], pressure=df['pressure'], Pref=976, L=130)\n",
"df['corrected'] = crnpy.incoming_flux_correction(df['corrected'], incoming_neutrons=df['incoming_flux'])\n"
"df['corrected'] = crnpy.humidity_correction(df['corrected'],humidity=df['RH'], temp=df['T'], Aref=0)\n",
"df['corrected'] = crnpy.pressure_correction(df['corrected'], pressure=df['pressure'], Pref=976, L=130)\n"
]
},
{
Expand All @@ -519,12 +506,12 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Mean corrected neutron count during sampling: 1538.0\n"
"Mean corrected neutron count during sampling: 1537\n"
]
}
],
"source": [
"print(f\"Mean corrected neutron count during sampling: {df['corrected'].mean().round()}\")\n"
"print(f\"Mean corrected neutron count during sampling: {int(df['corrected'].mean())}\")\n"
]
},
{
Expand All @@ -534,7 +521,7 @@
"source": [
"## Solving the equation for $N_0$\n",
"\n",
"Previous steps estimated the a field volumetric water content of `0.256` and an average neutron count of `1538`. Using [scipy.optimize.root()](https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.root.html) $N_0$ is estimated given the observed value of $\\theta_v$ and neutron counts."
"Previous steps estimated the a field volumetric water content of `0.263` and an average neutron count of `1537`. Using [`scipy.optimize.root()`](https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.root.html) $N_0$ is estimated given the observed value of $\\theta_v$ and neutron counts."
]
},
{
Expand Down
4 changes: 2 additions & 2 deletions docs/examples/rover/Hydroinnova_rover_example.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -317,7 +317,7 @@
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\jperaza\\AppData\\Local\\anaconda3\\envs\\crnpy\\lib\\site-packages\\crnpy\\crnpy.py:471: UserWarning: Reference incoming neutron flux not provided. Using first value of incoming neutron flux.\n",
"C:\\Users\\jperaza\\AppData\\Local\\anaconda3\\envs\\crnpy\\lib\\site-packages\\crnpy\\crnpy.py:473: UserWarning: Reference incoming neutron flux not provided. Using first value of incoming neutron flux.\n",
" warnings.warn('Reference incoming neutron flux not provided. Using first value of incoming neutron flux.')\n"
]
}
Expand Down Expand Up @@ -347,7 +347,7 @@
"source": [
"#### Atmospheric correction\n",
"\n",
"NaN values in the atmospheric data are filled. The neutron counts are then corrected for atmospheric variables and the incoming neutron flux. See [`humidity_correction()`](../../../reference/#crnpy.crnpy.humidity_correction) and [`pressure_correction()`](../../../reference/#crnpy.crnpy.pressure_correction)"
"NaN values in the atmospheric data are filled. The neutron counts are then corrected for atmospheric variables. See [`humidity_correction()`](../../../reference/#crnpy.crnpy.humidity_correction) and [`pressure_correction()`](../../../reference/#crnpy.crnpy.pressure_correction)"
]
},
{
Expand Down
41 changes: 22 additions & 19 deletions docs/examples/stationary/example_RDT_station.ipynb
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

setuptools.setup(
name="crnpy",
version="0.4.0",
version="0.4.1",
packages=['crnpy'],
package_dir = {"": "src"},
description="A Python package for the estimation and processing of soil moisture data from cosmic-ray neutron counts.",
Expand Down
Loading

0 comments on commit 387a201

Please sign in to comment.