Merge pull request #50 from NCAR/develop

Update master for 0.4 tag

.gitignore

@@ -35,4 +35,5 @@ test_regression
 test_calendar
 test_qm
 test_config
-downscale
+test_random
+gard

.travis.yml

@@ -1,19 +1,27 @@
-sudo: false
+sudo: false # use container-based build
 language: fortran
-compiler:
- - gfortran
-os:
- - linux
+notifications:
+ email: false
+
+compiler: gfortran-6
+os: linux
+env: TESTID='gard_linux'
 addons:
  apt:
- source:
+ sources:
  - ubuntu-toolchain-r-test
  packages:
- - gfortran
- - libnetcdf-dev
  - liblapack-dev
+ - libnetcdf-dev
+ - gfortran-6
+before_install:
+ - source ci/gard_install_utils
+ - gard_before_install
+install:
+ - gard_install
 script:
- - sed -i "s|NCDF_PATH = /usr/local|NCDF_PATH = /usr|" src/makefile
- - sed -i "s|LAPACK_PATH = /usr/local|LAPACK_PATH = /usr|" src/makefile
- - make -C src -j4 test
- - make -C src -j4
+ - gard_script
+after_success:
+ - gard_after_success
+after_failure:
+ - gard_after_failure

ci/gard_install_utils

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+export CC=/usr/bin/gcc-6
+export FC=/usr/bin/gfortran-6
+
+if [ -z "$WORKDIR" ]; then
+ export WORKDIR=$HOME/workdir
+ mkdir -p $WORKDIR
+fi
+
+if [ -z "$INSTALLDIR" ]; then
+ export INSTALLDIR=$HOME/installdir
+ mkdir -p $INSTALLDIR
+fi
+
+function install_szip {
+ echo install_szip
+ cd $WORKDIR
+ wget --no-check-certificate -q http://www.hdfgroup.org/ftp/lib-external/szip/2.1/src/szip-2.1.tar.gz
+ tar -xzf szip-2.1.tar.gz
+ cd szip-2.1
+ ./configure --prefix=$INSTALLDIR &> config.log
+ make &> make.log
+ make install
+ export CPPFLAGS="$CPPFLAGS -I${INSTALLDIR}/include"
+ export LDFLAGS="$LDFLAGS -L${INSTALLDIR}/lib"
+}
+
+function install_hdf5 {
+ echo install_hdf5
+ cd $WORKDIR
+ wget --no-check-certificate -q http://www.hdfgroup.org/ftp/HDF5/current/src/hdf5-1.10.0-patch1.tar.gz
+ tar -xzf hdf5-1.10.0-patch1.tar.gz
+ cd hdf5-1.10.0-patch1
+ ./configure --prefix=$INSTALLDIR &> config.log
+ make &> make.log
+ make install
+ export LIBDIR=${INSTALLDIR}/lib
+}
+
+function install_netcdf_c {
+ echo install_netcdf_c
+ cd $WORKDIR
+ wget --no-check-certificate -q ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.4.1.tar.gz
+ tar -xzf netcdf-4.4.1.tar.gz
+ cd netcdf-4.4.1
+ ./configure --prefix=$INSTALLDIR &> config.log
+ make &> make.log
+ make install
+ export LD_LIBRARY_PATH=${INSTALLDIR}/lib
+}
+
+function install_netcdf_fortran {
+ echo install_netcdf_fortran
+ cd $WORKDIR
+ wget --no-check-certificate -q ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-fortran-4.4.4.tar.gz
+ tar -xzf netcdf-fortran-4.4.4.tar.gz
+ cd netcdf-fortran-4.4.4
+ ./configure --prefix=$INSTALLDIR &> config.log
+ make &> make.log
+ make install
+}
+
+function gard_before_install {
+ echo gard_before_install
+ # Install szip (used by hdf5)
+ install_szip
+ # Install HDF5
+ install_hdf5
+ # Install NetCDF-C
+ install_netcdf_c
+ # Install NetCDF fortran
+ install_netcdf_fortran
+}
+
+function gard_install {
+ echo gard_install
+ cd ${TRAVIS_BUILD_DIR}
+ sed -i "s|NCDF_PATH = /usr/local|NCDF_PATH = ${INSTALLDIR}|" src/makefile
+ sed -i "s|LAPACK_PATH = /usr/local|LAPACK_PATH = /usr|" src/makefile
+ make -C src clean; make -C src -j4 test
+ make -C src clean; make -C src -j4 MODE=debugslow
+ make -C src clean; make -C src -j4 MODE=debug
+ make -C src clean; make -C src -j4 MODE=debugompslow
+ make -C src clean; make -C src -j4 MODE=debugomp
+ make -C src clean; make -C src -j4 MODE=profile
+ make -C src clean; make -C src -j4 MODE=fast
+ make -C src clean; make -C src -j4
+ echo "GARD install succeeded"
+}
+
+function gard_script {
+ cd ./src
+ ./gard --version
+ ./gard -h
+ ./test_calendar
+ # ./test_random
+ ./test_regression
+ # ./test_config ../run/downscale_options.txt
+ cd ../
+ echo "GARD script succeeded"
+}
+
+function gard_after_success {
+ echo gard_after_success
+ echo "GARD build succeeded"
+}
+
+function gard_after_failure {
+ echo gard_after_failure
+ echo "GARD build failed"
+}

docs/running.md

@@ -13,6 +13,8 @@ GARD has the following dependencies:
 1. LAPACK — Linear Algebra PACKage.
 1. netCDF4 - Network Common Data Form.
 
+*Note: GARD allocates memory to the stack. Users should set the "The maximum stack size." to "unlimited" prior to building/running GARD. `ulimit -s unlimited`*
+
 ## Building GARD
 
 GARD is built using a standard `makefile`. From the command line, simply run the following command:
@@ -43,3 +45,10 @@ After building GARD, it is run on the command line following this syntax:
 Use the following to generate a list of e.g. GEFS precipitation files for input.
 
  ls -1 gefs/2010/*/apcp_sfc_*_mean.nc | sed 's/*//g;s/$/"/g;s/^/"/g'>gefs_pr_file.txt
+
+## Common Errors
+
+1. Segmentation Fault
+ - GARD allocates memory to the stack. Users should set the "The maximum stack size" to "unlimited" prior to building/running GARD. `ulimit -s unlimited`
+2. Random errors (e.g. debug not staying set at False)
+ - Make sure all filenames in the namelist are in quotations. 

run/downscale_options.txt

@@ -4,7 +4,7 @@
  ! observation_file = "downscale_options.nml"
  ! prediction_file = "downscale_options.nml"
 
- output_file = "log_reg_output/"
+ output_file = "output/gard_out_" ! prefix for output files
 
  start_date = "2000-01-01 00:00:00" ! start time for the output data (only needs to exist in the prediction dataset)
  end_date = "2000-01-31 23:00:00" ! end time for the output data
@@ -15,23 +15,35 @@
  start_transform = "1990-01-01 00:00:00" ! start time for any transformations (e.g. quantile mapping) (must exist in both the training and the prediction datasets)
  end_transform = "1999-01-01 23:00:00" ! end time for the transformation period
 
+ start_post = "1990-01-01 00:00:00" ! start time for any post processing transformations (e.g. quantile mapping) (must exist in the prediction dataset)
+ end_post = "1999-01-01 23:00:00" ! end time for the post-proc transformation period
+
+
  ! model types
- pure_regression = False ! set to true (and others to false) to compute a single regression (no analogs) between training variables and observations to be applied to all predictor data
- pure_analog = False ! set to true (and others to false) to use the selected analogs (no regression) to predict the output
- analog_regression = True ! set to true (and others to false) to perform a regression only on the selected analog days when predicting the output
+ pure_regression = False ! set to true (and others to false) to compute a single regression (no analogs) between training variables and observations to be applied to all predictor data
+ pure_analog = False ! set to true (and others to false) to use the selected analogs (no regression) to predict the output
+ analog_regression = True ! set to true (and others to false) to perform a regression only on the selected analog days when predicting the output
+
+ ! Non-model type
+ pass_through = False ! set to true to simply pass a predictor variable through instead of computing regressions or analogs
+ pass_through_var = 1 ! Option to select which predictor variable is passed through
 
  ! analog selection parameters
- n_analogs = 200 ! set the number of analogs to find for each time step
- ! n_log_analogs = 20 ! set to the number of analogs to use for the logistic_from_analog_exceedance calculation if using something other than n_analogs
- ! analog_threshold = 0.25 ! set to use a threshold distance when selecting analogs instead of a fixed number (in units of standard deviations averaged across input variables)
+ n_analogs  = 200  ! set the number of analogs to find for each time step
+ ! n_log_analogs  = 20  ! set to the number of analogs to use for the logistic_from_analog_exceedance calculation if using something other than n_analogs
+ ! analog_threshold = 0.25  ! set to use a threshold distance when selecting analogs instead of a fixed number (in units of standard deviations averaged across input variables)
 
  ! model options
  sample_analog = False ! when using pure_analog this will sample randomly from the selection of analogs instead of using the mean
  logistic_from_analog_exceedance = False ! when true the logistic probability is calculated from the number of analogs that exceed the threshold instead of from a logistic regression
  logistic_threshold = 0 ! when set, the probability of exceeding this value will be calculated and the standard output will be trained only for the data that exceed this value
  weight_analogs = True ! when calculating analog_mean, analog_error, or logistic_from_analog, weight each analog by it's inverse square distance from the current input
 
+ ! output post processing transformation
+ post_correction_transform = 0 ! post processing to apply (e.g. quantile mapping to entire input obs period) (one entry for every output variable)
+
  debug = False ! prints more output at runtime and outputs files including the coefficients used in each analog regression (or analog values) as well as the predictor data
+ interactive = False ! Print downscaling status as a percentage on the command line
 /
 
 ! define the input atmospheric model data to be used when training the model
@@ -45,6 +57,13 @@
  lon_name = "XLONG" ! name of a variable in the input data that contains the longitude of each grid cell (can be a 1D or 2D variable)
  time_name = "XTIME" ! name of a variable in the input data that contains the time data (must be a number ideally with a units attribute such as "days since YYYY-MM-DD hh:mm:ss")
  nfiles = 21 ! the number of files to be read
+ selected_level = -1 ! If the input data have 4 dimensions (e.g. one dimension is elevation) this is the index that will be used
+ ! this assumes that the z dimension is the 2nd (time, z, y, x) in the NetCDF file
+ ! if -1, the first level will be used.
+
+ ! Controls normalization applied to training data.
+ ! 0 = don't normalize, 1 = normalize based on the training data mean and standard deviation
+ normalization_method = 1
 
  ! the following are arrays with one element for each input variable
  input_transformations = 0, 0, 3, 3 ! transformations to apply to each input variable (0=none, 1=qm?, 2=log, 3=cuberoot, 4=fifthroot)
@@ -55,6 +74,13 @@
 
  ! calendar to be used when interpreting the time variable, "gregorian", "standard", "noleap", "365-day","360-day" all work
  calendar = "gregorian"
+ calendar_start_year = 1900 ! set this to the year the time data use as time 0, will be read from the NetCDF units attribute if possible
+ timezone_offset = 0 ! offset (in hours) to add to the time data to permit a better match with the local obs. data
+
+ ! the following options are primarily for forecasting
+ ! GEFS data have n time-steps per file, corresponding to the n-forecast lead times
+ selected_time = -1 ! if set, only this time step will be read from each input file
+ time_indices = -1, -1, -1 ! this can be a list of time steps to read and average over instead
 /
 
 ! Define the input atmospheric model data to be used when applying the model for future predictions
@@ -72,13 +98,23 @@
 
  input_transformations = 0, 0, 3, 3
 
- ! this is the main difference between training and prediction data, permits an additional transformation to e.g. quantile map (transformation=1) each variable to match the training data
+ ! Here is the main difference between training and prediction data
+ ! This permits an additional transformation to e.g. quantile map (transformation=1) each variable to match the training data
+ ! this is primarily for climate simulations
  transformations = 0, 0, 0, 0
+ ! Also, if normalization_method = 2, the means and standard deviations from the training data will be used to normalize the predictors
+ ! this is primarily for forecasting applications. 1=Normalize based on prediction data, 0=no normalization applied
+ normalization_method = 1
 
  var_names = "T2", "PSFC", "PREC_ACC_NC", "PREC_ACC_C"
  file_list = "filelists/erai_files_200x.txt", "filelists/erai_files_200x.txt", "filelists/erai_files_200x.txt", "filelists/erai_files_200x.txt"
 
  calendar = "gregorian"
+ calendar_start_year = 1900
+ timezone_offset = 0
+
+ selected_time = -1
+ time_indices = -1, -1, -1
 /
 
 ! Define the input observation data to be used when training the model
@@ -99,4 +135,10 @@
  file_list = "filelists/obs_files_complete.txt"
 
  calendar = "gregorian"
+ calendar_start_year = 1900
+
+ ! specify a variable to use to find which grid cells should be masked
+ mask_variable = 1
+ ! specify a value to use to in that variable to define masked gridcells
+ mask_value = 1e20
 /