From acb7f5ffc9d207e4dcab5081cffb923f918f16fe Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Fri, 10 May 2019 09:43:48 -0600 Subject: [PATCH 01/10] And initial travisCI testing For now, the continuous integration just tests the build with python 2.7 and python 3.7 -- expect this first commit to pass the 2.7 testing and fail the 3.7 testing. Once the build works with 3.7, I'll add a script to run the regression tests as well. --- .travis.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..a991fe141 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,12 @@ +language: python +sudo: required +before_install: + - sudo apt-get install gfortran mpich libnetcdf-dev netcdf-bin +python: + - '2.7' + - '3.7' +script: + - cd src; make +branches: + only: + - master From 2aecdcd6d4b298fa7ba3e511b65b2bc791c7736b Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Fri, 10 May 2019 09:57:46 -0600 Subject: [PATCH 02/10] Need Makefile to pass arguments to cvmix_setup For testing, we need to specify the compiler (gfortran) and the netcdf root when we call make otherwise cvmix_setup will hang while waiting for keyboard input. --- .travis.yml | 2 +- src/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a991fe141..03599d73a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ python: - '2.7' - '3.7' script: - - cd src; make + - cd src; make SETUP_ARGS="gfortran $(dirname $(dirname $(which nc-config)))" branches: only: - master diff --git a/src/Makefile b/src/Makefile index f3f2c1479..2d00eb7ee 100644 --- a/src/Makefile +++ b/src/Makefile @@ -165,7 +165,7 @@ $(DEP_FILE): $(MAKE_DEP) $(SRC_DIR)/*.F90 $(SRC_DRIVE_DIR)/*.F90 # CVMix_env. If the file doesn't exist, then the python script "cvmix_setup" # is run to generate the file. $(ENV_FILE): - @(cd $(BLD_DIR); ./cvmix_setup) + @(cd $(BLD_DIR); ./cvmix_setup $(SETUP_ARGS)) $(MAKE) -f $(SRC_DIR)/Makefile .PHONY: exe netcdf depends check lib remove_exe clean libclean allclean \ From 4e38daa2c33e9cdda00f9d04a07504360bbefdc5 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Fri, 10 May 2019 10:30:55 -0600 Subject: [PATCH 03/10] cvmix_setup is now python3 compatible Switched from print statements to the logging package, and mapped my_input to raw_input() in py2 or input() in py3. --- bld/cvmix_setup | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/bld/cvmix_setup b/bld/cvmix_setup index 2dd95af9a..940905152 100755 --- a/bld/cvmix_setup +++ b/bld/cvmix_setup @@ -2,11 +2,24 @@ # Need to parse command line import sys + +# Use logger rather than print statements +import logging +logging.basicConfig(format='(cvmix_setup): %(message)s', level=logging.INFO) +logger = logging.getLogger(__name__) + +# Need input to work for python 2 and python 3 +# Achieve this my copying correct function to my_input +try: + my_input = raw_input +except NameError: + my_input = input + # Allow tab completion try: import readline,glob except ImportError: - print "Can not find readline or glob package, so tab-complete will not work" + logger.info("Can not find readline or glob package, so tab-complete will not work") else: def complete(text, state): return(glob.glob(text+'*')+[None])[state] @@ -34,26 +47,26 @@ filename = ".CVMix_env" # Print standard information about this tool if have_compiler + have_netcdf != 2: - print "This utility needs to be run before the included Makefile will" - print "successfully compile CVMix. It collects information about your compiler" - print "and associated libraries, and saves it in the .CVMix_env file. Note that" - print "it only needs to be run once, though you should run it again if anything" - print "changes.\n" + logger.info("This utility needs to be run before the included Makefile will") + logger.info("successfully compile CVMix. It collects information about your compiler") + logger.info("and associated libraries, and saves it in the .CVMix_env file. Note that") + logger.info("it only needs to be run once, though you should run it again if anything") + logger.info("changes.\n") # For now, just need fortran compiler and netCDF location / flags if have_compiler == 0: - compiler = raw_input('Fortran compiler (mpi not necessary): ') + compiler = my_input('Fortran compiler (mpi not necessary): ') netcdf_bin = "" netcdf_inc = "" netcdf_lib = "" if have_netcdf == 0: - netcdf_dir = raw_input('Directory containing netcdf configuration tool nc-config (or enter "no-nc-config" to enter location of netcdf include and netcdf lib directories): ') + netcdf_dir = my_input('Directory containing netcdf configuration tool nc-config (or enter "no-nc-config" to enter location of netcdf include and netcdf lib directories): ') if netcdf_dir == "no-nc-config": - netcdf_dir = raw_input('Directory containing netcdf (enter "need_both" to enter netcdf include directory and netcdf lib directory separately): ') + netcdf_dir = my_input('Directory containing netcdf (enter "need_both" to enter netcdf include directory and netcdf lib directory separately): ') if netcdf_dir == "need_both": - netcdf_inc = raw_input('netCDF/include location: ') - netcdf_lib = raw_input('netCDF/lib location: ') + netcdf_inc = my_input('netCDF/include location: ') + netcdf_lib = my_input('netCDF/lib location: ') else: netcdf_inc = netcdf_dir+"/include" netcdf_lib = netcdf_dir+"/lib" @@ -65,7 +78,7 @@ else: netcdf_bin = netcdf_dir+"/bin" if have_compiler + have_netcdf != 2: - print "Writing environment settings to " + filename + logger.info("Writing environment settings to %s", filename) fid = open(filename, 'w') fid.write("FC = " + compiler+"\n") From 970bb568588ce403a60268d906e896a935763f59 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Fri, 10 May 2019 10:36:58 -0600 Subject: [PATCH 04/10] Test with python 3.6, not 3.7 Was getting an error "3.7 is not installed" from TravisCI --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 03599d73a..ab58dd248 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ before_install: - sudo apt-get install gfortran mpich libnetcdf-dev netcdf-bin python: - '2.7' - - '3.7' + - '3.6' script: - cd src; make SETUP_ARGS="gfortran $(dirname $(dirname $(which nc-config)))" branches: From 4da6b241b7c5986e4328bbe5f949a247b36d9a19 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Fri, 10 May 2019 10:42:47 -0600 Subject: [PATCH 05/10] Add netCDF build to Travis tests --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ab58dd248..bd16d53fd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ python: - '3.6' script: - cd src; make SETUP_ARGS="gfortran $(dirname $(dirname $(which nc-config)))" + - make USE_NETCDF=TRUE branches: only: - master From 7fee4de954b8a4914cb0591cf468a0add06dbcda Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Fri, 10 May 2019 10:55:10 -0600 Subject: [PATCH 06/10] Use logging package for makedep.py as well Rather than use logging for cvmix_setup and __future__.print_function for makedep.py, I switched makedep.py to the logging framework as well. This drops the unicode_literals import as well, but that seems okay --- src/shared/makedep.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/shared/makedep.py b/src/shared/makedep.py index 0aeeef340..d277cabc4 100755 --- a/src/shared/makedep.py +++ b/src/shared/makedep.py @@ -6,11 +6,9 @@ # Read in every file in $SRC_DIR and $SRC_DIR2 (arguments 3 and 4) # Only depend on modules located in $SRC_DIR or $SRC_DIR2 -from __future__ import print_function, unicode_literals - -import os -import sys -import re +import os, sys, re, logging +logger = logging.getLogger(__name__) +logging.basicConfig(format='(makedep.py): %(message)s', level=logging.INFO) try: dep_file = sys.argv[1] @@ -57,17 +55,17 @@ # so we need to strip away the , to get the module name file_used = line_array[1].split(',')[0] if file_used+'.F90' in files_in_src_dir: - print(file_name+'.o depends on '+file_used+'.o') + logger.info('%s.o depends on %s.o', file_name, file_used) fout.write(obj_dir+'/'+file_name+'.o: '+obj_dir+'/'+file_used+'.o\n') else: if inc_dir != 'NONE': if file_used+'.mod' in files_in_inc_dir: - print(file_name+'.o depends on '+file_used+'.mod') + logger.info('%s.o depends on %s.mod', file_name, file_used) fout.write(obj_dir+'/'+file_name+'.o: '+inc_dir+'/'+file_used+'.mod\n') else: # Check for upper case file_used = file_used.upper() if file_used+'.mod' in files_in_inc_dir: - print(file_name+'.o depends on '+file_used+'.mod') + logger.info('%s.o depends on %s.mod', file_name, file_used) fout.write(obj_dir+'/'+file_name+'.o: '+inc_dir+'/'+file_used+'.mod\n') fin.close From 198b1f4212a99b61ca79980d2a91c4868b6a2959 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Fri, 10 May 2019 11:34:13 -0600 Subject: [PATCH 07/10] Copy MARBL's code_consistency check to CVMix This script will be run by TravisCI to ensure small coding mistakes like trailing whitespace and hard tabs don't creep into the code. The first time I ran it, it detected a few small issues to fix. --- CVMix_tools/code_consistency.py | 274 ++++++++++++++++++++++++++++ src/drivers/cvmix_bgrnd_BL.F90 | 2 +- src/drivers/cvmix_ddiff_drv.F90 | 2 +- src/drivers/cvmix_kpp_drv.F90 | 2 +- src/drivers/cvmix_shear_drv.F90 | 2 +- src/drivers/cvmix_tidal_Simmons.F90 | 2 +- templates/cvmix_MODULE.F90 | 10 +- 7 files changed, 284 insertions(+), 10 deletions(-) create mode 100755 CVMix_tools/code_consistency.py diff --git a/CVMix_tools/code_consistency.py b/CVMix_tools/code_consistency.py new file mode 100755 index 000000000..759ea85bd --- /dev/null +++ b/CVMix_tools/code_consistency.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python + +""" +This script flags lines that do not conform to CVMix's coding practices +""" + +import os +import sys +import logging +from collections import deque # Faster pop / append than standard lists +from collections import OrderedDict +UNIBLOCK = u"\u2588" + +############## + +class ConsistencyTestClass(object): + """ + This class contains all the Fortran consistency check tests. + The logs dictionary contains a deque object with a list of all source lines + that fail a particular test (given as the logs key) + """ + def __init__(self): + self.logs = OrderedDict() + + ############## + + def process(self): + """ + Check results from all tests that have been run: + 1. For each test: + i. log (as info) test description and number of lines of code that fail the test + ii. log (as info) all lines of code that do not conform to standards + 2. return total number of errors across all tests + """ + tot_err_cnt = 0 + while self.logs: + desc, log = self.logs.popitem(last=False) + err_cnt = len(log) + LOGGER.info("* %s: %d error(s) found", desc, err_cnt) + while log: + msg = log.popleft() + LOGGER.info(" %s", msg) + tot_err_cnt += err_cnt + return tot_err_cnt + ############## + + def init_log(self, description): + """ + If self.logs does not already have a key for description, set + self.logs[description] to an empty deque list. + """ + if description not in self.logs.keys(): + self.logs[description] = deque([]) + + ############## + + def check_for_hard_tabs(self, file_and_line, line): + """ + Log any lines containing hard tabs + """ + test_desc = 'Check for hard tabs' + self.init_log(test_desc) + if "\t" in line: + self.logs[test_desc].append("%s: %s" % (file_and_line, + line.replace("\t", 2*UNIBLOCK))) + + ############## + + def check_for_trailing_whitespace(self, file_and_line, line): + """ + Log any lines containing trailing whitespace + """ + test_desc = 'Check for trailing white space' + self.init_log(test_desc) + full_line_len = len(line) + no_trailing_space_len = len(line.rstrip(" ")) + if no_trailing_space_len < full_line_len: + self.logs[test_desc].append("%s: %s" % (file_and_line, + line.rstrip(" ") + + (full_line_len - no_trailing_space_len) * + UNIBLOCK)) + + ############## + + def check_line_length(self, file_and_line, line, comment_char="!", max_len=132): + """ + Log any lines exceeding max_len + Currently ignores comment characters / anything following + """ + test_desc = 'Check length of lines' + self.init_log(test_desc) + line_len = len(line.split(comment_char)[0].rstrip(" ")) + if line_len > max_len: + self.logs[test_desc].append("%s: %s" % (file_and_line, + line[:max_len] + + (line_len-max_len) * UNIBLOCK)) + + ############## + + def check_case_in_module_statements(self, file_and_line, line, comment_char="!"): + """ + The following module statements should be all lowercase: + * implicit none + * public + * private + * save + Note that at some point we may want to remove "save" altogether, + since it is implicit in a module + """ + test_desc = 'Check for case sensitive statements' + self.init_log(test_desc) + statements = ["implicit none", "public", "private", "save"] + # ignore comments, and strip all white space + line_without_comments = line.split(comment_char)[0].strip(" ") + if line_without_comments.lower() in statements: + if line_without_comments not in statements: + self.logs[test_desc].append("%s: %s" % (file_and_line, line)) + + ############## + + def check_for_spaces(self, file_and_line, line, comment_char="!"): + """ + The following statements should all include spaces: + * else if + * end if + * else where + * end where + * end do + * end module + """ + test_desc = 'Check for spaces in statements' + self.init_log(test_desc) + statements = ["elseif", "endif", "elsewhere", "endwhere", "enddo", "endmodule"] + # ignore comments, and strip all white space + line_without_comments = line.split(comment_char)[0].strip(" ") + for bad_statement in statements: + if line_without_comments.lower().startswith(bad_statement): + self.logs[test_desc].append("%s: %s" % (file_and_line, line_without_comments)) + break + + ############## + + def check_for_double_quotes(self, file_and_line, line): + """ + All Fortran strings should appear as 'string', not "string" + """ + test_desc = 'Check for double quotes in statements' + self.init_log(test_desc) + if '"' in line: + self.logs[test_desc].append("%s: %s" % (file_and_line, line)) + + ############## + + def check_logical_statements(self, file_and_line, line): + """ + Use symbols, not words, for logical operators: + * >=, not .ge. + * >, not .gt. + * <=, not .le. + * <, not .lt. + * ==, not .eq. + * /=, not .ne. + """ + test_desc = 'Check for unwanted logical operators' + self.init_log(test_desc) + operators = ['.ge.', '.gt.', '.le.', '.lt.', '.eq.', '.ne.'] + for operator in operators: + if operator in line: + self.logs[test_desc].append("%s: %s" % (file_and_line, line)) + break + + ############## + + def check_r8_settings(self, file_and_line, line, comment_char="!"): + """ + Make sure all real numbers are cast as r8 + """ + test_desc = 'Check for r8' + self.init_log(test_desc) + import re + # Looking for decimal numbers that do not end in _r8 + # Edge cases: + # 1. ignore decimals in comments + # 2. Ignore decimals in format strings (e.g. E24.16) + # 3. Allow 1.0e-2_r8 + line_without_comments = line.split(comment_char)[0].strip(" ") + + # Regex notes + # 1. (?=1) followed + # by a decimal point + # 3. (\d+[eE]([+-])?)? -- Optionally match a number followed by either + # e or E (optionally followed by + or -) + # 4. \d+ -- match N consecutive decimal digits (for N>=1) + # 5. (?!\d+|_|[eE]) -- do not match a decimal, an underscore, an e, + # or an E + # [regex refers to this as a negative lookahead] + regex = r'(? 0: + LOGGER.info("\nTotal error count: %d", FORTRAN_ERROR_COUNT + PYTHON_ERROR_COUNT) + sys.exit(1) + + LOGGER.info("\nNo errors found!") diff --git a/src/drivers/cvmix_bgrnd_BL.F90 b/src/drivers/cvmix_bgrnd_BL.F90 index 7950e34ff..a7947e3de 100644 --- a/src/drivers/cvmix_bgrnd_BL.F90 +++ b/src/drivers/cvmix_bgrnd_BL.F90 @@ -32,7 +32,7 @@ Subroutine cvmix_BL_driver(nlev, max_nlev, ocn_depth) #endif cvmix_io_close - Implicit None + implicit none ! !INPUT PARAMETERS: integer, intent(in) :: nlev, &! number of levels for column diff --git a/src/drivers/cvmix_ddiff_drv.F90 b/src/drivers/cvmix_ddiff_drv.F90 index 43519dfbd..713ee4636 100644 --- a/src/drivers/cvmix_ddiff_drv.F90 +++ b/src/drivers/cvmix_ddiff_drv.F90 @@ -26,7 +26,7 @@ Subroutine cvmix_ddiff_driver(nlev, max_nlev) #endif cvmix_io_close - Implicit None + implicit none ! !INPUT PARAMETERS: integer, intent(in) :: nlev, &! number of levels for column diff --git a/src/drivers/cvmix_kpp_drv.F90 b/src/drivers/cvmix_kpp_drv.F90 index 96c4666dc..46d702896 100644 --- a/src/drivers/cvmix_kpp_drv.F90 +++ b/src/drivers/cvmix_kpp_drv.F90 @@ -34,7 +34,7 @@ Subroutine cvmix_kpp_driver() #endif cvmix_io_close - Implicit None + implicit none !EOP !BOC diff --git a/src/drivers/cvmix_shear_drv.F90 b/src/drivers/cvmix_shear_drv.F90 index 22e9f59b3..fce415200 100644 --- a/src/drivers/cvmix_shear_drv.F90 +++ b/src/drivers/cvmix_shear_drv.F90 @@ -32,7 +32,7 @@ Subroutine cvmix_shear_driver(nlev, max_nlev) #endif cvmix_io_close - Implicit None + implicit none ! !INPUT PARAMETERS: integer, intent(in) :: nlev, &! number of levels for column diff --git a/src/drivers/cvmix_tidal_Simmons.F90 b/src/drivers/cvmix_tidal_Simmons.F90 index 702534df2..e16fa75ed 100644 --- a/src/drivers/cvmix_tidal_Simmons.F90 +++ b/src/drivers/cvmix_tidal_Simmons.F90 @@ -33,7 +33,7 @@ Subroutine cvmix_tidal_driver() cvmix_output_write_att, & cvmix_io_close - Implicit None + implicit none !EOP !BOC diff --git a/templates/cvmix_MODULE.F90 b/templates/cvmix_MODULE.F90 index 9822717b8..01aa6c88c 100644 --- a/templates/cvmix_MODULE.F90 +++ b/templates/cvmix_MODULE.F90 @@ -4,7 +4,7 @@ module cvmix_MODULE !\newpage ! !MODULE: cvmix_MODULE ! -! !AUTHOR: +! !AUTHOR: ! [your name], [your affiliation] ([your email address]) ! ! !DESCRIPTION: @@ -120,7 +120,7 @@ subroutine cvmix_put_MODULE_real(CVmix_MODULE_params, varname, val) !\\ ! !USES: -! Only those used by entire module. +! Only those used by entire module. ! !INPUT PARAMETERS: character(len=*), intent(in) :: varname @@ -137,7 +137,7 @@ subroutine cvmix_put_MODULE_real(CVmix_MODULE_params, varname, val) case DEFAULT print*, "ERROR: ", trim(varname), " not a valid choice!" stop 1 - + end select !EOC @@ -159,7 +159,7 @@ function cvmix_get_VARNAME_real(varname, CVmix_MODULE_params_user) !\\ ! !USES: -! Only those used by entire module. +! Only those used by entire module. ! !INPUT PARAMETERS: character(len=*), intent(in) :: varname @@ -186,7 +186,7 @@ function cvmix_get_VARNAME_real(varname, CVmix_MODULE_params_user) case DEFAULT print*, "ERROR: ", trim(varname), " not a valid choice!" stop 1 - + end select !EOC From 00f0cf878964e2d0ba75139523da23ccc2f55575 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Fri, 10 May 2019 11:52:38 -0600 Subject: [PATCH 08/10] Add some more tests to the TravisCI suite Added the script CVMix_tools/run_test_suite.sh that runs code consistency checks, builds the Fortran code (with and without netCDF) and then runs the Bryan-Lewis regression test. Future commits on this branch will add more tests to the suite, I just want to see how TravisCI does with this first. --- .travis.yml | 4 +- CVMix_tools/pylintrc | 549 ++++++++++++++++++++++++++++++++++ CVMix_tools/run_test_suite.sh | 112 +++++++ 3 files changed, 663 insertions(+), 2 deletions(-) create mode 100644 CVMix_tools/pylintrc create mode 100755 CVMix_tools/run_test_suite.sh diff --git a/.travis.yml b/.travis.yml index bd16d53fd..add8fc6d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,8 @@ python: - '2.7' - '3.6' script: - - cd src; make SETUP_ARGS="gfortran $(dirname $(dirname $(which nc-config)))" - - make USE_NETCDF=TRUE + - cd bld; ./cvmix_setup gfortran $(dirname $(dirname $(which nc-config))) + - cd ../CVMix_tools; ./run_test_suite.sh --already-ran-setup branches: only: - master diff --git a/CVMix_tools/pylintrc b/CVMix_tools/pylintrc new file mode 100644 index 000000000..6030306e9 --- /dev/null +++ b/CVMix_tools/pylintrc @@ -0,0 +1,549 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. +jobs=1 + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Specify a configuration file. +#rcfile= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=print-statement, + parameter-unpacking, + unpacking-in-except, + old-raise-syntax, + backtick, + long-suffix, + old-ne-operator, + old-octal-literal, + import-star-module-level, + non-ascii-bytes-literal, + invalid-unicode-literal, + raw-checker-failed, + bad-inline-option, + locally-disabled, + locally-enabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + apply-builtin, + basestring-builtin, + buffer-builtin, + cmp-builtin, + coerce-builtin, + execfile-builtin, + file-builtin, + long-builtin, + raw_input-builtin, + reduce-builtin, + standarderror-builtin, + unicode-builtin, + xrange-builtin, + coerce-method, + delslice-method, + getslice-method, + setslice-method, + no-absolute-import, + old-division, + dict-iter-method, + dict-view-method, + next-method-called, + metaclass-assignment, + indexing-exception, + raising-string, + reload-builtin, + oct-method, + hex-method, + nonzero-method, + cmp-method, + input-builtin, + round-builtin, + intern-builtin, + unichr-builtin, + map-builtin-not-iterating, + zip-builtin-not-iterating, + range-builtin-not-iterating, + filter-builtin-not-iterating, + using-cmp-argument, + eq-without-hash, + div-method, + idiv-method, + rdiv-method, + exception-message-attribute, + invalid-str-codec, + sys-max-int, + bad-python3-import, + deprecated-string-function, + deprecated-str-translate-call, + deprecated-itertools-function, + deprecated-types-field, + next-method-defined, + dict-items-not-iterating, + dict-keys-not-iterating, + dict-values-not-iterating, + deprecated-operator-function, + deprecated-urllib-function, + xreadlines-attribute, + deprecated-sys-function, + exception-escape, + comprehension-escape + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[REPORTS] + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio).You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=optparse.Values,sys.exit + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,io,builtins + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module +max-module-lines=1000 + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma, + dict-separator + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[BASIC] + +# Naming style matching correct argument names +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style +#argument-rgx= + +# Naming style matching correct attribute names +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Naming style matching correct class attribute names +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style +#class-attribute-rgx= + +# Naming style matching correct class names +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming-style +#class-rgx= + +# Naming style matching correct constant names +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma +good-names=i, + j, + k, + ex, + Run, + _ + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# Naming style matching correct inline iteration names +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style +#inlinevar-rgx= + +# Naming style matching correct method names +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style +#method-rgx= + +# Naming style matching correct module names +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style +#variable-rgx= + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of statements in function / method body +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[IMPORTS] + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub, + TERMIOS, + Bastion, + rexec + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/CVMix_tools/run_test_suite.sh b/CVMix_tools/run_test_suite.sh new file mode 100755 index 000000000..ee4401c11 --- /dev/null +++ b/CVMix_tools/run_test_suite.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +# Convert shell return code to "PASS" or "FAIL" +# (0 = PASS, all other return codes = FAIL) +function check_return() { + if [ $1 -eq 0 ]; then + echo "PASS" + else + echo "FAIL" + fi + +} + +################################################# + +# Output test results +function print_status() { + TEST_CNT=$((TEST_CNT+1)) + if [ "${STATUS}" == "FAIL" ]; then + FAIL_CNT=$((FAIL_CNT+1)) + fi + echo "${TEST_CNT}. $1: ${STATUS}" +} + +################################################# + +############### +# Global Vars # +############### + +CVMIX_ROOT=`(cd ..; pwd -P)` +OUTFILE=${CVMIX_ROOT}/.test.out +TEST_CNT=0 +FAIL_CNT=0 +echo "Test Results:" > $OUTFILE + +######### +# TESTS # +######### + +# Check for --already-ran-setup +if [ "$1" == "--already-ran-setup" ]; then + echo "Checking for existence of bld/.CVMix_env" + if [ ! -f ${CVMIX_ROOT}/bld/.CVMix_env ]; then + echo "ERROR: must run cvmix_setup prior to this script" + exit 1 + fi +fi + +# Code consistency check +cd ${CVMIX_ROOT}/CVMix_tools +echo "$ ./code_consistency.py" +./code_consistency.py +STATUS=$(check_return $?) +print_status "CodeConsistency.py" >> $OUTFILE + +# Run pylint (if installed) +command -v pylint &> /dev/null +if [ $? -eq 0 ]; then + cd ${CVMIX_ROOT}/CVMix_tools + echo "$ pylint --rcfile=pylintrc code_consistency.py" + pylint --rcfile=pylintrc code_consistency.py + STATUS=$(check_return $?) + print_status "pylint" >> $OUTFILE +fi + +# Clean Fortran Code +cd ${CVMIX_ROOT}/src +echo "$ make allclean" +make allclean +STATUS=$(check_return $?) +print_status "make allclean" >> $OUTFILE + +# Build libcvmix.a +cd ${CVMIX_ROOT}/src +echo "$ make lib" +make lib +STATUS=$(check_return $?) +print_status "make libcvmix.a" >> $OUTFILE + +# Build stand-alone executable (only if library built successfully) +if [ "${STATUS}" == "PASS" ]; then + cd ${CVMIX_ROOT}/src + echo "$ make USE_NETCDF=TRUE" + make USE_NETCDF=TRUE + STATUS=$(check_return $?) + print_status "make cvmix.exe with netcdf" >> $OUTFILE + + echo "$ make" + make + STATUS=$(check_return $?) + print_status "make cvmix.exe without netcdf" >> $OUTFILE +fi + +# Only test Fortran executable if build was successful +if [ "${STATUS}" == "PASS" ]; then + # Bryan-Lewis test + cd ${CVMIX_ROOT}/reg_tests/Bryan-Lewis + echo "$ ./BL_test.sh" + ./BL_test.sh + STATUS=$(check_return $?) + print_status "BL_test.sh" >> $OUTFILE + +fi + +echo "----" +cat $OUTFILE +rm -f $OUTFILE +echo "" +echo "${TEST_CNT} tests were run, and $FAIL_CNT failed." +exit ${FAIL_CNT} + From ce75e3736da464b81229a177f0a64eed72737843 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Fri, 10 May 2019 12:24:01 -0600 Subject: [PATCH 09/10] More tests for TravisCI Every test in reg_tests/ is run (note that the tidal mixing test is only run if the model can successfully build with netCDF) --- CVMix_tools/run_test_suite.sh | 52 +++++++++++++++---- .../{BL_test.sh => Bryan-Lewis-test.sh} | 0 2 files changed, 43 insertions(+), 9 deletions(-) rename reg_tests/Bryan-Lewis/{BL_test.sh => Bryan-Lewis-test.sh} (100%) diff --git a/CVMix_tools/run_test_suite.sh b/CVMix_tools/run_test_suite.sh index ee4401c11..b5af9036a 100755 --- a/CVMix_tools/run_test_suite.sh +++ b/CVMix_tools/run_test_suite.sh @@ -80,12 +80,6 @@ print_status "make libcvmix.a" >> $OUTFILE # Build stand-alone executable (only if library built successfully) if [ "${STATUS}" == "PASS" ]; then - cd ${CVMIX_ROOT}/src - echo "$ make USE_NETCDF=TRUE" - make USE_NETCDF=TRUE - STATUS=$(check_return $?) - print_status "make cvmix.exe with netcdf" >> $OUTFILE - echo "$ make" make STATUS=$(check_return $?) @@ -96,13 +90,53 @@ fi if [ "${STATUS}" == "PASS" ]; then # Bryan-Lewis test cd ${CVMIX_ROOT}/reg_tests/Bryan-Lewis - echo "$ ./BL_test.sh" - ./BL_test.sh + echo "$ ./Bryan-Lewis-test.sh" + ./Bryan-Lewis-test.sh + STATUS=$(check_return $?) + print_status "Bryan Lewis" >> $OUTFILE + + # Double Diffusion test + cd ${CVMIX_ROOT}/reg_tests/double_diff + echo "$ ./double_diff-test.sh" + ./double_diff-test.sh + STATUS=$(check_return $?) + print_status "Double Diffusion" >> $OUTFILE + + # Shear test + cd ${CVMIX_ROOT}/reg_tests/shear + echo "$ ./shear-test.sh" + ./shear-test.sh + STATUS=$(check_return $?) + print_status "Shear" >> $OUTFILE + + # KPP test + cd ${CVMIX_ROOT}/reg_tests/kpp + echo "$ ./kpp-test.sh" + ./kpp-test.sh STATUS=$(check_return $?) - print_status "BL_test.sh" >> $OUTFILE + print_status "KPP.sh" >> $OUTFILE + + # Build stand-alone executable with netCDF + cd ${CVMIX_ROOT}/src + echo "$ make USE_NETCDF=TRUE" + make USE_NETCDF=TRUE + STATUS=$(check_return $?) + print_status "make cvmix.exe with netcdf" >> $OUTFILE + + # Only test Fortran executable with netcdf if build was successful + if [ "${STATUS}" == "PASS" ]; then + # Tidal (Simmons) test + cd ${CVMIX_ROOT}/reg_tests/tidal-Simmons + echo "$ ./Simmons-test.sh -nc" + ./Simmons-test.sh -nc + STATUS=$(check_return $?) + print_status "Tidal (requires netCDF)" >> $OUTFILE + fi fi + + echo "----" cat $OUTFILE rm -f $OUTFILE diff --git a/reg_tests/Bryan-Lewis/BL_test.sh b/reg_tests/Bryan-Lewis/Bryan-Lewis-test.sh similarity index 100% rename from reg_tests/Bryan-Lewis/BL_test.sh rename to reg_tests/Bryan-Lewis/Bryan-Lewis-test.sh From 29449dc48d5da2210ca73e5465bb58766a759b58 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Fri, 10 May 2019 12:33:33 -0600 Subject: [PATCH 10/10] One import per line in makedep.py Accidentally reverted this code cleanup that was introduced in 2c5842c --- src/shared/makedep.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/shared/makedep.py b/src/shared/makedep.py index d277cabc4..c4b5c29dc 100755 --- a/src/shared/makedep.py +++ b/src/shared/makedep.py @@ -6,7 +6,10 @@ # Read in every file in $SRC_DIR and $SRC_DIR2 (arguments 3 and 4) # Only depend on modules located in $SRC_DIR or $SRC_DIR2 -import os, sys, re, logging +import os +import sys +import re +import logging logger = logging.getLogger(__name__) logging.basicConfig(format='(makedep.py): %(message)s', level=logging.INFO)