diff --git a/icesat2_toolkit/time.py b/icesat2_toolkit/time.py index 045d827..7328eae 100644 --- a/icesat2_toolkit/time.py +++ b/icesat2_toolkit/time.py @@ -43,7 +43,6 @@ import re import copy import logging -import pathlib import warnings import datetime import traceback diff --git a/scripts/convert_ICESat2_format.py b/scripts/convert_ICESat2_format.py index c9d70ca..08b122b 100644 --- a/scripts/convert_ICESat2_format.py +++ b/scripts/convert_ICESat2_format.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" convert_ICESat2_format.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (09/2023) Converts ICESat-2 HDF5 datafiles to zarr or rechunked HDF5 datafiles @@ -56,6 +56,7 @@ https://pandas.pydata.org/ UPDATE HISTORY: + Updated 09/2023: generalized regular expressions for non-entered cases Updated 12/2022: single implicit import of altimetry tools Updated 06/2022: use explicit import of convert functions Updated 05/2022: use argparse descriptions within sphinx documentation @@ -82,7 +83,7 @@ # PURPOSE: convert the ICESat-2 elevation data from HDF5 to zarr # or rechunked HDF5 formats def convert_ICESat2_format(DIRECTORY, PRODUCTS, RELEASE, VERSIONS, GRANULES, - TRACKS, YEARS=None, SUBDIRECTORY=None, FORMAT=None, CHUNKS=None, + TRACKS, YEARS=None, SUBDIRECTORY=None, CYCLES=None, FORMAT=None, CHUNKS=None, PROCESSES=0, CLOBBER=False, VERBOSE=False, MODE=0o775): # create logger @@ -91,11 +92,24 @@ def convert_ICESat2_format(DIRECTORY, PRODUCTS, RELEASE, VERSIONS, GRANULES, # regular expression operator for finding files of a particular granule # find ICESat-2 HDF5 files in the subdirectory for product and release - regex_track = '|'.join([rf'{T:04d}' for T in TRACKS]) - regex_granule = '|'.join([rf'{G:02d}' for G in GRANULES]) - regex_version = '|'.join([rf'{V:02d}' for V in VERSIONS]) - file_regex_pattern = (r'{0}(-\d{{2}})?_(\d{{4}})(\d{{2}})(\d{{2}})(\d{{2}})' - r'(\d{{2}})(\d{{2}})_({1})(\d{{2}})({2})_({3})_({4})(.*?).(h5)$') + if TRACKS: + regex_track = r'|'.join([rf'{T:04d}' for T in TRACKS]) + else: + regex_track = r'\d{4}' + if CYCLES: + regex_cycle = r'|'.join([rf'{C:02d}' for C in CYCLES]) + else: + regex_cycle = r'\d{2}' + if GRANULES: + regex_granule = r'|'.join([rf'{G:02d}' for G in GRANULES]) + else: + regex_granule = r'\d{2}' + if VERSIONS: + regex_version = r'|'.join([rf'{V:02d}' for V in VERSIONS]) + else: + regex_version = r'\d{2}' + regex_pattern = (r'(processed_)?({0})(-\d{{2}})?_(\d{{4}})(\d{{2}})(\d{{2}})' + r'(\d{{2}})(\d{{2}})(\d{{2}})_({1})({2})({3})_({4})_({5})(.*?).h5$') # regular expression operator for finding subdirectories if SUBDIRECTORY: @@ -103,8 +117,8 @@ def convert_ICESat2_format(DIRECTORY, PRODUCTS, RELEASE, VERSIONS, GRANULES, R2 = re.compile(r'('+r'|'.join(SUBDIRECTORY)+r')', re.VERBOSE) elif YEARS: # convert particular years for product - regex_pattern = '|'.join(rf'{y:d}' for y in YEARS) - R2 = re.compile(rf'({regex_pattern}).(\d+).(\d+)', re.VERBOSE) + regex_years = '|'.join(rf'{y:d}' for y in YEARS) + R2 = re.compile(rf'({regex_years}).(\d+).(\d+)', re.VERBOSE) else: # convert all available subdirectories for product R2 = re.compile(r'(\d+).(\d+).(\d+)', re.VERBOSE) @@ -118,8 +132,8 @@ def convert_ICESat2_format(DIRECTORY, PRODUCTS, RELEASE, VERSIONS, GRANULES, ddir = os.path.join(DIRECTORY,f'{p}.{RELEASE}') subdirectories = [sd for sd in os.listdir(ddir) if R2.match(sd)] # compile regular expression operator for product, release and version - args = (p,regex_track,regex_granule,RELEASE,regex_version) - R1 = re.compile(file_regex_pattern.format(*args), re.VERBOSE) + args = (p,regex_track,regex_cycle,regex_granule,RELEASE,regex_version) + R1 = re.compile(regex_pattern.format(*args), re.VERBOSE) # for each subdirectory for sd in subdirectories: # find matching files (for granule, release, version, track) @@ -257,13 +271,17 @@ def arguments(): help='ICESat-2 Data Release') # ICESat-2 data version parser.add_argument('--version','-v', - type=int, nargs='+', default=range(1,10), + type=int, nargs='+', help='ICESat-2 Data Version') # ICESat-2 granule region parser.add_argument('--granule','-g', metavar='REGION', type=int, nargs='+', choices=range(1,15), default=range(1,15), help='ICESat-2 Granule Region') + # ICESat-2 orbital cycle + parser.add_argument('--cycle','-c', + type=int, nargs='+', default=None, + help='ICESat-2 orbital cycles to convert') # ICESat-2 reference ground tracks parser.add_argument('--track','-t', metavar='RGT', type=int, nargs='+', @@ -305,9 +323,9 @@ def main(): # convert HDF5 files for each data product convert_ICESat2_format(args.directory, args.products, args.release, args.version, args.granule, args.track, YEARS=args.year, - SUBDIRECTORY=args.subdirectory, FORMAT=args.format, - CHUNKS=args.chunks, PROCESSES=args.np, CLOBBER=args.clobber, - VERBOSE=args.verbose, MODE=args.mode) + SUBDIRECTORY=args.subdirectory, CYCLES=args.cycle, + FORMAT=args.format, CHUNKS=args.chunks, PROCESSES=args.np, + CLOBBER=args.clobber, VERBOSE=args.verbose, MODE=args.mode) # run main program if __name__ == '__main__': diff --git a/scripts/copy_scf_ICESat2_files.py b/scripts/copy_scf_ICESat2_files.py index 2d27861..a32216c 100644 --- a/scripts/copy_scf_ICESat2_files.py +++ b/scripts/copy_scf_ICESat2_files.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" copy_scf_ICESat2_files.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (09/2023) Copies ICESat-2 HDF5 files from the SCF server CALLING SEQUENCE: @@ -33,6 +33,7 @@ https://github.com/paramiko/paramiko UPDATE HISTORY: + Updated 09/2023: generalized regular expressions for non-entered cases Updated 12/2022: single implicit import of altimetry tools Updated 05/2022: use argparse descriptions within sphinx documentation Updated 10/2021: using python logging for handling verbose output @@ -105,7 +106,7 @@ def arguments(): help='ICESat-2 data release to copy') # ICESat-2 data version parser.add_argument('--version','-v', - type=int, nargs='+', default=range(1,10), + type=int, nargs='+', help='ICESat-2 data versions to copy') # ICESat-2 granule region parser.add_argument('--granule','-g', @@ -115,7 +116,6 @@ def arguments(): # ICESat-2 orbital cycle parser.add_argument('--cycle','-c', type=int, nargs='+', - default=range(1,10), help='ICESat-2 orbital cycles to copy') # ICESat-2 reference ground tracks parser.add_argument('--track','-t', @@ -207,13 +207,22 @@ def copy_scf_files(client, client_ftp, base_dir, scf_incoming, scf_outgoing, PRODUCT, RELEASE, VERSIONS, GRANULES, CYCLES, TRACKS, CLOBBER=False, LIST=False, MODE=0o775): # find ICESat-2 HDF5 files in the subdirectory for product and release - TRACKS = np.arange(1,1388) if not np.any(TRACKS) else TRACKS - CYCLES = np.arange(1,3) if not np.any(CYCLES) else CYCLES - GRANULES = np.arange(1,15) if not np.any(GRANULES) else GRANULES - regex_track = r'|'.join([rf'{T:04d}' for T in TRACKS]) - regex_cycle = r'|'.join([rf'{C:02d}' for C in CYCLES]) - regex_granule = r'|'.join([rf'{G:02d}' for G in GRANULES]) - regex_version = r'|'.join([rf'{V:02d}' for V in VERSIONS]) + if TRACKS: + regex_track = r'|'.join([rf'{T:04d}' for T in TRACKS]) + else: + regex_track = r'\d{4}' + if CYCLES: + regex_cycle = r'|'.join([rf'{C:02d}' for C in CYCLES]) + else: + regex_cycle = r'\d{2}' + if GRANULES: + regex_granule = r'|'.join([rf'{G:02d}' for G in GRANULES]) + else: + regex_granule = r'\d{2}' + if VERSIONS: + regex_version = r'|'.join([rf'{V:02d}' for V in VERSIONS]) + else: + regex_version = r'\d{2}' # compile regular expression operator for extracting data from files args = (PRODUCT,regex_track,regex_cycle,regex_granule,RELEASE,regex_version) regex_pattern = (r'(processed_)?({0})(-\d{{2}})?_(\d{{4}})(\d{{2}})(\d{{2}})' diff --git a/scripts/nsidc_icesat2_convert.py b/scripts/nsidc_icesat2_convert.py index c74e6a5..4e64608 100644 --- a/scripts/nsidc_icesat2_convert.py +++ b/scripts/nsidc_icesat2_convert.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" nsidc_icesat2_convert.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (09/2023) Acquires ICESat-2 datafiles from NSIDC and directly converts to zarr datafiles or rechunked HDF5 files @@ -85,6 +85,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 09/2023: generalized regular expressions for non-entered cases Updated 12/2022: single implicit import of altimetry tools Updated 05/2022: use argparse descriptions within sphinx documentation Updated 03/2022: use attempt login function to check credentials @@ -159,8 +160,14 @@ def nsidc_icesat2_convert(DIRECTORY, PRODUCTS, RELEASE, VERSIONS, GRANULES, TRAC regex_cycle = r'|'.join([rf'{C:02d}' for C in CYCLES]) else: regex_cycle = r'\d{2}' - regex_granule = r'|'.join([rf'{G:02d}' for G in GRANULES]) - regex_version = r'|'.join([rf'{V:02d}' for V in VERSIONS]) + if GRANULES: + regex_granule = r'|'.join([rf'{G:02d}' for G in GRANULES]) + else: + regex_granule = r'\d{2}' + if VERSIONS: + regex_version = r'|'.join([rf'{V:02d}' for V in VERSIONS]) + else: + regex_version = r'\d{2}' regex_suffix = r'(.*?)' if AUXILIARY else r'(h5)' remote_regex_pattern=(r'{0}(-\d{{2}})?_(\d{{4}})(\d{{2}})(\d{{2}})(\d{{2}})' r'(\d{{2}})(\d{{2}})_({1})({2})({3})_({4})_({5})(.*?).{6}$') @@ -441,7 +448,7 @@ def arguments(): help='ICESat-2 Data Release') # ICESat-2 data version parser.add_argument('--version','-v', - type=int, nargs='+', default=range(1,10), + type=int, nargs='+', help='ICESat-2 Data Version') # ICESat-2 granule region parser.add_argument('--granule','-g', diff --git a/scripts/nsidc_icesat2_dragann.py b/scripts/nsidc_icesat2_dragann.py index 0bde465..28ebee2 100644 --- a/scripts/nsidc_icesat2_dragann.py +++ b/scripts/nsidc_icesat2_dragann.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" nsidc_icesat2_dragann.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (09/2023) Acquires the ATL03 geolocated photon height product and appends the ATL08 DRAGANN classifications from NSIDC @@ -57,6 +57,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 09/2023: generalized regular expressions for non-entered cases Updated 12/2022: single implicit import of altimetry tools Updated 05/2022: use argparse descriptions within sphinx documentation Updated 03/2022: use attempt login function to check credentials @@ -127,8 +128,14 @@ def nsidc_icesat2_dragann(DIRECTORY, RELEASE, VERSIONS, GRANULES, TRACKS, regex_cycle = r'|'.join([rf'{C:02d}' for C in CYCLES]) else: regex_cycle = r'\d{2}' - regex_granule = r'|'.join([rf'{G:02d}' for G in GRANULES]) - regex_version = r'|'.join([rf'{V:02d}' for V in VERSIONS]) + if GRANULES: + regex_granule = r'|'.join([rf'{G:02d}' for G in GRANULES]) + else: + regex_granule = r'\d{2}' + if VERSIONS: + regex_version = r'|'.join([rf'{V:02d}' for V in VERSIONS]) + else: + regex_version = r'\d{2}' regex_suffix = r'(h5)' remote_regex_pattern=(r'({0})_(\d{{4}})(\d{{2}})(\d{{2}})(\d{{2}})' r'(\d{{2}})(\d{{2}})_({1})({2})({3})_({4})_({5})(.*?).{6}$') @@ -410,7 +417,7 @@ def arguments(): help='ICESat-2 Data Release') # ICESat-2 data version parser.add_argument('--version','-v', - type=int, nargs='+', default=range(1,10), + type=int, nargs='+', help='ICESat-2 Data Version') # ICESat-2 granule region parser.add_argument('--granule','-g', diff --git a/scripts/nsidc_icesat2_sync.py b/scripts/nsidc_icesat2_sync.py index 9f5cb1d..8214635 100644 --- a/scripts/nsidc_icesat2_sync.py +++ b/scripts/nsidc_icesat2_sync.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" nsidc_icesat2_sync.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (09/2023) Acquires ICESat-2 datafiles from the National Snow and Ice Data Center (NSIDC) @@ -69,6 +69,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 09/2023: generalized regular expressions for non-entered cases Updated 12/2022: single implicit import of altimetry tools Updated 05/2022: use argparse descriptions within sphinx documentation Updated 03/2022: use attempt login function to check credentials @@ -152,8 +153,14 @@ def nsidc_icesat2_sync(DIRECTORY, PRODUCTS, RELEASE, VERSIONS, GRANULES, regex_cycle = r'|'.join([rf'{C:02d}' for C in CYCLES]) else: regex_cycle = r'\d{2}' - regex_granule = r'|'.join([rf'{G:02d}' for G in GRANULES]) - regex_version = r'|'.join([rf'{V:02d}' for V in VERSIONS]) + if GRANULES: + regex_granule = r'|'.join([rf'{G:02d}' for G in GRANULES]) + else: + regex_granule = r'\d{2}' + if VERSIONS: + regex_version = r'|'.join([rf'{V:02d}' for V in VERSIONS]) + else: + regex_version = r'\d{2}' regex_suffix = r'(.*?)' if AUXILIARY else r'(h5|nc)' default_pattern = (r'{0}(-\d{{2}})?_(\d{{4}})(\d{{2}})(\d{{2}})(\d{{2}})' r'(\d{{2}})(\d{{2}})_({1})({2})({3})_({4})_({5})(.*?).{6}$') @@ -471,7 +478,7 @@ def arguments(): help='ICESat-2 Data Release') # ICESat-2 data version parser.add_argument('--version','-v', - type=int, nargs='+', default=range(1,10), + type=int, nargs='+', help='ICESat-2 Data Version') # ICESat-2 granule region region = parser.add_mutually_exclusive_group(required=False) diff --git a/scripts/nsidc_icesat2_sync_s3.py b/scripts/nsidc_icesat2_sync_s3.py index 1a67bda..4d3ed1d 100644 --- a/scripts/nsidc_icesat2_sync_s3.py +++ b/scripts/nsidc_icesat2_sync_s3.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" nsidc_icesat2_sync_s3.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (09/2023) Acquires ICESat-2 datafiles from the National Snow and Ice Data Center (NSIDC) and transfers to an AWS S3 bucket using a local machine as pass through @@ -72,6 +72,7 @@ utilities.py: download and management utilities for syncing files UPDATE HISTORY: + Updated 09/2023: generalized regular expressions for non-entered cases Updated 12/2022: single implicit import of altimetry tools Updated 05/2022: use argparse descriptions within sphinx documentation Updated 03/2022: use attempt login function to check credentials @@ -157,8 +158,14 @@ def nsidc_icesat2_sync_s3(aws_access_key_id, aws_secret_access_key, regex_cycle = r'|'.join([rf'{C:02d}' for C in CYCLES]) else: regex_cycle = r'\d{2}' - regex_granule = r'|'.join([rf'{G:02d}' for G in GRANULES]) - regex_version = r'|'.join([rf'{V:02d}' for V in VERSIONS]) + if GRANULES: + regex_granule = r'|'.join([rf'{G:02d}' for G in GRANULES]) + else: + regex_granule = r'\d{2}' + if VERSIONS: + regex_version = r'|'.join([rf'{V:02d}' for V in VERSIONS]) + else: + regex_version = r'\d{2}' regex_suffix = r'(.*?)' if AUXILIARY else r'(h5|nc)' default_pattern = (r'{0}(-\d{{2}})?_(\d{{4}})(\d{{2}})(\d{{2}})(\d{{2}})' r'(\d{{2}})(\d{{2}})_({1})({2})({3})_({4})_({5})(.*?).{6}$') @@ -441,7 +448,7 @@ def arguments(): help='ICESat-2 Data Release') # ICESat-2 data version parser.add_argument('--version','-v', - type=int, nargs='+', default=range(1,10), + type=int, nargs='+', help='ICESat-2 Data Version') # ICESat-2 granule region region = parser.add_mutually_exclusive_group(required=False) diff --git a/scripts/scp_ICESat2_files.py b/scripts/scp_ICESat2_files.py index a3bf6ff..bfafa27 100644 --- a/scripts/scp_ICESat2_files.py +++ b/scripts/scp_ICESat2_files.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" scp_ICESat2_files.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (09/2023) Copies ICESat-2 HDF5 data from between a local host and a remote host can switch between pushing and pulling to/from remote PUSH to remote: s.put(local_file, remote_file) @@ -38,6 +38,7 @@ https://github.com/jbardin/scp.py UPDATE HISTORY: + Updated 09/2023: generalized regular expressions for non-entered cases Updated 12/2022: use f-strings for ascii and verbose outputs Updated 05/2022: use argparse descriptions within sphinx documentation Updated 10/2021: using python logging for handling verbose output @@ -108,7 +109,7 @@ def arguments(): help='ICESat-2 data release to copy') # ICESat-2 data version parser.add_argument('--version','-v', - type=int, nargs='+', default=range(1,10), + type=int, nargs='+', help='ICESat-2 data versions to copy') # ICESat-2 granule region parser.add_argument('--granule','-g', @@ -118,7 +119,6 @@ def arguments(): # ICESat-2 orbital cycle parser.add_argument('--cycle','-c', type=int, nargs='+', - default=range(1,10), help='ICESat-2 orbital cycles to copy') # ICESat-2 reference ground tracks parser.add_argument('--track','-t', @@ -243,14 +243,22 @@ def scp_ICESat2_files(client, client_ftp, DIRECTORY, REMOTE, PRODUCT, RELEASE, VERSIONS, GRANULES, CYCLES, TRACKS, CLOBBER=False, PUSH=False, LIST=False, MODE=0o775): # find ICESat-2 HDF5 files in the subdirectory for product and release - TRACKS = np.arange(1,1388) if not np.any(TRACKS) else TRACKS - CYCLES = np.arange(1,3) if not np.any(CYCLES) else CYCLES - GRANULES = np.arange(1,15) if not np.any(GRANULES) else GRANULES - VERSIONS = np.arange(1,10) if not np.any(VERSIONS) else VERSIONS - regex_track = '|'.join([rf'{T:04d}' for T in TRACKS]) - regex_cycle = '|'.join([rf'{C:02d}' for C in CYCLES]) - regex_granule = '|'.join([rf'{G:02d}' for G in GRANULES]) - regex_version = '|'.join([rf'{V:02d}' for V in VERSIONS]) + if TRACKS: + regex_track = r'|'.join([rf'{T:04d}' for T in TRACKS]) + else: + regex_track = r'\d{4}' + if CYCLES: + regex_cycle = r'|'.join([rf'{C:02d}' for C in CYCLES]) + else: + regex_cycle = r'\d{2}' + if GRANULES: + regex_granule = r'|'.join([rf'{G:02d}' for G in GRANULES]) + else: + regex_granule = r'\d{2}' + if VERSIONS: + regex_version = r'|'.join([rf'{V:02d}' for V in VERSIONS]) + else: + regex_version = r'\d{2}' # compile regular expression operator for finding subdirectories # and extracting date information from the subdirectory rx1 = re.compile(r'(\d+)\.(\d+)\.(\d+)',re.VERBOSE) diff --git a/scripts/scp_scf_ICESat2_files.py b/scripts/scp_scf_ICESat2_files.py index 56e4e55..90d9a0c 100644 --- a/scripts/scp_scf_ICESat2_files.py +++ b/scripts/scp_scf_ICESat2_files.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" scp_scf_ICESat2_files.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (09/2023) Copies ICESat-2 HDF5 files from the SCF server to a remote host using the SCF-authorized local computer as a proxy server @@ -40,6 +40,7 @@ https://github.com/jbardin/scp.py UPDATE HISTORY: + Updated 09/2023: generalized regular expressions for non-entered cases Updated 12/2022: use f-strings for ascii and verbose outputs Updated 05/2022: use argparse descriptions within sphinx documentation Updated 10/2021: using python logging for handling verbose output @@ -111,7 +112,7 @@ def arguments(): help='ICESat-2 data release to copy') # ICESat-2 data version parser.add_argument('--version','-v', - type=int, nargs='+', default=range(1,10), + type=int, nargs='+', help='ICESat-2 data versions to copy') # ICESat-2 granule region parser.add_argument('--granule','-g', @@ -121,7 +122,6 @@ def arguments(): # ICESat-2 orbital cycle parser.add_argument('--cycle','-c', type=int, nargs='+', - default=range(1,10), help='ICESat-2 orbital cycles to copy') # ICESat-2 reference ground tracks parser.add_argument('--track','-t', @@ -231,14 +231,22 @@ def scp_scf_files(client, client_ftp, scf_client, scf_client_ftp, remote_dir, scf_incoming, scf_outgoing, PRODUCT, RELEASE, VERSIONS, GRANULES, CYCLES, TRACKS, CLOBBER=False, LIST=False, MODE=0o775): # find ICESat-2 HDF5 files in the subdirectory for product and release - TRACKS = np.arange(1,1388) if not np.any(TRACKS) else TRACKS - CYCLES = np.arange(1,3) if not np.any(CYCLES) else CYCLES - GRANULES = np.arange(1,15) if not np.any(GRANULES) else GRANULES - VERSIONS = np.arange(1,10) if not np.any(VERSIONS) else VERSIONS - regex_track = '|'.join([rf'{T:04d}' for T in TRACKS]) - regex_cycle = '|'.join([rf'{C:02d}' for C in CYCLES]) - regex_granule = '|'.join([rf'{G:02d}' for G in GRANULES]) - regex_version = '|'.join([rf'{V:02d}' for V in VERSIONS]) + if TRACKS: + regex_track = r'|'.join([rf'{T:04d}' for T in TRACKS]) + else: + regex_track = r'\d{4}' + if CYCLES: + regex_cycle = r'|'.join([rf'{C:02d}' for C in CYCLES]) + else: + regex_cycle = r'\d{2}' + if GRANULES: + regex_granule = r'|'.join([rf'{G:02d}' for G in GRANULES]) + else: + regex_granule = r'\d{2}' + if VERSIONS: + regex_version = r'|'.join([rf'{V:02d}' for V in VERSIONS]) + else: + regex_version = r'\d{2}' # compile regular expression operator for extracting data from files args = (PRODUCT,regex_track,regex_cycle,regex_granule,RELEASE,regex_version) regex_pattern = (r'(processed_)?({0})(-\d{{2}})?_(\d{{4}})(\d{{2}})(\d{{2}})' diff --git a/scripts/symbolic_ICESat2_files.py b/scripts/symbolic_ICESat2_files.py index 2af2884..4743a92 100644 --- a/scripts/symbolic_ICESat2_files.py +++ b/scripts/symbolic_ICESat2_files.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" symbolic_ICESat2_files.py -Written by Tyler Sutterley (12/2022) +Written by Tyler Sutterley (09/2023) Creates symbolic links for ICESat-2 HDF5 files organized by date CALLING SEQUENCE: @@ -25,6 +25,7 @@ -M X, --mode X: permission mode of directories UPDATE HISTORY: + Updated 09/2023: generalized regular expressions for non-entered cases Updated 12/2022: use f-strings for ascii and verbose outputs Updated 11/2022: added option if SCF outgoing is a flattened structure Updated 05/2022: use argparse descriptions within sphinx documentation @@ -79,7 +80,7 @@ def arguments(): help='ICESat-2 data release to create symbolic links') # ICESat-2 data version parser.add_argument('--version','-v', - type=int, nargs='+', default=range(1,10), + type=int, nargs='+', help='ICESat-2 data versions to create symbolic links') # ICESat-2 granule region parser.add_argument('--granule','-g', @@ -89,7 +90,6 @@ def arguments(): # ICESat-2 orbital cycle parser.add_argument('--cycle','-c', type=int, nargs='+', - default=range(1,10), help='ICESat-2 orbital cycles to create symbolic links') # ICESat-2 reference ground tracks parser.add_argument('--track','-t', @@ -137,14 +137,22 @@ def main(): def symbolic_ICESat2_files(base_dir, scf_incoming, scf_outgoing, PRODUCT, RELEASE, VERSIONS, GRANULES, CYCLES, TRACKS, FLATTEN=True, MODE=0o775): # find ICESat-2 HDF5 files in the subdirectory for product and release - TRACKS = np.arange(1,1388) if not np.any(TRACKS) else TRACKS - CYCLES = np.arange(1,10) if not np.any(CYCLES) else CYCLES - GRANULES = np.arange(1,15) if not np.any(GRANULES) else GRANULES - VERSIONS = np.arange(1,10) if not np.any(VERSIONS) else VERSIONS - regex_track = '|'.join([rf'{T:04d}' for T in TRACKS]) - regex_cycle = '|'.join([rf'{C:02d}' for C in CYCLES]) - regex_granule = '|'.join([rf'{G:02d}' for G in GRANULES]) - regex_version = '|'.join([rf'{V:02d}' for V in VERSIONS]) + if TRACKS: + regex_track = r'|'.join([rf'{T:04d}' for T in TRACKS]) + else: + regex_track = r'\d{4}' + if CYCLES: + regex_cycle = r'|'.join([rf'{C:02d}' for C in CYCLES]) + else: + regex_cycle = r'\d{2}' + if GRANULES: + regex_granule = r'|'.join([rf'{G:02d}' for G in GRANULES]) + else: + regex_granule = r'\d{2}' + if VERSIONS: + regex_version = r'|'.join([rf'{V:02d}' for V in VERSIONS]) + else: + regex_version = r'\d{2}' # compile regular expression operator for extracting data from files args = (PRODUCT,regex_track,regex_cycle,regex_granule,RELEASE,regex_version) regex_pattern = (r'(processed_)?({0})(-\d{{2}})?_(\d{{4}})(\d{{2}})(\d{{2}})'