From dcdc04eebffe8eaffc2842222cc9ce86fe92bc74 Mon Sep 17 00:00:00 2001 From: Claire Porter Date: Mon, 21 Aug 2023 13:30:52 -0500 Subject: [PATCH 01/13] Add creation option for fileGDBs to bypass UTC warning --- index_setsm.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/index_setsm.py b/index_setsm.py index 57d442b..456cea2 100755 --- a/index_setsm.py +++ b/index_setsm.py @@ -527,7 +527,13 @@ def write_to_ogr_dataset(ogr_driver_str, ogrDriver, dst_ds, dst_lyr, groups, pai if not layer: logger.info("Creating table...") - layer = ds.CreateLayer(dst_lyr, tgt_srs, ogr.wkbMultiPolygon) + # FileGDB will throw a warning when inserting datetimes without this + if ogr_driver_str in ['FileGDB', 'OpenFileGDB']: + co = ['TIME_IN_UTC=NO'] + else: + co = [] + + layer = ds.CreateLayer(dst_lyr, tgt_srs, ogr.wkbMultiPolygon, options=co) if layer: for field_def in fld_defs: fname = fld_def_short_to_long_dict[field_def.fname] if args.long_fieldnames else field_def.fname @@ -1102,19 +1108,6 @@ def write_to_ogr_dataset(ogr_driver_str, ogrDriver, dst_ds, dst_lyr, groups, pai else: raise layer.CommitTransaction() - elif ogr_driver_str in ['OpenFileGDB', 'FileGDB']: - utils.GDAL_ERROR_HANDLER.reset_error_state() - try: - # Writing a date to a datetime field in GDB (date fields are not allowed) - # results in an error "Attempt at writing a datetime with a unknown time - # zone or local time in a layer that expects dates to be convertible to - # UTC. It will be written as if it was expressed in UTC." This happens - # even if the datetime string is expressed in proper ISO UTC syntax. - # Therefore, we have to refrain from catching warnings when writing to GDB. - with utils.GdalAllowWarnings(): - layer.CreateFeature(feat) - except Exception as e: - raise else: layer.CreateFeature(feat) From de348d1ca464798189a44a21f7dc5bf7197f19ef Mon Sep 17 00:00:00 2001 From: Claire Porter Date: Mon, 21 Aug 2023 13:48:19 -0500 Subject: [PATCH 02/13] Change strip DEM field types to boolean and datetime. --- index_setsm.py | 10 +++++----- lib/dem.py | 4 +++- lib/utils.py | 24 +++++++++++------------- package_setsm.py | 10 +++++----- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/index_setsm.py b/index_setsm.py index 456cea2..1dc9758 100755 --- a/index_setsm.py +++ b/index_setsm.py @@ -767,11 +767,11 @@ def write_to_ogr_dataset(ogr_driver_str, ogrDriver, dst_ds, dst_lyr, groups, pai 'AVGACQTM2': record.avg_acqtime2.strftime("%Y-%m-%d %H:%M:%S") if record.avg_acqtime2 is not None else None, 'CATALOGID1': record.catid1, 'CATALOGID2': record.catid2, - 'IS_LSF': int(record.is_lsf), - 'IS_XTRACK': int(record.is_xtrack), - 'EDGEMASK': int(record.mask_tuple[0]), - 'WATERMASK': int(record.mask_tuple[1]), - 'CLOUDMASK': int(record.mask_tuple[2]), + 'IS_LSF': record.is_lsf, + 'IS_XTRACK': record.is_xtrack, + 'EDGEMASK': record.mask_tuple[0], + 'WATERMASK': record.mask_tuple[1], + 'CLOUDMASK': record.mask_tuple[2], 'ALGM_VER': record.algm_version, 'S2S_VER': record.s2s_version, 'RMSE': record.rmse, diff --git a/lib/dem.py b/lib/dem.py index 39ce502..7dc753f 100644 --- a/lib/dem.py +++ b/lib/dem.py @@ -597,6 +597,8 @@ def __init__(self, filepath, md=None): self.rmse = -9999 self._set_density_and_stats_attribs() self._set_group_attribs_from_scenes() + self.is_xtrack = bool(self.is_xtrack) + self.is_lsf = bool(self.is_lsf) else: self.srcfp = filepath @@ -670,7 +672,7 @@ def __init__(self, filepath, md=None): self.release_version = groups[k] break - self.is_xtrack = 1 if xtrack_sensor_pattern.match(self.sensor1) else 0 + self.is_xtrack = True if xtrack_sensor_pattern.match(self.sensor1) else False self.is_dsp = False # Todo modify when dsp strips are a thing self.rmse = -9999 # if present, the metadata file value will overwrite this self.min_elev_value = None diff --git a/lib/utils.py b/lib/utils.py index 0a4b379..1cb6703 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -230,10 +230,10 @@ def __init__(self, print_warnings=None): StandardAttribute("PAIRNAME", "", ogr.OFTString, 64, 0), StandardAttribute("SENSOR1", "", ogr.OFTString, 8, 0), StandardAttribute("SENSOR2", "", ogr.OFTString, 8, 0), - StandardAttribute("ACQDATE1", "", ogr.OFTString, 32, 0), - StandardAttribute("ACQDATE2", "", ogr.OFTString, 32, 0), - StandardAttribute("AVGACQTM1", "", ogr.OFTString, 32, 0), - StandardAttribute("AVGACQTM2", "", ogr.OFTString, 32, 0), + StandardAttribute("ACQDATE1", "", ogr.OFTDateTime, 32, 0), + StandardAttribute("ACQDATE2", "", ogr.OFTDateTime, 32, 0), + StandardAttribute("AVGACQTM1", "", ogr.OFTDateTime, 32, 0), + StandardAttribute("AVGACQTM2", "", ogr.OFTDateTime, 32, 0), StandardAttribute("CATALOGID1", "", ogr.OFTString, 32, 0), StandardAttribute("CATALOGID2", "", ogr.OFTString, 32, 0), StandardAttribute("CENT_LAT", "", ogr.OFTReal, 0, 0), @@ -246,15 +246,14 @@ def __init__(self, print_warnings=None): StandardAttribute("PROJ4", "", ogr.OFTString, 100, 0), StandardAttribute("ND_VALUE", "", ogr.OFTReal, 0, 0), StandardAttribute("DEM_RES", "", ogr.OFTReal, 0, 0), - StandardAttribute("CR_DATE", "", ogr.OFTString, 32, 0), - # StandardAttribute("CR_DATE", "", ogr.OFTDateTime, 32, 0), + StandardAttribute("CR_DATE", "", ogr.OFTDateTime, 32, 0), StandardAttribute("ALGM_VER", "", ogr.OFTString, 32, 0), StandardAttribute("S2S_VER", "", ogr.OFTString, 32, 0), - StandardAttribute("IS_LSF", "", ogr.OFTInteger, 8, 8), - StandardAttribute("IS_XTRACK", "", ogr.OFTInteger, 8, 8), - StandardAttribute("EDGEMASK", "", ogr.OFTInteger, 8, 8), - StandardAttribute("WATERMASK", "", ogr.OFTInteger, 8, 8), - StandardAttribute("CLOUDMASK", "", ogr.OFTInteger, 8, 8), + StandardAttribute("IS_LSF", "", ogr.OFSTBoolean, 0, 0), + StandardAttribute("IS_XTRACK", "", ogr.OFSTBoolean, 0, 0), + StandardAttribute("EDGEMASK", "", ogr.OFSTBoolean, 0, 0), + StandardAttribute("WATERMASK", "", ogr.OFSTBoolean, 0, 0), + StandardAttribute("CLOUDMASK", "", ogr.OFSTBoolean, 0, 0), StandardAttribute("MASK_DENS", "", ogr.OFTReal, 0, 0), StandardAttribute("VALID_DENS", "", ogr.OFTReal, 0, 0), StandardAttribute("VALID_AREA", "", ogr.OFTReal, 0, 0), @@ -285,8 +284,7 @@ def __init__(self, print_warnings=None): StandardAttribute("FILESZ_MT", "", ogr.OFTReal, 0, 0), StandardAttribute("FILESZ_OR", "", ogr.OFTReal, 0, 0), StandardAttribute("FILESZ_OR2", "", ogr.OFTReal, 0, 0), - StandardAttribute("INDEX_DATE", "", ogr.OFTString, 32, 0), - # StandardAttribute("INDEX_DATE", "", ogr.OFTDateTime, 32, 0), + StandardAttribute("INDEX_DATE", "", ogr.OFTDateTime, 32, 0), ] DEM_ATTRIBUTE_DEFINITION_RELVER = [ diff --git a/package_setsm.py b/package_setsm.py index c33a4e4..a581ab1 100755 --- a/package_setsm.py +++ b/package_setsm.py @@ -495,11 +495,11 @@ def build_archive(raster, scratch, args): 'DEM_RES': (raster.xres + raster.yres) / 2.0, 'ALGM_VER': raster.algm_version, 'S2S_VER': raster.s2s_version, - 'IS_LSF': int(raster.is_lsf), - 'IS_XTRACK': int(raster.is_xtrack), - 'EDGEMASK': int(raster.mask_tuple[0]), - 'WATERMASK': int(raster.mask_tuple[1]), - 'CLOUDMASK': int(raster.mask_tuple[2]), + 'IS_LSF': raster.is_lsf, + 'IS_XTRACK': raster.is_xtrack, + 'EDGEMASK': raster.mask_tuple[0], + 'WATERMASK': raster.mask_tuple[1], + 'CLOUDMASK': raster.mask_tuple[2], 'RMSE': raster.rmse } From e200281447c05ee66552653b6fb8c4123c374f78 Mon Sep 17 00:00:00 2001 From: Claire Porter Date: Mon, 21 Aug 2023 13:48:52 -0500 Subject: [PATCH 03/13] Remove errant semicolon --- lib/dem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dem.py b/lib/dem.py index 7dc753f..ade7c0b 100644 --- a/lib/dem.py +++ b/lib/dem.py @@ -710,7 +710,7 @@ def get_geom_wgs84(self): ctf = osr.CoordinateTransformation(srs, srs_wgs84) geom.Transform(ctf) - return geom; + return geom def get_geocell(self): if not self.geocell: From 246ddbb1f36de2575732e2a6db45ca4c505b77d0 Mon Sep 17 00:00:00 2001 From: Claire Porter Date: Mon, 21 Aug 2023 13:50:04 -0500 Subject: [PATCH 04/13] Remove option to include registration sources --- index_setsm.py | 34 ++++++++++++++++------------------ lib/dem.py | 6 +++--- lib/utils.py | 34 ++++++++++++++++++++++++++++++++-- 3 files changed, 51 insertions(+), 23 deletions(-) diff --git a/index_setsm.py b/index_setsm.py index 1dc9758..8c7c5fc 100755 --- a/index_setsm.py +++ b/index_setsm.py @@ -115,10 +115,9 @@ def main(): )) parser.add_argument('--status', help='custom value for status field') parser.add_argument('--status-dsp-record-mode-orig', help='custom value for status field when dsp-record-mode is set to "orig"') - parser.add_argument('--include-registration', action='store_true', default=False, - help='include registration info if present (mode=strip and tile only)') - parser.add_argument('--include-relver', action='store_true', default=False, - help='include release version info if present (mode=strip and tile only)') + # DEPRECATED + # parser.add_argument('--include-registration', action='store_true', default=False, + # help='include registration info if present (mode=strip and tile only)') parser.add_argument('--use-release-fields', action='store_true', default=False, help="use field definitions for tile release indices (mode=tile only)") parser.add_argument('--long-fieldnames', action='store_true', default=False, @@ -398,9 +397,8 @@ def main(): fld_defs_base = utils.TILE_DEM_ATTRIBUTE_DEFINITIONS_RELEASE if args.mode == 'strip' and args.search_masked: suffix = mask_strip_suffixes + tuple([suffix]) - fld_defs = fld_defs_base + reg_fld_defs if args.include_registration else fld_defs_base - if args.include_relver and not (args.mode == 'tile' and args.use_release_fields): - fld_defs = fld_defs + utils.DEM_ATTRIBUTE_DEFINITION_RELVER + # fld_defs = fld_defs_base + reg_fld_defs if args.include_registration else fld_defs_base - DEPRECATED + fld_defs = fld_defs_base src_fps = [] records = [] logger.info('Source: {}'.format(src)) @@ -807,17 +805,17 @@ def write_to_ogr_dataset(ogr_driver_str, ogrDriver, dst_ds, dst_lyr, groups, pai val = getattr(record, a) attrib_map[f] = round(val, 6) if val is not None else -9999 - ## If registration info exists - if args.include_registration: - if len(record.reginfo_list) > 0: - for reginfo in record.reginfo_list: - if reginfo.name == 'ICESat': - attrib_map["DX"] = reginfo.dx - attrib_map["DY"] = reginfo.dy - attrib_map["DZ"] = reginfo.dz - attrib_map["REG_SRC"] = 'ICESat' - attrib_map["NUM_GCPS"] = reginfo.num_gcps - attrib_map["MEANRESZ"] = reginfo.mean_resid_z + ## If registration info exists - DEPRECATED + # if args.include_registration: + # if len(record.reginfo_list) > 0: + # for reginfo in record.reginfo_list: + # if reginfo.name == 'ICESat': + # attrib_map["DX"] = reginfo.dx + # attrib_map["DY"] = reginfo.dy + # attrib_map["DZ"] = reginfo.dz + # attrib_map["REG_SRC"] = 'ICESat' + # attrib_map["NUM_GCPS"] = reginfo.num_gcps + # attrib_map["MEANRESZ"] = reginfo.mean_resid_z ## Set path folders for use if path_prefix specified if path_prefix: diff --git a/lib/dem.py b/lib/dem.py index ade7c0b..235ef1f 100644 --- a/lib/dem.py +++ b/lib/dem.py @@ -1772,7 +1772,7 @@ def __init__(self, srcfp, md=None): if match: groups = match.groupdict() self.tilename = groups['tile'] - self.res = groups['res'] + self.res_str = groups['res'] # In case release version is in the file name and not the meta.txt self.release_version = groups['relversion'].strip('v') if groups['relversion'] else None self.subtile = groups['subtile'] @@ -1795,10 +1795,10 @@ def __init__(self, srcfp, md=None): self.regmetapath = os.path.join(self.srcdir, self.tileid + '_reg.txt') if self.scheme: - self.supertile_id = '_'.join([self.scheme,self.tilename,self.res]) + self.supertile_id = '_'.join([self.scheme,self.tilename,self.res_str]) self.supertile_id_no_res = '_'.join([self.scheme,self.tilename]) else: - self.supertile_id = '_'.join([self.tilename,self.res]) + self.supertile_id = '_'.join([self.tilename,self.res_str]) self.supertile_id_no_res = self.tilename self.tile_id_no_res = '_'.join([self.supertile_id_no_res, self.subtile]) if self.subtile else self.supertile_id_no_res self.density = None diff --git a/lib/utils.py b/lib/utils.py index 1cb6703..87d8725 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -287,8 +287,38 @@ def __init__(self, print_warnings=None): StandardAttribute("INDEX_DATE", "", ogr.OFTDateTime, 32, 0), ] -DEM_ATTRIBUTE_DEFINITION_RELVER = [ - StandardAttribute("REL_VER", "", ogr.OFTString, 20, 0) +DEM_ATTRIBUTE_DEFINITIONS_RELEASE = [ + StandardAttribute("DEM_ID", "", ogr.OFTString, 254, 0), + StandardAttribute("PAIRNAME", "", ogr.OFTString, 64, 0), + StandardAttribute("STRIPDEMID", "", ogr.OFTString, 254, 0), + StandardAttribute("SENSOR1", "", ogr.OFTString, 8, 0), + StandardAttribute("SENSOR2", "", ogr.OFTString, 8, 0), + StandardAttribute("CATALOGID1", "", ogr.OFTString, 32, 0), + StandardAttribute("CATALOGID2", "", ogr.OFTString, 32, 0), + StandardAttribute("ACQDATE1", "", ogr.OFTDateTime, 32, 0), + StandardAttribute("ACQDATE2", "", ogr.OFTDateTime, 32, 0), + StandardAttribute("GSD", "", ogr.OFTReal, 0, 0), + StandardAttribute("EPSG", "", ogr.OFTInteger, 8, 8), + StandardAttribute("SETSM_VER", "", ogr.OFTString, 32, 0), + StandardAttribute("S2S_VER", "", ogr.OFTString, 32, 0), + StandardAttribute("CR_DATE", "CREATIONDATE", ogr.OFTDateTime, 32, 0), + StandardAttribute("GEOCELL", "", ogr.OFTString, 10, 0), + StandardAttribute("IS_LSF", "", ogr.OFSTBoolean, 0, 0), + StandardAttribute("IS_XTRACK", "", ogr.OFSTBoolean, 0, 0), + StandardAttribute("VALID_DENS", "VALID_AREA_MATCHTAG_DENSITY", ogr.OFTReal, 0, 0), + StandardAttribute("VALID_AREA", "VALID_AREA_SQKM", ogr.OFTReal, 0, 0), + StandardAttribute("VALID_PERC", "VALID_AREA_PERCENT", ogr.OFTReal, 0, 0), + StandardAttribute("WATER_AREA", "WATER_AREA_SQKM", ogr.OFTReal, 0, 0), + StandardAttribute("WATER_PERC", "WATER_AREA_PERCENT", ogr.OFTReal, 0, 0), + StandardAttribute("CLOUD_AREA", "CLOUD_AREA_SQKM", ogr.OFTReal, 0, 0), + StandardAttribute("CLOUD_PERC", "CLOUD_AREA_PERCENT", ogr.OFTReal, 0, 0), + StandardAttribute("AVGCONVANG", "AVG_CONVERGENCE_ANGLE", ogr.OFTReal, 0, 0), + StandardAttribute("AVG_HT_ACC", "AVG_EXPECTED_HEIGHT_ACCURACY", ogr.OFTReal, 0, 0), + StandardAttribute("AVG_SUNEL1", "AVG_SUN_ELEV1", ogr.OFTReal, 0, 0), + StandardAttribute("AVG_SUNEL2", "AVG_SUN_ELEV2", ogr.OFTReal, 0, 0), + StandardAttribute("RMSE", "", ogr.OFTReal, 0, 0), + StandardAttribute("FILEURL", "", ogr.OFTString, 254, 0), + StandardAttribute("S3URL", "", ogr.OFTString, 254, 0), ] SCENE_ATTRIBUTE_DEFINITIONS_BASIC = [ From 9e34674168053d1e71038d4d14fcbf798dc539e6 Mon Sep 17 00:00:00 2001 From: Claire Porter Date: Mon, 21 Aug 2023 14:43:44 -0500 Subject: [PATCH 05/13] Make option to create release-style indexes for DEM strips --- index_setsm.py | 125 ++++++++++++++++++++++++++++++------------------- lib/utils.py | 5 ++ 2 files changed, 83 insertions(+), 47 deletions(-) diff --git a/index_setsm.py b/index_setsm.py index 8c7c5fc..46b252e 100755 --- a/index_setsm.py +++ b/index_setsm.py @@ -56,6 +56,7 @@ recordid_map = { 'scene': '{SCENEDEMID}|{STRIPDEMID}|{IS_DSP}|{LOCATION}|{INDEX_DATE}', 'strip': '{DEM_ID}|{STRIPDEMID}|{LOCATION}|{INDEX_DATE}', + 'strip_release': '{DEM_ID}|{STRIPDEMID}|{FILEURL}|{CR_DATE}', 'tile': '{DEM_ID}|{TILE}|{LOCATION}|{INDEX_DATE}', 'tile_release': '{DEM_ID}|{TILE}|{FILEURL}|{CR_DATE}', } @@ -78,6 +79,11 @@ 'both': 'write a record for each' } +dem_type_folder_lookup = { + 'strip': 'strips', + 'tile': 'mosaics', +} + DEFAULT_DSP_OPTION = 'dsp' # handle unicode in Python 3 @@ -147,15 +153,15 @@ def main(): parser.add_argument("--write-pickle", help="store region lookup in a pickle file. skipped if --write-json is used") parser.add_argument("--read-pickle", help='read region lookup from a pickle file. skipped if --write-json is used') parser.add_argument("--custom-paths", choices=custom_path_prefixes.keys(), help='Use custom path schema') - parser.add_argument('--project', choices=PROJECTS.keys(), help='project name (required when writing tiles)') + parser.add_argument('--project', choices=utils.PROJECTS.keys(), help='project name (required when writing tiles)') parser.add_argument('--debug', action='store_true', default=False, help='print DEBUG level logger messages to terminal') parser.add_argument('--dryrun', action='store_true', default=False, help='run script without inserting records') parser.add_argument('--np', action='store_true', default=False, help='do not print progress bar') parser.add_argument('--version', action='version', version=f"Current version: {SHORT_VERSION}", help='print version and exit') - parser.add_argument('--release-fileurl', type=str, default="https://data.pgc.umn.edu/elev/dem/setsm//mosaic/v///.tar.gz", + parser.add_argument('--release-fileurl', type=str, default="https://data.pgc.umn.edu/elev/dem/setsm//////.tar.gz", help="template for release field 'fileurl' (--use-release-fields only)") - parser.add_argument('--release-s3url', type=str, default="https://polargeospatialcenter.github.io/stac-browser/#/external/pgc-opendata-dems.s3.us-west-2.amazonaws.com//mosaics/v///.json", + parser.add_argument('--release-s3url', type=str, default="https://polargeospatialcenter.github.io/stac-browser/#/external/pgc-opendata-dems.s3.us-west-2.amazonaws.com//////.json", help="template for release field 's3url' (--use-release-fields only)") #### Parse Arguments args = parser.parse_args() @@ -189,6 +195,12 @@ def main(): if args.mode == 'tile' and not args.project: parser.error("--project option is required if when mode=tile") + if args.mode == 'strip' and args.use_release_fields and not args.project: + parser.error("--project option is required if when mode=strip wit--use-release-fields") + + if args.mode == 'scene' and args.use_release_fields: + parser.error("--use-release-fields option is not applicable to mode=scene") + ## Todo add Bp region lookup via API instead of Danco? if args.skip_region_lookup and (args.custom_paths == 'PGC' or args.custom_paths == 'BP'): parser.error('--skip-region-lookup is not compatible with --custom-paths = PGC or BP') @@ -395,6 +407,8 @@ def main(): dem_class, suffix, groupid_fld, fld_defs_base, reg_fld_defs = MODES[args.mode] if args.mode == 'tile' and args.use_release_fields: fld_defs_base = utils.TILE_DEM_ATTRIBUTE_DEFINITIONS_RELEASE + if args.mode == 'strip' and args.use_release_fields: + fld_defs_base = utils.DEM_ATTRIBUTE_DEFINITIONS_RELEASE if args.mode == 'strip' and args.search_masked: suffix = mask_strip_suffixes + tuple([suffix]) # fld_defs = fld_defs_base + reg_fld_defs if args.include_registration else fld_defs_base - DEPRECATED @@ -639,7 +653,7 @@ def write_to_ogr_dataset(ogr_driver_str, ogrDriver, dst_ds, dst_lyr, groups, pai for k in record.filesz_attrib_map: attrib_map[k.upper()] = getattr(record,'{}{}'.format(attr_pfx,k)) - # TODO revisit after all incorrect 50cminfo.txt files are ingested + # TODO revisit after all incorrect 50cminfo.txt files are ingested # Overwrite original res dsp filesz values will Null if dsp_mode == 'orig': for k in record.filesz_attrib_map: @@ -719,7 +733,7 @@ def write_to_ogr_dataset(ogr_driver_str, ogrDriver, dst_ds, dst_lyr, groups, pai valid_record = False else: - pretty_project = PROJECTS[region.split('_')[0]] + pretty_project = utils.PROJECTS[region.split('_')[0]] custom_path = '/'.join([ path_prefix, @@ -854,7 +868,7 @@ def write_to_ogr_dataset(ogr_driver_str, ogrDriver, dst_ds, dst_lyr, groups, pai valid_record = False else: - pretty_project = PROJECTS[region.split('_')[0]] + pretty_project = utils.PROJECTS[region.split('_')[0]] custom_path = '/'.join([ path_prefix, @@ -996,46 +1010,63 @@ def write_to_ogr_dataset(ogr_driver_str, ogrDriver, dst_ds, dst_lyr, groups, pai mp_geom = ogr.ForceToMultiPolygon(temp_geom) feat_geom = mp_geom - ## Fields for tile DEM - if args.mode == 'tile' and args.use_release_fields: - tile_to_general_attrib_name = { - 'GSD': 'DEM_RES', - 'RELEASEVER': 'REL_VER', - 'DATA_PERC': 'DENSITY', - } - remove_attrib_names = sorted(list(set.difference( - set([attr.fname for attr in utils.TILE_DEM_ATTRIBUTE_DEFINITIONS]), - set([attr.fname for attr in utils.TILE_DEM_ATTRIBUTE_DEFINITIONS_RELEASE]), - ))) - - for tname, gname in tile_to_general_attrib_name.items(): - if gname in attrib_map: - attrib_map[tname] = attrib_map[gname] - del attrib_map[gname] - for fname in remove_attrib_names: - if fname in attrib_map: - del attrib_map[fname] - - if args.release_fileurl: - filurl = args.release_fileurl - pretty_project = PROJECTS[args.project] - pretty_res = '{}m'.format(str(int(attrib_map['GSD']))) - filurl = filurl.replace('', pretty_project) - filurl = filurl.replace('', record.release_version) - filurl = filurl.replace('', pretty_res) - filurl = filurl.replace('', record.supertile_id_no_res) - filurl = filurl.replace('', record.tileid) - attrib_map['FILEURL'] = filurl - - if args.release_s3url: - s3url = args.release_s3url - pretty_res = '{}m'.format(str(int(attrib_map['GSD']))) - s3url = s3url.replace('', args.project) - s3url = s3url.replace('', record.release_version) - s3url = s3url.replace('', pretty_res) - s3url = s3url.replace('', record.supertile_id_no_res) - s3url = s3url.replace('', record.tileid) - attrib_map['S3URL'] = s3url + ## Convert fields for tile and strip DEM to release format + if args.use_release_fields: + tile_to_general_attrib_name = { + 'GSD': 'DEM_RES', + 'RELEASEVER': 'REL_VER', + 'DATA_PERC': 'DENSITY', + 'ACQDATE1': 'AVGACQTM1', + 'ACQDATE2': 'AVGACQTM2', + } + + if args.mode == 'tile': + fdefs1 = utils.TILE_DEM_ATTRIBUTE_DEFINITIONS + fdefs2 = utils.TILE_DEM_ATTRIBUTE_DEFINITIONS_RELEASE + version = f'v{record.release_version}' + group = record.supertile_id_no_res + elif args.mode == 'strip': + fdefs1 = utils.DEM_ATTRIBUTE_DEFINITIONS + fdefs2 = utils.DEM_ATTRIBUTE_DEFINITIONS_RELEASE + version = record.release_version + group = record.geocell + else: + msg = '--use-release-fields used with an incompatible mode' + raise RuntimeError(msg) + + remove_attrib_names = sorted(list(set.difference( + set([attr.fname for attr in fdefs1]), + set([attr.fname for attr in fdefs2]), + ))) + + for tname, gname in tile_to_general_attrib_name.items(): + if gname in attrib_map: + attrib_map[tname] = attrib_map[gname] + del attrib_map[gname] + for fname in remove_attrib_names: + if fname in attrib_map: + del attrib_map[fname] + + if args.release_fileurl: + filurl = args.release_fileurl + pretty_project = utils.PROJECTS[args.project] + filurl = filurl.replace('', pretty_project) + filurl = filurl.replace('', dem_type_folder_lookup[args.mode]) + filurl = filurl.replace('', version) + filurl = filurl.replace('', record.res_str) + filurl = filurl.replace('', group) + filurl = filurl.replace('', record.id) + attrib_map['FILEURL'] = filurl + + if args.release_s3url: + s3url = args.release_s3url + s3url = s3url.replace('', args.project) + s3url = s3url.replace('', dem_type_folder_lookup[args.mode]) + s3url = s3url.replace('', version) + s3url = s3url.replace('', record.res_str) + s3url = s3url.replace('', group) + s3url = s3url.replace('', record.id) + attrib_map['S3URL'] = s3url ## Write feature if valid_record: @@ -1079,7 +1110,7 @@ def write_to_ogr_dataset(ogr_driver_str, ogrDriver, dst_ds, dst_lyr, groups, pai else: if not args.dryrun: # Store record identifiers for later checking - recordid_mode = 'tile_release' if args.mode == 'tile' and args.use_release_fields else args.mode + recordid_mode = args.mode + '_release' if args.use_release_fields else args.mode recordids.append(recordid_map[recordid_mode].format(**attrib_map)) # Append record diff --git a/lib/utils.py b/lib/utils.py index 87d8725..ae91236 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -152,6 +152,11 @@ def __init__(self, print_warnings=None): # at the top of only the scripts in which you want to handle warnings. setup_gdal_error_handler(catch_warnings=True, print_uncaught_warnings=True) +PROJECTS = { + 'arcticdem': 'ArcticDEM', + 'rema': 'REMA', + 'earthdem': 'EarthDEM', +} # Copy DEM global vars deliv_suffixes = ( From 67a9011d3f48b9f6138de2833c0afcc4fe418cbc Mon Sep 17 00:00:00 2001 From: Claire Porter Date: Mon, 21 Aug 2023 17:11:42 -0500 Subject: [PATCH 06/13] Change package index shps to use release-style field names --- index_setsm.py | 2 - package_setsm.py | 103 ++++++++++++++++++++++++----------------- package_setsm_tiles.py | 86 +++++++++++++++++++++++----------- 3 files changed, 120 insertions(+), 71 deletions(-) diff --git a/index_setsm.py b/index_setsm.py index 46b252e..f784768 100755 --- a/index_setsm.py +++ b/index_setsm.py @@ -559,8 +559,6 @@ def write_to_ogr_dataset(ogr_driver_str, ogrDriver, dst_ds, dst_lyr, groups, pai ftype = ogr.OFTInteger fstype = field_def.ftype fwidth = field_def.fwidth - # elif field_def.ftype == ogr.OFTDateTime and ogr_driver_str in ['FileGDB', 'OpenFileGDB']: - # ftype = ogr.OFTString else: ftype = field_def.ftype fwidth = field_def.fwidth diff --git a/package_setsm.py b/package_setsm.py index a581ab1..0dfd4d2 100755 --- a/package_setsm.py +++ b/package_setsm.py @@ -40,21 +40,28 @@ def main(): #### Positional Arguments parser.add_argument('src', help="source directory, text file of file paths, or dem") parser.add_argument('scratch', help="scratch space to build index shps") - + parser.add_argument('project', choices=utils.PROJECTS.keys(), help='project name') + #### Optional Arguments parser.add_argument('--skip-cog', action='store_true', default=False, help="skip COG conversion and build archive with existing tiffs") parser.add_argument('--skip-archive', action='store_true', default=False, help="build mdf and readme files and convert rasters to COG, do not archive") - parser.add_argument('--rasterproxy-prefix', - help="build rasterProxy .mrf files using this s3 bucket and path prefix\ - for the source data path with geocell folder and dem tif appended (must start with s3://)") + parser.add_argument('--build-rasterproxies', action='store_true', default=False, + help='build rasterproxy .mrf files') parser.add_argument('--filter-dems', action='store_true', default=False, help="remove dems with valid (masked) area < {} sqkm or masked density < {}".format( VALID_AREA_THRESHOLD, DENSITY_THRESHOLD)) parser.add_argument('--force-filter-dems', action='store_true', default=False, help="remove already-packaged DEMs with valid (masked) area < {} sqkm or masked density < {}".format( VALID_AREA_THRESHOLD, DENSITY_THRESHOLD)) + parser.add_argument('--rasterproxy-prefix', + default="s3://pgc-opendata-dems//////", + help="template for rasterproxy .mrf file s3 path") + parser.add_argument('--release-fileurl', type=str, default="https://data.pgc.umn.edu/elev/dem/setsm//////.tar.gz", + help="template for release field 'fileurl'") + parser.add_argument('--release-s3url', type=str, default="https://polargeospatialcenter.github.io/stac-browser/#/external/pgc-opendata-dems.s3.us-west-2.amazonaws.com//////.json", + help="template for release field 's3url'") parser.add_argument('-v', action='store_true', default=False, help="verbose output") parser.add_argument('--overwrite', action='store_true', default=False, help="overwrite existing index") @@ -88,8 +95,8 @@ def main(): log_level = logging.INFO # Check raster proxy prefix is well-formed - if args.rasterproxy_prefix and not args.rasterproxy_prefix.startswith('s3://'): - parser.error('--rasterproxy-prefix (e.g. s3://pgc-opendata-dems/arcticdem/strips/s2s041/2m)') + if args.build_rasterproxies and not args.rasterproxy_prefix.startswith('s3://'): + parser.error('--rasterproxy-prefix must start with s3:// (e.g. s3://pgc-opendata-dems/arcticdem/strips/s2s041/2m)') lsh = logging.StreamHandler() lsh.setLevel(log_level) @@ -327,7 +334,14 @@ def build_archive(raster, scratch, args): mrf_tifs = [c for c in components if c[0].endswith(('dem.tif', 'bitmask.tif'))] if args.rasterproxy_prefix: logger.info("Creating raster proxy files") - rasterproxy_prefix_parts = args.rasterproxy_prefix.split('/') + s3url = args.rasterproxy_prefix + s3url = s3url.replace('', args.project) + s3url = s3url.replace('', 'strips') + s3url = s3url.replace('', raster.release_version) + s3url = s3url.replace('', raster.res_str) + s3url = s3url.replace('', raster.geocell) + s3url = s3url.replace('', raster.id) + rasterproxy_prefix_parts = s3url.split('/') bucket = rasterproxy_prefix_parts[2] bpath = '/'.join(rasterproxy_prefix_parts[3:]).strip(r'/') sourceprefix = '/vsicurl/http://{}.s3.us-west-2.amazonaws.com/{}'.format(bucket, bpath) @@ -336,18 +350,8 @@ def build_archive(raster, scratch, args): suffix = tif[len(raster.stripid):-4] # eg "_dem" mrf = '{}{}.mrf'.format(raster.stripid, suffix) if not os.path.isfile(mrf): - sourcepath = '{}/{}/{}{}.tif'.format( - sourceprefix, - raster.geocell, - raster.stripid, - suffix - ) - datapath = '{}/{}/{}{}.mrfcache'.format( - dataprefix, - raster.geocell, - raster.stripid, - suffix - ) + sourcepath = f'{sourceprefix}{suffix}.tif' + datapath = f'{dataprefix}{suffix}.mrfcache' static_args = '-q -of MRF -co BLOCKSIZE=512 -co "UNIFORM_SCALE=2" -co COMPRESS=LERC -co NOCOPY=TRUE' cmd = 'gdal_translate {0} -co INDEXNAME={1} -co DATANAME={1} -co CACHEDSOURCE={2} {3} {4}'.format( static_args, @@ -455,18 +459,26 @@ def build_archive(raster, scratch, args): ds = ogrDriver.CreateDataSource(index) if ds is not None: - lyr = ds.CreateLayer(index_lyr, tgt_srs, ogr.wkbPolygon) + lyr = ds.CreateLayer(index_lyr, tgt_srs, ogr.wkbMultiPolygon) if lyr is not None: - for field_def in utils.DEM_ATTRIBUTE_DEFINITIONS_BASIC: + for field_def in utils.DEM_ATTRIBUTE_DEFINITIONS_RELEASE: + fname = field_def.fname.lower() + fstype = None if field_def.ftype == ogr.OFTDateTime: ftype = ogr.OFTString fwidth = 28 + elif field_def.ftype == ogr.OFSTBoolean: + ftype = ogr.OFTInteger + fstype = field_def.ftype + fwidth = field_def.fwidth else: ftype = field_def.ftype fwidth = field_def.fwidth - field = ogr.FieldDefn(field_def.fname, ftype) + field = ogr.FieldDefn(fname, ftype) + if fstype: + field.SetSubType(fstype) field.SetWidth(fwidth) field.SetPrecision(field_def.fprecision) lyr.CreateField(field) @@ -477,29 +489,21 @@ def build_archive(raster, scratch, args): ## Set fields attrib_map = { 'DEM_ID': raster.stripid, - 'STRIPDEMID': raster.stripdemid, 'PAIRNAME': raster.pairname, + 'STRIPDEMID': raster.stripdemid, 'SENSOR1': raster.sensor1, 'SENSOR2': raster.sensor2, - 'ACQDATE1': raster.acqdate1.strftime('%Y-%m-%d'), - 'ACQDATE2': raster.acqdate2.strftime('%Y-%m-%d'), - 'AVGACQTM1': raster.avg_acqtime1.strftime("%Y-%m-%d %H:%M:%S"), - 'AVGACQTM2': raster.avg_acqtime2.strftime("%Y-%m-%d %H:%M:%S"), 'CATALOGID1': raster.catid1, 'CATALOGID2': raster.catid2, - 'GEOCELL': raster.geocell, - - 'PROJ4': raster.proj4, + 'ACQDATE1': raster.avg_acqtime1.strftime("%Y-%m-%d %H:%M:%S"), + 'ACQDATE2': raster.avg_acqtime2.strftime("%Y-%m-%d %H:%M:%S"), + 'GSD': (raster.xres + raster.yres) / 2.0, 'EPSG': raster.epsg, - 'ND_VALUE': raster.ndv, - 'DEM_RES': (raster.xres + raster.yres) / 2.0, - 'ALGM_VER': raster.algm_version, + 'SETSM_VER': raster.algm_version, 'S2S_VER': raster.s2s_version, + 'GEOCELL': raster.geocell, 'IS_LSF': raster.is_lsf, 'IS_XTRACK': raster.is_xtrack, - 'EDGEMASK': raster.mask_tuple[0], - 'WATERMASK': raster.mask_tuple[1], - 'CLOUDMASK': raster.mask_tuple[2], 'RMSE': raster.rmse } @@ -509,7 +513,27 @@ def build_archive(raster, scratch, args): for f, a in utils.field_attrib_map.items(): val = getattr(raster, a) - attrib_map[f] = round(val, 6) if val is not None else -9999 + if not f in ['MASK_DENS']: + attrib_map[f] = round(val, 6) if val is not None else -9999 + + filurl = args.release_fileurl + pretty_project = utils.PROJECTS[args.project] + filurl = filurl.replace('', pretty_project) + filurl = filurl.replace('', 'strips') + filurl = filurl.replace('', raster.release_version) + filurl = filurl.replace('', raster.res_str) + filurl = filurl.replace('', raster.geocell) + filurl = filurl.replace('', raster.id) + attrib_map['FILEURL'] = filurl + + s3url = args.release_s3url + s3url = s3url.replace('', args.project) + s3url = s3url.replace('', 'strips') + s3url = s3url.replace('', raster.release_version) + s3url = s3url.replace('', raster.res_str) + s3url = s3url.replace('', raster.geocell) + s3url = s3url.replace('', raster.id) + attrib_map['S3URL'] = s3url ## transform and write geom src_srs = utils.osr_srs_preserve_axis_order(osr.SpatialReference()) @@ -528,11 +552,6 @@ def build_archive(raster, scratch, args): valid_record = False else: - ## Get centroid coordinates - centroid = temp_geom.Centroid() - attrib_map['CENT_LAT'] = centroid.GetY() - attrib_map['CENT_LON'] = centroid.GetX() - ## If srs is geographic and geom crosses 180, split geom into 2 parts if tgt_srs.IsGeographic: diff --git a/package_setsm_tiles.py b/package_setsm_tiles.py index f604cc6..adf8783 100755 --- a/package_setsm_tiles.py +++ b/package_setsm_tiles.py @@ -35,17 +35,26 @@ def main(): #### Positional Arguments parser.add_argument('src', help="source directory or dem") parser.add_argument('scratch', help="scratch space to build index shps") + parser.add_argument('project', choices=utils.PROJECTS.keys(), help='project name') #### Optionsl Arguments parser.add_argument('--epsg', type=int, default=default_epsg, help="egsg code for output index projection (default epsg:{})".format(default_epsg)) parser.add_argument('--skip-archive', action='store_true', default=False, - help="build mdf and readme files and convert rasters to COG, do not archive") - parser.add_argument('--rasterproxy-prefix', - help="build rasterProxy .mrf files using this s3 bucket and path prefix\ - for the source data path with geocell folder and dem tif appended (must start with s3://)") + help="build mdf and readme files and convert rasters to COG, do not build archive") + parser.add_argument('--build-rasterproxies', action='store_true', default=False, + help='build rasterproxy .mrf files') parser.add_argument('--convert-to-cog', action='store_true', default=False, help="convert dem files to COG before building the archive") + parser.add_argument('--rasterproxy-prefix', + default="s3://pgc-opendata-dems//////", + help="template for rasterproxy .mrf file s3 path") + parser.add_argument('--release-fileurl', type=str, + default="https://data.pgc.umn.edu/elev/dem/setsm//////.tar.gz", + help="template for release field 'fileurl'") + parser.add_argument('--release-s3url', type=str, + default="https://polargeospatialcenter.github.io/stac-browser/#/external/pgc-opendata-dems.s3.us-west-2.amazonaws.com//////.json", + help="template for release field 's3url'") parser.add_argument('-v', action='store_true', default=False, help="verbose output") parser.add_argument('--overwrite', action='store_true', default=False, help="overwrite existing index") @@ -261,6 +270,13 @@ def build_archive(raster, scratch, args): mrf_tifs = [c for c in components + optional_components if c.endswith('dem.tif') and os.path.isfile(c)] if args.rasterproxy_prefix: logger.info("Creating raster proxy files") + s3url = args.rasterproxy_prefix + s3url = s3url.replace('', args.project) + s3url = s3url.replace('', 'mosaics') + s3url = s3url.replace('', f'v{raster.release_version}') + s3url = s3url.replace('', raster.res_str) + s3url = s3url.replace('', raster.supertile_id_no_res) + s3url = s3url.replace('', raster.id) rasterproxy_prefix_parts = args.rasterproxy_prefix.split('/') bucket = rasterproxy_prefix_parts[2] bpath = '/'.join(rasterproxy_prefix_parts[3:]).strip(r'/') @@ -270,18 +286,8 @@ def build_archive(raster, scratch, args): suffix = tif[len(raster.tileid):-4] # eg "_dem" mrf = '{}{}.mrf'.format(raster.tileid, suffix) if not os.path.isfile(mrf): - sourcepath = '{}/{}/{}{}.tif'.format( - sourceprefix, - raster.supertile_id_no_res, - raster.tileid, - suffix - ) - datapath = '{}/{}/{}{}.mrfcache'.format( - dataprefix, - raster.supertile_id_no_res, - raster.tileid, - suffix - ) + sourcepath = f'{sourceprefix}{suffix}.tif' + datapath = f'{dataprefix}{suffix}.mrfcache' static_args = '-q -of MRF -co BLOCKSIZE=512 -co "UNIFORM_SCALE=2" -co COMPRESS=LERC -co NOCOPY=TRUE' cmd = 'gdal_translate {0} -co INDEXNAME={1} -co DATANAME={1} -co CACHEDSOURCE={2} {3} {4}'.format( static_args, @@ -391,18 +397,26 @@ def build_archive(raster, scratch, args): tgt_srs = osr.SpatialReference() tgt_srs.ImportFromEPSG(args.epsg) - lyr = ds.CreateLayer(index_lyr, tgt_srs, ogr.wkbPolygon) + lyr = ds.CreateLayer(index_lyr, tgt_srs, ogr.wkbMultiPolygon) if lyr is not None: - for field_def in utils.TILE_DEM_ATTRIBUTE_DEFINITIONS_BASIC + utils.DEM_ATTRIBUTE_DEFINITION_RELVER: + for field_def in utils.DEM_ATTRIBUTE_DEFINITIONS_RELEASE: + fname = field_def.fname.lower() + fstype = None if field_def.ftype == ogr.OFTDateTime: ftype = ogr.OFTString fwidth = 28 + elif field_def.ftype == ogr.OFSTBoolean: + ftype = ogr.OFTInteger + fstype = field_def.ftype + fwidth = field_def.fwidth else: ftype = field_def.ftype fwidth = field_def.fwidth - field = ogr.FieldDefn(field_def.fname, ftype) + field = ogr.FieldDefn(fname, ftype) + if fstype: + field.SetSubType(fstype) field.SetWidth(fwidth) field.SetPrecision(field_def.fprecision) lyr.CreateField(field) @@ -413,20 +427,38 @@ def build_archive(raster, scratch, args): ## Set fields attrib_map = { "DEM_ID": raster.tileid, - "TILE": raster.supertile_id_no_res, - "ND_VALUE": raster.ndv, - "DEM_RES" : (raster.xres + raster.yres) / 2.0, - "DENSITY": raster.density, - "NUM_COMP": raster.num_components + "TILE": raster.tile_id_no_res, + "SUPERTILE": raster.supertile_id_no_res, + "GSD": (raster.xres + raster.yres) / 2.0, + 'EPSG': raster.epsg, + "RELEASEVER": raster.release_version, + "DATA_PERC": raster.density, + "NUM_COMP": raster.num_components, } - if raster.release_version: - attrib_map["REL_VER"] = raster.release_version - #### Set fields if populated (will not be populated if metadata file is not found) if raster.creation_date: attrib_map["CR_DATE"] = raster.creation_date.strftime("%Y-%m-%d") + filurl = args.release_fileurl + pretty_project = utils.PROJECTS[args.project] + filurl = filurl.replace('', pretty_project) + filurl = filurl.replace('', 'strips') + filurl = filurl.replace('', f'v{raster.release_version}') + filurl = filurl.replace('', raster.res_str) + filurl = filurl.replace('', raster.geocell) + filurl = filurl.replace('', raster.id) + attrib_map['FILEURL'] = filurl + + s3url = args.release_s3url + s3url = s3url.replace('', args.project) + s3url = s3url.replace('', 'strips') + s3url = s3url.replace('', raster.release_version) + s3url = s3url.replace('', raster.res_str) + s3url = s3url.replace('', raster.supertile_id_no_res) + s3url = s3url.replace('', raster.id) + attrib_map['S3URL'] = s3url + ## transform and write geom src_srs = utils.osr_srs_preserve_axis_order(osr.SpatialReference()) src_srs.ImportFromWkt(raster.proj) From ce07cc92b9742d471e592aecafd822cd113bb0ef Mon Sep 17 00:00:00 2001 From: Claire Porter Date: Wed, 23 Aug 2023 09:18:30 -0500 Subject: [PATCH 07/13] Make project arg non-positional to allow passing in to scheduler bash scripts --- package_setsm.py | 3 ++- package_setsm_tiles.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/package_setsm.py b/package_setsm.py index 0dfd4d2..0c782ab 100755 --- a/package_setsm.py +++ b/package_setsm.py @@ -40,7 +40,8 @@ def main(): #### Positional Arguments parser.add_argument('src', help="source directory, text file of file paths, or dem") parser.add_argument('scratch', help="scratch space to build index shps") - parser.add_argument('project', choices=utils.PROJECTS.keys(), help='project name') + parser.add_argument('--project', required=True, choices=utils.PROJECTS.keys(), + help='project name') #### Optional Arguments parser.add_argument('--skip-cog', action='store_true', default=False, diff --git a/package_setsm_tiles.py b/package_setsm_tiles.py index adf8783..e74d448 100755 --- a/package_setsm_tiles.py +++ b/package_setsm_tiles.py @@ -35,7 +35,8 @@ def main(): #### Positional Arguments parser.add_argument('src', help="source directory or dem") parser.add_argument('scratch', help="scratch space to build index shps") - parser.add_argument('project', choices=utils.PROJECTS.keys(), help='project name') + parser.add_argument('--project', required=True, choices=utils.PROJECTS.keys(), + help='project name') #### Optionsl Arguments parser.add_argument('--epsg', type=int, default=default_epsg, From 41a9cd5fe6948fb3443522e33b165f04854f8838 Mon Sep 17 00:00:00 2001 From: Claire Porter Date: Wed, 23 Aug 2023 14:42:10 -0500 Subject: [PATCH 08/13] Remove deprecated option in index_setsm.py --- index_setsm.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/index_setsm.py b/index_setsm.py index 52303bf..e305a9f 100755 --- a/index_setsm.py +++ b/index_setsm.py @@ -926,12 +926,12 @@ def write_to_ogr_dataset(ogr_driver_str, ogrDriver, dst_ds, dst_lyr, groups, pai attrib_map['DENSITY'] = record.density if record.density is not None else -9999 - if args.include_registration: - if record.reg_src: - attrib_map["REG_SRC"] = record.reg_src - attrib_map["NUM_GCPS"] = record.num_gcps - if record.mean_resid_z: - attrib_map["MEANRESZ"] = record.mean_resid_z + # if args.include_registration: --DEPRECATED + # if record.reg_src: + # attrib_map["REG_SRC"] = record.reg_src + # attrib_map["NUM_GCPS"] = record.num_gcps + # if record.mean_resid_z: + # attrib_map["MEANRESZ"] = record.mean_resid_z ## Set path folders for use if db_path_prefix specified if path_prefix: From 44adb1262266585a5b4cc89a69b91de80eacf75a Mon Sep 17 00:00:00 2001 From: Claire Porter Date: Thu, 24 Aug 2023 14:49:01 -0500 Subject: [PATCH 09/13] Bug fixes in package_setsm_tiles.py --- lib/dem.py | 2 ++ package_setsm_tiles.py | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/lib/dem.py b/lib/dem.py index dfb6c20..4ae5a28 100644 --- a/lib/dem.py +++ b/lib/dem.py @@ -1774,6 +1774,7 @@ def __init__(self, srcfp, md=None): groups = match.groupdict() self.tilename = groups['tile'] self.res_str = groups['res'] + self.res = groups['res'] # In case release version is in the file name and not the meta.txt self.release_version = groups['relversion'].strip('v') if groups['relversion'] else None self.subtile = groups['subtile'] @@ -2079,6 +2080,7 @@ def _rebuild_scene_from_dict(self, md): 'proj4', 'regmetapath', 'res', + 'res_str', 'srcdir', 'srcfn', 'srcfp', diff --git a/package_setsm_tiles.py b/package_setsm_tiles.py index e74d448..5e2c3b5 100755 --- a/package_setsm_tiles.py +++ b/package_setsm_tiles.py @@ -271,14 +271,14 @@ def build_archive(raster, scratch, args): mrf_tifs = [c for c in components + optional_components if c.endswith('dem.tif') and os.path.isfile(c)] if args.rasterproxy_prefix: logger.info("Creating raster proxy files") - s3url = args.rasterproxy_prefix - s3url = s3url.replace('', args.project) - s3url = s3url.replace('', 'mosaics') - s3url = s3url.replace('', f'v{raster.release_version}') - s3url = s3url.replace('', raster.res_str) - s3url = s3url.replace('', raster.supertile_id_no_res) - s3url = s3url.replace('', raster.id) - rasterproxy_prefix_parts = args.rasterproxy_prefix.split('/') + rp = args.rasterproxy_prefix + rp = rp.replace('', args.project) + rp = rp.replace('', 'mosaics') + rp = rp.replace('', f'v{raster.release_version}') + rp = rp.replace('', raster.res_str) + rp = rp.replace('', raster.supertile_id_no_res) + rp = rp.replace('', raster.id) + rasterproxy_prefix_parts = rp.split('/') bucket = rasterproxy_prefix_parts[2] bpath = '/'.join(rasterproxy_prefix_parts[3:]).strip(r'/') sourceprefix = '/vsicurl/http://{}.s3.us-west-2.amazonaws.com/{}'.format(bucket, bpath) @@ -402,7 +402,7 @@ def build_archive(raster, scratch, args): if lyr is not None: - for field_def in utils.DEM_ATTRIBUTE_DEFINITIONS_RELEASE: + for field_def in utils.TILE_DEM_ATTRIBUTE_DEFINITIONS_RELEASE: fname = field_def.fname.lower() fstype = None if field_def.ftype == ogr.OFTDateTime: @@ -447,7 +447,7 @@ def build_archive(raster, scratch, args): filurl = filurl.replace('', 'strips') filurl = filurl.replace('', f'v{raster.release_version}') filurl = filurl.replace('', raster.res_str) - filurl = filurl.replace('', raster.geocell) + filurl = filurl.replace('', raster.supertile_id_no_res) filurl = filurl.replace('', raster.id) attrib_map['FILEURL'] = filurl From 0c93edb5324b153051a06989cfc8735800d93250 Mon Sep 17 00:00:00 2001 From: Claire Porter Date: Thu, 24 Aug 2023 14:49:40 -0500 Subject: [PATCH 10/13] Update tests --- tests/func_test_index.py | 30 +++--- tests/func_test_package.py | 193 +++++++++++++++++++++++++++++++++++++ 2 files changed, 209 insertions(+), 14 deletions(-) create mode 100644 tests/func_test_package.py diff --git a/tests/func_test_index.py b/tests/func_test_index.py index 8cd33f4..be5fd4f 100644 --- a/tests/func_test_index.py +++ b/tests/func_test_index.py @@ -636,7 +636,8 @@ def testStrip(self): 'WARNING- Strip DEM avg acquisition times not found'), # test rebuild from mdf (self.stripmasked_dir, self.test_str, '--overwrite --check', self.stripmasked_count, 'Done'), # test index of masked strips (self.stripmasked_dir, self.test_str, '--overwrite --search-masked', self.stripmasked_count * 5, 'Done'), # test index of masked strips - (self.striprenamed_dir, self.test_str, '--overwrite --include-relver', self.strip_renamed_count, 'Done') + (self.striprenamed_dir, self.test_str, '--overwrite --project arcticdem --use-release-fields --lowercase-fieldnames', + self.strip_renamed_count, 'Done') ) strip_masks = { @@ -668,24 +669,23 @@ def testStrip(self): self.assertIsNotNone(layer) cnt = layer.GetFeatureCount() self.assertEqual(cnt, result_cnt) + location_field = 'FILEURL' if '--use-release-fields' in options else 'LOCATION' for feat in layer: - srcfp = feat.GetField('LOCATION') + srcfp = feat.GetField(location_field) srcdir, srcfn = os.path.split(srcfp) srcfn_minus_prefix = '_'.join(srcfn.split('_')[2:]) if srcfn.startswith('SETSM_s2s') else srcfn - stripdemid = feat.GetField('STRIPDEMID') - folder_stripdemid = os.path.basename(srcdir).replace('_lsf','') - if len(folder_stripdemid.split('_')) > 5: - self.assertEqual(folder_stripdemid,stripdemid) dem_suffix = srcfn[srcfn.find('_dem'):] - masks = strip_masks[dem_suffix] - self.assertEqual(feat.GetField('EDGEMASK'), masks[0]) - self.assertEqual(feat.GetField('WATERMASK'), masks[1]) - self.assertEqual(feat.GetField('CLOUDMASK'), masks[2]) - is_xtrack = 0 if srcfn_minus_prefix.startswith(('WV', 'GE', 'QB')) else 1 + if not '--use-release-fields' in options: + stripdemid = feat.GetField('STRIPDEMID') + folder_stripdemid = os.path.basename(srcdir).replace('_lsf', '') + if len(folder_stripdemid.split('_')) > 5: + self.assertEqual(folder_stripdemid, stripdemid) + masks = strip_masks[dem_suffix] + self.assertEqual(feat.GetField('EDGEMASK'), masks[0]) + self.assertEqual(feat.GetField('WATERMASK'), masks[1]) + self.assertEqual(feat.GetField('CLOUDMASK'), masks[2]) + is_xtrack = False if srcfn_minus_prefix.startswith(('WV', 'GE', 'QB')) else True self.assertEqual(feat.GetField('IS_XTRACK'), is_xtrack) - if 'include-relver' in options: - s2sver = feat.GetField('REL_VER') - self.assertEqual(s2sver, 's2s041') ds, layer = None, None ## Test if stdout has proper error @@ -879,6 +879,7 @@ def testTileJson(self): ) p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (so, se) = p.communicate() + # print(cmd) # print(se) # print(so) @@ -900,6 +901,7 @@ def testTileJson(self): ) p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (so, se) = p.communicate() + # print(cmd) # print(se) # print(so) diff --git a/tests/func_test_package.py b/tests/func_test_package.py new file mode 100644 index 0000000..65cb834 --- /dev/null +++ b/tests/func_test_package.py @@ -0,0 +1,193 @@ +import argparse +import glob +import os +import shutil +import subprocess +import sys +import unittest + +from osgeo import ogr, gdal, gdalconst + +try: + import ConfigParser +except ImportError: + import configparser as ConfigParser + +script_dir = os.path.dirname(os.path.abspath(sys.argv[0])) +testdata_dir = os.path.join(script_dir, 'testdata') +root_dir = os.path.dirname(script_dir) + +# logger = logging.getLogger("logger") +# lso = logging.StreamHandler() +# lso.setLevel(logging.ERROR) +# formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S') +# lso.setFormatter(formatter) +# logger.addHandler(lso) + + +class TestPackagerStrips(unittest.TestCase): + + def setUp(self): + self.script_name = 'package_setsm.py' + self.source_data_dn = 'setsm_strip_packager' + self.source_data = os.path.join(testdata_dir, self.source_data_dn) + self.output_dir = os.path.join(testdata_dir, 'output') + + def tearDown(self): + ## Clean up output + for f in os.listdir(self.output_dir): + fp = os.path.join(self.output_dir, f) + if os.path.isfile(fp): + os.remove(fp) + else: + shutil.rmtree(fp) + + # @unittest.skip("test") + def testOutput(self): + + o_list = [ + 'SETSM_s2s041_WV01_20221223_10200100D0A07B00_10200100D1CD1900_2m_seg1', + 'SETSM_s2s041_WV02_20221220_10300100DF1F1500_10300100DF27F000_2m_seg1', + 'SETSM_s2s041_WV02_20221220_10300100DF1F1500_10300100DF27F000_2m_seg2', + 'SETSM_s2s041_WV03_20220927_104001007C72BD00_104001007D43CB00_2m_lsf_seg1' + ] + + test_param_list = ( + # input, [expected outputs], args, message + (self.source_data, o_list, '--project arcticdem --build-rasterproxies', None), + ) + + for i, o_list, opts, msg in test_param_list: + o_dir = os.path.join(self.output_dir, self.source_data_dn) + # clean up test area if needed + try: + shutil.rmtree(o_dir) + except: + pass + # link source to test area + shutil.copytree(self.source_data, o_dir, copy_function=os.link) + # run test + cmd = f'python {root_dir}/{self.script_name} {o_dir} {o_dir} {opts}' + + p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (so, se) = p.communicate() + # print(se) + # print(so) + + ## Test assertions + for o in o_list: + ob = os.path.join(o_dir, o) + ## Test that tar and dem.mrf exist + self.assertTrue(os.path.isfile(f'{ob}.tar.gz')) + self.assertTrue(os.path.isfile(f'{ob}_dem.mrf')) + + ## Test that tifs are COGs + for tif in glob.glob(f'{ob}*tif'): + if 'dem_10m.tif' not in tif: + ds = gdal.Open(tif, gdalconst.GA_ReadOnly) + self.assertIn('LAYOUT=COG', ds.GetMetadata_List('IMAGE_STRUCTURE')) + + ## Test if stdout has proper text + if msg: + try: + self.assertIn(msg, so.decode()) + except AssertionError as e: + self.assertIn(msg, se.decode()) + + +class TestPackagerTiles(unittest.TestCase): + + def setUp(self): + self.script_name = 'package_setsm_tiles.py' + self.source_data_dn = 'setsm_tile_packager' + self.source_data = os.path.join(testdata_dir, self.source_data_dn) + self.output_dir = os.path.join(testdata_dir, 'output') + + def tearDown(self): + ## Clean up output + for f in os.listdir(self.output_dir): + fp = os.path.join(self.output_dir, f) + if os.path.isfile(fp): + os.remove(fp) + else: + shutil.rmtree(fp) + + # @unittest.skip("test") + def testOutput(self): + + o_list = [ + '10_27_1_1_2m_v4.1', + '10_27_2_1_2m_v4.1', + '10_27_2_2_2m_v4.1', + '59_57_1_1_2m', + '59_57_1_2_2m', + '59_57_2_1_2m', + '59_57_2_2_2m', + ] + + test_param_list = ( + # input, [expected outputs], args, message + (self.source_data, o_list, + '--project arcticdem --epsg 3413 --build-rasterproxies', None), + ) + + for i, o_list, opts, msg in test_param_list: + o_dir = os.path.join(self.output_dir, self.source_data_dn) + # clean up test area if needed + try: + shutil.rmtree(o_dir) + except: + pass + # link source to test area + shutil.copytree(self.source_data, o_dir, copy_function=os.link) + # run test + cmd = f'python {root_dir}/{self.script_name} {o_dir} {o_dir} {opts}' + + p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (so, se) = p.communicate() + # print(se) + # print(so) + + ## Test assertions + for o in o_list: + ob = os.path.join(o_dir, o) + ## Test that tar exists + self.assertTrue(os.path.isfile(f'{ob}.tar.gz')) + self.assertTrue(os.path.isfile(f'{ob}_dem.mrf')) + + ## Test that tifs are COGs + for tif in glob.glob(f'{ob}*tif'): + if 'countmt.tif' not in tif: + ds = gdal.Open(tif, gdalconst.GA_ReadOnly) + self.assertIn('LAYOUT=COG', ds.GetMetadata_List('IMAGE_STRUCTURE')) + + ## Test if stdout has proper text + if msg: + try: + self.assertIn(msg, so.decode()) + except AssertionError as e: + self.assertIn(msg, se.decode()) + + +if __name__ == '__main__': + + #### Set Up Arguments + parser = argparse.ArgumentParser( + description="Functional test for index_setsm" + ) + + #### Parse Arguments + args = parser.parse_args() + + test_cases = [ + TestPackagerStrips, + TestPackagerTiles, + ] + + suites = [] + for test_case in test_cases: + suite = unittest.TestLoader().loadTestsFromTestCase(test_case) + suites.append(suite) + + alltests = unittest.TestSuite(suites) + unittest.TextTestRunner(verbosity=2).run(alltests) From df85d89a7489aa68d8bd1485fb099cd9c180146f Mon Sep 17 00:00:00 2001 From: Claire Porter Date: Thu, 24 Aug 2023 16:25:10 -0500 Subject: [PATCH 11/13] Align package_setsm_tiles.py args with package_setsm.py --- package_setsm_tiles.py | 14 ++++++++++---- tests/func_test_package.py | 7 +------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/package_setsm_tiles.py b/package_setsm_tiles.py index 5e2c3b5..7f87242 100755 --- a/package_setsm_tiles.py +++ b/package_setsm_tiles.py @@ -41,12 +41,12 @@ def main(): #### Optionsl Arguments parser.add_argument('--epsg', type=int, default=default_epsg, help="egsg code for output index projection (default epsg:{})".format(default_epsg)) + parser.add_argument('--skip-cog', action='store_true', default=False, + help="skip converting dem files to COG before building the archive") parser.add_argument('--skip-archive', action='store_true', default=False, help="build mdf and readme files and convert rasters to COG, do not build archive") parser.add_argument('--build-rasterproxies', action='store_true', default=False, help='build rasterproxy .mrf files') - parser.add_argument('--convert-to-cog', action='store_true', default=False, - help="convert dem files to COG before building the archive") parser.add_argument('--rasterproxy-prefix', default="s3://pgc-opendata-dems//////", help="template for rasterproxy .mrf file s3 path") @@ -166,7 +166,7 @@ def main(): expected_outputs = [ #raster.readme ] - if args.convert_to_cog: + if not args.skip_cog: expected_outputs.append(cog_sem) if not args.skip_archive: expected_outputs.append(raster.archive) @@ -263,8 +263,14 @@ def build_archive(raster, scratch, args): ] cog_params = { + # ( path, lzw predictor, resample strategy) os.path.basename(raster.srcfp): ('YES', 'BILINEAR'), # dem os.path.basename(raster.browse): ('YES', 'CUBIC'), # browse + os.path.basename(raster.count): ('NO', 'NEAREST'), + os.path.basename(raster.mad): ('YES', 'BILINEAR'), + os.path.basename(raster.mindate): ('NO', 'NEAREST'), + os.path.basename(raster.maxdate): ('NO', 'NEAREST'), + os.path.basename(raster.datamask): ('NO', 'NEAREST'), } ## create rasterproxy MRF file @@ -315,7 +321,7 @@ def build_archive(raster, scratch, args): ## Convert all rasters to COG in place (should no longer be needed) tifs = [c for c in components + optional_components if c.endswith('.tif') and os.path.isfile(c)] - if args.convert_to_cog: + if not args.skip_cog: cog_sem = raster.tileid + '.cogfin' if os.path.isfile(cog_sem) and not args.overwrite: logger.info('COG conversion already complete') diff --git a/tests/func_test_package.py b/tests/func_test_package.py index 65cb834..9dc02da 100644 --- a/tests/func_test_package.py +++ b/tests/func_test_package.py @@ -116,13 +116,8 @@ def tearDown(self): def testOutput(self): o_list = [ - '10_27_1_1_2m_v4.1', - '10_27_2_1_2m_v4.1', '10_27_2_2_2m_v4.1', '59_57_1_1_2m', - '59_57_1_2_2m', - '59_57_2_1_2m', - '59_57_2_2_2m', ] test_param_list = ( @@ -173,7 +168,7 @@ def testOutput(self): #### Set Up Arguments parser = argparse.ArgumentParser( - description="Functional test for index_setsm" + description="Functional tests for package_setsm.py and package_setsm_tiles.py" ) #### Parse Arguments From c7763fcf9a4ff684095e4fba21941a87213747f6 Mon Sep 17 00:00:00 2001 From: Claire Porter Date: Fri, 25 Aug 2023 15:31:51 -0500 Subject: [PATCH 12/13] Scene DEM object rebuild from json --- lib/dem.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/dem.py b/lib/dem.py index 4ae5a28..4e966f0 100644 --- a/lib/dem.py +++ b/lib/dem.py @@ -175,8 +175,10 @@ def __init__(self,metapath,md=None): setattr(self, p, None) # Note: this approach will not work for DSP dems being used as proxies for 50cm - self.has_lsf = self.filesz_lsf > 0 - self.has_nonlsf = self.filesz_dem > 0 + if not hasattr(self,'has_lsf'): + self.has_lsf = self.filesz_lsf > 0 + if not hasattr(self, 'has_nonlsf'): + self.has_nonlsf = self.filesz_dem > 0 self.is_xtrack = bool(self.is_xtrack) else: @@ -2120,6 +2122,7 @@ def _set_component_attribs(self): self.acqdate_min = acqdate_min self.acqdate_max = acqdate_max + class RegInfo(object): def __init__(self, dx, dy, dz, num_gcps, mean_resid_z, src, name=None): From e4137c5990f57c42abd65e310b9039f5172c47eb Mon Sep 17 00:00:00 2001 From: Claire Porter Date: Fri, 25 Aug 2023 15:32:12 -0500 Subject: [PATCH 13/13] Update tests --- tests/func_test_index.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/func_test_index.py b/tests/func_test_index.py index be5fd4f..cafcb29 100644 --- a/tests/func_test_index.py +++ b/tests/func_test_index.py @@ -33,6 +33,7 @@ class TestIndexerScenes(unittest.TestCase): def setUp(self): self.scene_dir = os.path.join(testdata_dir, 'setsm_scene') + self.scene_json_dir = os.path.join(testdata_dir, 'setsm_scene_json') self.scene50cm_dir = os.path.join(testdata_dir, 'setsm_scene_50cm') self.scenedsp_dir = os.path.join(testdata_dir, 'setsm_scene_2mdsp') self.output_dir = os.path.join(testdata_dir, 'output') @@ -40,6 +41,7 @@ def setUp(self): self.pg_test_str = 'PG:sandwich:test_pgcdemtools' self.scene_count = 52 + self.scene_json_count = 35 self.scene50cm_count = 14 self.scenedsp_count = 102 @@ -66,6 +68,8 @@ def testOutputShp(self): (self.scene_dir, self.test_str, '--skip-region-lookup --overwrite --check', self.scene_count, 'Done'), # test check (self.scene_dir, self.test_str, '--dsp-record-mode both --skip-region-lookup --overwrite', self.scene_count, 'Done'), # test dsp-record-mode both has no effect when record is not dsp + (self.scene_json_dir, self.test_str, '--skip-region-lookup --overwrite --read-json', self.scene_json_count, + 'Done'), # test old jsons ) for i, o, options, result_cnt, msg in test_param_list: @@ -93,17 +97,11 @@ def testOutputShp(self): self.assertEqual(feat.GetField('IS_XTRACK'), is_xtrack) self.assertIsNotNone(feat.GetField('PROD_VER')) record_res = feat.GetField('DEM_RES') - has_lsf = feat.GetField("HAS_LSF") - has_nonlsf = feat.GetField("HAS_NONLSF") + has_lsf = bool(feat.GetField("HAS_LSF")) + has_nonlsf = bool(feat.GetField("HAS_NONLSF")) if record_res == 0.5: self.assertTrue(has_nonlsf) self.assertFalse(has_lsf) - elif record_res == 2.0: - self.assertTrue(has_lsf) - if feat.GetField("CENT_LAT") < -60: - self.assertFalse(has_lsf) - else: - self.assertTrue(has_nonlsf) ds, layer = None, None ## Test if stdout has proper error @@ -631,6 +629,10 @@ def testStrip(self): (self.strip_dir, self.test_str, '', self.strip_count, 'Done'), # test creation (self.strip_json_dir, self.test_str, '--overwrite --read-json', self.strip_json_count, 'Done'), # test old json rebuild + (self.strip_json_dir, self.test_str, + '--overwrite --read-json --overwrite --project arcticdem --use-release-fields --lowercase-fieldnames', + self.strip_json_count, 'Done'), + # test old json rebuild with release fields (self.strip_mixedver_dir, self.test_str, '--overwrite', self.strip_mixedver_count, 'Done'), # test mixed version (self.strip_mdf_dir, self.test_str, '--overwrite', self.strip_count, 'WARNING- Strip DEM avg acquisition times not found'), # test rebuild from mdf