From 18af1fc628dfc21122283ac340c7cb851dffbe29 Mon Sep 17 00:00:00 2001 From: Jennifer Tran Date: Tue, 21 May 2024 13:52:46 -0700 Subject: [PATCH 1/3] fix: campire-data transfer config --- ingestion-data/transfer-config/campfire-nlcd.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingestion-data/transfer-config/campfire-nlcd.json b/ingestion-data/transfer-config/campfire-nlcd.json index e7b5d66..4496992 100644 --- a/ingestion-data/transfer-config/campfire-nlcd.json +++ b/ingestion-data/transfer-config/campfire-nlcd.json @@ -4,6 +4,6 @@ "origin_prefix": "campfire-data/", "target_bucket": "veda-data-store", "collection": "campfire-nlcd", - "filename_regex": "^(.*)CampFire_NLCD_2019" + "filename_regex": "^(.*)CampFire_NLCD_201\\d" } ] \ No newline at end of file From e49237d95b9c5a86ce14ebf701514f9532904fd6 Mon Sep 17 00:00:00 2001 From: Jennifer Tran Date: Wed, 22 May 2024 10:54:58 -0700 Subject: [PATCH 2/3] fix: lis-global-da-gpp-trend transfer-config and discovery-items --- .../lis-global-da-gpp-trend.json | 36 ++++++++++++++----- .../lis-global-da-gpp-trend.json | 16 ++++++++- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/ingestion-data/production/discovery-items/lis-global-da-gpp-trend.json b/ingestion-data/production/discovery-items/lis-global-da-gpp-trend.json index 59028f0..83b5b67 100644 --- a/ingestion-data/production/discovery-items/lis-global-da-gpp-trend.json +++ b/ingestion-data/production/discovery-items/lis-global-da-gpp-trend.json @@ -1,8 +1,28 @@ -{ - "collection": "lis-global-da-gpp-trend", - "bucket": "veda-data-store", - "prefix": "lis-global-da-gpp-trend/", - "filename_regex": "^(.*)DAGPP_STL_based_trend", - "discovery": "s3", - "upload": false -} \ No newline at end of file +[ + { + "collection": "lis-global-da-gpp-trend", + "bucket": "veda-data-store", + "prefix": "lis-global-da-gpp-trend/", + "filename_regex": "^(.*)DAGPP_STL_based_trend_v3_2003_2021.cog.tif", + "discovery": "s3", + "upload": false + }, + { + "collection": "lis-global-da-gpp-trend", + "bucket": "veda-data-store", + "prefix": "lis-global-da-gpp-trend/", + "filename_regex": "^(.*)DAGPP_STL_based_trendv2_2003_2021.cog.tif", + "discovery": "s3", + "upload": false + }, + { + "collection": "lis-global-da-gpp-trend", + "bucket": "veda-data-store", + "prefix": "lis-global-da-gpp-trend/", + "filename_regex": "^(.*)DAGPP_STL_based_trend.cog.tif", + "discovery": "s3", + "upload": false, + "start_datetime": "2003-01-01T00:00:00Z", + "end_datetime": "2021-12-31T23:59:59Z" + } +] \ No newline at end of file diff --git a/ingestion-data/transfer-config/lis-global-da-gpp-trend.json b/ingestion-data/transfer-config/lis-global-da-gpp-trend.json index 8c6c2a4..3273cf5 100644 --- a/ingestion-data/transfer-config/lis-global-da-gpp-trend.json +++ b/ingestion-data/transfer-config/lis-global-da-gpp-trend.json @@ -4,6 +4,20 @@ "origin_prefix": "EIS/COG/LIS_GLOBAL_DA/DA_Trends/", "target_bucket": "veda-data-store", "collection": "lis-global-da-gpp-trend", - "filename_regex": "^(.*)DAGPP_STL_based_trend" + "filename_regex": "^(.*)DAGPP_STL_based_trend.cog.tif" + }, + { + "origin_bucket": "veda-data-store-staging", + "origin_prefix": "EIS/COG/LIS_GLOBAL_DA/DA_Trends/", + "target_bucket": "veda-data-store", + "collection": "lis-global-da-gpp-trend", + "filename_regex": "^(.*)DAGPP_STL_based_trend_v3_2003_2021.cog.tif" + }, + { + "origin_bucket": "veda-data-store-staging", + "origin_prefix": "EIS/COG/LIS_GLOBAL_DA/DA_Trends/", + "target_bucket": "veda-data-store", + "collection": "lis-global-da-gpp-trend", + "filename_regex": "^(.*)DAGPP_STL_based_trendv2_2003_2021.cog.tif" } ] \ No newline at end of file From ca953b29bed38f487938e535f96f92a758b0d191 Mon Sep 17 00:00:00 2001 From: Jennifer Tran Date: Thu, 23 May 2024 10:22:38 -0700 Subject: [PATCH 3/3] fix: update snow-projection configs and lis-global-da-tws-trend --- .../production/discovery-items/lis-global-da-tws-trend.json | 4 +++- .../production/discovery-items/snow-projections-diff-585.json | 2 +- .../discovery-items/snow-projections-median-245.json | 2 +- .../discovery-items/snow-projections-median-585.json | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ingestion-data/production/discovery-items/lis-global-da-tws-trend.json b/ingestion-data/production/discovery-items/lis-global-da-tws-trend.json index 15922ef..d6ba153 100644 --- a/ingestion-data/production/discovery-items/lis-global-da-tws-trend.json +++ b/ingestion-data/production/discovery-items/lis-global-da-tws-trend.json @@ -21,6 +21,8 @@ "prefix": "lis-global-da-tws-trend/", "filename_regex":"^(.*)DATWS_STL_based_trend.cog.tif", "discovery": "s3", - "upload": false + "upload": false, + "start_datetime": "2003-01-01T00:00:00Z", + "end_datetime": "2021-12-31T23:59:59Z" } ] diff --git a/ingestion-data/production/discovery-items/snow-projections-diff-585.json b/ingestion-data/production/discovery-items/snow-projections-diff-585.json index 4b28aca..b516236 100644 --- a/ingestion-data/production/discovery-items/snow-projections-diff-585.json +++ b/ingestion-data/production/discovery-items/snow-projections-diff-585.json @@ -1,7 +1,7 @@ [ { "collection": "snow-projections-diff-585", - "prefix": "snow-projections-diff-245/", + "prefix": "snow-projections-diff-585/", "bucket": "veda-data-store", "filename_regex": "^(.*)ssp585(.*)percChange.cog.tif$", "discovery": "s3", diff --git a/ingestion-data/production/discovery-items/snow-projections-median-245.json b/ingestion-data/production/discovery-items/snow-projections-median-245.json index fa33b37..81d1752 100644 --- a/ingestion-data/production/discovery-items/snow-projections-median-245.json +++ b/ingestion-data/production/discovery-items/snow-projections-median-245.json @@ -1,7 +1,7 @@ [ { "collection": "snow-projections-median-245", - "prefix": "snow-projections-diff-245/", + "prefix": "snow-projections-median-245/", "bucket": "veda-data-store", "filename_regex": "^(.*)ssp245(.*)median.cog.tif$", "discovery": "s3", diff --git a/ingestion-data/production/discovery-items/snow-projections-median-585.json b/ingestion-data/production/discovery-items/snow-projections-median-585.json index 484291d..60de9d2 100644 --- a/ingestion-data/production/discovery-items/snow-projections-median-585.json +++ b/ingestion-data/production/discovery-items/snow-projections-median-585.json @@ -1,7 +1,7 @@ [ { "collection": "snow-projections-median-585", - "prefix": "snow-projections-diff-245/", + "prefix": "snow-projections-median-585/", "bucket": "veda-data-store", "filename_regex": "^(.*)ssp585(.*)median.cog.tif$", "discovery": "s3",