From 699341717810811a1f2d29bb5309a07a74539e85 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 29 Jul 2024 12:52:07 -0400 Subject: [PATCH] calculate todays date on archive ingest init --- data-registry-ingest.sh | 84 +++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/data-registry-ingest.sh b/data-registry-ingest.sh index 82397d9..a5e1e35 100644 --- a/data-registry-ingest.sh +++ b/data-registry-ingest.sh @@ -1,58 +1,62 @@ +#1/bin/bash + # The purpose of this script is to ingest all of the operational WOUDC Archive datasets and rename the generated ingest reports to avoid overwriting # Inital setup and table initialization of WOUDC Data Registry is required to run this script -# Replace YYYY-MM-DD with today's date before proceeding +# Replace $TODAY with today's date before proceeding + +TODAY=`date +"%Y-%m-%d"` -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/TotalOzone_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzone_1.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzone_1.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/TotalOzone_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/TotalOzone_1.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/TotalOzone_1.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/TotalOzone_2.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzone_2.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzone_2.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/TotalOzone_2.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/TotalOzone_2.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/TotalOzone_2.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Broad-band_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Broad-band_1.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Broad-band_1.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Broad-band_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/Broad-band_1.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/Broad-band_1.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Broad-band_2.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Broad-band_2.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Broad-band_2.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Broad-band_2.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/Broad-band_2.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/Broad-band_2.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Lidar_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Lidar_1.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Lidar_1.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Lidar_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/Lidar_1.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/Lidar_1.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Multi-band_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Multi-band_1.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Multi-band_1.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Multi-band_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/Multi-band_1.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/Multi-band_1.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/RocketSonde_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/RocketSonde_1.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/RocketSonde_1.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/RocketSonde_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/RocketSonde_1.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/RocketSonde_1.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/OzoneSonde_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/OzoneSonde_1.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/OzoneSonde_1.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/OzoneSonde_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/OzoneSonde_1.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/OzoneSonde_1.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/TotalOzoneObs_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzoneObs_1.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzoneObs_1.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/TotalOzoneObs_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/TotalOzoneObs_1.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/TotalOzoneObs_1.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Spectral_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Spectral_1.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Spectral_1.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Spectral_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/Spectral_1.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/Spectral_1.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Spectral_2.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Spectral_2.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Spectral_2.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Spectral_2.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/Spectral_2.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/Spectral_2.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/UmkehrN14_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/UmkehrN14_1.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/UmkehrN14_1.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/UmkehrN14_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/UmkehrN14_1.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/UmkehrN14_1.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/UmkehrN14_2.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/UmkehrN14_2.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/UmkehrN14_2.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/UmkehrN14_2.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/UmkehrN14_2.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/UmkehrN14_2.0_1_run_report ## Table generation process into registry for totalozone, uv-index and ozonesonde woudc-data-registry product totalozone generate -y /apps/data/web/woudc-archive/Archive-NewFormat/