diff --git a/data-registry-ingest.sh b/data-registry-ingest.sh index 82397d9..a5e1e35 100644 --- a/data-registry-ingest.sh +++ b/data-registry-ingest.sh @@ -1,58 +1,62 @@ +#1/bin/bash + # The purpose of this script is to ingest all of the operational WOUDC Archive datasets and rename the generated ingest reports to avoid overwriting # Inital setup and table initialization of WOUDC Data Registry is required to run this script -# Replace YYYY-MM-DD with today's date before proceeding +# Replace $TODAY with today's date before proceeding + +TODAY=`date +"%Y-%m-%d"` -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/TotalOzone_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzone_1.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzone_1.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/TotalOzone_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/TotalOzone_1.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/TotalOzone_1.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/TotalOzone_2.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzone_2.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzone_2.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/TotalOzone_2.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/TotalOzone_2.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/TotalOzone_2.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Broad-band_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Broad-band_1.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Broad-band_1.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Broad-band_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/Broad-band_1.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/Broad-band_1.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Broad-band_2.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Broad-band_2.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Broad-band_2.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Broad-band_2.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/Broad-band_2.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/Broad-band_2.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Lidar_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Lidar_1.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Lidar_1.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Lidar_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/Lidar_1.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/Lidar_1.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Multi-band_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Multi-band_1.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Multi-band_1.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Multi-band_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/Multi-band_1.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/Multi-band_1.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/RocketSonde_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/RocketSonde_1.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/RocketSonde_1.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/RocketSonde_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/RocketSonde_1.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/RocketSonde_1.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/OzoneSonde_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/OzoneSonde_1.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/OzoneSonde_1.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/OzoneSonde_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/OzoneSonde_1.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/OzoneSonde_1.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/TotalOzoneObs_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzoneObs_1.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzoneObs_1.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/TotalOzoneObs_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/TotalOzoneObs_1.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/TotalOzoneObs_1.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Spectral_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Spectral_1.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Spectral_1.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Spectral_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/Spectral_1.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/Spectral_1.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Spectral_2.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Spectral_2.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Spectral_2.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Spectral_2.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/Spectral_2.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/Spectral_2.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/UmkehrN14_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/UmkehrN14_1.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/UmkehrN14_1.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/UmkehrN14_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/UmkehrN14_1.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/UmkehrN14_1.0_1_run_report -woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/UmkehrN14_2.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD -mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/UmkehrN14_2.0_1.csv -mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/UmkehrN14_2.0_1_run_report +woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/UmkehrN14_2.0_1 -y -r /apps/data/wdr-ingest/$TODAY +mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/UmkehrN14_2.0_1.csv +mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/UmkehrN14_2.0_1_run_report ## Table generation process into registry for totalozone, uv-index and ozonesonde woudc-data-registry product totalozone generate -y /apps/data/web/woudc-archive/Archive-NewFormat/