Skip to content

Commit

Permalink
calculate todays date on archive ingest init
Browse files Browse the repository at this point in the history
  • Loading branch information
@tomkralidis
tomkralidis committed Jul 29, 2024
1 parent 00c365d commit 6d6fbb1
Showing 1 changed file with 43 additions and 40 deletions.
83 changes: 43 additions & 40 deletions data-registry-ingest.sh
View file
Original file line number Diff line number Diff line change
@@ -1,58 +1,61 @@
#1/bin/bash

# The purpose of this script is to ingest all of the operational WOUDC Archive datasets and rename the generated ingest reports to avoid overwriting
# Inital setup and table initialization of WOUDC Data Registry is required to run this script
# Replace YYYY-MM-DD with today's date before proceeding

woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/TotalOzone_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzone_1.0_1.csv
mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzone_1.0_1_run_report
TODAY=`date +"%Y-%m-%d"`

woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/TotalOzone_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY
mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/TotalOzone_1.0_1.csv
mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/TotalOzone_1.0_1_run_report

woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/TotalOzone_2.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzone_2.0_1.csv
mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzone_2.0_1_run_report
woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/TotalOzone_2.0_1 -y -r /apps/data/wdr-ingest/$TODAY
mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/TotalOzone_2.0_1.csv
mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/TotalOzone_2.0_1_run_report

woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Broad-band_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Broad-band_1.0_1.csv
mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Broad-band_1.0_1_run_report
woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Broad-band_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY
mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/Broad-band_1.0_1.csv
mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/Broad-band_1.0_1_run_report

woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Broad-band_2.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Broad-band_2.0_1.csv
mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Broad-band_2.0_1_run_report
woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Broad-band_2.0_1 -y -r /apps/data/wdr-ingest/$TODAY
mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/Broad-band_2.0_1.csv
mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/Broad-band_2.0_1_run_report

woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Lidar_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Lidar_1.0_1.csv
mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Lidar_1.0_1_run_report
woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Lidar_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY
mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/Lidar_1.0_1.csv
mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/Lidar_1.0_1_run_report

woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Multi-band_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Multi-band_1.0_1.csv
mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Multi-band_1.0_1_run_report
woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Multi-band_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY
mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/Multi-band_1.0_1.csv
mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/Multi-band_1.0_1_run_report

woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/RocketSonde_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/RocketSonde_1.0_1.csv
mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/RocketSonde_1.0_1_run_report
woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/RocketSonde_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY
mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/RocketSonde_1.0_1.csv
mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/RocketSonde_1.0_1_run_report

woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/OzoneSonde_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/OzoneSonde_1.0_1.csv
mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/OzoneSonde_1.0_1_run_report
woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/OzoneSonde_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY
mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/OzoneSonde_1.0_1.csv
mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/OzoneSonde_1.0_1_run_report

woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/TotalOzoneObs_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzoneObs_1.0_1.csv
mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/TotalOzoneObs_1.0_1_run_report
woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/TotalOzoneObs_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY
mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/TotalOzoneObs_1.0_1.csv
mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/TotalOzoneObs_1.0_1_run_report

woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Spectral_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Spectral_1.0_1.csv
mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Spectral_1.0_1_run_report
woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Spectral_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY
mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/Spectral_1.0_1.csv
mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/Spectral_1.0_1_run_report

woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Spectral_2.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/Spectral_2.0_1.csv
mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/Spectral_2.0_1_run_report
woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/Spectral_2.0_1 -y -r /apps/data/wdr-ingest/$TODAY
mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/Spectral_2.0_1.csv
mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/Spectral_2.0_1_run_report

woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/UmkehrN14_1.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/UmkehrN14_1.0_1.csv
mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/UmkehrN14_1.0_1_run_report
woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/UmkehrN14_1.0_1 -y -r /apps/data/wdr-ingest/$TODAY
mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/UmkehrN14_1.0_1.csv
mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/UmkehrN14_1.0_1_run_report

woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/UmkehrN14_2.0_1 -y -r /apps/data/wdr-ingest/YYYY-MM-DD
mv /apps/data/wdr-ingest/YYYY-MM-DD/operator-report.csv /apps/data/wdr-ingest/YYYY-MM-DD/UmkehrN14_2.0_1.csv
mv /apps/data/wdr-ingest/YYYY-MM-DD/run_report /apps/data/wdr-ingest/YYYY-MM-DD/UmkehrN14_2.0_1_run_report
woudc-data-registry data ingest /apps/data/web/woudc-archive/Archive-NewFormat/UmkehrN14_2.0_1 -y -r /apps/data/wdr-ingest/$TODAY
mv /apps/data/wdr-ingest/$TODAY/operator-report.csv /apps/data/wdr-ingest/$TODAY/UmkehrN14_2.0_1.csv
mv /apps/data/wdr-ingest/$TODAY/run_report /apps/data/wdr-ingest/$TODAY/UmkehrN14_2.0_1_run_report

## Table generation process into registry for totalozone, uv-index and ozonesonde
woudc-data-registry product totalozone generate -y /apps/data/web/woudc-archive/Archive-NewFormat/
Expand Down

0 comments on commit 6d6fbb1

Please sign in to comment.