From 3adf936f7887a34b368165ed57649fadedb46bda Mon Sep 17 00:00:00 2001 From: Pranav Anbarasu Date: Fri, 12 Jul 2024 01:07:37 +0000 Subject: [PATCH] Update progress statements and add total execution time timing --- pipeline/run-pipeline.R | 2 ++ scripts/egress/egress.R | 2 +- scripts/fetch-data/fetch_data.R | 2 +- scripts/process-data/fitbitactivitylogs.R | 2 +- scripts/process-data/fitbitdailydata.R | 2 +- scripts/process-data/fitbitecg.R | 2 +- scripts/process-data/fitbitintradaycombined.R | 2 +- scripts/process-data/fitbitsleeplogs.R | 2 +- scripts/process-data/healthkitv2electrocardiogram.R | 2 +- scripts/process-data/healthkitv2samples.R | 2 +- scripts/process-data/healthkitv2statistics.R | 2 +- scripts/process-data/participant_devices.R | 2 +- scripts/write-output/final-output-concepts.R | 2 +- 13 files changed, 14 insertions(+), 12 deletions(-) diff --git a/pipeline/run-pipeline.R b/pipeline/run-pipeline.R index a8915c6..4b9d2ea 100644 --- a/pipeline/run-pipeline.R +++ b/pipeline/run-pipeline.R @@ -1,3 +1,4 @@ +tictoc::tic(msg = "INFO: Total execution time") # Get config variables list2env( x = config::get(file = "config/config.yml", @@ -32,3 +33,4 @@ tictoc::toc() tictoc::tic(msg = "INFO: Store in Synapse") source("scripts/egress/egress.R") tictoc::toc() +tictoc::toc() diff --git a/scripts/egress/egress.R b/scripts/egress/egress.R index cb2ce59..68d84bf 100644 --- a/scripts/egress/egress.R +++ b/scripts/egress/egress.R @@ -38,4 +38,4 @@ rm(latest_commit, file_name ) -cat("\n----Finished egress----\n") +cat("----Finished----\n") diff --git a/scripts/fetch-data/fetch_data.R b/scripts/fetch-data/fetch_data.R index 1ede767..cc427ac 100644 --- a/scripts/fetch-data/fetch_data.R +++ b/scripts/fetch-data/fetch_data.R @@ -61,5 +61,5 @@ if (!dir.exists(outputConceptsDir)) { dir.create(outputConceptsDir) } -cat("\n----Finished----\n") +cat("----Finished----\n") diff --git a/scripts/process-data/fitbitactivitylogs.R b/scripts/process-data/fitbitactivitylogs.R index 84a9325..efc0475 100644 --- a/scripts/process-data/fitbitactivitylogs.R +++ b/scripts/process-data/fitbitactivitylogs.R @@ -93,7 +93,7 @@ output_concepts %>% write.csv(file.path(outputConceptsDir, glue::glue("{dataset}.csv")), row.names = F) cat(glue::glue("output_concepts written to {file.path(outputConceptsDir, paste0(dataset, '.csv'))}"),"\n") -cat(paste0("\n----", glue::glue("Finished transforming data for {dataset}"),"\n")) +cat("----Finished----\n") # Remove objects created here from the global environment rm(dataset, diff --git a/scripts/process-data/fitbitdailydata.R b/scripts/process-data/fitbitdailydata.R index efcda79..3c5ed07 100644 --- a/scripts/process-data/fitbitdailydata.R +++ b/scripts/process-data/fitbitdailydata.R @@ -112,7 +112,7 @@ output_concepts %>% write.csv(file.path(outputConceptsDir, glue::glue("{dataset}.csv")), row.names = F) cat(glue::glue("output_concepts written to {file.path(outputConceptsDir, paste0(dataset, '.csv'))}"), "\n") -cat(paste0("\n----", glue::glue("Finished transforming data for {dataset}"),"\n")) +cat("----Finished----\n") # Remove objects created here from the global environment rm(dataset, diff --git a/scripts/process-data/fitbitecg.R b/scripts/process-data/fitbitecg.R index 7dd3202..9033457 100644 --- a/scripts/process-data/fitbitecg.R +++ b/scripts/process-data/fitbitecg.R @@ -193,7 +193,7 @@ output_concepts %>% write.csv(file.path(outputConceptsDir, glue::glue("{dataset}.csv")), row.names = F) cat(glue::glue("output_concepts written to {file.path(outputConceptsDir, paste0(dataset, '.csv'))}"), "\n") -cat(paste0("\n----", glue::glue("Finished transforming data for {dataset}"),"\n")) +cat("----Finished----\n") # Remove objects created here from the global environment rm(ecg_stat_summarize, diff --git a/scripts/process-data/fitbitintradaycombined.R b/scripts/process-data/fitbitintradaycombined.R index 1fb0f8a..503b8a1 100644 --- a/scripts/process-data/fitbitintradaycombined.R +++ b/scripts/process-data/fitbitintradaycombined.R @@ -126,7 +126,7 @@ output_concepts %>% write.csv(file.path(outputConceptsDir, glue::glue("{dataset}.csv")), row.names = F) cat(glue::glue("output_concepts written to {file.path(outputConceptsDir, paste0(dataset, '.csv'))}"), "\n") -cat(paste0("\n----", glue::glue("Finished transforming data for {dataset}"),"\n")) +cat("----Finished----\n") # Remove objects created here from the global environment rm(dataset, diff --git a/scripts/process-data/fitbitsleeplogs.R b/scripts/process-data/fitbitsleeplogs.R index 8a236c4..7a475f4 100644 --- a/scripts/process-data/fitbitsleeplogs.R +++ b/scripts/process-data/fitbitsleeplogs.R @@ -432,7 +432,7 @@ output_concepts %>% write.csv(file.path(outputConceptsDir, glue::glue("{dataset}.csv")), row.names = F) cat(glue::glue("output_concepts written to {file.path(outputConceptsDir, paste0(dataset, '.csv'))}"),"\n") -cat(paste0("\n----", glue::glue("Finished transforming data for {dataset}"),"\n")) +cat("----Finished----\n") # Remove objects created here from the global environment rm(sleeplogs_stat_summarize, diff --git a/scripts/process-data/healthkitv2electrocardiogram.R b/scripts/process-data/healthkitv2electrocardiogram.R index 9c018d7..2c89565 100644 --- a/scripts/process-data/healthkitv2electrocardiogram.R +++ b/scripts/process-data/healthkitv2electrocardiogram.R @@ -196,7 +196,7 @@ output_concepts %>% write.csv(file.path(outputConceptsDir, glue::glue("{dataset}.csv")), row.names = F) cat(glue::glue("output_concepts written to {file.path(outputConceptsDir, paste0(dataset, '.csv'))}"),"\n") -cat(paste0("\n----", glue::glue("Finished transforming data for {dataset}"),"\n")) +cat("----Finished----\n") # Remove objects created here from the global environment rm(ecg_stat_summarize, diff --git a/scripts/process-data/healthkitv2samples.R b/scripts/process-data/healthkitv2samples.R index 0ba9b58..af4c9e5 100644 --- a/scripts/process-data/healthkitv2samples.R +++ b/scripts/process-data/healthkitv2samples.R @@ -116,7 +116,7 @@ output_concepts %>% write.csv(file.path(outputConceptsDir, glue::glue("{dataset}.csv")), row.names = F) cat(glue::glue("output_concepts written to {file.path(outputConceptsDir, paste0(dataset, '.csv'))}"),"\n") -cat(paste0("\n----", glue::glue("Finished transforming data for {dataset}"),"\n")) +cat("----Finished----\n") # Remove objects created here from the global environment rm(dataset, diff --git a/scripts/process-data/healthkitv2statistics.R b/scripts/process-data/healthkitv2statistics.R index 9a4c35a..1bf0a2d 100644 --- a/scripts/process-data/healthkitv2statistics.R +++ b/scripts/process-data/healthkitv2statistics.R @@ -96,7 +96,7 @@ output_concepts %>% write.csv(file.path(outputConceptsDir, glue::glue("{dataset}.csv")), row.names = F) cat(glue::glue("output_concepts written to {file.path(outputConceptsDir, paste0(dataset, '.csv'))}"),"\n") -cat(paste0("\n----", glue::glue("Finished transforming data for {dataset}"),"\n")) +cat("----Finished----\n") # Remove objects created here from the global environment rm(dataset, diff --git a/scripts/process-data/participant_devices.R b/scripts/process-data/participant_devices.R index a5b0945..1e569a7 100644 --- a/scripts/process-data/participant_devices.R +++ b/scripts/process-data/participant_devices.R @@ -91,7 +91,7 @@ output_concepts %>% write.csv(file.path(outputConceptsDir, "participant_devices.csv"), row.names = F) cat(glue::glue("participant_devices written to {file.path(outputConceptsDir, 'participant_devices.csv')}"), "\n") -cat(glue::glue("Finished tansforming device data for {dataset}"),"\n\n") +cat("----Finished----\n") # Remove objects created here from the global environment rm(dataset, diff --git a/scripts/write-output/final-output-concepts.R b/scripts/write-output/final-output-concepts.R index c565e95..67af71d 100644 --- a/scripts/write-output/final-output-concepts.R +++ b/scripts/write-output/final-output-concepts.R @@ -66,4 +66,4 @@ rm(datasets, valid_participants ) -cat("\n----Finished creating final output concepts----\n") +cat("----Finished----\n")