Improve handling and logging of unexpected errors (#44)

* Improve handling and logging of unexpected errors * Quarto files now write error info to driver log
COMPASS-DOE · Oct 10, 2023 · 3e9a32f · 3e9a32f
1 parent 13a2e1a
commit 3e9a32f
Show file tree

Hide file tree

Showing 5 changed files with 70 additions and 8 deletions.
diff --git a/portage/L0.qmd b/portage/L0.qmd
@@ -69,6 +69,7 @@ overwrites <- 0
 errors <- 0
 
 f <- function(fn, new_dir) {
+
  basefn <- basename(fn)
  message(Sys.time(), " Processing ", basefn)
  # Try to read in the data; if it doesn't work, we won't get a data frame back
@@ -119,7 +120,15 @@ f <- function(fn, new_dir) {
  Note = note)
 }
 
-out <- lapply(files_to_process, f, new_dir = L0)
+log_info("About to L0", logfile = params$logfile)
+tryCatch({
+ out <- lapply(files_to_process, f, new_dir = L0)
+},
+error = function(e) {
+ log_warning("L0: an error occurred!", logfile = params$logfile)
+ log_info(as.character(e), logfile = params$logfile)
+ stop(e)
+})
 ```
 
 ## Summary

diff --git a/portage/L1_normalize.qmd b/portage/L1_normalize.qmd
@@ -214,9 +214,17 @@ f <- function(fn, out_dir, design_table) {
  return(smry)
 }
 
+log_info("About to L1_normalize", logfile = params$logfile)
+tryCatch({
 out <- lapply(files_to_process, f, 
  out_dir = L1_NORMALIZE,
  design_table = dt_ex)
+},
+error = function(e) {
+ log_warning("L1_normalize: an error occurred!", logfile = params$logfile)
+ log_info(as.character(e), logfile = params$logfile)
+ stop(e)
+})
 ```
 
 ## Summary

diff --git a/portage/L1a.qmd b/portage/L1a.qmd
@@ -95,8 +95,16 @@ f <- function(dir_name, dirs_to_process, out_dir) {
  return(smry)
 }
 
+log_info("About to L1a", logfile = params$logfile)
+tryCatch({
 out <- lapply(names(dirs_to_process), f, 
  dirs_to_process = dirs_to_process, out_dir = L1A)
+},
+error = function(e) {
+ log_warning("L1a: an error occurred!", logfile = params$logfile)
+ log_info(as.character(e), logfile = params$logfile)
+ stop(e)
+})
 ```
 
 ## File summary

diff --git a/portage/L1b.qmd b/portage/L1b.qmd
@@ -109,8 +109,16 @@ f <- function(dir_name, dirs_to_process, out_dir) {
  return(smry)
 }
 
+log_info("About to L1b", logfile = params$logfile)
+tryCatch({
 out <- lapply(names(dirs_to_process), f, 
  dirs_to_process = dirs_to_process, out_dir = L1B)
+},
+error = function(e) {
+ log_warning("L1b: an error occurred!", logfile = params$logfile)
+ log_info(as.character(e), logfile = params$logfile)
+ stop(e)
+})
 ```
 
 ## Summary

diff --git a/portage/driver.R b/portage/driver.R
@@ -21,12 +21,31 @@ now_string <- function() format(Sys.time(), "%Y%m%d.%H%M")
 
 ROOT <- "./data_TEST"
 
+# Log file ----------------------------------------------------
+
 LOGS <- file.path(ROOT, "Logs/")
 
 # Main logfile
 LOGFILE <- file.path(LOGS, paste0("driver_log_", now_string(), ".txt"))
 if(file.exists(LOGFILE)) file.remove(LOGFILE)
 
+# Error handling ----------------------------------------------
+
+STOP_ON_ERROR <- TRUE
+ERROR_OCCURRED <- FALSE
+
+# driver_try: ensure that if an *unexpected* error occurs,
+# it's captured in the driver log file, and a flag is set
+driver_try <- function(...) {
+ tryCatch(eval(...),
+ error = function(e) {
+ ERROR_OCCURRED <<- TRUE
+ log_warning("Driver: an error occurred!")
+ log_info(as.character(e))
+ if(STOP_ON_ERROR) stop(e)
+ }
+ )
+}
 
 # Construct L0 data ---------------------------------------------
 # L0 data are raw but in CSV form, and with "Logger" and "Table" columns added
@@ -37,10 +56,12 @@ new_section("Starting L0")
 outfile <- paste0("L0_", now_string(), ".html")
 outfile <- file.path(LOGS, outfile)
 
-quarto_render("L0.qmd",
- execute_params = list(DATA_ROOT = ROOT,
- html_outfile = outfile,
- logfile = LOGFILE))
+driver_try(
+ quarto_render("L0.qmd",
+ execute_params = list(DATA_ROOT = ROOT,
+ html_outfile = outfile,
+ logfile = LOGFILE))
+)
 copy_output("L0.html", outfile)
 
 
@@ -55,10 +76,12 @@ dt <- file.path(ROOT, "design_table.csv")
 outfile <- paste0("L1_normalize_", now_string(), ".html")
 outfile <- file.path(LOGS, outfile)
 
-quarto_render("L1_normalize.qmd",
+driver_try(
+ quarto_render("L1_normalize.qmd",
  execute_params = list(DATA_ROOT = ROOT,
  html_outfile = outfile,
  logfile = LOGFILE))
+)
 copy_output("L1_normalize.html", outfile)
 
 
@@ -76,10 +99,12 @@ new_section("Starting L1a")
 outfile <- paste0("L1a_", now_string(), ".html")
 outfile <- file.path(LOGS, outfile)
 
-quarto_render("L1a.qmd",
+driver_try(
+ quarto_render("L1a.qmd",
  execute_params = list(DATA_ROOT = ROOT,
  html_outfile = outfile,
  logfile = LOGFILE))
+)
 copy_output("L1a.html", outfile)
 
 
@@ -106,11 +131,15 @@ pt <- file.path(ROOT, "plot_table.csv")
 outfile <- paste0("L1b_", now_string(), ".html")
 outfile <- file.path(LOGS, outfile)
 
-# quarto_render("L1b.qmd",
+# driver_try(
+# quarto_render("L1b.qmd",
 # execute_params = list(DATA_ROOT = ROOT,
 # html_outfile = outfile,
 # logfile = LOGFILE))
+# )
 # copy_output("L1b.html", outfile)
 
 
+if(ERROR_OCCURRED) warning ("One or more errors occurred!")
+
 message("All done.")