Draft: Workflow improvements to run independently of BETY

base/remote/R/check_qsub_status.R

@@ -24,7 +24,7 @@ qsub_run_finished <- function(run, host, qstat) {
     out <- remote.execute.cmd(host = host, cmd = check, stderr = TRUE)
   }
 
-  if (length(out) > 0 && substring(out, nchar(out) - 3) == "DONE") {
+  if (length(out) > 0 && all(substring(out, nchar(out) - 3) == "DONE")) {
     PEcAn.logger::logger.debug("Job", run, "for run", run_id_string, "finished")
     return(TRUE)
   } else {

base/workflow/R/run.write.configs.R

@@ -25,14 +25,25 @@
 run.write.configs <- function(settings, write = TRUE, ens.sample.method = "uniform", 
                               posterior.files = rep(NA, length(settings$pfts)), 
                               overwrite = TRUE) {
-  tryCatch({
-    con <- PEcAn.DB::db.open(settings$database$bety)
-    on.exit(PEcAn.DB::db.close(con), add = TRUE)
-  }, error = function(e) {
-    PEcAn.logger::logger.severe(
-      "Connection requested, but failed to open with the following error: ",
-      conditionMessage(e))
-  })
+  ## Skip database connection if settings$database is NULL or write is False
+  if (!isTRUE(write) && is.null(settings$database)) {
+    PEcAn.logger::logger.info("Not writing this run to database, so database connection skipped")
+    con <- NULL  # Set con to NULL to avoid errors in subsequent code
+  } else if(is.null(settings$database)) {
+    PEcAn.logger::logger.error(
+      "Database is NULL but writing is enabled. Provide valid database settings in pecan.xml."
+    )
+    stop("Database connection required but settings$database is NULL.")
+  } else{
+    tryCatch({
+      con <- PEcAn.DB::db.open(settings$database$bety)
+      on.exit(PEcAn.DB::db.close(con), add = TRUE)
+    }, error = function(e) {
+      PEcAn.logger::logger.severe(
+        "Connection requested, but failed to open with the following error: ",
+        conditionMessage(e))
+    })
+  }
 
   ## Which posterior to use?
   for (i in seq_along(settings$pfts)) {
@@ -81,7 +92,7 @@ run.write.configs <- function(settings, write = TRUE, ens.sample.method = "unifo
   model <- settings$model$type
   scipen <- getOption("scipen")
   options(scipen = 12)
-
+  
   PEcAn.uncertainty::get.parameter.samples(settings, posterior.files, ens.sample.method)
   samples.file <- file.path(settings$outdir, "samples.Rdata")
   if (file.exists(samples.file)) {
@@ -109,8 +120,8 @@ run.write.configs <- function(settings, write = TRUE, ens.sample.method = "unifo
   my.write.config <- paste0("write.config.",model)
   if (!exists(my.write.config)) {
     PEcAn.logger::logger.error(my.write.config, 
-                 "does not exist, please make sure that the model package contains a function called", 
-                 my.write.config)
+                               "does not exist, please make sure that the model package contains a function called", 
+                               my.write.config)
   }
 
   ## Prepare for model output.  Clean up any old config files (if exists)
@@ -134,29 +145,29 @@ run.write.configs <- function(settings, write = TRUE, ens.sample.method = "unifo
     ### Write out SA config files
     PEcAn.logger::logger.info("\n ----- Writing model run config files ----")
     sa.runs <- PEcAn.uncertainty::write.sa.configs(defaults = settings$pfts,
-                                quantile.samples = sa.samples, 
-                                settings = settings, 
-                                model = model,
-                                write.to.db = write)
+                                                   quantile.samples = sa.samples, 
+                                                   settings = settings, 
+                                                   model = model,
+                                                   write.to.db = write)
 
     # Store output in settings and output variables
     runs.samples$sa <- sa.run.ids <- sa.runs$runs
     settings$sensitivity.analysis$ensemble.id <- sa.ensemble.id <- sa.runs$ensemble.id
 
     # Save sensitivity analysis info
     fname <- PEcAn.uncertainty::sensitivity.filename(settings, "sensitivity.samples", "Rdata",
-                                  all.var.yr = TRUE, pft = NULL)
+                                                     all.var.yr = TRUE, pft = NULL)
     save(sa.run.ids, sa.ensemble.id, sa.samples, pft.names, trait.names, file = fname)
 
   }  ### End of SA
 
   ### Write ENSEMBLE
   if ("ensemble" %in% names(settings)) {
     ens.runs <- PEcAn.uncertainty::write.ensemble.configs(defaults = settings$pfts,
-                                       ensemble.samples = ensemble.samples, 
-                                       settings = settings,
-                                       model = model, 
-                                       write.to.db = write)
+                                                          ensemble.samples = ensemble.samples, 
+                                                          settings = settings,
+                                                          model = model, 
+                                                          write.to.db = write)
 
     # Store output in settings and output variables
     runs.samples$ensemble <- ens.run.ids <- ens.runs$runs

base/workflow/R/runModule.run.write.configs.R

@@ -31,4 +31,4 @@ runModule.run.write.configs <- function(settings, overwrite = TRUE) {
   } else {
     stop("runModule.run.write.configs only works with Settings or MultiSettings")
   }
-}
+ }

base/workflow/R/start_model_runs.R

@@ -100,8 +100,10 @@ start_model_runs <- function(settings, write = TRUE, stop.on.error = TRUE) {
   # launch each of the jobs
   for (run in run_list) {
     run_id_string <- format(run, scientific = FALSE)
-    # write start time to database
-    PEcAn.DB::stamp_started(con = dbcon, run = run)
+    if(write){
+      # write start time to database
+      PEcAn.DB::stamp_started(con = dbcon, run = run)
+    }
 
     # check to see if we use the model launcher
     if (is_rabbitmq) {
@@ -170,9 +172,10 @@ start_model_runs <- function(settings, write = TRUE, stop.on.error = TRUE) {
         )
       }
 
-      # write finished time to database
-      PEcAn.DB::stamp_finished(con = dbcon, run = run)
-
+      if (write){
+        # write finished time to database
+        PEcAn.DB::stamp_finished(con = dbcon, run = run)
+      }
       pbi <- pbi + 1
       utils::setTxtProgressBar(pb, pbi)
     }
@@ -236,12 +239,12 @@ start_model_runs <- function(settings, write = TRUE, stop.on.error = TRUE) {
       }
 
     } else {
-      out <- PEcAn.remote::start_serial(
-        run = run,
-        host = settings$host,
-        rundir = settings$rundir,
-        host_rundir = settings$host$rundir,
-        job_script = "launcher.sh")
+        out <- PEcAn.remote::start_serial(
+          run = run,
+          host = settings$host,
+          rundir = settings$rundir,
+          host_rundir = settings$host$rundir,
+          job_script = "launcher.sh")
 
       # check output to see if an error occurred during the model run
       PEcAn.remote::check_model_run(out = out, stop.on.error = TRUE)

modules/uncertainty/R/ensemble.R

@@ -222,7 +222,10 @@ write.ensemble.configs <- function(defaults, ensemble.samples, settings, model,
   }
 
   # See if we need to write to DB
-  write.to.db <- as.logical(settings$database$bety$write)
+  if (!is.null(settings$database$bety$write)) {
+    # specifying `write` in settings overrides write.to.db in fn args
+    write.to.db <- as.logical(settings$database$bety$write)
+  }
 
   if (write.to.db) {
     # Open connection to database so we can store all run/ensemble information
@@ -233,7 +236,7 @@ write.ensemble.configs <- function(defaults, ensemble.samples, settings, model,
     # If we fail to connect to DB then we set to NULL
     if (inherits(con, "try-error"))  {
       con <- NULL
-      PEcAn.logger::logger.warn("We were not able to successfully establish a connection with Bety ")
+      PEcAn.logger::logger.warn("We were not able to successfully establish a connection with BETYdb ")
     }
   }
 

modules/uncertainty/R/get.parameter.samples.R

@@ -15,10 +15,11 @@ get.parameter.samples <- function(settings,
   num.pfts  <- length(settings$pfts)
   pft.names <- list()
   outdirs   <- list()
+
   ## Open database connection
   con <- try(PEcAn.DB::db.open(settings$database$bety))
   on.exit(try(PEcAn.DB::db.close(con), silent = TRUE), add = TRUE)
-  
+
   # If we fail to connect to DB then we set to NULL
   if (inherits(con, "try-error"))  {
     con <- NULL
@@ -202,7 +203,7 @@ get.parameter.samples <- function(settings,
                                                env.samples, ens.sample.method, param.names)
     }
   }
-  
+
   save(ensemble.samples, trait.samples, sa.samples, runs.samples, env.samples, 
        file = file.path(settings$outdir, "samples.Rdata"))
 } # get.parameter.samples