From fc7fd71ba05f50c2e737156444ae84d1b1697e50 Mon Sep 17 00:00:00 2001 From: Yinghao Sun Date: Wed, 2 Oct 2024 14:43:15 -0400 Subject: [PATCH 1/8] Fix the problem where checking qsub status. --- base/remote/R/check_qsub_status.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/remote/R/check_qsub_status.R b/base/remote/R/check_qsub_status.R index 76d744de845..892dc37e895 100644 --- a/base/remote/R/check_qsub_status.R +++ b/base/remote/R/check_qsub_status.R @@ -24,7 +24,7 @@ qsub_run_finished <- function(run, host, qstat) { out <- remote.execute.cmd(host = host, cmd = check, stderr = TRUE) } - if (length(out) > 0 && substring(out, nchar(out) - 3) == "DONE") { + if (length(out) > 0 && all(substring(out, nchar(out) - 3) == "DONE")) { PEcAn.logger::logger.debug("Job", run, "for run", run_id_string, "finished") return(TRUE) } else { From 6f2ab9f6450cab0d021a25e98026d42f8a6b3c65 Mon Sep 17 00:00:00 2001 From: Yinghao Sun Date: Fri, 22 Nov 2024 08:58:20 -0500 Subject: [PATCH 2/8] Allow workflow to run independently of BETY database --- base/workflow/R/run.write.configs.R | 48 +++++++++++-------- base/workflow/R/runModule.run.write.configs.R | 3 +- base/workflow/R/start_model_runs.R | 25 +++++----- modules/uncertainty/R/ensemble.R | 2 + modules/uncertainty/R/get.parameter.samples.R | 10 ++-- 5 files changed, 51 insertions(+), 37 deletions(-) diff --git a/base/workflow/R/run.write.configs.R b/base/workflow/R/run.write.configs.R index 22ed6f3b729..394d6dde431 100644 --- a/base/workflow/R/run.write.configs.R +++ b/base/workflow/R/run.write.configs.R @@ -25,14 +25,20 @@ run.write.configs <- function(settings, write = TRUE, ens.sample.method = "uniform", posterior.files = rep(NA, length(settings$pfts)), overwrite = TRUE) { - tryCatch({ - con <- PEcAn.DB::db.open(settings$database$bety) - on.exit(PEcAn.DB::db.close(con), add = TRUE) - }, error = function(e) { - PEcAn.logger::logger.severe( - "Connection requested, but failed to open with the following error: ", - conditionMessage(e)) - }) + ## Skip database connection if settings$database is NULL + if (write == FALSE) { + PEcAn.logger::logger.info("Database connection skipped: No database settings provided.") + con <- NULL # Set con to NULL to avoid errors in subsequent code + } else { + tryCatch({ + con <- PEcAn.DB::db.open(settings$database$bety) + on.exit(PEcAn.DB::db.close(con), add = TRUE) + }, error = function(e) { + PEcAn.logger::logger.severe( + "Connection requested, but failed to open with the following error: ", + conditionMessage(e)) + }) + } ## Which posterior to use? for (i in seq_along(settings$pfts)) { @@ -81,7 +87,7 @@ run.write.configs <- function(settings, write = TRUE, ens.sample.method = "unifo model <- settings$model$type scipen <- getOption("scipen") options(scipen = 12) - + PEcAn.uncertainty::get.parameter.samples(settings, posterior.files, ens.sample.method) samples.file <- file.path(settings$outdir, "samples.Rdata") if (file.exists(samples.file)) { @@ -109,8 +115,8 @@ run.write.configs <- function(settings, write = TRUE, ens.sample.method = "unifo my.write.config <- paste0("write.config.",model) if (!exists(my.write.config)) { PEcAn.logger::logger.error(my.write.config, - "does not exist, please make sure that the model package contains a function called", - my.write.config) + "does not exist, please make sure that the model package contains a function called", + my.write.config) } ## Prepare for model output. Clean up any old config files (if exists) @@ -134,10 +140,10 @@ run.write.configs <- function(settings, write = TRUE, ens.sample.method = "unifo ### Write out SA config files PEcAn.logger::logger.info("\n ----- Writing model run config files ----") sa.runs <- PEcAn.uncertainty::write.sa.configs(defaults = settings$pfts, - quantile.samples = sa.samples, - settings = settings, - model = model, - write.to.db = write) + quantile.samples = sa.samples, + settings = settings, + model = model, + write.to.db = write) # Store output in settings and output variables runs.samples$sa <- sa.run.ids <- sa.runs$runs @@ -145,7 +151,7 @@ run.write.configs <- function(settings, write = TRUE, ens.sample.method = "unifo # Save sensitivity analysis info fname <- PEcAn.uncertainty::sensitivity.filename(settings, "sensitivity.samples", "Rdata", - all.var.yr = TRUE, pft = NULL) + all.var.yr = TRUE, pft = NULL) save(sa.run.ids, sa.ensemble.id, sa.samples, pft.names, trait.names, file = fname) } ### End of SA @@ -153,10 +159,10 @@ run.write.configs <- function(settings, write = TRUE, ens.sample.method = "unifo ### Write ENSEMBLE if ("ensemble" %in% names(settings)) { ens.runs <- PEcAn.uncertainty::write.ensemble.configs(defaults = settings$pfts, - ensemble.samples = ensemble.samples, - settings = settings, - model = model, - write.to.db = write) + ensemble.samples = ensemble.samples, + settings = settings, + model = model, + write.to.db = write) # Store output in settings and output variables runs.samples$ensemble <- ens.run.ids <- ens.runs$runs @@ -181,4 +187,4 @@ run.write.configs <- function(settings, write = TRUE, ens.sample.method = "unifo options(scipen = scipen) return(invisible(settings)) -} +} \ No newline at end of file diff --git a/base/workflow/R/runModule.run.write.configs.R b/base/workflow/R/runModule.run.write.configs.R index 265da4f6c1c..bde5e7f53b3 100644 --- a/base/workflow/R/runModule.run.write.configs.R +++ b/base/workflow/R/runModule.run.write.configs.R @@ -13,7 +13,6 @@ runModule.run.write.configs <- function(settings, overwrite = TRUE) { } return(PEcAn.settings::papply(settings, runModule.run.write.configs, overwrite = FALSE)) } else if (PEcAn.settings::is.Settings(settings)) { - write <- settings$database$bety$write # double check making sure we have method for parameter sampling if (is.null(settings$ensemble$samplingspace$parameters$method)) settings$ensemble$samplingspace$parameters$method <- "uniform" ens.sample.method <- settings$ensemble$samplingspace$parameters$method @@ -31,4 +30,4 @@ runModule.run.write.configs <- function(settings, overwrite = TRUE) { } else { stop("runModule.run.write.configs only works with Settings or MultiSettings") } -} + } diff --git a/base/workflow/R/start_model_runs.R b/base/workflow/R/start_model_runs.R index 12ef228704e..a088f4ab96e 100644 --- a/base/workflow/R/start_model_runs.R +++ b/base/workflow/R/start_model_runs.R @@ -100,8 +100,10 @@ start_model_runs <- function(settings, write = TRUE, stop.on.error = TRUE) { # launch each of the jobs for (run in run_list) { run_id_string <- format(run, scientific = FALSE) - # write start time to database - PEcAn.DB::stamp_started(con = dbcon, run = run) + if(write){ + # write start time to database + PEcAn.DB::stamp_started(con = dbcon, run = run) + } # check to see if we use the model launcher if (is_rabbitmq) { @@ -170,9 +172,10 @@ start_model_runs <- function(settings, write = TRUE, stop.on.error = TRUE) { ) } - # write finished time to database - PEcAn.DB::stamp_finished(con = dbcon, run = run) - + if (write){ + # write finished time to database + PEcAn.DB::stamp_finished(con = dbcon, run = run) + } pbi <- pbi + 1 utils::setTxtProgressBar(pb, pbi) } @@ -236,12 +239,12 @@ start_model_runs <- function(settings, write = TRUE, stop.on.error = TRUE) { } } else { - out <- PEcAn.remote::start_serial( - run = run, - host = settings$host, - rundir = settings$rundir, - host_rundir = settings$host$rundir, - job_script = "launcher.sh") + out <- PEcAn.remote::start_serial( + run = run, + host = settings$host, + rundir = settings$rundir, + host_rundir = settings$host$rundir, + job_script = "launcher.sh") # check output to see if an error occurred during the model run PEcAn.remote::check_model_run(out = out, stop.on.error = TRUE) diff --git a/modules/uncertainty/R/ensemble.R b/modules/uncertainty/R/ensemble.R index dc92e3985a9..e716af8b0f8 100644 --- a/modules/uncertainty/R/ensemble.R +++ b/modules/uncertainty/R/ensemble.R @@ -222,6 +222,7 @@ write.ensemble.configs <- function(defaults, ensemble.samples, settings, model, } # See if we need to write to DB + if (!is.null(settings$database) && !is.null(settings$database$bety) && !is.null(settings$database$bety$write)) { write.to.db <- as.logical(settings$database$bety$write) if (write.to.db) { @@ -236,6 +237,7 @@ write.ensemble.configs <- function(defaults, ensemble.samples, settings, model, PEcAn.logger::logger.warn("We were not able to successfully establish a connection with Bety ") } } + } diff --git a/modules/uncertainty/R/get.parameter.samples.R b/modules/uncertainty/R/get.parameter.samples.R index 03e69b29783..886ae14263f 100644 --- a/modules/uncertainty/R/get.parameter.samples.R +++ b/modules/uncertainty/R/get.parameter.samples.R @@ -15,16 +15,20 @@ get.parameter.samples <- function(settings, num.pfts <- length(settings$pfts) pft.names <- list() outdirs <- list() + if (settings$database$bety$write == FALSE) { + PEcAn.logger::logger.info("Skipping database access in get.parameter.samples because database settings are NULL.") + con <- NULL + } else { ## Open database connection con <- try(PEcAn.DB::db.open(settings$database$bety)) on.exit(try(PEcAn.DB::db.close(con), silent = TRUE), add = TRUE) - + # If we fail to connect to DB then we set to NULL if (inherits(con, "try-error")) { con <- NULL PEcAn.logger::logger.warn("We were not able to successfully establish a connection with Bety ") } - + } for (i.pft in seq_along(pfts)) { pft.names[i.pft] <- settings$pfts[[i.pft]]$name @@ -202,7 +206,7 @@ get.parameter.samples <- function(settings, env.samples, ens.sample.method, param.names) } } - + #ensemble.samples <- NULL save(ensemble.samples, trait.samples, sa.samples, runs.samples, env.samples, file = file.path(settings$outdir, "samples.Rdata")) } # get.parameter.samples From af15734cbf00395960962dbea9452abe18ffecb8 Mon Sep 17 00:00:00 2001 From: David LeBauer Date: Mon, 16 Dec 2024 12:14:54 -0700 Subject: [PATCH 3/8] Update base/workflow/R/run.write.configs.R Co-authored-by: Chris Black --- base/workflow/R/run.write.configs.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/workflow/R/run.write.configs.R b/base/workflow/R/run.write.configs.R index 394d6dde431..56e5cc93a5d 100644 --- a/base/workflow/R/run.write.configs.R +++ b/base/workflow/R/run.write.configs.R @@ -26,8 +26,8 @@ run.write.configs <- function(settings, write = TRUE, ens.sample.method = "unifo posterior.files = rep(NA, length(settings$pfts)), overwrite = TRUE) { ## Skip database connection if settings$database is NULL - if (write == FALSE) { - PEcAn.logger::logger.info("Database connection skipped: No database settings provided.") + if (!isTRUE(write)) { + PEcAn.logger::logger.info("Not writing this run to database, so database connection skipped") con <- NULL # Set con to NULL to avoid errors in subsequent code } else { tryCatch({ From 9c6ebe146742c127ad37ca98f292f4c1b9c06f78 Mon Sep 17 00:00:00 2001 From: David LeBauer Date: Mon, 16 Dec 2024 12:15:51 -0700 Subject: [PATCH 4/8] Update modules/uncertainty/R/ensemble.R --- modules/uncertainty/R/ensemble.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/uncertainty/R/ensemble.R b/modules/uncertainty/R/ensemble.R index e716af8b0f8..9e9d7b14ba9 100644 --- a/modules/uncertainty/R/ensemble.R +++ b/modules/uncertainty/R/ensemble.R @@ -234,7 +234,7 @@ write.ensemble.configs <- function(defaults, ensemble.samples, settings, model, # If we fail to connect to DB then we set to NULL if (inherits(con, "try-error")) { con <- NULL - PEcAn.logger::logger.warn("We were not able to successfully establish a connection with Bety ") + PEcAn.logger::logger.warn("We were not able to successfully establish a connection with BETYdb ") } } } From cf855feb93800c2f057b83e5f37a9c0ae6704e57 Mon Sep 17 00:00:00 2001 From: Yinghao Sun Date: Mon, 16 Dec 2024 15:59:59 -0500 Subject: [PATCH 5/8] Remove unused commented-out line --- modules/uncertainty/R/get.parameter.samples.R | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/uncertainty/R/get.parameter.samples.R b/modules/uncertainty/R/get.parameter.samples.R index 886ae14263f..ff447864bb8 100644 --- a/modules/uncertainty/R/get.parameter.samples.R +++ b/modules/uncertainty/R/get.parameter.samples.R @@ -206,7 +206,6 @@ get.parameter.samples <- function(settings, env.samples, ens.sample.method, param.names) } } - #ensemble.samples <- NULL save(ensemble.samples, trait.samples, sa.samples, runs.samples, env.samples, file = file.path(settings$outdir, "samples.Rdata")) } # get.parameter.samples From e8b87628871aa6c2c095f1c409d32ba7499ca65f Mon Sep 17 00:00:00 2001 From: Yinghao Sun Date: Mon, 16 Dec 2024 16:09:17 -0500 Subject: [PATCH 6/8] Update runModule.run.write.configs.R Re-add the `write <- settings$database$bety$write` that was mistakenly deleted earlier. --- base/workflow/R/runModule.run.write.configs.R | 1 + 1 file changed, 1 insertion(+) diff --git a/base/workflow/R/runModule.run.write.configs.R b/base/workflow/R/runModule.run.write.configs.R index bde5e7f53b3..e7bc4b05fb4 100644 --- a/base/workflow/R/runModule.run.write.configs.R +++ b/base/workflow/R/runModule.run.write.configs.R @@ -13,6 +13,7 @@ runModule.run.write.configs <- function(settings, overwrite = TRUE) { } return(PEcAn.settings::papply(settings, runModule.run.write.configs, overwrite = FALSE)) } else if (PEcAn.settings::is.Settings(settings)) { + write <- settings$database$bety$write # double check making sure we have method for parameter sampling if (is.null(settings$ensemble$samplingspace$parameters$method)) settings$ensemble$samplingspace$parameters$method <- "uniform" ens.sample.method <- settings$ensemble$samplingspace$parameters$method From 0080e8bf4c989b9fe50c8ea1936e3c88b0b18e25 Mon Sep 17 00:00:00 2001 From: Yinghao Sun Date: Fri, 27 Dec 2024 19:49:04 -0500 Subject: [PATCH 7/8] Update modules/uncertainty/R/ensemble.R Co-authored-by: Chris Black --- modules/uncertainty/R/ensemble.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/uncertainty/R/ensemble.R b/modules/uncertainty/R/ensemble.R index 9e9d7b14ba9..b6e51298947 100644 --- a/modules/uncertainty/R/ensemble.R +++ b/modules/uncertainty/R/ensemble.R @@ -222,7 +222,7 @@ write.ensemble.configs <- function(defaults, ensemble.samples, settings, model, } # See if we need to write to DB - if (!is.null(settings$database) && !is.null(settings$database$bety) && !is.null(settings$database$bety$write)) { + if (!is.null(settings$database$bety$write)) { write.to.db <- as.logical(settings$database$bety$write) if (write.to.db) { From dcefee38633e0fe726eab592136ed17366e5c373 Mon Sep 17 00:00:00 2001 From: Yinghao Sun Date: Fri, 27 Dec 2024 19:55:44 -0500 Subject: [PATCH 8/8] Update get.parameter.samples.R Re-add an empty line before the final statement in the get.parameter.samples function to align with common R style guidelines. --- modules/uncertainty/R/get.parameter.samples.R | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/uncertainty/R/get.parameter.samples.R b/modules/uncertainty/R/get.parameter.samples.R index ff447864bb8..bb2a420851b 100644 --- a/modules/uncertainty/R/get.parameter.samples.R +++ b/modules/uncertainty/R/get.parameter.samples.R @@ -206,6 +206,7 @@ get.parameter.samples <- function(settings, env.samples, ens.sample.method, param.names) } } + save(ensemble.samples, trait.samples, sa.samples, runs.samples, env.samples, file = file.path(settings$outdir, "samples.Rdata")) } # get.parameter.samples