From 6ca24de9c9a0f80216867406d9747edc4c959fcb Mon Sep 17 00:00:00 2001 From: Ben Bond-Lamberty Date: Wed, 25 Oct 2023 19:25:52 -0400 Subject: [PATCH] Rename things to new level scheme: Raw-L0-L1_normalize-L1-L2 (#47) --- portage/{L1a.qmd => L1.qmd} | 18 +++++------ portage/{L1b.qmd => L2.qmd} | 0 portage/data_TEST/{L1a => L1}/.gitignore | 0 portage/data_TEST/{L1a => L1}/README.md | 0 portage/data_TEST/{L1b => L2}/.gitignore | 0 portage/data_TEST/{L1b => L2}/README.md | 0 .../README.md | 0 .../teros.csv | 0 .../trees.csv | 0 .../{metadata_L1a.csv => metadata_L1.csv} | 0 portage/driver.R | 32 +++++++++---------- portage/helpers.R | 26 +++++++-------- 12 files changed, 38 insertions(+), 38 deletions(-) rename portage/{L1a.qmd => L1.qmd} (90%) rename portage/{L1b.qmd => L2.qmd} (100%) rename portage/data_TEST/{L1a => L1}/.gitignore (100%) rename portage/data_TEST/{L1a => L1}/README.md (100%) rename portage/data_TEST/{L1b => L2}/.gitignore (100%) rename portage/data_TEST/{L1b => L2}/README.md (100%) rename portage/data_TEST/{L1b_output_templates => L2_output_templates}/README.md (100%) rename portage/data_TEST/{L1b_output_templates => L2_output_templates}/teros.csv (100%) rename portage/data_TEST/{L1b_output_templates => L2_output_templates}/trees.csv (100%) rename portage/data_TEST/{metadata_L1a.csv => metadata_L1.csv} (100%) diff --git a/portage/L1a.qmd b/portage/L1.qmd similarity index 90% rename from portage/L1a.qmd rename to portage/L1.qmd index 4259f47..7b9c168 100644 --- a/portage/L1a.qmd +++ b/portage/L1.qmd @@ -1,12 +1,12 @@ --- -title: "L1a" +title: "L1" author: "COMPASS workflows team" title-block-banner: true params: - html_outfile: "L1a.html" + html_outfile: "L1.html" DATA_ROOT: "data_TEST/" L1_NORMALIZE: "L1_normalize/" - L1A: "L1a/" + L1: "L1/" debug: false remove_input_files: false logfile: "" @@ -40,13 +40,13 @@ source("helpers.R") L1_NORMALIZE <- file.path(params$DATA_ROOT, params$L1_NORMALIZE) dirs_to_process <- scan_folders(L1_NORMALIZE) -L1A <- file.path(params$DATA_ROOT, params$L1A) +L1 <- file.path(params$DATA_ROOT, params$L1) ``` I see `r length(dirs_to_process)` directories to process in `r L1_NORMALIZE`. -Output directory is `r L1A`. +Output directory is `r L1`. HTML outfile is "`r params$html_outfile`". @@ -91,19 +91,19 @@ f <- function(dir_name, dirs_to_process, out_dir) { write_to_folders(dat, root_dir = out_dir, - data_level = "L1a", + data_level = "L1", site = dat$Site[1]) return(smry) } -log_info("About to L1a", logfile = params$logfile) +log_info("About to L1", logfile = params$logfile) tryCatch({ out <- lapply(names(dirs_to_process), f, - dirs_to_process = dirs_to_process, out_dir = L1A) + dirs_to_process = dirs_to_process, out_dir = L1) }, error = function(e) { - log_warning("L1a: an error occurred!", logfile = params$logfile) + log_warning("L1: an error occurred!", logfile = params$logfile) log_info(as.character(e), logfile = params$logfile) stop(e) }) diff --git a/portage/L1b.qmd b/portage/L2.qmd similarity index 100% rename from portage/L1b.qmd rename to portage/L2.qmd diff --git a/portage/data_TEST/L1a/.gitignore b/portage/data_TEST/L1/.gitignore similarity index 100% rename from portage/data_TEST/L1a/.gitignore rename to portage/data_TEST/L1/.gitignore diff --git a/portage/data_TEST/L1a/README.md b/portage/data_TEST/L1/README.md similarity index 100% rename from portage/data_TEST/L1a/README.md rename to portage/data_TEST/L1/README.md diff --git a/portage/data_TEST/L1b/.gitignore b/portage/data_TEST/L2/.gitignore similarity index 100% rename from portage/data_TEST/L1b/.gitignore rename to portage/data_TEST/L2/.gitignore diff --git a/portage/data_TEST/L1b/README.md b/portage/data_TEST/L2/README.md similarity index 100% rename from portage/data_TEST/L1b/README.md rename to portage/data_TEST/L2/README.md diff --git a/portage/data_TEST/L1b_output_templates/README.md b/portage/data_TEST/L2_output_templates/README.md similarity index 100% rename from portage/data_TEST/L1b_output_templates/README.md rename to portage/data_TEST/L2_output_templates/README.md diff --git a/portage/data_TEST/L1b_output_templates/teros.csv b/portage/data_TEST/L2_output_templates/teros.csv similarity index 100% rename from portage/data_TEST/L1b_output_templates/teros.csv rename to portage/data_TEST/L2_output_templates/teros.csv diff --git a/portage/data_TEST/L1b_output_templates/trees.csv b/portage/data_TEST/L2_output_templates/trees.csv similarity index 100% rename from portage/data_TEST/L1b_output_templates/trees.csv rename to portage/data_TEST/L2_output_templates/trees.csv diff --git a/portage/data_TEST/metadata_L1a.csv b/portage/data_TEST/metadata_L1.csv similarity index 100% rename from portage/data_TEST/metadata_L1a.csv rename to portage/data_TEST/metadata_L1.csv diff --git a/portage/driver.R b/portage/driver.R index 2bdc4ca..6ed85aa 100644 --- a/portage/driver.R +++ b/portage/driver.R @@ -1,7 +1,7 @@ # Driver script for data workflow # # This calls the quarto (*.qmd) files that handle data processing for -# each step (raw data to L0, L0 to L1a, etc). +# each step (raw data to L0, L0 to L1, etc). library(quarto) @@ -85,27 +85,27 @@ driver_try( copy_output("L1_normalize.html", outfile) -# Construct L1a data -------------------------------------------- +# Construct L1 data -------------------------------------------- # Unit conversion and bounds checks performed -# L1a data are long form but without any plot (experimental) info +# L1 data are long form but without any plot (experimental) info # This step will use a 'units_bounds.csv' file or something like that # This step also sorts data into folders based on site, year, and month; # see write_to_folders() in helpers.R -message("Running L1a.qmd") -new_section("Starting L1a") +message("Running L1.qmd") +new_section("Starting L1") -outfile <- paste0("L1a_", now_string(), ".html") +outfile <- paste0("L1_", now_string(), ".html") outfile <- file.path(LOGS, outfile) driver_try( - quarto_render("L1a.qmd", + quarto_render("L1.qmd", execute_params = list(DATA_ROOT = ROOT, html_outfile = outfile, logfile = LOGFILE)) ) -copy_output("L1a.html", outfile) +copy_output("L1.html", outfile) # Manual QA/QC step --------------------------------------------- @@ -114,30 +114,30 @@ copy_output("L1a.html", outfile) # from this step -# Summary report data at L1a stage ------------------------------ +# Summary report data at L1 stage ------------------------------ # Seems like it? So that we move files out of this stage by hand # (i.e. when month folder is complete)? -# Construct L1b data -------------------------------------------- -# L1b data are semi-wide form, organized around experimental units for +# Construct L2 data -------------------------------------------- +# L2 data are semi-wide form, organized around experimental units for # each timestamp. They have been matched with plot/experimental info, and ready for analysis -message("Running L1b.qmd") -new_section("Starting L1b") +message("Running L2.qmd") +new_section("Starting L2") pt <- file.path(ROOT, "plot_table.csv") -outfile <- paste0("L1b_", now_string(), ".html") +outfile <- paste0("L2_", now_string(), ".html") outfile <- file.path(LOGS, outfile) # driver_try( -# quarto_render("L1b.qmd", +# quarto_render("L2.qmd", # execute_params = list(DATA_ROOT = ROOT, # html_outfile = outfile, # logfile = LOGFILE)) # ) -# copy_output("L1b.html", outfile) +# copy_output("L2.html", outfile) if(ERROR_OCCURRED) warning ("One or more errors occurred!") diff --git a/portage/helpers.R b/portage/helpers.R index e094e9e..0a45a85 100644 --- a/portage/helpers.R +++ b/portage/helpers.R @@ -19,8 +19,8 @@ new_section <- function(name, logfile = LOGFILE, root = ROOT) { list_directories(list(file.path(root, "Raw/"), file.path(root, "L0/"), file.path(root, "L1_normalize/"), - file.path(root, "L1a/"), - file.path(root, "L1b")), + file.path(root, "L1/"), + file.path(root, "L2")), list_files = TRUE, outfile = logfile) } @@ -95,10 +95,10 @@ write_to_folders <- function(x, root_dir, data_level, site, if(data_level == "L1_normalize") { folder <- file.path(root_dir, paste(site, y, m, sep = "_")) filename <- paste0(paste(logger, table, y, m, sep = "_"), ".csv") - } else if(data_level == "L1a") { + } else if(data_level == "L1") { folder <- file.path(root_dir, paste(site, y, m, sep = "_")) filename <- paste0(paste(site, tr, data_level, sep = "_"), ".csv") - } else if(data_level == "L1b") { + } else if(data_level == "L1") { folder <- file.path(root_dir, paste(site, y, sep = "_")) filename <- paste0(paste(site, y, m, data_level, table, data_level, sep = "_"), ".csv") } else { @@ -134,7 +134,7 @@ write_to_folders <- function(x, root_dir, data_level, site, # Reset the system by removing all intermediate files in L0, L1_normalize, -# L1a, L1b, and Logs folders +# L1, L2, and Logs folders reset <- function(root = here::here("portage/data")) { message("root is ", root) items <- list.files(file.path(root, "L0/"), pattern = "*.csv", @@ -147,23 +147,23 @@ reset <- function(root = here::here("portage/data")) { message("Removing ", length(items), " files in L1_normalize") lapply(items, file.remove) - items <- list.files(file.path(root, "L1a/"), recursive = TRUE, + items <- list.files(file.path(root, "L1/"), recursive = TRUE, include.dirs = FALSE, full.names = TRUE) items <- items[basename(items) != "README.md"] - message("Removing ", length(items), " files in L1a") + message("Removing ", length(items), " files in L1") lapply(items, file.remove) - items <- list.files(file.path(root, "L1a/"), recursive = TRUE, + items <- list.files(file.path(root, "L1/"), recursive = TRUE, include.dirs = TRUE, full.names = TRUE) items <- items[basename(items) != "README.md"] - message("Removing ", length(items), " directories in L1a") + message("Removing ", length(items), " directories in L1") lapply(items, file.remove) - items <- list.files(file.path(root, "L1b/"), recursive = TRUE, + items <- list.files(file.path(root, "L2/"), recursive = TRUE, include.dirs = FALSE, full.names = TRUE) items <- items[basename(items) != "README.md"] - message("Removing ", length(items), " files in L1b") + message("Removing ", length(items), " files in L2") lapply(items, file.remove) - items <- list.files(file.path(root, "L1a/"), recursive = TRUE, + items <- list.files(file.path(root, "L2/"), recursive = TRUE, include.dirs = TRUE, full.names = TRUE) items <- items[basename(items) != "README.md"] message("Removing ", length(items), " directories in L1a") @@ -181,7 +181,7 @@ reset <- function(root = here::here("portage/data")) { # Print a nicely-formatted directory tree and its files # Usage: # list_directories(list("portage/Raw/", "portage/L0/", "portage/L1_normalize/", -# "portage/L1a/", "portage/L1b")) +# "portage/L1/", "portage/L2")) list_directories <- function(dir_list, outfile = "", prefix = "", pattern = NULL, list_files = TRUE) {