Rename things to new level scheme: Raw-L0-L1_normalize-L1-L2 (#47)

portage/L1a.qmd → portage/L1.qmd

@@ -1,12 +1,12 @@
 ---
-title: "L1a"
+title: "L1"
 author: "COMPASS workflows team"
 title-block-banner: true
 params:
- html_outfile: "L1a.html"
+ html_outfile: "L1.html"
  DATA_ROOT: "data_TEST/"
  L1_NORMALIZE: "L1_normalize/"
- L1A: "L1a/"
+ L1: "L1/"
  debug: false
  remove_input_files: false
  logfile: ""
@@ -40,13 +40,13 @@ source("helpers.R")
 L1_NORMALIZE <- file.path(params$DATA_ROOT, params$L1_NORMALIZE)
 dirs_to_process <- scan_folders(L1_NORMALIZE)
 
-L1A <- file.path(params$DATA_ROOT, params$L1A)
+L1 <- file.path(params$DATA_ROOT, params$L1)
 
 ```
 
 I see `r length(dirs_to_process)` directories to process in `r L1_NORMALIZE`.
 
-Output directory is `r L1A`.
+Output directory is `r L1`.
 
 HTML outfile is "`r params$html_outfile`".
 
@@ -91,19 +91,19 @@ f <- function(dir_name, dirs_to_process, out_dir) {
  
  write_to_folders(dat, 
  root_dir = out_dir, 
- data_level = "L1a",
+ data_level = "L1",
  site = dat$Site[1])
  
  return(smry)
 }
 
-log_info("About to L1a", logfile = params$logfile)
+log_info("About to L1", logfile = params$logfile)
 tryCatch({
 out <- lapply(names(dirs_to_process), f, 
- dirs_to_process = dirs_to_process, out_dir = L1A)
+ dirs_to_process = dirs_to_process, out_dir = L1)
 },
 error = function(e) {
- log_warning("L1a: an error occurred!", logfile = params$logfile)
+ log_warning("L1: an error occurred!", logfile = params$logfile)
  log_info(as.character(e), logfile = params$logfile)
  stop(e)
 })

portage/driver.R

@@ -1,7 +1,7 @@
 # Driver script for data workflow
 #
 # This calls the quarto (*.qmd) files that handle data processing for
-# each step (raw data to L0, L0 to L1a, etc).
+# each step (raw data to L0, L0 to L1, etc).
 
 library(quarto)
 
@@ -85,27 +85,27 @@ driver_try(
 copy_output("L1_normalize.html", outfile)
 
 
-# Construct L1a data --------------------------------------------
+# Construct L1 data --------------------------------------------
 # Unit conversion and bounds checks performed
-# L1a data are long form but without any plot (experimental) info
+# L1 data are long form but without any plot (experimental) info
 
 # This step will use a 'units_bounds.csv' file or something like that
 # This step also sorts data into folders based on site, year, and month;
 # see write_to_folders() in helpers.R
 
-message("Running L1a.qmd")
-new_section("Starting L1a")
+message("Running L1.qmd")
+new_section("Starting L1")
 
-outfile <- paste0("L1a_", now_string(), ".html")
+outfile <- paste0("L1_", now_string(), ".html")
 outfile <- file.path(LOGS, outfile)
 
 driver_try(
- quarto_render("L1a.qmd",
+ quarto_render("L1.qmd",
  execute_params = list(DATA_ROOT = ROOT,
  html_outfile = outfile,
  logfile = LOGFILE))
 )
-copy_output("L1a.html", outfile)
+copy_output("L1.html", outfile)
 
 
 # Manual QA/QC step ---------------------------------------------
@@ -114,30 +114,30 @@ copy_output("L1a.html", outfile)
 # from this step
 
 
-# Summary report data at L1a stage ------------------------------
+# Summary report data at L1 stage ------------------------------
 
 # Seems like it? So that we move files out of this stage by hand
 # (i.e. when month folder is complete)?
 
 
-# Construct L1b data --------------------------------------------
-# L1b data are semi-wide form, organized around experimental units for
+# Construct L2 data --------------------------------------------
+# L2 data are semi-wide form, organized around experimental units for
 # each timestamp. They have been matched with plot/experimental info, and ready for analysis
 
-message("Running L1b.qmd")
-new_section("Starting L1b")
+message("Running L2.qmd")
+new_section("Starting L2")
 
 pt <- file.path(ROOT, "plot_table.csv")
-outfile <- paste0("L1b_", now_string(), ".html")
+outfile <- paste0("L2_", now_string(), ".html")
 outfile <- file.path(LOGS, outfile)
 
 # driver_try(
-# quarto_render("L1b.qmd",
+# quarto_render("L2.qmd",
 # execute_params = list(DATA_ROOT = ROOT,
 # html_outfile = outfile,
 # logfile = LOGFILE))
 # )
-# copy_output("L1b.html", outfile)
+# copy_output("L2.html", outfile)
 
 
 if(ERROR_OCCURRED) warning ("One or more errors occurred!")

portage/helpers.R

@@ -19,8 +19,8 @@ new_section <- function(name, logfile = LOGFILE, root = ROOT) {
  list_directories(list(file.path(root, "Raw/"),
  file.path(root, "L0/"),
  file.path(root, "L1_normalize/"),
- file.path(root, "L1a/"),
- file.path(root, "L1b")),
+ file.path(root, "L1/"),
+ file.path(root, "L2")),
  list_files = TRUE, outfile = logfile)
 }
 
@@ -95,10 +95,10 @@ write_to_folders <- function(x, root_dir, data_level, site,
  if(data_level == "L1_normalize") {
  folder <- file.path(root_dir, paste(site, y, m, sep = "_"))
  filename <- paste0(paste(logger, table, y, m, sep = "_"), ".csv")
- } else if(data_level == "L1a") {
+ } else if(data_level == "L1") {
  folder <- file.path(root_dir, paste(site, y, m, sep = "_"))
  filename <- paste0(paste(site, tr, data_level, sep = "_"), ".csv")
- } else if(data_level == "L1b") {
+ } else if(data_level == "L1") {
  folder <- file.path(root_dir, paste(site, y, sep = "_"))
  filename <- paste0(paste(site, y, m, data_level, table, data_level, sep = "_"), ".csv")
  } else {
@@ -134,7 +134,7 @@ write_to_folders <- function(x, root_dir, data_level, site,
 
 
 # Reset the system by removing all intermediate files in L0, L1_normalize,
-# L1a, L1b, and Logs folders
+# L1, L2, and Logs folders
 reset <- function(root = here::here("portage/data")) {
  message("root is ", root)
  items <- list.files(file.path(root, "L0/"), pattern = "*.csv",
@@ -147,23 +147,23 @@ reset <- function(root = here::here("portage/data")) {
  message("Removing ", length(items), " files in L1_normalize")
  lapply(items, file.remove)
 
- items <- list.files(file.path(root, "L1a/"), recursive = TRUE,
+ items <- list.files(file.path(root, "L1/"), recursive = TRUE,
  include.dirs = FALSE, full.names = TRUE)
  items <- items[basename(items) != "README.md"]
- message("Removing ", length(items), " files in L1a")
+ message("Removing ", length(items), " files in L1")
  lapply(items, file.remove)
- items <- list.files(file.path(root, "L1a/"), recursive = TRUE,
+ items <- list.files(file.path(root, "L1/"), recursive = TRUE,
  include.dirs = TRUE, full.names = TRUE)
  items <- items[basename(items) != "README.md"]
- message("Removing ", length(items), " directories in L1a")
+ message("Removing ", length(items), " directories in L1")
  lapply(items, file.remove)
 
- items <- list.files(file.path(root, "L1b/"), recursive = TRUE,
+ items <- list.files(file.path(root, "L2/"), recursive = TRUE,
  include.dirs = FALSE, full.names = TRUE)
  items <- items[basename(items) != "README.md"]
- message("Removing ", length(items), " files in L1b")
+ message("Removing ", length(items), " files in L2")
  lapply(items, file.remove)
- items <- list.files(file.path(root, "L1a/"), recursive = TRUE,
+ items <- list.files(file.path(root, "L2/"), recursive = TRUE,
  include.dirs = TRUE, full.names = TRUE)
  items <- items[basename(items) != "README.md"]
  message("Removing ", length(items), " directories in L1a")
@@ -181,7 +181,7 @@ reset <- function(root = here::here("portage/data")) {
 # Print a nicely-formatted directory tree and its files
 # Usage:
 # list_directories(list("portage/Raw/", "portage/L0/", "portage/L1_normalize/",
-# "portage/L1a/", "portage/L1b"))
+# "portage/L1/", "portage/L2"))
 list_directories <- function(dir_list, outfile = "", prefix = "",
  pattern = NULL, list_files = TRUE) {