Add data version info and overall version README

synoptic/L1.qmd

@@ -8,6 +8,7 @@ params:
  L1_NORMALIZE: "L1_normalize/"
  L1: "L1/"
  L1_METADATA: "L1_metadata/"
+ L1_VERSION: "???"
  debug: false
  remove_input_files: false
  write_plots: true
@@ -49,7 +50,6 @@ L1_NORMALIZE <- file.path(params$DATA_ROOT, params$L1_NORMALIZE)
 dirs_to_process <- scan_folders(L1_NORMALIZE)
 
 L1 <- file.path(params$DATA_ROOT, params$L1)
-
 ```
 
 I see `r length(dirs_to_process)` directories to process in `r L1_NORMALIZE`.
@@ -130,6 +130,17 @@ error = function(e) {
 L1 metadata template directory is `r params$L1_METADATA`.
 
 ```{r metadata}
+# Write the overall README
+readme_fn <- file.path(params$DATA_ROOT, params$L1_METADATA, 
+ paste0("README_v", params$L1_VERSION, ".txt"))
+if(!file.exists(readme_fn)) stop("Couln't find ", readme_fn)
+readme <- readLines(readme_fn)
+readme <- gsub("[DATESTAMP]", format(Sys.time(), "%Y-%m-%d"), readme, fixed = TRUE)
+readme <- gsub("[VERSION]", params$L1_VERSION, readme, fixed = TRUE)
+readme <- gsub("[GIT_COMMIT]", GIT_COMMIT, readme, fixed = TRUE)
+readme_outfn <- file.path(L1, basename(readme_fn))
+message("Writing overall README ", readme_outfn, "...")
+writeLines(readme, readme_outfn)
 
 # Get the L1 template file
 template_file <- file.path(params$DATA_ROOT,
@@ -160,16 +171,22 @@ message("Main template has ", length(L1_metadata_template), " lines")
 message("Column metadata info has ", length(col_md_for_insert), " lines")
 message("Variable metadata info has ", length(var_md_for_insert), " lines")
 
-# Identify the main data directories in L1/, which are <site>_<year>
-data_dirs <- list.files(file.path(params$DATA_ROOT, params$L1), 
- pattern = "^[a-zA-Z]+_[0-4]{4}$")
+message("Inserting data version string")
+L1_metadata_template <- gsub("[VERSION]", params$L1_VERSION,
+ L1_metadata_template, fixed = TRUE)
+
+# Identify the main data directories in L1/{version}/, which are <site>_<year>
+data_dirs <- list.files(L1, pattern = "^[a-zA-Z]+_[0-4]{4}$")
 
 for(dd in data_dirs) {
- dd_full <- file.path(params$DATA_ROOT, params$L1, dd)
+ dd_full <- file.path(L1, dd)
  message("Generating metadata for ", dd_full)
 
+ message("\tInserting verion")
+ md <- gsub("[VERSION]", params$L1_VERSION, L1_metadata_template, fixed = TRUE)
+
  message("\tInserting timestamp and folder name")
- md <- gsub("[TIMESTAMP]", date(), L1_metadata_template, fixed = TRUE)
+ md <- gsub("[TIMESTAMP]", date(), md, fixed = TRUE)
  md <- gsub("[FOLDER_NAME]", dd, md, fixed = TRUE)
 
  # File info
@@ -232,7 +249,7 @@ for(dd in data_dirs) {
  # Write the final metadata file
  mdfn <- paste0(dd, "_metadata.txt")
  message("\tWriting ", mdfn, "...")
- writeLines(md, file.path(params$DATA_ROOT, params$L1, dd, mdfn))
+ writeLines(md, file.path(L1, dd, mdfn))
 }
 ```
 

synoptic/data_TEST/L1_metadata/L1_metadata_template.txt

@@ -1,4 +1,5 @@
-COMPASS-FME L1 data documentation file
+COMPASS-FME Level 1 documentation file
+Data version [VERSION]
 [FOLDER_NAME]
 [TIMESTAMP]
 
@@ -8,8 +9,8 @@ General information
 Level 1 (L1) data are close to raw, although they have been units-transformed
 and have out-of-instrument-bounds and out-of-service flags added.
 Duplicates are removed but otherwise these data are not filtered,
-and have not been subject to any algorithmic or human QA/QC.
-Any use of L1 data for science analyses should be done with care.
+and have not been subject to any additional algorithmic or human QA/QC.
+Any use of L1 data for science analyses should be performed with care.
 
 
 Site information:

synoptic/data_TEST/L1_metadata/README_v???.txt

@@ -0,0 +1,27 @@
+COMPASS-FME Level 1 data
+Version: [VERSION]
+Date: [DATESTAMP]
+Git commit: [GIT_COMMIT]
+
+DESCRIPTION
+—----------------------------------
+Level 1 (L1) data are close to raw, although they have been units-transformed
+and have out-of-instrument-bounds and out-of-service flags added.
+Duplicates are removed but otherwise these data are not filtered,
+and have not been subject to any additional algorithmic or human QA/QC.
+Any use of L1 data for science analyses should be performed with care.
+
+CONTACT
+—----------------------------------
+Project: https://compass.pnnl.gov
+Data lead: Stephanie Pennington, stephanie.pennington@pnnl.gov
+
+HOW TO CITE THESE DATA
+—----------------------------------
+Pennington, Rich, Cheng, and Bond-Lamberty (2024). COMPASS-FME Level 1 Sensor Data
+(version [VERSION]), downloaded YYYY-MM-DD, https://compass.pnnl.gov.
+
+CHANGELOG
+—----------------------------------
+Version ??? released YYYY-MM-DD
+* Release notes here

synoptic/driver.R

@@ -20,6 +20,7 @@ source("helpers.R")
 now_string <- function() format(Sys.time(), "%Y%m%d.%H%M")
 
 ROOT <- "./data_TEST"
+VERSION <- "???"
 
 # Log file ----------------------------------------------------
 
@@ -102,6 +103,7 @@ outfile <- file.path(LOGS, outfile)
 driver_try(
  quarto_render("L1.qmd",
  execute_params = list(DATA_ROOT = ROOT,
+ L1_VERSION = VERSION,
  html_outfile = outfile,
  logfile = LOGFILE))
 )