diff --git a/synoptic/L1.qmd b/synoptic/L1.qmd index 1a1dffd..00547df 100644 --- a/synoptic/L1.qmd +++ b/synoptic/L1.qmd @@ -8,6 +8,7 @@ params: L1_NORMALIZE: "L1_normalize/" L1: "L1/" L1_METADATA: "L1_metadata/" + L1_VERSION: "???" debug: false remove_input_files: false write_plots: true @@ -49,7 +50,6 @@ L1_NORMALIZE <- file.path(params$DATA_ROOT, params$L1_NORMALIZE) dirs_to_process <- scan_folders(L1_NORMALIZE) L1 <- file.path(params$DATA_ROOT, params$L1) - ``` I see `r length(dirs_to_process)` directories to process in `r L1_NORMALIZE`. @@ -130,6 +130,17 @@ error = function(e) { L1 metadata template directory is `r params$L1_METADATA`. ```{r metadata} +# Write the overall README +readme_fn <- file.path(params$DATA_ROOT, params$L1_METADATA, + paste0("README_v", params$L1_VERSION, ".txt")) +if(!file.exists(readme_fn)) stop("Couln't find ", readme_fn) +readme <- readLines(readme_fn) +readme <- gsub("[DATESTAMP]", format(Sys.time(), "%Y-%m-%d"), readme, fixed = TRUE) +readme <- gsub("[VERSION]", params$L1_VERSION, readme, fixed = TRUE) +readme <- gsub("[GIT_COMMIT]", GIT_COMMIT, readme, fixed = TRUE) +readme_outfn <- file.path(L1, basename(readme_fn)) +message("Writing overall README ", readme_outfn, "...") +writeLines(readme, readme_outfn) # Get the L1 template file template_file <- file.path(params$DATA_ROOT, @@ -160,16 +171,22 @@ message("Main template has ", length(L1_metadata_template), " lines") message("Column metadata info has ", length(col_md_for_insert), " lines") message("Variable metadata info has ", length(var_md_for_insert), " lines") -# Identify the main data directories in L1/, which are _ -data_dirs <- list.files(file.path(params$DATA_ROOT, params$L1), - pattern = "^[a-zA-Z]+_[0-4]{4}$") +message("Inserting data version string") +L1_metadata_template <- gsub("[VERSION]", params$L1_VERSION, + L1_metadata_template, fixed = TRUE) + +# Identify the main data directories in L1/{version}/, which are _ +data_dirs <- list.files(L1, pattern = "^[a-zA-Z]+_[0-4]{4}$") for(dd in data_dirs) { - dd_full <- file.path(params$DATA_ROOT, params$L1, dd) + dd_full <- file.path(L1, dd) message("Generating metadata for ", dd_full) + message("\tInserting verion") + md <- gsub("[VERSION]", params$L1_VERSION, L1_metadata_template, fixed = TRUE) + message("\tInserting timestamp and folder name") - md <- gsub("[TIMESTAMP]", date(), L1_metadata_template, fixed = TRUE) + md <- gsub("[TIMESTAMP]", date(), md, fixed = TRUE) md <- gsub("[FOLDER_NAME]", dd, md, fixed = TRUE) # File info @@ -232,7 +249,7 @@ for(dd in data_dirs) { # Write the final metadata file mdfn <- paste0(dd, "_metadata.txt") message("\tWriting ", mdfn, "...") - writeLines(md, file.path(params$DATA_ROOT, params$L1, dd, mdfn)) + writeLines(md, file.path(L1, dd, mdfn)) } ``` diff --git a/synoptic/data_TEST/L1_metadata/L1_metadata_template.txt b/synoptic/data_TEST/L1_metadata/L1_metadata_template.txt index 68ee286..06e282b 100644 --- a/synoptic/data_TEST/L1_metadata/L1_metadata_template.txt +++ b/synoptic/data_TEST/L1_metadata/L1_metadata_template.txt @@ -1,4 +1,5 @@ -COMPASS-FME L1 data documentation file +COMPASS-FME Level 1 documentation file +Data version [VERSION] [FOLDER_NAME] [TIMESTAMP] @@ -8,8 +9,8 @@ General information Level 1 (L1) data are close to raw, although they have been units-transformed and have out-of-instrument-bounds and out-of-service flags added. Duplicates are removed but otherwise these data are not filtered, -and have not been subject to any algorithmic or human QA/QC. -Any use of L1 data for science analyses should be done with care. +and have not been subject to any additional algorithmic or human QA/QC. +Any use of L1 data for science analyses should be performed with care. Site information: diff --git a/synoptic/data_TEST/L1_metadata/README_v???.txt b/synoptic/data_TEST/L1_metadata/README_v???.txt new file mode 100644 index 0000000..9cec797 --- /dev/null +++ b/synoptic/data_TEST/L1_metadata/README_v???.txt @@ -0,0 +1,27 @@ +COMPASS-FME Level 1 data +Version: [VERSION] +Date: [DATESTAMP] +Git commit: [GIT_COMMIT] + +DESCRIPTION +—---------------------------------- +Level 1 (L1) data are close to raw, although they have been units-transformed +and have out-of-instrument-bounds and out-of-service flags added. +Duplicates are removed but otherwise these data are not filtered, +and have not been subject to any additional algorithmic or human QA/QC. +Any use of L1 data for science analyses should be performed with care. + +CONTACT +—---------------------------------- +Project: https://compass.pnnl.gov +Data lead: Stephanie Pennington, stephanie.pennington@pnnl.gov + +HOW TO CITE THESE DATA +—---------------------------------- +Pennington, Rich, Cheng, and Bond-Lamberty (2024). COMPASS-FME Level 1 Sensor Data +(version [VERSION]), downloaded YYYY-MM-DD, https://compass.pnnl.gov. + +CHANGELOG +—---------------------------------- +Version ??? released YYYY-MM-DD +* Release notes here diff --git a/synoptic/driver.R b/synoptic/driver.R index 298a641..210313a 100644 --- a/synoptic/driver.R +++ b/synoptic/driver.R @@ -20,6 +20,7 @@ source("helpers.R") now_string <- function() format(Sys.time(), "%Y%m%d.%H%M") ROOT <- "./data_TEST" +VERSION <- "???" # Log file ---------------------------------------------------- @@ -102,6 +103,7 @@ outfile <- file.path(LOGS, outfile) driver_try( quarto_render("L1.qmd", execute_params = list(DATA_ROOT = ROOT, + L1_VERSION = VERSION, html_outfile = outfile, logfile = LOGFILE)) )