Add section comments

Sage-Bionetworks · Feb 28, 2024 · 6e5a680 · 6e5a680
1 parent d0c3344
commit 6e5a680
Show file tree

Hide file tree

Showing 10 changed files with 99 additions and 10 deletions.
diff --git a/scripts/egress/egress.R b/scripts/egress/egress.R
@@ -1,5 +1,6 @@
 cat("Beginning egress: storing output concepts, input concept map, and input variable list in Synapse\n")
 
+# Write the following to Synapse: 1) the final output concepts data, 2) the input data used in this pipeline
 latest_commit <- 
   gh::gh(
     endpoint = "/repos/:owner/:repo/commits/main", 

diff --git a/scripts/fetch-data/fetch_data.R b/scripts/fetch-data/fetch_data.R
@@ -4,6 +4,7 @@ library(dplyr)
 
 cat("Fetching data\n")
 
+# Get config variables
 config::get(
   file = "config/config.yml", 
   config = "prod"
@@ -12,6 +13,7 @@ config::get(
 
 synLogin()
 
+# Get input files from synapse
 concept_map <- 
   syn_file_to_df(ontologyFileID, "CONCEPT_CD") %>% 
   filter(CONCEPT_CD!="<null>")
@@ -21,7 +23,11 @@ selected_vars <-
   mutate(Lower_Bound = suppressWarnings(as.numeric(Lower_Bound)),
          Upper_Bound = suppressWarnings(as.numeric(Upper_Bound)))
 
-dataset_name_filter <- selected_vars %>% dplyr::pull(Export) %>% unique()
+# Get list of which datasets to use
+dataset_name_filter <- 
+  selected_vars %>% 
+  dplyr::pull(Export) %>% 
+  unique()
 
 # Sync S3 bucket to local
 token <- synapser::synGetStsStorageToken(
@@ -43,11 +49,13 @@ if (deleteExistingDir==TRUE) {
   unlink(downloadLocation, recursive = T, force = T)
 }
 
+# Only sync the bucket folders containing the datasets we need
 inclusions <- paste0("--include \"*",dataset_name_filter,"*\"", collapse = " ")
 sync_cmd <- glue::glue('aws s3 sync {base_s3_uri} {downloadLocation} --exclude "*" {inclusions}')
 system(sync_cmd)
 rm(sync_cmd)
 
+# For use in process-data steps
 concept_replacements_reversed <- vec_reverse(concept_replacements)
 
 if (!dir.exists(outputConceptsDir)) {

diff --git a/scripts/process-data/fitbitactivitylogs.R b/scripts/process-data/fitbitactivitylogs.R
@@ -4,18 +4,21 @@ dataset <- "fitbitactivitylogs"
 
 cat(glue::glue("Transforming data for {dataset}"),"\n")
 
+# Get variables for this dataset
 vars <- 
   selected_vars %>% 
   filter(grepl(dataset, Export, ignore.case = TRUE)) %>% 
   pull(Variable)
 
+# Load the desired subset of this dataset in memory
 df <- 
   arrow::open_dataset(file.path(downloadLocation, glue::glue("dataset_{dataset}"))) %>% 
   select(all_of(vars)) %>% 
   collect()
 
 colnames(df) <- tolower(colnames(df))
 
+# Create lists for ID variables and i2b2 concept variables
 excluded_concepts <- c("participantidentifier", "startdate", "enddate")
 
 approved_concepts_summarized <- 
@@ -26,6 +29,7 @@ approved_concepts_summarized <-
 
 df[approved_concepts_summarized] <- lapply(df[approved_concepts_summarized], as.numeric)
 
+# Get QA/QC ranges for variables and exclude values outside the ranges
 bounds <- 
   selected_vars %>% 
   filter(grepl(dataset, Export, ignore.case = TRUE),
@@ -46,6 +50,7 @@ for (col_name in names(df_filtered)) {
   }
 }
 
+# Pivot data frame from long to wide
 df_melted_filtered <- 
   df_filtered %>% 
   recoverSummarizeR::melt_df(excluded_concepts = excluded_concepts) %>% 
@@ -57,13 +62,15 @@ df_melted_filtered <-
   mutate(value = as.numeric(value))
 cat("recoverSummarizeR::melt_df() completed.\n")
 
+# Generate i2b2 summaries
 df_summarized <- 
   df_melted_filtered %>% 
   select(all_of(c("participantidentifier", "startdate", "enddate", "concept", "value"))) %>% 
   recoverSummarizeR::stat_summarize() %>% 
   distinct()
 cat("recoverSummarizeR::stat_summarize() completed.\n")
 
+# Add i2b2 columns from concept map (ontology file) and clean the output
 output_concepts <- 
   process_df(df_summarized, concept_map, concept_replacements_reversed, concept_map_concepts = "CONCEPT_CD", concept_map_units = "UNITS_CD") %>% 
   dplyr::mutate(nval_num = signif(nval_num, 9)) %>% 
@@ -73,12 +80,14 @@ output_concepts <-
   dplyr::filter(nval_num != "<null>" | tval_char != "<null>")
 cat("recoverSummarizeR::process_df() completed.\n")
 
+# Write the output
 output_concepts %>% 
   write.csv(file.path(outputConceptsDir, glue::glue("{dataset}.csv")), row.names = F)
 cat(glue::glue("output_concepts written to {file.path(outputConceptsDir, paste0(dataset, '.csv'))}"),"\n")
 
 cat(glue::glue("Finished transforming data for {dataset}"),"\n\n")
 
+# Remove objects created here from the global environment
 rm(dataset,
    vars, 
    df, 

diff --git a/scripts/process-data/fitbitdailydata.R b/scripts/process-data/fitbitdailydata.R
@@ -4,23 +4,25 @@ dataset <- "fitbitdailydata"
 
 cat(glue::glue("Transforming data for {dataset}"),"\n")
 
+# Get variables for this dataset
 vars <- 
   selected_vars %>% 
   filter(grepl(dataset, Export, ignore.case = TRUE)) %>% 
   pull(Variable)
 
+# Load the desired subset of this dataset in memory
 df <- 
   arrow::open_dataset(file.path(downloadLocation, glue::glue("dataset_{dataset}"))) %>% 
   mutate(Tracker_Steps = as.numeric(Tracker_Steps),
          HeartRateIntradayMinuteCount = as.numeric(HeartRateIntradayMinuteCount)) %>% 
-  # filter(Tracker_Steps > 0, HeartRateIntradayMinuteCount > 0) %>%
   filter(Tracker_Steps != 0, 
          HeartRateIntradayMinuteCount != 0 | !is.na(HeartRateIntradayMinuteCount)) %>% 
   select(all_of(vars)) %>% 
   collect()
 
 colnames(df) <- tolower(colnames(df))
 
+# Create lists for ID variables and i2b2 concept variables
 excluded_concepts <- c("participantidentifier", "date")
 
 approved_concepts_summarized <- 
@@ -31,6 +33,7 @@ approved_concepts_summarized <-
 
 df[approved_concepts_summarized] <- lapply(df[approved_concepts_summarized], as.numeric)
 
+# Get QA/QC ranges for variables and exclude values outside the ranges
 bounds <- 
   selected_vars %>% 
   filter(grepl(dataset, Export, ignore.case = TRUE),
@@ -51,6 +54,7 @@ for (col_name in names(df_filtered)) {
   }
 }
 
+# Pivot data frame from long to wide
 df_melted_filtered <- 
   df_filtered %>% 
   recoverSummarizeR::melt_df(excluded_concepts = excluded_concepts) %>% 
@@ -62,6 +66,7 @@ df_melted_filtered <-
   mutate(value = as.numeric(value))
 cat("recoverSummarizeR::melt_df() completed.\n")
 
+# Generate i2b2 summaries
 df_summarized <- 
   df_melted_filtered %>% 
   rename(startdate = dplyr::any_of(c("date", "datetime"))) %>% 
@@ -71,21 +76,28 @@ df_summarized <-
   distinct()
 cat("recoverSummarizeR::stat_summarize() completed.\n")
 
+# Add i2b2 columns from concept map (ontology file) and clean the output
 output_concepts <- 
-  process_df(df_summarized, concept_map, concept_replacements_reversed, concept_map_concepts = "CONCEPT_CD", concept_map_units = "UNITS_CD") %>% 
+  process_df(df_summarized, 
+             concept_map, 
+             concept_replacements_reversed, 
+             concept_map_concepts = "CONCEPT_CD", 
+             concept_map_units = "UNITS_CD") %>% 
   dplyr::mutate(nval_num = signif(nval_num, 9)) %>% 
   dplyr::arrange(concept) %>% 
   dplyr::mutate(dplyr::across(.cols = dplyr::everything(), .fns = as.character)) %>% 
   replace(is.na(.), "<null>") %>% 
   dplyr::filter(nval_num != "<null>" | tval_char != "<null>")
 cat("recoverSummarizeR::process_df() completed.\n")
 
+# Write the output
 output_concepts %>% 
   write.csv(file.path(outputConceptsDir, glue::glue("{dataset}.csv")), row.names = F)
 cat(glue::glue("output_concepts written to {file.path(outputConceptsDir, paste0(dataset, '.csv'))}"), "\n")
 
 cat(glue::glue("Finished transforming data for {dataset}"),"\n\n")
 
+# Remove objects created here from the global environment
 rm(dataset,
    vars, 
    df, 

diff --git a/scripts/process-data/fitbitintradaycombined.R b/scripts/process-data/fitbitintradaycombined.R
@@ -4,11 +4,13 @@ dataset <- "fitbitintradaycombined"
 
 cat(glue::glue("Transforming data for {dataset}"),"\n")
 
+# Get variables for this dataset
 vars <- 
   selected_vars %>% 
   filter(grepl(dataset, Export, ignore.case = TRUE)) %>% 
   pull(Variable)
 
+# Load the desired subset of this dataset in memory
 df <- 
   arrow::open_dataset(file.path(downloadLocation, glue::glue("dataset_{dataset}"))) %>% 
   select(all_of(vars)) %>% 
@@ -39,6 +41,7 @@ df <-
 
 colnames(df) <- tolower(colnames(df))
 
+# Create lists for ID variables and i2b2 concept variables
 excluded_concepts <- c("participantidentifier", "datetime")
 
 approved_concepts_summarized <- 
@@ -49,6 +52,7 @@ approved_concepts_summarized <-
 
 df[approved_concepts_summarized] <- lapply(df[approved_concepts_summarized], as.numeric)
 
+# Get QA/QC ranges for variables and exclude values outside the ranges
 bounds <- 
   selected_vars %>% 
   filter(grepl(dataset, Export, ignore.case = TRUE),
@@ -69,6 +73,7 @@ for (col_name in names(df_filtered)) {
   }
 }
 
+# Pivot data frame from long to wide
 df_melted_filtered <- 
   df_filtered %>% 
   recoverSummarizeR::melt_df(excluded_concepts = excluded_concepts) %>% 
@@ -80,6 +85,7 @@ df_melted_filtered <-
   mutate(value = as.numeric(value))
 cat("recoverSummarizeR::melt_df() completed.\n")
 
+# Generate i2b2 summaries
 df_summarized <- 
   df_melted_filtered %>% 
   rename(startdate = dplyr::any_of(c("date", "datetime"))) %>% 
@@ -89,6 +95,7 @@ df_summarized <-
   distinct()
 cat("recoverSummarizeR::stat_summarize() completed.\n")
 
+# Add i2b2 columns from concept map (ontology file) and clean the output
 output_concepts <- 
   process_df(df_summarized, concept_map, concept_replacements_reversed, concept_map_concepts = "CONCEPT_CD", concept_map_units = "UNITS_CD") %>% 
   dplyr::mutate(nval_num = signif(nval_num, 9)) %>% 
@@ -98,12 +105,14 @@ output_concepts <-
   dplyr::filter(nval_num != "<null>" | tval_char != "<null>")
 cat("recoverSummarizeR::process_df() completed.\n")
 
+# Write the output
 output_concepts %>% 
   write.csv(file.path(outputConceptsDir, glue::glue("{dataset}.csv")), row.names = F)
 cat(glue::glue("output_concepts written to {file.path(outputConceptsDir, paste0(dataset, '.csv'))}"), "\n")
 
 cat(glue::glue("Finished transforming data for {dataset}"),"\n\n")
 
+# Remove objects created here from the global environment
 rm(dataset,
    vars, 
    df,