Merge pull request #5 from mrustl/master

Get Michael's (I assume genius) changes
KWB-R · Jun 11, 2018 · cdd4163 · cdd4163
2 parents 537eef1 + f81be82
commit cdd4163
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 9 deletions.
diff --git a/R/copy_xlsx_files.R b/R/copy_xlsx_files.R
@@ -13,7 +13,7 @@ copy_xlsx_files <- function(
 
   to_paths <- gsub(from_dir, to_dir, from_paths, fixed = TRUE)
 
-  fs::dir_create(normalizePath(dirname(from_paths)), recursive = TRUE)
+  fs::dir_create(dirname(to_paths), recursive = TRUE)
 
   for (i in seq_along(from_paths)) {
 

diff --git a/R/read_bwb_data.R b/R/read_bwb_data.R
@@ -53,7 +53,7 @@ read_bwb_header1_meta <- function(
     metadata <- all_metadata[all_metadata$Sheet == sheet, ]
 
     # Load the data from the current sheet
-    tmp_data <- readxl::read_excel(file, sheet)
+    tmp_data <- readxl::read_excel(file, sheet, guess_max = 2^20)
 
     # Safely select the original column names
     columns_orig <- kwb.utils::selectColumns(metadata, "OriginalName")
@@ -65,7 +65,7 @@ read_bwb_header1_meta <- function(
     keep <- stringr::str_detect(columns_clean, keep_pattern)
 
     # Convert the data from wide to long format
-    gather_and_join_1(tmp_data, columns_clean[keep], metadata)
+    gather_and_join_1(tmp_data, columns_clean[keep], metadata, dbg = TRUE)
   })
 
   # Merge all data frames in long format  
@@ -110,8 +110,8 @@ read_bwb_header2 <- function(
   site_id_pattern = "^[0-9]{1,4}", dbg = TRUE
 )
 {
-  # Define helper functions
-  read_from_excel <- function(...) readxl::read_excel(..., col_names = FALSE)
+  # Define helper functions, 2^20 = max number of rows in xlsx
+  read_from_excel <- function(...) readxl::read_excel(..., col_names = FALSE, guess_max = 2^20)
 
   sheets <- readxl::excel_sheets(file)
 
@@ -165,7 +165,7 @@ read_bwb_header4 <- function(
 {
   # Define helper functions
   read_from_excel <- function(...) {
-    readxl::read_xlsx(..., col_names = FALSE)
+    readxl::read_xlsx(..., col_names = FALSE, guess_max = 2^20)
   }
 
   sheets <- readxl::excel_sheets(file)
@@ -232,11 +232,11 @@ stop_on_missing_or_inform_on_extra_sheets <- function(has_site_id, file, sheets)
 
   if (! all(has_site_id)) {
 
-    warning(crayon::blue(sprintf(
+    crayon::blue(sprintf(
       "FROM: %s\nIgnoring the following (%d/%d) sheet(s):\n%s\n", 
       file, sum(! has_site_id), length(sheets), 
       kwb.utils::stringList(sheets[! has_site_id])
-    )))
+    ))
   }
 }
 

diff --git a/main.R b/main.R
@@ -51,7 +51,9 @@ paths <- list(
   downloads = "<home>/Downloads",
   input_dir = "<drive>/02_Daten_Labor_Aufbereitung_02",
   export_dir = "<drive>/03_ANALYSIS_R/tmp",
-  export_dir_allg = "<export_dir>/K-TL_LSW-Altdaten-Werke Teil 1/Werke Teil 1/Allgemein",
+  sel_folder = "K-TL_LSW-Altdaten-Werke Teil 1/Werke Teil 1/Buch",
+  input_dir_sel = "<input_dir>/<sel_folder>",
+  export_dir_sel = "<export_dir>/<sel_folder>",
   home = get_homedir()
 )
 
@@ -68,9 +70,16 @@ if (FALSE)
   # Get location of excelcnv.exe
   get_excelcnv_exe()
 
+
   # Convert xls to xlsx Excel files
   convert_xls_as_xlsx(input_dir, export_dir)
 
+  if (FALSE) {
+  convert_xls_as_xlsx(input_dir = paths$input_dir_sel, 
+                      export_dir = paths$export_dir_sel)
+  }
+
+
   # Copy remaining already existing .xlsx files in same directory
   copy_xlsx_files(input_dir, export_dir, overwrite = TRUE)
 }
@@ -165,8 +174,14 @@ if (FALSE)
 
   View(labor_header4_df)
 
+  options(warn = 2)
+  for (i in 11:20) {
+  print(sprintf("File: %s", files_to_import[i]))
+  labor_tmp <- read_bwb_data(files = files_to_import[i])
+  }
 
   labor <- read_bwb_data(files = files_to_import)
+
   View(head(labor))
 
   ### Problems: