Merge branch 'develop' into fix_data_remote

PecanProject · Oct 21, 2023 · 6dfd7ae · 6dfd7ae
2 parents 0e84cc8 + b3b3191
commit 6dfd7ae
Show file tree

Hide file tree

Showing 5 changed files with 81 additions and 76 deletions.
diff --git a/book_source/_bookdown.yml b/book_source/_bookdown.yml
@@ -1,2 +1,2 @@
 rmd_subdir: true
-edit: https://github.com/tonygardella/pecan/edit/release/vtonydoc/book_source/%s
+edit: https://github.com/PecanProject/pecan/edit/develop/book_source/%s
diff --git a/modules/data.atmosphere/inst/integrationTests/test.download.AmerifluxLBL.R b/modules/data.atmosphere/inst/integrationTests/test.download.AmerifluxLBL.R
@@ -1,4 +1,5 @@
 library(testthat)
+library(PEcAn.DB)
 
 test_download_AmerifluxLBL <- function(start_date, end_date, sitename, lat.in, lon.in) {
   # putting logger to debug mode
@@ -7,13 +8,13 @@ test_download_AmerifluxLBL <- function(start_date, end_date, sitename, lat.in, l
   PEcAn.logger::logger.setLevel("DEBUG")
 
   # mocking functions
-  mockery::stub(PEcAn.DB::convert_input, 'dbfile.input.check', data.frame())
-  mockery::stub(PEcAn.DB::convert_input, 'db.query', data.frame(id = 1))
+  mockery::stub(convert_input, 'dbfile.input.check', data.frame())
+  mockery::stub(convert_input, 'db.query', data.frame(id = 1))
 
   withr::with_dir(tempdir(), {
     tmpdir <- getwd()
     # calling download function
-    res <- PEcAn.DB::convert_input(
+    res <- convert_input(
       input.id = NA,
       outfolder = tmpdir,
       formatname = NULL,
@@ -31,23 +32,23 @@ test_download_AmerifluxLBL <- function(start_date, end_date, sitename, lat.in, l
       lon.in = lon.in,
       sitename = sitename
     )
-  })
-
-  # checking if the file is downloaded
-  test_that("Downloaded files are present at the desired location", {
-    expect_true(file.exists(paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")))
-  })
 
-  test_that("Downloaded data files have the right format", {
-    firstline <- system(paste0("head -4 ", paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")), intern = TRUE)
-    lastline <- system(paste0("tail -1 ", paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")), intern = TRUE)
-
-    # checking if first line of CSV has the sitename
-    expect_true(grepl(sitename, firstline[1]))
-
-    # fourth and last row checked to contain non-alphabetical data since these are used to verify start and end dates
-    expect_false(grepl("[A-Za-z]", firstline[4]))
-    expect_false(grepl("[A-Za-z]", lastline[1]))
+    # checking if the file is downloaded
+    test_that("Downloaded files are present at the desired location", {
+      expect_true(file.exists(paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")))
+    })
+
+    test_that("Downloaded data files have the right format", {
+      firstline <- system(paste0("head -4 ", paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")), intern = TRUE)
+      lastline <- system(paste0("tail -1 ", paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")), intern = TRUE)
+
+      # checking if first line of CSV has the sitename
+      expect_true(grepl(sitename, firstline[1]))
+
+      # fourth and last row checked to contain non-alphabetical data since these are used to verify start and end dates
+      expect_false(grepl("[A-Za-z]", firstline[4]))
+      expect_false(grepl("[A-Za-z]", lastline[1]))
+    })
   })
 }
 

diff --git a/modules/data.atmosphere/inst/integrationTests/test.download.CRUNCEP.R b/modules/data.atmosphere/inst/integrationTests/test.download.CRUNCEP.R
@@ -1,5 +1,6 @@
 library(testthat)
 library(ncdf4)
+library(PEcAn.DB)
 
 test_download_CRUNCEP <- function(start_date, end_date, lat.in, lon.in, method, maxErrors, sleep) {
   # putting logger to debug mode
@@ -8,12 +9,12 @@ test_download_CRUNCEP <- function(start_date, end_date, lat.in, lon.in, method,
   PEcAn.logger::logger.setLevel("DEBUG")
 
   # mocking functions
-  mockery::stub(PEcAn.DB::convert_input, 'dbfile.input.check', data.frame())
-  mockery::stub(PEcAn.DB::convert_input, 'db.query', data.frame(id = 1))
+  mockery::stub(convert_input, 'dbfile.input.check', data.frame())
+  mockery::stub(convert_input, 'db.query', data.frame(id = 1))
 
   withr::with_dir(tempdir(), {
     tmpdir <- getwd()
-    PEcAn.DB::convert_input(
+    convert_input(
       input.id = NA,
       outfolder = tmpdir,
       formatname = NULL,

diff --git a/modules/data.atmosphere/inst/integrationTests/test.download.ERA5.R b/modules/data.atmosphere/inst/integrationTests/test.download.ERA5.R
@@ -1,5 +1,6 @@
 library(testthat)
 library(ncdf4)
+library(PEcAn.DB)
 
 test_download_ERA5 <- function(start_date, end_date, lat.in, lon.in, product_types, reticulate_python) {
   # putting logger to debug mode
@@ -9,16 +10,16 @@ test_download_ERA5 <- function(start_date, end_date, lat.in, lon.in, product_typ
 
 
   # mocking functions
-  mockery::stub(PEcAn.DB::convert_input, 'dbfile.input.check', data.frame())
-  mockery::stub(PEcAn.DB::convert_input, 'db.query', data.frame(id = 1))
+  mockery::stub(convert_input, 'dbfile.input.check', data.frame())
+  mockery::stub(convert_input, 'db.query', data.frame(id = 1))
 
   # additional mocks needed since download.ERA5 does not return data as other download functions
-  mockery::stub(PEcAn.DB::convert_input, 'length', 2)
-  mockery::stub(PEcAn.DB::convert_input, 'purrr::map_dfr', data.frame(missing = c(FALSE), empty = c(FALSE)))
+  mockery::stub(convert_input, 'length', 2)
+  mockery::stub(convert_input, 'purrr::map_dfr', data.frame(missing = c(FALSE), empty = c(FALSE)))
 
   withr::with_dir(tempdir(), {
     tmpdir <- getwd()
-    PEcAn.DB::convert_input(
+    convert_input(
       input.id = NA,
       outfolder = tmpdir,
       formatname = NULL,
@@ -37,51 +38,51 @@ test_download_ERA5 <- function(start_date, end_date, lat.in, lon.in, product_typ
       product_types = product_types,
       reticulate_python = reticulate_python
     )
-  })
-
-  test_that("All the required files are downloaded and stored at desired location", { 
-    expect_true(file.exists(paste0(tmpdir, "/era5.2m_dewpoint_temperature.nc")))
-    expect_true(file.exists(paste0(tmpdir, "/era5.2m_temperature.nc")))
-    expect_true(file.exists(paste0(tmpdir, "/era5.10m_u_component_of_wind.nc")))
-    expect_true(file.exists(paste0(tmpdir, "/era5.10m_v_component_of_wind.nc")))
-    expect_true(file.exists(paste0(tmpdir, "/era5.surface_pressure.nc")))
-    expect_true(file.exists(paste0(tmpdir, "/era5.surface_solar_radiation_downwards.nc")))
-    expect_true(file.exists(paste0(tmpdir, "/era5.surface_thermal_radiation_downwards.nc")))
-    expect_true(file.exists(paste0(tmpdir, "/era5.total_precipitation.nc")))
-  })
+
+    test_that("All the required files are downloaded and stored at desired location", { 
+      expect_true(file.exists(paste0(tmpdir, "/era5.2m_dewpoint_temperature.nc")))
+      expect_true(file.exists(paste0(tmpdir, "/era5.2m_temperature.nc")))
+      expect_true(file.exists(paste0(tmpdir, "/era5.10m_u_component_of_wind.nc")))
+      expect_true(file.exists(paste0(tmpdir, "/era5.10m_v_component_of_wind.nc")))
+      expect_true(file.exists(paste0(tmpdir, "/era5.surface_pressure.nc")))
+      expect_true(file.exists(paste0(tmpdir, "/era5.surface_solar_radiation_downwards.nc")))
+      expect_true(file.exists(paste0(tmpdir, "/era5.surface_thermal_radiation_downwards.nc")))
+      expect_true(file.exists(paste0(tmpdir, "/era5.total_precipitation.nc")))
+    })
 
-  test_that("All ERA5 data files have the correct variable units", {
-    nc <- nc_open(paste0(tmpdir, "/era5.2m_dewpoint_temperature.nc"))
-    expect_equal(nc$var$d2m$units, "K")
-    nc_close(nc)
+    test_that("All ERA5 data files have the correct variable units", {
+      nc <- nc_open(paste0(tmpdir, "/era5.2m_dewpoint_temperature.nc"))
+      expect_equal(nc$var$d2m$units, "K")
+      nc_close(nc)
 
-    nc <- nc_open(paste0(tmpdir, "/era5.2m_temperature.nc"))
-    expect_equal(nc$var$t2m$units, "K")
-    nc_close(nc)
+      nc <- nc_open(paste0(tmpdir, "/era5.2m_temperature.nc"))
+      expect_equal(nc$var$t2m$units, "K")
+      nc_close(nc)
 
-    nc <- nc_open(paste0(tmpdir, "/era5.10m_u_component_of_wind.nc"))
-    expect_equal(nc$var$u10$units, "m s**-1")
-    nc_close(nc)
+      nc <- nc_open(paste0(tmpdir, "/era5.10m_u_component_of_wind.nc"))
+      expect_equal(nc$var$u10$units, "m s**-1")
+      nc_close(nc)
 
-    nc <- nc_open(paste0(tmpdir, "/era5.10m_v_component_of_wind.nc"))
-    expect_equal(nc$var$v10$units, "m s**-1")
-    nc_close(nc)
+      nc <- nc_open(paste0(tmpdir, "/era5.10m_v_component_of_wind.nc"))
+      expect_equal(nc$var$v10$units, "m s**-1")
+      nc_close(nc)
 
-    nc <- nc_open(paste0(tmpdir, "/era5.surface_pressure.nc"))
-    expect_equal(nc$var$sp$units, "Pa")
-    nc_close(nc)
+      nc <- nc_open(paste0(tmpdir, "/era5.surface_pressure.nc"))
+      expect_equal(nc$var$sp$units, "Pa")
+      nc_close(nc)
 
-    nc <- nc_open(paste0(tmpdir, "/era5.surface_solar_radiation_downwards.nc"))
-    expect_equal(nc$var$ssrd$units, "J m**-2")
-    nc_close(nc)
+      nc <- nc_open(paste0(tmpdir, "/era5.surface_solar_radiation_downwards.nc"))
+      expect_equal(nc$var$ssrd$units, "J m**-2")
+      nc_close(nc)
 
-    nc <- nc_open(paste0(tmpdir, "/era5.surface_thermal_radiation_downwards.nc"))
-    expect_equal(nc$var$strd$units, "J m**-2")
-    nc_close(nc)
+      nc <- nc_open(paste0(tmpdir, "/era5.surface_thermal_radiation_downwards.nc"))
+      expect_equal(nc$var$strd$units, "J m**-2")
+      nc_close(nc)
 
-    nc <- nc_open(paste0(tmpdir, "/era5.total_precipitation.nc"))
-    expect_equal(nc$var$tp$units, "m")
-    nc_close(nc)
+      nc <- nc_open(paste0(tmpdir, "/era5.total_precipitation.nc"))
+      expect_equal(nc$var$tp$units, "m")
+      nc_close(nc)
+    })
   })
 }
 

diff --git a/scripts/check_with_errors.R b/scripts/check_with_errors.R
@@ -7,6 +7,8 @@ log_level <- Sys.getenv("LOGLEVEL", unset = NA)
 die_level <- Sys.getenv("DIELEVEL", unset = NA)
 redocument <- as.logical(Sys.getenv("REBUILD_DOCS", unset = NA))
 runtests <- as.logical(Sys.getenv("RUN_TESTS", unset = TRUE))
+resave <- as.logical(Sys.getenv("RESAVE_CHECKS", unset = FALSE))
+if (resave) die_level <- "never"
 
 old_file <- file.path(pkg, "tests", "Rcheck_reference.log")
 if (file.exists(old_file)) {
@@ -71,7 +73,7 @@ if (log_notes && n_notes > 0) {
 # such that it's not yet practical to break the build on every warning.
 # Cleaning this up is a long-term goal, but will take time.
 # Meanwhile, we compare against a cached historic check output to enforce that
-# no *new* warnings are added. As historic warnings are removed, we will update
+# no *new* warnings are added. As historic warnings are removed, we update
 # the cached results to ensure they stay gone.
 #
 # To compare checks, we take a two-level approach:
@@ -83,16 +85,16 @@ if (log_notes && n_notes > 0) {
 ###
 # To update reference files after fixing an old warning:
 # * Run check_with_errors.R to be sure the check is currently passing
-# * Delete the file you want to update
-# * Uncomment this section
-# * run `DIELEVEL=never Rscript scripts/check_with_errors.R path/to/package`
-# * recomment this section
-# * Commit updated file
-# if (!file.exists(old_file)) {
-#     cat("No reference check file found. Saving current results as the new standard\n")
-#     cat(chk$stdout, file = old_file)
-#     quit("no")
-# }
+# * run `RESAVE_CHECKS=true Rscript scripts/check_with_errors.R path/to/package`
+# * Commit updated <pkgname>/tests/Rcheck_reference.log file
+if (resave) {
+    cat("Saving current check results as the new standard\n")
+    if (file.exists(old_file)) {
+        cat("**Overwriting** existing saved check output\n")
+    }
+    cat(chk$stdout, file = old_file)
+    quit("no")
+}
 ###
 
 # everything beyond this point is comparing to old version