diff --git a/NAMESPACE b/NAMESPACE index 70b62d0..3549ab3 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -6,6 +6,7 @@ export(set_window) export(tar_stars) export(tar_stars_proxy) export(tar_terra_rast) +export(tar_terra_sds) export(tar_terra_sprc) export(tar_terra_tiles) export(tar_terra_vect) diff --git a/R/tar-terra-sprc.R b/R/tar-terra-sprc.R index c39d076..dc03472 100644 --- a/R/tar-terra-sprc.R +++ b/R/tar-terra-sprc.R @@ -66,87 +66,268 @@ tar_terra_sprc <- function(name, retrieval = targets::tar_option_get("retrieval"), cue = targets::tar_option_get("cue"), description = targets::tar_option_get("description")) { - check_pkg_installed("terra") + check_pkg_installed("terra") #ensure that user-passed `resources` doesn't include `custom_format` if ("custom_format" %in% names(resources)) { cli::cli_abort("{.val custom_format} cannot be supplied to targets created with {.fn tar_terra_sprc}") } - gdal <- gdal %||% character(0) - filetype <- filetype %||% "GTiff" - - # check that filetype option is available - drv <- get_gdal_available_driver_list("raster") - filetype <- rlang::arg_match0(filetype, drv$name) - - name <- targets::tar_deparse_language(substitute(name)) - - envir <- targets::tar_option_get("envir") - - command <- targets::tar_tidy_eval( - expr = as.expression(substitute(command)), - envir = envir, - tidy_eval = tidy_eval - ) - - pattern <- targets::tar_tidy_eval( - expr = as.expression(substitute(pattern)), - envir = envir, - tidy_eval = tidy_eval - ) - - .write_terra_rasters_sprc <- - function(object, path) { - for (i in seq(object)) { - if (i > 1) { - opt <- "APPEND_SUBDATASET=YES" - } else { - opt <- "" - } - terra::writeRaster( - x = object[i], - filename = path, - filetype = Sys.getenv("GEOTARGETS_GDAL_RASTER_DRIVER"), - overwrite = (i == 1), - gdal = opt - ) - } - } - - targets::tar_target_raw( - name = name, - command = command, - pattern = pattern, - packages = packages, - library = library, - format = targets::tar_format( - read = function(path) terra::sprc(path), - write = .write_terra_rasters_sprc, - marshal = function(object) terra::wrap(object), - unmarshal = function(object) terra::unwrap(object) - ), - repository = repository, - iteration = "list", - error = error, - memory = memory, - garbage_collection = garbage_collection, - deployment = deployment, - priority = priority, - resources = utils::modifyList( - targets::tar_resources( - custom_format = targets::tar_resources_custom_format( - #these envvars are used in write function of format - envvars = c( - "GEOTARGETS_GDAL_RASTER_DRIVER" = filetype, - "GEOTARGETS_GDAL_RASTER_CREATION_OPTIONS" = ( - paste0(gdal, collapse = ";") + gdal <- gdal %||% character(0) + filetype <- filetype %||% "GTiff" + + # check that filetype option is available + drv <- get_gdal_available_driver_list("raster") + filetype <- rlang::arg_match0(filetype, drv$name) + + name <- targets::tar_deparse_language(substitute(name)) + + envir <- targets::tar_option_get("envir") + + command <- targets::tar_tidy_eval( + expr = as.expression(substitute(command)), + envir = envir, + tidy_eval = tidy_eval + ) + + pattern <- targets::tar_tidy_eval( + expr = as.expression(substitute(pattern)), + envir = envir, + tidy_eval = tidy_eval + ) + + tar_terra_collection_raw( + name = name, + command = command, + pattern = pattern, + filetype = filetype, + gdal = gdal, + packages = packages, + library = library, + format = format_terra_collections(type = "sprc"), + repository = repository, + error = error, + memory = memory, + garbage_collection = garbage_collection, + deployment = deployment, + priority = priority, + resources = resources, + storage = storage, + retrieval = retrieval, + cue = cue, + description = description + ) +} + + +#' Create a terra _SpatRasterDataset_ target +#' +#' Provides a target format for [terra::SpatRasterDataset] objects, +#' which hold sub-datasets, each a `SpatRaster` that can have multiple layers. +#' +#' @param filetype character. File format expressed as GDAL driver names passed +#' to [terra::writeRaster()] +#' @param gdal character. GDAL driver specific datasource creation options +#' passed to [terra::writeRaster()] +#' @param ... Additional arguments not yet used +#' +#' @inheritParams targets::tar_target +#' +#' @note The `iteration` argument is unavailable because it is hard-coded to +#' `"list"`, the only option that works currently. +#' +#' @returns target class "tar_stem" for use in a target pipeline +#' @seealso [targets::tar_target_raw()], [tar_terra_sprc()] +#' @author Andrew Gene Brown +#' @author Nicholas Tierney +#' @author Eric R. Scott +#' @export +#' @examples +#' if (Sys.getenv("TAR_LONG_EXAMPLES") == "true") { +#' targets::tar_dir({ # tar_dir() runs code from a temporary directory. +#' library(geotargets) +#' targets::tar_script({ +#' elev_scale <- function(z = 1) { +#' terra::rast(system.file("ex", "elev.tif", package = "terra")) * z +#' } +#' list( +#' tar_terra_sds( +#' raster_elevs, +#' # two rasters, one unaltered, one scaled by factor of 2 +#' command = terra::sds(list( +#' elev_scale(1), +#' elev_scale(2) +#' )) +#' ) +#' ) +#' }) +#' targets::tar_make() +#' targets::tar_read(raster_elevs) +#' }) +#' } +tar_terra_sds <- function(name, + command, + pattern = NULL, + filetype = geotargets_option_get("gdal.raster.driver"), + gdal = geotargets_option_get("gdal.raster.creation.options"), + ..., + tidy_eval = targets::tar_option_get("tidy_eval"), + packages = targets::tar_option_get("packages"), + library = targets::tar_option_get("library"), + repository = targets::tar_option_get("repository"), + error = targets::tar_option_get("error"), + memory = targets::tar_option_get("memory"), + garbage_collection = targets::tar_option_get("garbage_collection"), + deployment = targets::tar_option_get("deployment"), + priority = targets::tar_option_get("priority"), + resources = targets::tar_option_get("resources"), + storage = targets::tar_option_get("storage"), + retrieval = targets::tar_option_get("retrieval"), + cue = targets::tar_option_get("cue"), + description = targets::tar_option_get("description")) { + check_pkg_installed("terra") + #ensure that user-passed `resources` doesn't include `custom_format` + if ("custom_format" %in% names(resources)) { + cli::cli_abort("{.val custom_format} cannot be supplied to targets created with {.fn tar_terra_sprc}") + } + + gdal <- gdal %||% character(0) + filetype <- filetype %||% "GTiff" + + # check that filetype option is available + drv <- get_gdal_available_driver_list("raster") + filetype <- rlang::arg_match0(filetype, drv$name) + + name <- targets::tar_deparse_language(substitute(name)) + + envir <- targets::tar_option_get("envir") + + command <- targets::tar_tidy_eval( + expr = as.expression(substitute(command)), + envir = envir, + tidy_eval = tidy_eval + ) + + pattern <- targets::tar_tidy_eval( + expr = as.expression(substitute(pattern)), + envir = envir, + tidy_eval = tidy_eval + ) + + tar_terra_collection_raw( + name = name, + command = command, + pattern = pattern, + filetype = filetype, + gdal = gdal, + packages = packages, + library = library, + format = format_terra_collections(type = "sds"), + repository = repository, + error = error, + memory = memory, + garbage_collection = garbage_collection, + deployment = deployment, + priority = priority, + resources = resources, + storage = storage, + retrieval = retrieval, + cue = cue, + description = description + ) +} + + + + +#' @noRd +tar_terra_collection_raw <- function( + name, + command, + filetype = geotargets_option_get("gdal.raster.driver"), + gdal = geotargets_option_get("gdal.raster.creation.options"), + pattern = NULL, + packages = targets::tar_option_get("packages"), + library = targets::tar_option_get("library"), + format = format_terra_collections(), + repository = targets::tar_option_get("repository"), + error = targets::tar_option_get("error"), + memory = targets::tar_option_get("memory"), + garbage_collection = targets::tar_option_get("garbage_collection"), + deployment = targets::tar_option_get("deployment"), + priority = targets::tar_option_get("priority"), + resources = targets::tar_option_get("resources"), + storage = targets::tar_option_get("storage"), + retrieval = targets::tar_option_get("retrieval"), + cue = targets::tar_option_get("cue"), + description = targets::tar_option_get("description") +) { + targets::tar_target_raw( + name = name, + command = command, + pattern = pattern, + packages = packages, + library = library, + format = format, + repository = repository, + iteration = "list", + error = error, + memory = memory, + garbage_collection = garbage_collection, + deployment = deployment, + priority = priority, + resources = utils::modifyList( + targets::tar_resources( + custom_format = targets::tar_resources_custom_format( + #these envvars are used in write function of format + envvars = c( + "GEOTARGETS_GDAL_RASTER_DRIVER" = filetype, + "GEOTARGETS_GDAL_RASTER_CREATION_OPTIONS" = ( + paste0(gdal, collapse = ";") + ) ) ) - ) - ), resources), - storage = storage, - retrieval = retrieval, - cue = cue, - description = description - ) + ), resources), + storage = storage, + retrieval = retrieval, + cue = cue, + description = description + ) +} + + + + +#' Format function for sprc and sds +#' @noRd +format_terra_collections <- function(type = c("sprc", "sds")) { + type <- match.arg(type) + targets::tar_format( + read = switch(type, + "sprc" = function(path) terra::sprc(path), + "sds" = function(path) terra::sds(path) + ), + write = function(object, path) { + gdal <- strsplit( + Sys.getenv("GEOTARGETS_GDAL_RASTER_CREATION_OPTIONS", + unset = ";"), + ";")[[1]] + + for (i in seq(object)) { + if (i > 1) { + opt <- "APPEND_SUBDATASET=YES" + } else { + opt <- "" + } + terra::writeRaster( + x = object[i], + filename = path, + filetype = Sys.getenv("GEOTARGETS_GDAL_RASTER_DRIVER"), + overwrite = (i == 1), + gdal = c(opt, gdal) + ) + } + }, + marshal = function(object) terra::wrap(object), + unmarshal = function(object) terra::unwrap(object) + ) } diff --git a/man/tar_terra_sds.Rd b/man/tar_terra_sds.Rd new file mode 100644 index 0000000..35bb05c --- /dev/null +++ b/man/tar_terra_sds.Rd @@ -0,0 +1,254 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/tar-terra-sprc.R +\name{tar_terra_sds} +\alias{tar_terra_sds} +\title{Create a terra \emph{SpatRasterDataset} target} +\usage{ +tar_terra_sds( + name, + command, + pattern = NULL, + filetype = geotargets_option_get("gdal.raster.driver"), + gdal = geotargets_option_get("gdal.raster.creation.options"), + ..., + tidy_eval = targets::tar_option_get("tidy_eval"), + packages = targets::tar_option_get("packages"), + library = targets::tar_option_get("library"), + repository = targets::tar_option_get("repository"), + error = targets::tar_option_get("error"), + memory = targets::tar_option_get("memory"), + garbage_collection = targets::tar_option_get("garbage_collection"), + deployment = targets::tar_option_get("deployment"), + priority = targets::tar_option_get("priority"), + resources = targets::tar_option_get("resources"), + storage = targets::tar_option_get("storage"), + retrieval = targets::tar_option_get("retrieval"), + cue = targets::tar_option_get("cue"), + description = targets::tar_option_get("description") +) +} +\arguments{ +\item{name}{Symbol, name of the target. A target +name must be a valid name for a symbol in R, and it +must not start with a dot. Subsequent targets +can refer to this name symbolically to induce a dependency relationship: +e.g. \code{tar_target(downstream_target, f(upstream_target))} is a +target named \code{downstream_target} which depends on a target +\code{upstream_target} and a function \code{f()}. In addition, a target's +name determines its random number generator seed. In this way, +each target runs with a reproducible seed so someone else +running the same pipeline should get the same results, +and no two targets in the same pipeline share the same seed. +(Even dynamic branches have different names and thus different seeds.) +You can recover the seed of a completed target +with \code{tar_meta(your_target, seed)} and run \code{\link[targets:tar_seed_set]{tar_seed_set()}} +on the result to locally recreate the target's initial RNG state.} + +\item{command}{R code to run the target.} + +\item{pattern}{Language to define branching for a target. +For example, in a pipeline with numeric vector targets \code{x} and \code{y}, +\code{tar_target(z, x + y, pattern = map(x, y))} implicitly defines +branches of \code{z} that each compute \code{x[1] + y[1]}, \code{x[2] + y[2]}, +and so on. See the user manual for details.} + +\item{filetype}{character. File format expressed as GDAL driver names passed +to \code{\link[terra:writeRaster]{terra::writeRaster()}}} + +\item{gdal}{character. GDAL driver specific datasource creation options +passed to \code{\link[terra:writeRaster]{terra::writeRaster()}}} + +\item{...}{Additional arguments not yet used} + +\item{tidy_eval}{Logical, whether to enable tidy evaluation +when interpreting \code{command} and \code{pattern}. If \code{TRUE}, you can use the +"bang-bang" operator \verb{!!} to programmatically insert +the values of global objects.} + +\item{packages}{Character vector of packages to load right before +the target runs or the output data is reloaded for +downstream targets. Use \code{tar_option_set()} to set packages +globally for all subsequent targets you define.} + +\item{library}{Character vector of library paths to try +when loading \code{packages}.} + +\item{repository}{Character of length 1, remote repository for target +storage. Choices: +\itemize{ +\item \code{"local"}: file system of the local machine. +\item \code{"aws"}: Amazon Web Services (AWS) S3 bucket. Can be configured +with a non-AWS S3 bucket using the \code{endpoint} argument of +\code{\link[targets:tar_resources_aws]{tar_resources_aws()}}, but versioning capabilities may be lost +in doing so. +See the cloud storage section of +\url{https://books.ropensci.org/targets/data.html} +for details for instructions. +\item \code{"gcp"}: Google Cloud Platform storage bucket. +See the cloud storage section of +\url{https://books.ropensci.org/targets/data.html} +for details for instructions. +} + +Note: if \code{repository} is not \code{"local"} and \code{format} is \code{"file"} +then the target should create a single output file. +That output file is uploaded to the cloud and tracked for changes +where it exists in the cloud. The local file is deleted after +the target runs.} + +\item{error}{Character of length 1, what to do if the target +stops and throws an error. Options: +\itemize{ +\item \code{"stop"}: the whole pipeline stops and throws an error. +\item \code{"continue"}: the whole pipeline keeps going. +\item \code{"abridge"}: any currently running targets keep running, +but no new targets launch after that. +(Visit \url{https://books.ropensci.org/targets/debugging.html} +to learn how to debug targets using saved workspaces.) +\item \code{"null"}: The errored target continues and returns \code{NULL}. +The data hash is deliberately wrong so the target is not +up to date for the next run of the pipeline. +}} + +\item{memory}{Character of length 1, memory strategy. +If \code{"persistent"}, the target stays in memory +until the end of the pipeline (unless \code{storage} is \code{"worker"}, +in which case \code{targets} unloads the value from memory +right after storing it in order to avoid sending +copious data over a network). +If \code{"transient"}, the target gets unloaded +after every new target completes. +Either way, the target gets automatically loaded into memory +whenever another target needs the value. +For cloud-based dynamic files +(e.g. \code{format = "file"} with \code{repository = "aws"}), +this memory strategy applies to the +temporary local copy of the file: +\code{"persistent"} means it remains until the end of the pipeline +and is then deleted, +and \code{"transient"} means it gets deleted as soon as possible. +The former conserves bandwidth, +and the latter conserves local storage.} + +\item{garbage_collection}{Logical, whether to run \code{base::gc()} +just before the target runs.} + +\item{deployment}{Character of length 1. If \code{deployment} is +\code{"main"}, then the target will run on the central controlling R process. +Otherwise, if \code{deployment} is \code{"worker"} and you set up the pipeline +with distributed/parallel computing, then +the target runs on a parallel worker. For more on distributed/parallel +computing in \code{targets}, please visit +\url{https://books.ropensci.org/targets/crew.html}.} + +\item{priority}{Numeric of length 1 between 0 and 1. Controls which +targets get deployed first when multiple competing targets are ready +simultaneously. Targets with priorities closer to 1 get dispatched earlier +(and polled earlier in \code{\link[targets:tar_make_future]{tar_make_future()}}).} + +\item{resources}{Object returned by \code{tar_resources()} +with optional settings for high-performance computing +functionality, alternative data storage formats, +and other optional capabilities of \code{targets}. +See \code{tar_resources()} for details.} + +\item{storage}{Character of length 1, only relevant to +\code{\link[targets:tar_make_clustermq]{tar_make_clustermq()}} and \code{\link[targets:tar_make_future]{tar_make_future()}}. +Must be one of the following values: +\itemize{ +\item \code{"main"}: the target's return value is sent back to the +host machine and saved/uploaded locally. +\item \code{"worker"}: the worker saves/uploads the value. +\item \code{"none"}: almost never recommended. It is only for +niche situations, e.g. the data needs to be loaded +explicitly from another language. If you do use it, +then the return value of the target is totally ignored +when the target ends, but +each downstream target still attempts to load the data file +(except when \code{retrieval = "none"}). + +If you select \code{storage = "none"}, then +the return value of the target's command is ignored, +and the data is not saved automatically. +As with dynamic files (\code{format = "file"}) it is the +responsibility of the user to write to +the data store from inside the target. + +The distinguishing feature of \code{storage = "none"} +(as opposed to \code{format = "file"}) +is that in the general case, +downstream targets will automatically try to load the data +from the data store as a dependency. As a corollary, \code{storage = "none"} +is completely unnecessary if \code{format} is \code{"file"}. +}} + +\item{retrieval}{Character of length 1, only relevant to +\code{\link[targets:tar_make_clustermq]{tar_make_clustermq()}} and \code{\link[targets:tar_make_future]{tar_make_future()}}. +Must be one of the following values: +\itemize{ +\item \code{"main"}: the target's dependencies are loaded on the host machine +and sent to the worker before the target runs. +\item \code{"worker"}: the worker loads the targets dependencies. +\item \code{"none"}: the dependencies are not loaded at all. +This choice is almost never recommended. It is only for +niche situations, e.g. the data needs to be loaded +explicitly from another language. +}} + +\item{cue}{An optional object from \code{tar_cue()} to customize the +rules that decide whether the target is up to date.} + +\item{description}{Character of length 1, a custom free-form human-readable +text description of the target. Descriptions appear as target labels +in functions like \code{\link[targets:tar_manifest]{tar_manifest()}} and \code{\link[targets:tar_visnetwork]{tar_visnetwork()}}, +and they let you select subsets of targets for the \code{names} argument of +functions like \code{\link[targets:tar_make]{tar_make()}}. For example, +\code{tar_manifest(names = tar_described_as(starts_with("survival model")))} +lists all the targets whose descriptions start with the character +string \code{"survival model"}.} +} +\value{ +target class "tar_stem" for use in a target pipeline +} +\description{ +Provides a target format for \link[terra:SpatRaster-class]{terra::SpatRasterDataset} objects, +which hold sub-datasets, each a \code{SpatRaster} that can have multiple layers. +} +\note{ +The \code{iteration} argument is unavailable because it is hard-coded to +\code{"list"}, the only option that works currently. +} +\examples{ +if (Sys.getenv("TAR_LONG_EXAMPLES") == "true") { + targets::tar_dir({ # tar_dir() runs code from a temporary directory. + library(geotargets) + targets::tar_script({ + elev_scale <- function(z = 1) { + terra::rast(system.file("ex", "elev.tif", package = "terra")) * z + } + list( + tar_terra_sds( + raster_elevs, + # two rasters, one unaltered, one scaled by factor of 2 + command = terra::sds(list( + elev_scale(1), + elev_scale(2) + )) + ) + ) + }) + targets::tar_make() + targets::tar_read(raster_elevs) + }) +} +} +\seealso{ +\code{\link[targets:tar_target_raw]{targets::tar_target_raw()}}, \code{\link[=tar_terra_sprc]{tar_terra_sprc()}} +} +\author{ +Andrew Gene Brown + +Nicholas Tierney + +Eric R. Scott +} diff --git a/tests/testthat/_snaps/tar-terra-sprc.md b/tests/testthat/_snaps/tar-terra-sprc.md index 3aaf79d..f29f87e 100644 --- a/tests/testthat/_snaps/tar-terra-sprc.md +++ b/tests/testthat/_snaps/tar-terra-sprc.md @@ -12,3 +12,18 @@ crs (first) : lon/lat WGS 84 (EPSG:4326) names : raster_elevs, raster_elevs +# tar_terra_sds() works + + Code + x + Output + class : SpatRasterDataset + subdatasets : 2 + dimensions : 90, 95 (nrow, ncol) + nlyr : 1, 1 + resolution : 0.008333333, 0.008333333 (x, y) + extent : 5.741667, 6.533333, 49.44167, 50.19167 (xmin, xmax, ymin, ymax) + coord. ref. : lon/lat WGS 84 (EPSG:4326) + source(s) : raster_elevs + names : raster_elevs, raster_elevs + diff --git a/tests/testthat/test-tar-terra-sprc.R b/tests/testthat/test-tar-terra-sprc.R index e3e5b73..a5b0e95 100644 --- a/tests/testthat/test-tar-terra-sprc.R +++ b/tests/testthat/test-tar-terra-sprc.R @@ -1,3 +1,4 @@ +# test_that() #Included to make RStudio recognize this file as a test targets::tar_test("tar_terra_sprc() works", { geotargets::geotargets_option_set( gdal_raster_creation_options = @@ -33,3 +34,35 @@ targets::tar_test("tar_terra_sprc() works", { expect_s4_class(x, "SpatRasterCollection") expect_snapshot(x) }) + +targets::tar_test("tar_terra_sds() works", { + geotargets::geotargets_option_set( + gdal_raster_creation_options = + c("COMPRESS=DEFLATE", "TFW=YES") + ) + targets::tar_script({ + elev_scale <- function(z = 1) { + terra::rast( + system.file( + "ex", + "elev.tif", + package = "terra" + ) + ) * z + } + list( + geotargets::tar_terra_sds( + raster_elevs, + # two rasters, one unaltered, one scaled by factor of 2 + command = terra::sprc(list( + elev_scale(1), + elev_scale(2) + )) + ) + ) + }) + targets::tar_make() + x <- targets::tar_read(raster_elevs) + expect_s4_class(x, "SpatRasterDataset") + expect_snapshot(x) +})