Merge pull request #28 from socialcopsdev/update

Bugs solved and issues resolved.

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: flyio
 Type: Package
 Title: Read or Write any Format from Anywhere
-Version: 0.1.0
+Version: 0.1.1
 Author: Himanshu Sikaria [aut, cre]
 Maintainer: Himanshu Sikaria <himanshu.sikaria@socialcops.com>
 Description: Perform input, output of files in R from data sources like Google Cloud Storage ('GCS') <https://cloud.google.com/storage/>, Amazon Web Services ('AWS S3') <https://aws.amazon.com/s3> or local drive.

NAMESPACE

@@ -19,6 +19,7 @@ export("import_rda")
 export("import_rds")
 export("import_shp")
 export("import_table")
+export("list_bucket")
 export("list_files")
 import("assertthat")
 import("aws.s3")

R/export_folder.R

@@ -61,14 +61,17 @@ export_folder <- function(localfolder, bucketpath, pattern = "*", overwrite = TR
   }
 
   # upload the file if everything is fine
+  pb <- txtProgressBar(min = 0, max = length(file_upload), style = 3)
   for(i in 1:length(file_upload)){
     bucketpath = gsub("\\/+$","",bucketpath)
     if(data_source == "gcs"){
       upload_return = gcs_upload(file = file_upload[i], name = paste0(bucketpath,"/", basename(file_upload[i])), bucket = bucket, ...)
     } else if(data_source == "s3"){
       upload_return = aws.s3::put_object(file = file_upload[i], bucket = bucket, object =  paste0(bucketpath,"/", basename(file_upload[i])), ...)
     }
+    setTxtProgressBar(pb, i)
   }
+  close(pb)
   return(invisible(bucketpath))
 }
 

R/export_shp.R

@@ -43,7 +43,7 @@ export_shp <- function(obj, pathshp, FUN = rgdal::writeOGR, dsnlayerbind = F, da
   if(dsnlayerbind == F){
     result = FUN1(obj, tempdir(), layer, ...)
   } else{
-    tmplayer = gsub("\\/+","/", paste0(tempdir(),"/",layer))
+    tmplayer = gsub("\\/+","/", paste0(tempdir(),"/",layer,".shp"))
     result = FUN1(obj, tmplayer, ...)
   }
   shpfiles = list.files(path = tempdir(), pattern = paste0(layer,"."))
@@ -53,10 +53,10 @@ export_shp <- function(obj, pathshp, FUN = rgdal::writeOGR, dsnlayerbind = F, da
     # a tempfile with the required extension
     temp <- paste0(tempdir(), "/", i)
     on.exit(unlink(temp))
-    # downloading the file
+    # uploading the file
     dsnlayer_i = gsub(paste0("\\.",tools::file_ext(dsnlayer),"$"), "", dsnlayer)
     downlogical = export_file(localfile = temp, bucketpath = paste0(dsnlayer_i, ".", tools::file_ext(i)),
-                                bucket = bucket)
+                                bucket = bucket, data_source = data_source)
   }
 }
 

R/flyio_auth.R

@@ -1,8 +1,9 @@
 #' Authenticate flyio
 #' @description Authenticate any of the cloud storage platforms to perform any I/O
-#' @param auth_list path to the json file or the system environment name in case of gcs. For s3 a vector for access_key, secret_access_key, region (optional; default us-east-1) and session_id (optional); this could also be a single comma-separated string.
+#' @param auth_list path to the json file or the system environment name in case of gcs. For s3 a vector for access_key, secret_access_key, region (optional; default us-east-1) and session_id (optional); this could also be a single comma-separated string. If left blank, then for s3 it will pick from ~/.aws/credentials file
 #' @param data_source default to local. Possible options : gcs, s3, local. Case insensitive
 #' @param scope the scope of the auth if gcs. Default: https://www.googleapis.com/auth/devstorage.full_control
+#' @param awsprofile if auth_list = "", which profile to use from ~/.aws/credentials
 #' @export "flyio_auth"
 #' @import "googleCloudStorageR" "stringr" "aws.s3" "assertthat" "utils" "tools"
 #' @examples
@@ -12,7 +13,8 @@
 
 
 flyio_auth <- function(auth_list = "", data_source = flyio_get_datasource(),
-                   scope = "https://www.googleapis.com/auth/devstorage.full_control"){
+                   scope = "https://www.googleapis.com/auth/devstorage.full_control",
+                   awsprofile = "default"){
 
   # checking if data_source input is valid
   invisible(assertthat::assert_that(stringr::str_to_lower(data_source) %in% c("local", "gcs", "s3"),
@@ -25,7 +27,7 @@ flyio_auth <- function(auth_list = "", data_source = flyio_get_datasource(),
   }
 
   # check the input for auth_list - split if comma present.
-  if(length(auth_list) == 1){
+  if(length(auth_list) == 1 & auth_list != ""){
     auth_list = stringr::str_trim(unlist(strsplit(auth_list, ",")))
   } else{
     invisible(assertthat::assert_that(!is.list(auth_list), msg = "Please input a vector in auth_list"))
@@ -40,7 +42,7 @@ flyio_auth <- function(auth_list = "", data_source = flyio_get_datasource(),
   if(str_to_lower(data_source) == "gcs"){
     auth_response = .gcsAuth(auth_list[1], scope)
   } else if(str_to_lower(data_source) == "s3"){
-    auth_response = .s3Auth(auth_list)
+    auth_response = .s3Auth(auth_list, awsprofile)
   }
   auth_response = assertthat::assert_that(isTRUE(auth_response), msg = "Authentication Failed!")
 }
@@ -63,7 +65,10 @@ flyio_auth <- function(auth_list = "", data_source = flyio_get_datasource(),
     })
 
 }
-.s3Auth <- function(auth_list){
+.s3Auth <- function(auth_list, awsprofile = "default"){
+  if(auth_list == ""){
+    auth_list = .awscred_profile(profile = awsprofile)
+  }
   invisible(assertthat::assert_that(length(auth_list)>=2, msg = "Input access key and secret key for S3"))
   auth_list <- switch (as.character(length(auth_list)),
                        "2" = c(auth_list, "us-east-1", ""),
@@ -77,3 +82,18 @@ flyio_auth <- function(auth_list = "", data_source = flyio_get_datasource(),
     cat("AWS S3 Authenticated!\n"); return(TRUE)}, error = function(err){})
 
 }
+
+.awscred_profile <- function(profile = "default"){
+  awscreds = readLines("~/.aws/credentials")
+  defaultprofile = grep(profile, awscreds)[1]
+  if(is.na(defaultprofile)){
+    return(list(aws_access_key_id="",
+                aws_secret_access_key=""))
+  }
+  assign(strsplit(awscreds[defaultprofile+1], " = ")[[1]][1],
+         strsplit(awscreds[defaultprofile+1], " = ")[[1]][2])
+  assign(strsplit(awscreds[defaultprofile+2], " = ")[[1]][1],
+         strsplit(awscreds[defaultprofile+2], " = ")[[1]][2])
+  return(c(aws_access_key_id, aws_secret_access_key))
+}
+

R/flyio_datasource.R

@@ -9,7 +9,7 @@
 flyio_set_datasource <- function(data_source){
   invisible(assert_that(is.string(data_source) && str_to_lower(data_source) %in% c("gcs", "s3", "local"), msg = "Enter a valid data source name"))
   data_source = str_to_lower(data_source)
-  Sys.setenv("flyioDataSource" = data_source)
+  Sys.setenv("CLOUD_STORAGE_NAME" = data_source)
   message("Default Data Source name set to '",data_source,"'")
 }
 
@@ -24,7 +24,13 @@ flyio_set_datasource <- function(data_source){
 #' # getting the data source
 #' flyio_get_datasource()
 flyio_get_datasource <- function(){
-  data_source = Sys.getenv("flyioDataSource")
+  data_source = Sys.getenv("CLOUD_STORAGE_NAME")
+  if(data_source == ""){
+    data_source = Sys.getenv("flyioDataSource")
+    if(data_source != ""){
+      cat("flyioDataSource env name is depreciated. Please use CLOUD_STORAGE_NAME.")
+    }
+  }
   invisible(assert_that(is.string(data_source) && data_source != "", msg = "No data source set. Use flyio_set_datasource to set the data source."))
   return(data_source)
 }

R/list_bucket.R

@@ -0,0 +1,37 @@
+#' List buckets for cloud storage
+#'
+#' @param data_source default to local. Possible options : gcs, s3, local. Case insensitive
+#' @param gcs_project Project containing buckets to list in Google Cloud Storage
+#'
+#' @return vector of bucket names
+#' @import "googleCloudStorageR" "aws.s3" "assertthat"
+#' @export "list_bucket"
+#'
+#' @examples
+#' \dontrun{
+#' flyio_set_datasource("s3")
+#' flyio_auth()
+#' list_bucket()
+#' }
+list_bucket <- function(data_source = flyio_get_datasource(), gcs_project = ""){
+
+  # checking if data_source input is valid
+  invisible(assertthat::assert_that(stringr::str_to_lower(data_source) %in% c("local", "gcs", "s3"),
+                                    msg = "data_source should be either local, gcs or s3"))
+
+  # if data source is local return
+  if(str_to_lower(data_source) == "local"){
+    cat("data_source is set to Local. No buckets.\n")
+    return(invisible(""))
+  }
+
+  # running authentication for set data source
+  if(str_to_lower(data_source) == "gcs"){
+    invisible(assertthat::assert_that(gcs_project != "",
+                                      msg = "Please specift the project containing buckets to list"))
+    buckets = gcs_list_buckets(projectId = gcs_project)
+  } else if(str_to_lower(data_source) == "s3"){
+    buckets = bucketlist()
+  }
+  return(buckets)
+}