support file as source for URLs

README.md

@@ -5,3 +5,6 @@
 ![vault-sync](https://user-images.githubusercontent.com/6597086/40061189-0d140284-5861-11e8-9e78-1d5d4f8d8e48.png)
 
 
+* Supports URL files as source
+
+Example: `vault-push -U /path/to/urls1.txt -U /path/to/urls2.txt ...`

cmd/vault-drop/main.go

@@ -5,10 +5,12 @@ import (
 	"os"
 	"github.com/reddec/vault"
 	"fmt"
+	"github.com/reddec/vault/utils"
 )
 
 var config struct {
-	URL []string `short:"u" long:"url" env:"URL" description:"storage url"`
+	URL      []string `short:"u" long:"url" env:"URL" description:"storage url"`
+	URLFiles []string `short:"U" long:"url-file" env:"URL_FILES" description:"file with lines of storage URLs"`
 	Args struct {
 		ID string
 	} `positional-args:"yes" required:"yes"`
@@ -41,6 +43,12 @@ func main() {
 		wr.Add(url)
 	}
 
+	err = utils.ReadUrlFiles(&wr, config.URLFiles)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to read urls file: %v\n", err)
+		os.Exit(2)
+	}
+
 	network := vault.SimpleNet(&wr)
 
 	err = vault.Delete(network, config.Args.ID)

cmd/vault-pull/main.go

@@ -5,10 +5,12 @@ import (
 	"os"
 	"github.com/reddec/vault"
 	"fmt"
+	"github.com/reddec/vault/utils"
 )
 
 var config struct {
-	URL []string `short:"u" long:"url" env:"URL" description:"storage url"`
+	URL      []string `short:"u" long:"url" env:"URL" description:"storage url"`
+	URLFiles []string `short:"U" long:"url-file" env:"URL_FILES" description:"file with lines of storage URLs"`
 	Args struct {
 		ID string
 	} `positional-args:"yes" required:"yes"`
@@ -39,12 +41,18 @@ func main() {
 		wr.Add(url)
 	}
 
+	err = utils.ReadUrlFiles(&wr, config.URLFiles)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to read urls file: %v\n", err)
+		os.Exit(2)
+	}
+
 	network := vault.SimpleNet(&wr)
 
 	amount, err := vault.Read(network, config.Args.ID, os.Stdout)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to read: %v\n", err)
-		os.Exit(2)
+		os.Exit(3)
 	}
 	fmt.Fprintln(os.Stderr, "read", amount, "bytes")
 }

cmd/vault-push/main.go

@@ -5,10 +5,12 @@ import (
 	"os"
 	"github.com/reddec/vault"
 	"fmt"
+	"github.com/reddec/vault/utils"
 )
 
 var config struct {
 	URL        []string `short:"u" long:"url" env:"URLS" description:"storage url"`
+	URLFiles   []string `short:"U" long:"url-file" env:"URL_FILES" description:"file with lines of storage URLs"`
 	ChunkSize  int      `short:"c" long:"chunk" env:"CHUNK" description:"chunk size in kb" default:"1024"`
 	Redundancy int      `short:"r" long:"redundancy" env:"REDUNDANCY" description:"number of copies" default:"3"`
 	Args struct {
@@ -26,7 +28,6 @@ Write stream from stdin to object with specified redundancy and chunk size
 `
 )
 
-
 func main() {
 	parser := flags.NewParser(&config, flags.Default)
 	parser.ShortDescription = fmt.Sprint(name, " part of VAULT tools - distributes master-less object storage")
@@ -42,12 +43,18 @@ func main() {
 		wr.Add(url)
 	}
 
+	err = utils.ReadUrlFiles(&wr, config.URLFiles)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to read urls file: %v\n", err)
+		os.Exit(2)
+	}
+
 	network := vault.SimpleNet(&wr)
 
 	amount, err := vault.WriteStream(network, config.Args.ID, config.Redundancy, config.ChunkSize*1024, os.Stdin)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to write: %v\n", err)
-		os.Exit(2)
+		os.Exit(3)
 	}
 	fmt.Println("written", amount, "bytes")
 }

cmd/vault-sync/main.go

@@ -5,10 +5,12 @@ import (
 	"os"
 	"github.com/reddec/vault"
 	"fmt"
+	"github.com/reddec/vault/utils"
 )
 
 var config struct {
 	URL        []string `short:"u" long:"url" env:"URLS" description:"storage url"`
+	URLFiles   []string `short:"U" long:"url-file" env:"URL_FILES" description:"file with lines of storage URLs"`
 	Redundancy int      `short:"r" long:"redundancy" env:"REDUNDANCY" description:"number of copies" default:"3"`
 }
 
@@ -37,12 +39,18 @@ func main() {
 		wr.Add(url)
 	}
 
+	err = utils.ReadUrlFiles(&wr, config.URLFiles)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to read urls file: %v\n", err)
+		os.Exit(2)
+	}
+
 	network := vault.SimpleNet(&wr)
 
 	items, err := network.Sync(config.Redundancy)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to sync: %v\n", err)
-		os.Exit(2)
+		os.Exit(3)
 	}
 	fmt.Println("fixed", items, "chunks")
 }

utils/cli.go

@@ -0,0 +1,24 @@
+package utils
+
+import (
+	"github.com/reddec/vault"
+	"io/ioutil"
+	"strings"
+)
+
+func ReadUrlFiles(wh *vault.SimpleWarehouse, files []string) error {
+	for _, file := range files {
+		content, err := ioutil.ReadFile(file)
+		if err != nil {
+			return err
+		}
+		for _, line := range strings.Split(string(content), "\n") {
+			line = strings.TrimSpace(line)
+			if len(line) == 0 || line[0] == '#' {
+				continue
+			}
+			wh.Add(line)
+		}
+	}
+	return nil
+}