diff --git a/README.md b/README.md index 0985ab3..2584cc7 100644 --- a/README.md +++ b/README.md @@ -5,3 +5,6 @@ ![vault-sync](https://user-images.githubusercontent.com/6597086/40061189-0d140284-5861-11e8-9e78-1d5d4f8d8e48.png) +* Supports URL files as source + +Example: `vault-push -U /path/to/urls1.txt -U /path/to/urls2.txt ...` \ No newline at end of file diff --git a/cmd/vault-drop/main.go b/cmd/vault-drop/main.go index c9308c6..be2bdc5 100644 --- a/cmd/vault-drop/main.go +++ b/cmd/vault-drop/main.go @@ -5,10 +5,12 @@ import ( "os" "github.com/reddec/vault" "fmt" + "github.com/reddec/vault/utils" ) var config struct { - URL []string `short:"u" long:"url" env:"URL" description:"storage url"` + URL []string `short:"u" long:"url" env:"URL" description:"storage url"` + URLFiles []string `short:"U" long:"url-file" env:"URL_FILES" description:"file with lines of storage URLs"` Args struct { ID string } `positional-args:"yes" required:"yes"` @@ -41,6 +43,12 @@ func main() { wr.Add(url) } + err = utils.ReadUrlFiles(&wr, config.URLFiles) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to read urls file: %v\n", err) + os.Exit(2) + } + network := vault.SimpleNet(&wr) err = vault.Delete(network, config.Args.ID) diff --git a/cmd/vault-pull/main.go b/cmd/vault-pull/main.go index a8aac0a..50b896e 100644 --- a/cmd/vault-pull/main.go +++ b/cmd/vault-pull/main.go @@ -5,10 +5,12 @@ import ( "os" "github.com/reddec/vault" "fmt" + "github.com/reddec/vault/utils" ) var config struct { - URL []string `short:"u" long:"url" env:"URL" description:"storage url"` + URL []string `short:"u" long:"url" env:"URL" description:"storage url"` + URLFiles []string `short:"U" long:"url-file" env:"URL_FILES" description:"file with lines of storage URLs"` Args struct { ID string } `positional-args:"yes" required:"yes"` @@ -39,12 +41,18 @@ func main() { wr.Add(url) } + err = utils.ReadUrlFiles(&wr, config.URLFiles) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to read urls file: %v\n", err) + os.Exit(2) + } + network := vault.SimpleNet(&wr) amount, err := vault.Read(network, config.Args.ID, os.Stdout) if err != nil { fmt.Fprintf(os.Stderr, "failed to read: %v\n", err) - os.Exit(2) + os.Exit(3) } fmt.Fprintln(os.Stderr, "read", amount, "bytes") } diff --git a/cmd/vault-push/main.go b/cmd/vault-push/main.go index 0c3e9f1..96aedbc 100644 --- a/cmd/vault-push/main.go +++ b/cmd/vault-push/main.go @@ -5,10 +5,12 @@ import ( "os" "github.com/reddec/vault" "fmt" + "github.com/reddec/vault/utils" ) var config struct { URL []string `short:"u" long:"url" env:"URLS" description:"storage url"` + URLFiles []string `short:"U" long:"url-file" env:"URL_FILES" description:"file with lines of storage URLs"` ChunkSize int `short:"c" long:"chunk" env:"CHUNK" description:"chunk size in kb" default:"1024"` Redundancy int `short:"r" long:"redundancy" env:"REDUNDANCY" description:"number of copies" default:"3"` Args struct { @@ -26,7 +28,6 @@ Write stream from stdin to object with specified redundancy and chunk size ` ) - func main() { parser := flags.NewParser(&config, flags.Default) parser.ShortDescription = fmt.Sprint(name, " part of VAULT tools - distributes master-less object storage") @@ -42,12 +43,18 @@ func main() { wr.Add(url) } + err = utils.ReadUrlFiles(&wr, config.URLFiles) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to read urls file: %v\n", err) + os.Exit(2) + } + network := vault.SimpleNet(&wr) amount, err := vault.WriteStream(network, config.Args.ID, config.Redundancy, config.ChunkSize*1024, os.Stdin) if err != nil { fmt.Fprintf(os.Stderr, "failed to write: %v\n", err) - os.Exit(2) + os.Exit(3) } fmt.Println("written", amount, "bytes") } diff --git a/cmd/vault-sync/main.go b/cmd/vault-sync/main.go index a545bcc..27ee5c4 100644 --- a/cmd/vault-sync/main.go +++ b/cmd/vault-sync/main.go @@ -5,10 +5,12 @@ import ( "os" "github.com/reddec/vault" "fmt" + "github.com/reddec/vault/utils" ) var config struct { URL []string `short:"u" long:"url" env:"URLS" description:"storage url"` + URLFiles []string `short:"U" long:"url-file" env:"URL_FILES" description:"file with lines of storage URLs"` Redundancy int `short:"r" long:"redundancy" env:"REDUNDANCY" description:"number of copies" default:"3"` } @@ -37,12 +39,18 @@ func main() { wr.Add(url) } + err = utils.ReadUrlFiles(&wr, config.URLFiles) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to read urls file: %v\n", err) + os.Exit(2) + } + network := vault.SimpleNet(&wr) items, err := network.Sync(config.Redundancy) if err != nil { fmt.Fprintf(os.Stderr, "failed to sync: %v\n", err) - os.Exit(2) + os.Exit(3) } fmt.Println("fixed", items, "chunks") } diff --git a/utils/cli.go b/utils/cli.go new file mode 100644 index 0000000..9d526ec --- /dev/null +++ b/utils/cli.go @@ -0,0 +1,24 @@ +package utils + +import ( + "github.com/reddec/vault" + "io/ioutil" + "strings" +) + +func ReadUrlFiles(wh *vault.SimpleWarehouse, files []string) error { + for _, file := range files { + content, err := ioutil.ReadFile(file) + if err != nil { + return err + } + for _, line := range strings.Split(string(content), "\n") { + line = strings.TrimSpace(line) + if len(line) == 0 || line[0] == '#' { + continue + } + wh.Add(line) + } + } + return nil +}