[WIP] Kadai3-2 ramenjuniti

kadai3/ramenjuniti/gdown/Makefile

@@ -0,0 +1,14 @@
+BIN := gdown
+
+.PHONY: test
+test:
+	go test -cover -v ./...
+
+.PHONY: build
+build: test
+	go build -o $(BIN) ./cmd/gdown
+
+.PHONY: clean
+clean:
+	rm $(BIN)
+	go clean

kadai3/ramenjuniti/gdown/README.md

@@ -0,0 +1,19 @@
+# gdown
+
+## Usage
+
+```bash
+./gdown -p {並列数} {URL}
+```
+
+## Build
+
+```bash
+make build
+```
+
+## Test
+
+```bash
+make test
+```

kadai3/ramenjuniti/gdown/cmd/gdown/main.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"runtime"
+
+	"github.com/gopherdojo/dojo5/kadai3/ramenjuniti/gdown"
+)
+
+func main() {
+	p := flag.Int("p", runtime.NumCPU(), "並列数")
+	flag.Parse()
+
+	if *p < 1 {
+		fmt.Fprintln(os.Stderr, "p cannot be less than 1")
+		os.Exit(1)
+	}
+
+	url := flag.Arg(0)
+	if url == "" {
+		fmt.Fprintln(os.Stderr, "please input URL")
+		os.Exit(1)
+	}
+
+	c, err := gdown.New(url, *p)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+
+	if err = c.Run(); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

kadai3/ramenjuniti/gdown/gdown.go

@@ -0,0 +1,173 @@
+package gdown
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+
+	"golang.org/x/sync/errgroup"
+)
+
+type RangeRequest struct {
+	URL      string
+	FileName string
+	Unit     int64
+	Ranges   []*Range
+}
+
+type Range struct {
+	start int64
+	end   int64
+}
+
+func New(rawurl string, p int) (*RangeRequest, error) {
+	u, err := url.Parse(rawurl)
+	if err != nil {
+		return nil, err
+	}
+
+	fn := getName(u.Path)
+
+	res, err := http.Head(rawurl)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	if res.Header.Get("Accept-Ranges") != "bytes" {
+		return &RangeRequest{URL: rawurl, FileName: fn, Ranges: nil}, nil
+	}
+
+	cl := res.ContentLength
+	unit := cl / int64(p)
+	ranges := make([]*Range, p)
+
+	for i := 0; i < p; i++ {
+		start := int64(i) * unit
+		end := start + unit - 1
+		if i == p-1 {
+			end = cl
+		}
+
+		ranges[i] = &Range{start: start, end: end}
+	}
+
+	return &RangeRequest{URL: rawurl, FileName: fn, Unit: unit, Ranges: ranges}, nil
+}
+
+func (r *RangeRequest) Run() error {
+	if r.Ranges == nil {
+		req, err := http.NewRequest(http.MethodGet, r.URL, nil)
+		if err != nil {
+			return err
+		}
+
+		res, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return err
+		}
+		defer res.Body.Close()
+
+		return save(r.FileName, res)
+
+	} else {
+		g, ctx := errgroup.WithContext(context.Background())
+
+		for i, ran := range r.Ranges {
+			i, ran := i, ran
+			tmpn := fmt.Sprintf("%s-%d", r.FileName, i)
+
+			if info, err := os.Stat(tmpn); err == nil {
+				size := info.Size()
+				if i == len(r.Ranges)-1 {
+					if size == ran.end-ran.start {
+						continue
+					}
+				} else if size == r.Unit {
+					continue
+				}
+				ran.start += size
+			}
+
+			g.Go(func() error {
+				req, err := makeRangeReqest(ctx, r.URL, ran.start, ran.end)
+				if err != nil {
+					return err
+				}
+
+				res, err := http.DefaultClient.Do(req)
+				if err != nil {
+					return err
+				}
+				defer res.Body.Close()
+
+				return save(tmpn, res)
+			})
+		}
+
+		if err := g.Wait(); err != nil {
+			return err
+		}
+
+		return merge(r.FileName, len(r.Ranges))
+	}
+}
+
+func makeRangeReqest(ctx context.Context, url string, start, end int64) (*http.Request, error) {
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req = req.WithContext(ctx)
+	req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))
+
+	return req, nil
+}
+
+func getName(url string) string {
+	us := strings.Split(url, "/")
+	return us[len(us)-1]
+}
+
+func save(fn string, res *http.Response) error {
+	f, err := os.OpenFile(fn, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	if _, err := io.Copy(f, res.Body); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func merge(fn string, p int) error {
+	f, err := os.Create(fn)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	for i := 0; i < p; i++ {
+		tmpn := fmt.Sprintf("%s-%d", fn, i)
+		tmp, err := os.Open(tmpn)
+		if err != nil {
+			return err
+		}
+
+		io.Copy(f, tmp)
+		tmp.Close()
+
+		if err := os.Remove(tmpn); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

kadai3/ramenjuniti/gdown/go.mod

@@ -0,0 +1,5 @@
+module github.com/gopherdojo/dojo5/kadai3/ramenjuniti/gdown
+
+go 1.12
+
+require golang.org/x/sync v0.0.0-20190423024810-112230192c58

kadai3/ramenjuniti/gdown/go.sum

@@ -0,0 +1,2 @@
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=