diff --git a/kadai3-2/lfcd85/.gitignore b/kadai3-2/lfcd85/.gitignore new file mode 100644 index 0000000..a03e7d8 --- /dev/null +++ b/kadai3-2/lfcd85/.gitignore @@ -0,0 +1 @@ +bin/mypget diff --git a/kadai3-2/lfcd85/Makefile b/kadai3-2/lfcd85/Makefile new file mode 100644 index 0000000..ec6cc3d --- /dev/null +++ b/kadai3-2/lfcd85/Makefile @@ -0,0 +1,14 @@ +build: + GO111MODULE=on go build -o bin/mypget cmd/main.go + +PHONY: fmt +fmt: + go fmt ./... + +PHONY: check +check: + GO111MODULE=on go test ./... -v + +PHONY: coverage +coverage: + GO111MODULE=on go test ./... -cover diff --git a/kadai3-2/lfcd85/README.md b/kadai3-2/lfcd85/README.md new file mode 100644 index 0000000..65cebb7 --- /dev/null +++ b/kadai3-2/lfcd85/README.md @@ -0,0 +1,30 @@ +# Split Downloader (my pget) + +A CLI for split downloading, kadai3-2 of Gopherdojo #5. + +Gopher道場 #5 課題3-2 `分割ダウンローダを作ろう` の実装です。 + +## Installation + +```bash +$ make build +``` + +## Usage + +URL to download is required as argument. + +ダウンロードするURLを引数として渡してください。 + + +```bash +$ bin/mypget https://domain.name/path/to/file +``` + +With `-n` option, you can specify the number of split ranges. Default number is `8` . The number must be less than the length of file to download. + +`-n` オプションで分割の数を指定できます。デフォルトの数は `8` です。分割数はダウンロード対象のファイルサイズよりも小さい必要があります。 + +```bash +$ bin/mypget -n 16 https://domain.name/path/to/file +``` diff --git a/kadai3-2/lfcd85/cmd/main.go b/kadai3-2/lfcd85/cmd/main.go new file mode 100644 index 0000000..ff764d3 --- /dev/null +++ b/kadai3-2/lfcd85/cmd/main.go @@ -0,0 +1,33 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "net/url" + "os" + + "github.com/gopherdojo/dojo5/kadai3-2/lfcd85/mypget" +) + +func main() { + splitNum := flag.Int("n", 8, "Number of splitted downloads") + flag.Parse() + urlStr := flag.Arg(0) + if urlStr == "" { + err := errors.New("URL is not inputted") + fmt.Fprintln(os.Stderr, "error: ", err) + os.Exit(1) + } + + url, err := url.Parse(urlStr) + if err != nil { + fmt.Fprintln(os.Stderr, "error: ", err) + os.Exit(1) + } + + if err := mypget.New(url, *splitNum).Execute(nil); err != nil { + fmt.Fprintln(os.Stderr, "error: ", err) + os.Exit(1) + } +} diff --git a/kadai3-2/lfcd85/go.mod b/kadai3-2/lfcd85/go.mod new file mode 100644 index 0000000..49ca55c --- /dev/null +++ b/kadai3-2/lfcd85/go.mod @@ -0,0 +1,5 @@ +module github.com/gopherdojo/dojo5/kadai3-2/lfcd85 + +go 1.12 + +require golang.org/x/sync v0.0.0-20190423024810-112230192c58 diff --git a/kadai3-2/lfcd85/go.sum b/kadai3-2/lfcd85/go.sum new file mode 100644 index 0000000..6eae930 --- /dev/null +++ b/kadai3-2/lfcd85/go.sum @@ -0,0 +1,2 @@ +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/kadai3-2/lfcd85/mypget/export_test.go b/kadai3-2/lfcd85/mypget/export_test.go new file mode 100644 index 0000000..79f9275 --- /dev/null +++ b/kadai3-2/lfcd85/mypget/export_test.go @@ -0,0 +1,5 @@ +package mypget + +func (d *Downloader) ExportOutputPath() string { + return d.outputPath +} diff --git a/kadai3-2/lfcd85/mypget/mypget.go b/kadai3-2/lfcd85/mypget/mypget.go new file mode 100644 index 0000000..5ef538d --- /dev/null +++ b/kadai3-2/lfcd85/mypget/mypget.go @@ -0,0 +1,200 @@ +package mypget + +import ( + "context" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + + "golang.org/x/sync/errgroup" +) + +const ( + tempDirName = "partials" + tempFilePrefix = "partial" +) + +// Downloader stores the information used for split downloading. +type Downloader struct { + url *url.URL + splitNum int + ranges []string + outputPath string +} + +// New creates a Downloader struct. +func New(url *url.URL, splitNum int) *Downloader { + return &Downloader{ + url: url, + splitNum: splitNum, + } +} + +// Execute do the split download. +func (d *Downloader) Execute(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + req, err := http.NewRequest(http.MethodGet, d.url.String(), nil) + if err != nil { + return err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if !acceptBytesRanges(resp) { + return errors.New("split download is not supported in this response") + } + + length := int(resp.ContentLength) + if length < d.splitNum { + return errors.New("the number of split ranges is larger than file length") + } + d.splitToRanges(length) + + tempDir, err := ioutil.TempDir("", tempDirName) + if err != nil { + return err + } + defer os.RemoveAll(tempDir) + + if err := d.downloadByRanges(ctx, tempDir); err != nil { + return err + } + + if err := d.combine(tempDir); err != nil { + return err + } + + fmt.Printf("Download completed! saved at: %v\n", d.outputPath) + + return nil +} + +func acceptBytesRanges(resp *http.Response) bool { + return resp.Header.Get("Accept-Ranges") == "bytes" +} + +func (d *Downloader) splitToRanges(length int) { + var ranges []string + var rangeStart, rangeEnd int + rangeLength := length / d.splitNum + + for i := 0; i < d.splitNum; i++ { + if i != 0 { + rangeStart = rangeEnd + 1 + } + rangeEnd = rangeStart + rangeLength + + if i == d.splitNum-1 && rangeEnd != length { + rangeEnd = length + } + + ranges = append(ranges, fmt.Sprintf("bytes=%d-%d", rangeStart, rangeEnd)) + } + d.ranges = ranges +} + +func (d *Downloader) downloadByRanges(ctx context.Context, tempDir string) error { + eg, ctx := errgroup.WithContext(ctx) + + for i, r := range d.ranges { + i, r := i, r + eg.Go(func() error { + req, err := http.NewRequest("GET", d.url.String(), nil) + if err != nil { + return err + } + req = req.WithContext(ctx) + req.Header.Set("Range", r) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if err := validateStatusPartialContent(resp); err != nil { + return err + } + + partialPath := generatePartialPath(tempDir, i) + fmt.Printf("Downloading range %v / %v (%v) ...\n", i+1, len(d.ranges), r) + + f, err := os.Create(partialPath) + if err != nil { + return err + } + defer f.Close() + + if _, err = io.Copy(f, resp.Body); err != nil { + return err + } + return nil + }) + } + return eg.Wait() +} + +func validateStatusPartialContent(resp *http.Response) error { + validStatusCode := http.StatusPartialContent + if resp.StatusCode != validStatusCode { + return fmt.Errorf("status code must be %d: actually was %d", validStatusCode, resp.StatusCode) + } + return nil +} + +func generatePartialPath(tempDir string, i int) string { + base := strings.Join([]string{tempFilePrefix, strconv.Itoa(i)}, "_") + return strings.Join([]string{tempDir, base}, "/") +} + +func (d *Downloader) combine(tempDir string) error { + d.outputPath = d.getOutputFileName() + f, err := os.Create(d.outputPath) + if err != nil { + return err + } + defer f.Close() + + fmt.Printf("Combining partials to %v ...\n", d.outputPath) + + for i, _ := range d.ranges { + partialPath := generatePartialPath(tempDir, i) + partial, err := os.Open(partialPath) + if err != nil { + return err + } + + _, err = io.Copy(f, partial) + partial.Close() + if err != nil { + return err + } + } + return nil +} + +func (d *Downloader) getOutputFileName() string { + base := filepath.Base(d.url.EscapedPath()) + switch base { + case "/", ".", "": + return "output" + default: + return base + } +} diff --git a/kadai3-2/lfcd85/mypget/mypget_test.go b/kadai3-2/lfcd85/mypget/mypget_test.go new file mode 100644 index 0000000..8e26c0e --- /dev/null +++ b/kadai3-2/lfcd85/mypget/mypget_test.go @@ -0,0 +1,121 @@ +package mypget_test + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strconv" + "strings" + "testing" + + "github.com/gopherdojo/dojo5/kadai3-2/lfcd85/mypget" +) + +type testServerFile struct { + testDataPath string +} + +func TestDownloader_Execute(t *testing.T) { + cases := []struct { + testDataPath string + splitNum int + }{ + {"../testdata/tower.jpg", 8}, + {"../testdata/lorem_ipsum.txt", 4}, + } + + for _, c := range cases { + c := c + t.Run(c.testDataPath, func(t *testing.T) { + tsf := testServerFile{c.testDataPath} + + ts, closeTs := initTestServer(t, tsf.testServerHandler) + defer closeTs() + + d := initDownloader(t, ts.URL, c.splitNum) + if err := d.Execute(nil); err != nil { + t.Errorf("failed to execute split downloader: %v", err) + } + deleteOutputFile(t, d) + }) + } +} + +func initDownloader(t *testing.T, urlStr string, splitNum int) *mypget.Downloader { + t.Helper() + + url, err := url.Parse(urlStr) + if err != nil { + t.Errorf("failed to parse URL for testing: %v", err) + } + + return mypget.New(url, splitNum) +} + +func deleteOutputFile(t *testing.T, d *mypget.Downloader) { + t.Helper() + + outputPath := d.ExportOutputPath() + if outputPath != "" { + if err := os.Remove(outputPath); err != nil { + t.Errorf("failed to remove output file: %v", err) + } + } +} + +func initTestServer(t *testing.T, handler func(t *testing.T, w http.ResponseWriter, r *http.Request)) (*httptest.Server, func()) { + t.Helper() + + ts := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + handler(t, w, r) + }, + )) + + return ts, func() { ts.Close() } +} + +func (tsf *testServerFile) testServerHandler(t *testing.T, w http.ResponseWriter, r *http.Request) { + t.Helper() + + w.Header().Set("Accept-Ranges", "bytes") + headerRange := r.Header.Get("Range") + + body := func() []byte { + testDataBytes, err := ioutil.ReadFile(tsf.testDataPath) + if err != nil { + t.Errorf("failed to read the test file in test server: %v", err) + } + + if headerRange == "" { + return testDataBytes + } + + rangeItems := strings.Split(headerRange, "=") + if rangeItems[0] != "bytes" { + t.Errorf("range in test server should have bytes value, but actually does not") + } + rangeValues := strings.Split(rangeItems[1], "-") + + rangeStart, err := strconv.Atoi(rangeValues[0]) + if err != nil { + t.Errorf("failed to get the start of the range in test server: %v", err) + } + + rangeEnd, err := strconv.Atoi(rangeValues[1]) + if err != nil { + t.Errorf("failed to get the end of the range in test server: %v", err) + } + + return testDataBytes[rangeStart:rangeEnd] + }() + + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + w.WriteHeader(http.StatusPartialContent) + + if _, err := w.Write(body); err != nil { + t.Errorf("failed to write the response body in test server: %v", err) + } +} diff --git a/kadai3-2/lfcd85/testdata/lorem_ipsum.txt b/kadai3-2/lfcd85/testdata/lorem_ipsum.txt new file mode 100644 index 0000000..1b37687 --- /dev/null +++ b/kadai3-2/lfcd85/testdata/lorem_ipsum.txt @@ -0,0 +1 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. diff --git a/kadai3-2/lfcd85/testdata/tower.jpg b/kadai3-2/lfcd85/testdata/tower.jpg new file mode 100644 index 0000000..fa3f4f8 Binary files /dev/null and b/kadai3-2/lfcd85/testdata/tower.jpg differ