code-smells reduced, x/net/html version bump (#11)

s0rg · Dec 19, 2021 · cc3eba7 · cc3eba7
1 parent 079d02b
commit cc3eba7
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 14 deletions.
diff --git a/go.mod b/go.mod
@@ -2,4 +2,4 @@ module github.com/s0rg/crawley
 
 go 1.17
 
-require golang.org/x/net v0.0.0-20211209124913-491a49abca63
+require golang.org/x/net v0.0.0-20211216030914-fe4d6282115f
diff --git a/go.sum b/go.sum
@@ -1,5 +1,5 @@
-golang.org/x/net v0.0.0-20211209124913-491a49abca63 h1:iocB37TsdFuN6IBRZ+ry36wrkoV51/tl5vOWqkcPGvY=
-golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM=
+golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

diff --git a/pkg/client/http.go b/pkg/client/http.go
@@ -102,6 +102,7 @@ func (h *HTTP) request(req *http.Request) (body io.ReadCloser, hdrs http.Header,
 	return resp.Body, resp.Header, err
 }
 
+// Discard read all contents from ReaderCloser, closing it afterwards.
 func Discard(rc io.ReadCloser) {
 	_, _ = io.Copy(io.Discard, rc)
 	_ = rc.Close()

diff --git a/pkg/crawler/crawl.go b/pkg/crawler/crawl.go
@@ -39,8 +39,11 @@ const (
 type taskFlag byte
 
 const (
+	// TaskDefault marks result for printing only.
 	TaskDefault taskFlag = iota
+	// TaskCrawl marks result as can-be-crawled.
 	TaskCrawl
+	// TaskDone marks result as final - crawling end here.
 	TaskDone
 )
 
@@ -156,20 +159,13 @@ func (c *Crawler) crawl(base *url.URL, t *crawlResult) (yes bool) {
 		return
 	}
 
-	if !canCrawl(base, u, c.cfg.Depth) {
-		return
-	}
-
-	if c.robots.Forbidden(u.Path) {
-		return
-	}
-
-	if c.cfg.Dirs == DirsOnly && isResorce(u.Path) {
+	switch {
+	case !canCrawl(base, u, c.cfg.Depth), c.robots.Forbidden(u.Path), c.cfg.Dirs == DirsOnly && isResorce(u.Path):
 		return
+	default:
+		go func(r *url.URL) { c.crawlCh <- r }(u)
 	}
 
-	go func(r *url.URL) { c.crawlCh <- r }(u)
-
 	return true
 }