Support ignoring URL params #90

README.md

@@ -107,6 +107,8 @@ possible flags with default values:
     show version
 -workers int
       number of workers (default - number of CPU cores)
+-ignore-query
+    ignore query parameters in URL
 ```
 
 # flags autocompletion

cmd/crawley/main.go

@@ -42,6 +42,7 @@ var (
 	fSkipSSL, fScanJS       bool
 	fScanCSS, fScanALL      bool
 	fSubdomains             bool
+	fIgnoreQuery            bool
 	fDirsPolicy, fProxyAuth string
 	fRobotsPolicy, fUA      string
 	fDelay                  time.Duration
@@ -165,6 +166,7 @@ func parseFlags() (rv []crawler.Option, err error) {
 		crawler.WithProxyAuth(fProxyAuth),
 		crawler.WithTimeout(fTimeout),
 		crawler.WithSubdomains(fSubdomains),
+		crawler.WithIgnoreQueryParams(fIgnoreQuery),
 	}
 
 	return rv, nil
@@ -193,6 +195,7 @@ func setupFlags() {
 	flag.BoolVar(&fSkipSSL, "skip-ssl", false, "skip ssl verification")
 	flag.BoolVar(&fSilent, "silent", false, "suppress info and error messages in stderr")
 	flag.BoolVar(&fVersion, "version", false, "show version")
+	flag.BoolVar(&fIgnoreQuery, "ignore-query", false, "ignore query parameters in URL comparison")
 
 	flag.StringVar(&fDirsPolicy, "dirs", crawler.DefaultDirsPolicy,
 		"policy for non-resource urls: show / hide / only")
@@ -236,8 +239,11 @@ func main() {
 	if fSilent {
 		log.SetOutput(io.Discard)
 	}
-
-	if err := crawl(flag.Arg(0), opts...); err != nil {
+	uri := flag.Arg(0)
+	if fIgnoreQuery {
+		uri = crawler.NormalizeURL(uri)
+	}
+	if err := crawl(uri, opts...); err != nil {
 		// forcing back stderr in case of errors, otherwise, if 'silent' is on - no one will knows what happened.
 		log.SetOutput(os.Stderr)
 		log.Fatal("[-] crawler:", err)

internal/crawler/config.go

@@ -18,18 +18,19 @@ const (
 )
 
 type config struct {
-	AlowedTags []string
-	Ignored    []string
-	Client     client.Config
-	Delay      time.Duration
-	Depth      int
-	Robots     RobotsPolicy
-	Dirs       DirsPolicy
-	Brute      bool
-	NoHEAD     bool
-	ScanJS     bool
-	ScanCSS    bool
-	Subdomains bool
+	AlowedTags  []string
+	Ignored     []string
+	Client      client.Config
+	Delay       time.Duration
+	Depth       int
+	Robots      RobotsPolicy
+	Dirs        DirsPolicy
+	Brute       bool
+	NoHEAD      bool
+	ScanJS      bool
+	ScanCSS     bool
+	Subdomains  bool
+	IgnoreQuery bool
 }
 
 func (c *config) validate() {
@@ -64,5 +65,9 @@ func (c *config) String() (rv string) {
 		sb.WriteString(" +subdomains")
 	}
 
+	if c.IgnoreQuery {
+		sb.WriteString(" +ignore-query")
+	}
+
 	return sb.String()
 }

internal/crawler/options.go

@@ -134,3 +134,10 @@ func WithSubdomains(v bool) Option {
 		c.Subdomains = v
 	}
 }
+
+// WithIgnoreQueryParams strips query parameters from uri.
+func WithIgnoreQueryParams(v bool) Option {
+	return func(c *config) {
+		c.IgnoreQuery = v
+	}
+}

internal/crawler/util.go

@@ -241,3 +241,13 @@ func resolveRef(base, uri string) (rv string, ok bool) {
 
 	return rv, true
 }
+
+func NormalizeURL(rawURL string) string {
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return rawURL
+	}
+	u.RawQuery = ""
+
+	return u.String()
+}