From 8bfa2d02f66b8db546ba63839d522f9d242559b6 Mon Sep 17 00:00:00 2001 From: Tom Hudson Date: Wed, 3 Jan 2018 14:23:32 +0000 Subject: [PATCH] Switches prefix/suffix teminology to host/path; breaking change; fixes #21 --- .gitignore | 2 ++ README.mkd | 28 ++++++++++++++-------------- args.go | 36 ++++++++++++++++++------------------ main.go | 32 ++++++++++++++++---------------- rawhttp.go | 4 ++-- request.go | 8 ++++---- response.go | 2 +- 7 files changed, 57 insertions(+), 55 deletions(-) diff --git a/.gitignore b/.gitignore index 10b16c9..3cb1b47 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ meg prefixes +hosts suffixes +paths domains out *.swp diff --git a/README.mkd b/README.mkd index 28437a1..1641729 100644 --- a/README.mkd +++ b/README.mkd @@ -22,7 +22,7 @@ put it somewhere in your `$PATH`. ## Basic Usage -Given a file full of *suffixes*: +Given a file full of paths: ``` /robots.txt @@ -30,7 +30,7 @@ Given a file full of *suffixes*: /package.json ``` -And a file full of *prefixes*: +And a file full of hosts (with a protocol): ``` http://example.com @@ -38,10 +38,10 @@ https://example.com http://example.net ``` -`meg` will request each *suffix* for every *prefix*: +`meg` will request each *path* for every *host*: ``` -▶ meg --verbose suffixes prefixes +▶ meg --verbose paths hosts out/example.com/45ed6f717d44385c5e9c539b0ad8dc71771780e0 http://example.com/robots.txt (404 Not Found) out/example.com/61ac5fbb9d3dd054006ae82630b045ba730d8618 https://example.com/robots.txt (404 Not Found) out/example.net/1432c16b671043271eab84111242b1fe2a28eb98 http://example.net/robots.txt (404 Not Found) @@ -75,8 +75,8 @@ http://example.com/robots.txt ``` -Without any arguments, meg will read suffixes from a file called `suffixes`, -and prefixes from a file called `prefixes`. There will also be no output: +Without any arguments, meg will read paths from a file called `./paths`, +and hosts from a file called `./hosts`. There will also be no output: ``` ▶ meg @@ -100,7 +100,7 @@ out/example.com/61ac5fbb9d3dd054006ae82630b045ba730d8618:14:< Server: ECS (lga/1 out/example.com/bd8d9f4c470ffa0e6ec8cfa8ba1c51d62289b6dd:16:< Server: ECS (lga/13A3) ``` -If you want to request just one suffix, you can specify it directly as an argument: +If you want to request just one path, you can specify it directly as an argument: ``` ▶ meg /admin.php @@ -112,10 +112,10 @@ meg's help output tries to actually be helpful: ``` ▶ meg --help -Request many paths (suffixes) for many hosts (prefixes) +Request many paths for many hosts Usage: - meg [suffix|suffixFile] [prefixFile] [outputDir] + meg [path|pathsFile] [hostsFile] [outputDir] Options: -c, --concurrency Set the concurrency level (defaut: 20) @@ -127,16 +127,16 @@ Options: -X, --method HTTP method (default: GET) Defaults: - suffixFile: ./suffixes - prefixFile: ./prefixes + pathsFile: ./paths + hostsFile: ./hosts outputDir: ./out -Suffix file format: +Paths file format: /robots.txt /package.json /security.txt -Prefix file format: +Hosts file format: http://example.com https://example.edu https://example.net @@ -156,7 +156,7 @@ with the `-c` or `--concurrency` option: ``` It's not very friendly to keep the concurrency level higher than the number of -prefixes - you may end up sending lots of requests to one host at once. +hosts - you may end up sending lots of requests to one host at once. ### Delay By default meg will wait 5000 milliseconds between requests to the same host. diff --git a/args.go b/args.go index 91cd541..26e43f8 100644 --- a/args.go +++ b/args.go @@ -26,8 +26,8 @@ type config struct { saveStatus int requester requester verbose bool - suffix string - prefix string + paths string + hosts string output string } @@ -74,16 +74,16 @@ func processArgs() config { flag.Parse() - // suffixes might be in a file, or it might be a single value - suffix := flag.Arg(0) - if suffix == "" { - suffix = "suffixes" + // paths might be in a file, or it might be a single value + paths := flag.Arg(0) + if paths == "" { + paths = "paths" } - // prefixes are always in a file - prefix := flag.Arg(1) - if prefix == "" { - prefix = "prefixes" + // hosts are always in a file + hosts := flag.Arg(1) + if hosts == "" { + hosts = "hosts" } // default the output directory to ./out @@ -106,18 +106,18 @@ func processArgs() config { saveStatus: saveStatus, requester: requesterFn, verbose: verbose, - suffix: suffix, - prefix: prefix, + paths: paths, + hosts: hosts, output: output, } } func init() { flag.Usage = func() { - h := "Request many paths (suffixes) for many hosts (prefixes)\n\n" + h := "Request many paths for many hosts\n\n" h += "Usage:\n" - h += " meg [suffix|suffixFile] [prefixFile] [outputDir]\n\n" + h += " meg [path|pathsFile] [hostsFile] [outputDir]\n\n" h += "Options:\n" h += " -c, --concurrency Set the concurrency level (defaut: 20)\n" @@ -129,16 +129,16 @@ func init() { h += " -X, --method HTTP method (default: GET)\n\n" h += "Defaults:\n" - h += " suffixFile: ./suffixes\n" - h += " prefixFile: ./prefixes\n" + h += " pathsFile: ./paths\n" + h += " hostsFile: ./hosts\n" h += " outputDir: ./out\n\n" - h += "Suffix file format:\n" + h += "Paths file format:\n" h += " /robots.txt\n" h += " /package.json\n" h += " /security.txt\n\n" - h += "Prefix file format:\n" + h += "Hosts file format:\n" h += " http://example.com\n" h += " https://example.edu\n" h += " https://example.net\n\n" diff --git a/main.go b/main.go index 981fa88..8255ef4 100644 --- a/main.go +++ b/main.go @@ -19,24 +19,24 @@ func main() { // get the config struct c := processArgs() - // if the suffix argument is a file, read it; otherwise + // if the paths argument is a file, read it; otherwise // treat it as a literal value - var suffixes []string - if isFile(c.suffix) { - lines, err := readLines(c.suffix) + var paths []string + if isFile(c.paths) { + lines, err := readLines(c.paths) if err != nil { - fmt.Fprintf(os.Stderr, "failed to open suffix file: %s\n", err) + fmt.Fprintf(os.Stderr, "failed to open paths file: %s\n", err) os.Exit(1) } - suffixes = lines - } else if c.suffix != "suffixes" { - suffixes = []string{c.suffix} + paths = lines + } else if c.paths != "paths" { + paths = []string{c.paths} } - // read the prefix file - prefixes, err := readLines(c.prefix) + // read the hosts file + hosts, err := readLines(c.hosts) if err != nil { - fmt.Fprintf(os.Stderr, "failed to open prefix file: %s\n", err) + fmt.Fprintf(os.Stderr, "failed to open hosts file: %s\n", err) os.Exit(1) } @@ -106,14 +106,14 @@ func main() { owg.Done() }() - // send requests for each suffix for every prefix - for _, suffix := range suffixes { - for _, prefix := range prefixes { + // send requests for each path for every host + for _, path := range paths { + for _, host := range hosts { requests <- request{ method: c.method, - prefix: prefix, - suffix: suffix, + host: host, + path: path, headers: c.headers, } } diff --git a/rawhttp.go b/rawhttp.go index 41d3082..aa35340 100644 --- a/rawhttp.go +++ b/rawhttp.go @@ -8,12 +8,12 @@ import ( ) func rawRequest(r request) response { - req, err := rawhttp.FromURL(r.method, r.prefix) + req, err := rawhttp.FromURL(r.method, r.host) if err != nil { return response{request: r, err: err} } - req.Path = r.suffix + req.Path = r.path r.headers = append(r.headers, "Connection: close") diff --git a/request.go b/request.go index ff58fa5..abf7b25 100644 --- a/request.go +++ b/request.go @@ -8,14 +8,14 @@ import ( // a request is a wrapper for a URL that we want to request type request struct { method string - prefix string - suffix string + path string + host string headers []string } // Hostname returns the hostname part of the request func (r request) Hostname() string { - u, err := url.Parse(r.prefix) + u, err := url.Parse(r.host) // the hostname part is used only for the rate // limiting and the @@ -27,7 +27,7 @@ func (r request) Hostname() string { // URL returns the full URL to request func (r request) URL() string { - return r.prefix + r.suffix + return r.host + r.path } // hasHeader returns true if the request diff --git a/response.go b/response.go index 4975b4d..36105bb 100644 --- a/response.go +++ b/response.go @@ -28,7 +28,7 @@ func (r response) String() string { b.WriteString(r.request.URL()) b.WriteString("\n\n") - b.WriteString(fmt.Sprintf("> %s %s HTTP/1.1\n", r.request.method, r.request.suffix)) + b.WriteString(fmt.Sprintf("> %s %s HTTP/1.1\n", r.request.method, r.request.path)) // request headers for _, h := range r.request.headers {