diff --git a/Dockerfile b/Dockerfile index 196ba439..33474c3f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Base -FROM golang:1.20.6-alpine AS builder +FROM golang:1.21.4-alpine AS builder RUN apk add --no-cache git build-base gcc musl-dev WORKDIR /app @@ -8,8 +8,8 @@ RUN go mod download RUN go build ./cmd/httpx FROM alpine:3.18.2 -RUN apk -U upgrade --no-cache \ +RUN apk upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates chromium COPY --from=builder /app/httpx /usr/local/bin/ -ENTRYPOINT ["httpx"] \ No newline at end of file +ENTRYPOINT ["httpx"] diff --git a/README.md b/README.md index c20a76d6..a4622530 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Features • Installation • Usage • - Running httpx • + Documentation • Notes • Join Discord

@@ -62,12 +62,14 @@ # Installation Instructions -`httpx` requires **go1.20** to install successfully. Run the following command to get the repo: +`httpx` requires **go1.21** to install successfully. Run the following command to get the repo: ```sh go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest ``` +To learn more about installing httpx, see https://docs.projectdiscovery.io/tools/httpx/install. + | :exclamation: **Disclaimer** | |---------------------------------| | **This project is in active development**. Expect breaking changes with releases. Review the changelog before updating. | @@ -118,8 +120,10 @@ PROBES: HEADLESS: -ss, -screenshot enable saving screenshot of the page using headless browser -system-chrome enable using local installed chrome for screenshot + -ho, -headless-options string[] start headless chrome with additional options -esb, -exclude-screenshot-bytes enable excluding screenshot bytes from json output -ehb, -exclude-headless-body enable excluding headless header from json output + -st, -screenshot-timeout int set timeout for screenshot in seconds (default 10) MATCHERS: -mc, -match-code string match response with specified status code (-mc 200,302) @@ -135,7 +139,7 @@ MATCHERS: EXTRACTOR: -er, -extract-regex string[] display response content with matched regex - -ep, -extract-preset string[] display response content matched by a pre-defined regex (ipv4,mail,url) + -ep, -extract-preset string[] display response content matched by a pre-defined regex (url,ipv4,mail) FILTERS: -fc, -filter-code string filter response with specified status code (-fc 403,401) @@ -230,8 +234,7 @@ OPTIMIZATIONS: -nf, -no-fallback display both probed protocol (HTTPS and HTTP) -nfs, -no-fallback-scheme probe with protocol scheme specified in input -maxhr, -max-host-error int max error count per host before skipping remaining path/s (default 30) - -ec, -exclude-cdn skip full port scans for CDN/WAF (only checks for 80,443) - -eph, -exclude-private-hosts skip any hosts which have a private ip address + -e, -exclude string[] exclude host matching specified filter ('cdn', 'private-ips', cidr, ip, regex) -retries int number of retries -timeout int timeout in seconds (default 10) -delay value duration between each http request (eg: 200ms, 1s) (default -1ns) @@ -239,348 +242,9 @@ OPTIMIZATIONS: -rstr, -response-size-to-read int max response size to read in bytes (default 2147483647) ``` -# Running httpX - -### URL Probe - -This will run the tool against all the hosts and subdomains in `hosts.txt` and returns URLs running HTTP webserver. - -```console -cat hosts.txt | httpx - - __ __ __ _ __ - / /_ / /_/ /_____ | |/ / - / __ \/ __/ __/ __ \| / - / / / / /_/ /_/ /_/ / | -/_/ /_/\__/\__/ .___/_/|_| v1.1.1 - /_/ - - projectdiscovery.io - -[WRN] Use with caution. You are responsible for your actions -[WRN] Developers assume no liability and are not responsible for any misuse or damage. - -https://mta-sts.managed.hackerone.com -https://mta-sts.hackerone.com -https://mta-sts.forwarding.hackerone.com -https://docs.hackerone.com -https://www.hackerone.com -https://resources.hackerone.com -https://api.hackerone.com -https://support.hackerone.com -``` - -### File Input - -This will run the tool with the `-probe` flag against all the hosts in **hosts.txt** and return URLs with probed status. - -```console -httpx -list hosts.txt -silent -probe - -http://ns.hackerone.com [FAILED] -https://docs.hackerone.com [SUCCESS] -https://mta-sts.hackerone.com [SUCCESS] -https://mta-sts.managed.hackerone.com [SUCCESS] -http://email.hackerone.com [FAILED] -https://mta-sts.forwarding.hackerone.com [SUCCESS] -http://links.hackerone.com [FAILED] -https://api.hackerone.com [SUCCESS] -https://www.hackerone.com [SUCCESS] -http://events.hackerone.com [FAILED] -https://support.hackerone.com [SUCCESS] -https://gslink.hackerone.com [SUCCESS] -http://o1.email.hackerone.com [FAILED] -http://info.hackerone.com [FAILED] -https://resources.hackerone.com [SUCCESS] -http://o2.email.hackerone.com [FAILED] -http://o3.email.hackerone.com [FAILED] -http://go.hackerone.com [FAILED] -http://a.ns.hackerone.com [FAILED] -http://b.ns.hackerone.com [FAILED] -``` - -### CIDR Input - -```console -echo 173.0.84.0/24 | httpx -silent - -https://173.0.84.29 -https://173.0.84.43 -https://173.0.84.31 -https://173.0.84.44 -https://173.0.84.12 -https://173.0.84.4 -https://173.0.84.36 -https://173.0.84.45 -https://173.0.84.14 -https://173.0.84.25 -https://173.0.84.46 -https://173.0.84.24 -https://173.0.84.32 -https://173.0.84.9 -https://173.0.84.13 -https://173.0.84.6 -https://173.0.84.16 -https://173.0.84.34 -``` -### AS Number Input -```console -echo AS14421 | httpx -silent - -https://216.101.17.248 -https://216.101.17.249 -https://216.101.17.250 -https://216.101.17.251 -https://216.101.17.252 -``` - -### Tool Chain - - -```console -subfinder -d hackerone.com -silent| httpx -title -tech-detect -status-code - - __ __ __ _ __ - / /_ / /_/ /_____ | |/ / - / __ \/ __/ __/ __ \| / - / / / / /_/ /_/ /_/ / | -/_/ /_/\__/\__/ .___/_/|_| - /_/ v1.1.1 - - projectdiscovery.io - -Use with caution. You are responsible for your actions -Developers assume no liability and are not responsible for any misuse or damage. -https://mta-sts.managed.hackerone.com [404] [Page not found · GitHub Pages] [Varnish,GitHub Pages,Ruby on Rails] -https://mta-sts.hackerone.com [404] [Page not found · GitHub Pages] [Varnish,GitHub Pages,Ruby on Rails] -https://mta-sts.forwarding.hackerone.com [404] [Page not found · GitHub Pages] [GitHub Pages,Ruby on Rails,Varnish] -https://docs.hackerone.com [200] [HackerOne Platform Documentation] [Ruby on Rails,jsDelivr,Gatsby,React,webpack,Varnish,GitHub Pages] -https://support.hackerone.com [301,302,301,200] [HackerOne] [Cloudflare,Ruby on Rails,Ruby] -https://resources.hackerone.com [301,301,404] [Sorry, no Folders found.] -``` - -### Error Page Classifier and Filtering - -The Error Page Classifier and Filtering feature aims to add intelligence to the tool by enabling it to classify and filter out common error pages returned by web applications. It is an enhancement to the existing httpx capabilities and is geared towards reducing the noise in the results and helping users focus on what matters most. - -```console -httpx -l urls.txt -path /v1/api -fep - - __ __ __ _ __ - / /_ / /_/ /_____ | |/ / - / __ \/ __/ __/ __ \| / - / / / / /_/ /_/ /_/ / | -/_/ /_/\__/\__/ .___/_/|_| - /_/ - - projectdiscovery.io - -[INF] Current httpx version v1.3.3 (latest) -https://scanme.sh/v1/api -``` - -Filtered error pages are stored to predefined file `filtered_error_page.json` in jsonline format when `-filter-error-page` option is used. - -### Favicon Hash - - -```console -subfinder -d hackerone.com -silent | httpx -favicon - - __ __ __ _ __ - / /_ / /_/ /_____ | |/ / - / __ \/ __/ __/ __ \| / - / / / / /_/ /_/ /_/ / | -/_/ /_/\__/\__/ .___/_/|_| - /_/ v1.1.5 - - projectdiscovery.io - -Use with caution. You are responsible for your actions. -Developers assume no liability and are not responsible for any misuse or damage. -https://docs.hackerone.com/favicon.ico [595148549] -https://hackerone.com/favicon.ico [595148549] -https://mta-sts.managed.hackerone.com/favicon.ico [-1700323260] -https://mta-sts.forwarding.hackerone.com/favicon.ico [-1700323260] -https://support.hackerone.com/favicon.ico [-1279294674] -https://gslink.hackerone.com/favicon.ico [1506877856] -https://resources.hackerone.com/favicon.ico [-1840324437] -https://api.hackerone.com/favicon.ico [566218143] -https://mta-sts.hackerone.com/favicon.ico [-1700323260] -https://www.hackerone.com/favicon.ico [778073381] -``` - -### [JARM Fingerprint](https://github.com/salesforce/jarm) - - -```console -subfinder -d hackerone.com -silent | httpx -jarm - __ __ __ _ __ - / /_ / /_/ /_____ | |/ / - / __ \/ __/ __/ __ \| / - / / / / /_/ /_/ /_/ / | -/_/ /_/\__/\__/ .___/_/|_| - /_/ v1.2.1 - - projectdiscovery.io - -Use with caution. You are responsible for your actions. -Developers assume no liability and are not responsible for any misuse or damage. -https://www.hackerone.com [29d3dd00029d29d00042d43d00041d5de67cc9954cc85372523050f20b5007] -https://mta-sts.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af] -https://mta-sts.managed.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af] -https://docs.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af] -https://support.hackerone.com [29d3dd00029d29d00029d3dd29d29d5a74e95248e58a6162e37847a24849f7] -https://api.hackerone.com [29d3dd00029d29d00042d43d00041d5de67cc9954cc85372523050f20b5007] -https://mta-sts.forwarding.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af] -https://resources.hackerone.com [2ad2ad0002ad2ad0002ad2ad2ad2ad043bfbd87c13813505a1b60adf4f6ff5] -``` - -### ASN Fingerprint - - -```console -subfinder -d hackerone.com -silent | httpx -asn - __ __ __ _ __ - / /_ / /_/ /_____ | |/ / - / __ \/ __/ __/ __ \| / - / / / / /_/ /_/ /_/ / | -/_/ /_/\__/\__/ .___/_/|_| - /_/ v1.2.1 - - projectdiscovery.io - -Use with caution. You are responsible for your actions. -Developers assume no liability and are not responsible for any misuse or damage. -https://mta-sts.managed.hackerone.com [AS54113, FASTLY, US] -https://gslink.hackerone.com [AS16509, AMAZON-02, US] -https://www.hackerone.com [AS13335, CLOUDFLARENET, US] -https://mta-sts.forwarding.hackerone.com [AS54113, FASTLY, US] -https://resources.hackerone.com [AS16509, AMAZON-02, US] -https://support.hackerone.com [AS13335, CLOUDFLARENET, US] -https://mta-sts.hackerone.com [AS54113, FASTLY, US] -https://docs.hackerone.com [AS54113, FASTLY, US] -https://api.hackerone.com [AS13335, CLOUDFLARENET, US] -``` - - -### File/Path Bruteforce - - -```console -httpx -l urls.txt -path /v1/api -sc - - __ __ __ _ __ - / /_ / /_/ /_____ | |/ / - / __ \/ __/ __/ __ \| / - / / / / /_/ /_/ /_/ / | -/_/ /_/\__/\__/ .___/_/|_| - /_/ v1.1.5 - - projectdiscovery.io - -Use with caution. You are responsible for your actions. -Developers assume no liability and are not responsible for any misuse or damage. -https://mta-sts.managed.hackerone.com/v1/api [404] -https://mta-sts.hackerone.com/v1/api [404] -https://mta-sts.forwarding.hackerone.com/v1/api [404] -https://docs.hackerone.com/v1/api [404] -https://api.hackerone.com/v1/api [401] -https://hackerone.com/v1/api [302] -https://support.hackerone.com/v1/api [404] -https://resources.hackerone.com/v1/api [301] -https://gslink.hackerone.com/v1/api [404] -http://www.hackerone.com/v1/api [301] -``` - -### Docker Run - -```console -cat sub_domains.txt | docker run -i projectdiscovery/httpx - - __ __ __ _ __ - / /_ / /_/ /_____ | |/ / - / __ \/ __/ __/ __ \| / - / / / / /_/ /_/ /_/ / | -/_/ /_/\__/\__/ .___/_/|_| - /_/ v1.1.2 - - projectdiscovery.io - -Use with caution. You are responsible for your actions -Developers assume no liability and are not responsible for any misuse or damage. -https://mta-sts.forwarding.hackerone.com -https://mta-sts.hackerone.com -https://mta-sts.managed.hackerone.com -https://www.hackerone.com -https://api.hackerone.com -https://gslink.hackerone.com -https://resources.hackerone.com -https://docs.hackerone.com -https://support.hackerone.com -``` - -### Screenshot - -Latest addition to the project, the addition of the `-screenshot` option in httpx, a powerful new feature that allows users to take screenshots of target URLs, pages, or endpoints along with the rendered DOM. This functionality enables the **visual content discovery process**, providing a comprehensive view of the target's visual appearance. - -Rendered DOM body is also included in json line output when `-screenshot` option is used with `-json` option. - -#### 🚩 Usage - -To use the screenshot feature, simply add the `-screenshot` flag to your httpx command: - -```console -httpx -screenshot -u https://example.com -``` - -🎯 Domain, Subdomain, and Path Support -The `-screenshot` option is versatile and can be used to capture screenshots for domains, subdomains, and even specific paths when used in conjunction with the `-path` option: - -```console -httpx -screenshot -u example.com -httpx -screenshot -u https://example.com/login -httpx -screenshot -path fuzz_path.txt -u https://example.com -``` - -Using with other tools: - -```console -subfinder -d example.com | httpx -screenshot -``` - -#### 🌐 System Chrome - -By default, httpx will use the go-rod library to install and manage Chrome for taking screenshots. However, if you prefer to use your locally installed system Chrome, add the `-system-chrome` flag: - -```console -httpx -screenshot -system-chrome -u https://example.com -``` - -#### 📁 Output Directory - -Screenshots are stored in the output/screenshot directory by default. To specify a custom output directory, use the `-srd` option: - -```console -httpx -screenshot -srd /path/to/custom/directory -u https://example.com -``` - -### Body Preview -Body preview shows first N characters of response. And strip html tags in response. - -```console -httpx -u https://example.com -silent -body-preview -https://example.com [Example Domain This domain is for use in illustrative examples in documents. You may use this domai] -``` - -```console -httpx -u https://example.com -silent -body-preview=200 -strip=html -https://example.com [Example Domain This domain is for use in illustrative examples in documents. You may use this domain in literature without prior coordination or asking for permission. More information...] -``` - -#### ⏳ Performance Considerations +# Running httpx -Please note that since screenshots are captured using a headless browser, httpx runs will be slower when using the `-screenshot` option. +For details about running httpx, see https://docs.projectdiscovery.io/tools/httpx/running. ### Using `httpx` as a library `httpx` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Once validated, the struct should be passed to a runner instance (to be closed at the end of the program) and the `RunEnumeration` method should be called. A minimal example of how to do it is in the [examples](examples/) folder diff --git a/cmd/functional-test/testcases.txt b/cmd/functional-test/testcases.txt index 8efc8e3c..34a53b4c 100644 --- a/cmd/functional-test/testcases.txt +++ b/cmd/functional-test/testcases.txt @@ -13,7 +13,7 @@ scanme.sh {{binary}} -silent -tls-grab scanme.sh {{binary}} -silent -unsafe scanme.sh {{binary}} -silent -x all scanme.sh {{binary}} -silent -body 'a=b' -scanme.sh {{binary}} -silent -exclude-cdn +scanme.sh {{binary}} -silent -exclude cdn scanme.sh {{binary}} -silent -ports https:443 scanme.sh {{binary}} -silent -ztls scanme.sh {{binary}} -silent -jarm diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 46a9f51a..5d723fba 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -43,7 +43,13 @@ type HTTPX struct { func New(options *Options) (*HTTPX, error) { httpx := &HTTPX{} fastdialerOpts := fastdialer.DefaultOptions - fastdialerOpts.EnableFallback = true + + // if the user specified any custom resolver disables system resolvers and syscall lookup fallback + if len(options.Resolvers) > 0 { + fastdialerOpts.ResolversFile = false + fastdialerOpts.EnableFallback = false + } + fastdialerOpts.Deny = options.Deny fastdialerOpts.Allow = options.Allow fastdialerOpts.WithDialerHistory = true @@ -105,8 +111,8 @@ func New(options *Options) (*HTTPX, error) { httpx.setCustomCookies(redirectedRequest) // Check if we get a redirect to a different host - var newHost = redirectedRequest.URL.Host - var oldHost = previousRequests[0].Host + var newHost = redirectedRequest.URL.Hostname() + var oldHost = previousRequests[0].URL.Hostname() if oldHost == "" { oldHost = previousRequests[0].URL.Host } diff --git a/common/httpx/option.go b/common/httpx/option.go index 5e3a98b5..f1de106d 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -8,14 +8,13 @@ import ( // Options contains configuration options for the client type Options struct { - RandomAgent bool - DefaultUserAgent string - HTTPProxy string - SocksProxy string - Threads int - CdnCheck bool - ExcludeCdn bool - ExcludePrivateHosts bool + RandomAgent bool + DefaultUserAgent string + HTTPProxy string + SocksProxy string + Threads int + CdnCheck bool + ExcludeCdn bool // Timeout is the maximum time to wait for the request Timeout time.Duration // RetryMax is the maximum number of retries @@ -49,15 +48,14 @@ type Options struct { // DefaultOptions contains the default options var DefaultOptions = Options{ - RandomAgent: true, - Threads: 25, - Timeout: 30 * time.Second, - RetryMax: 5, - MaxRedirects: 10, - Unsafe: false, - CdnCheck: true, - ExcludeCdn: false, - ExcludePrivateHosts: false, + RandomAgent: true, + Threads: 25, + Timeout: 30 * time.Second, + RetryMax: 5, + MaxRedirects: 10, + Unsafe: false, + CdnCheck: true, + ExcludeCdn: false, // VHOSTs options VHostIgnoreStatusCode: false, VHostIgnoreContentLength: true, diff --git a/common/httpx/pipeline.go b/common/httpx/pipeline.go index 84f508b4..f4130ef3 100644 --- a/common/httpx/pipeline.go +++ b/common/httpx/pipeline.go @@ -18,7 +18,7 @@ func (h *HTTPX) SupportPipeline(protocol, method, host string, port int) bool { port = 443 } } - if port > 0 { + if _, _, err := net.SplitHostPort(host); err != nil && port > 0 { addr = fmt.Sprintf("%s:%d", host, port) } // dummy method while awaiting for full rawhttp implementation diff --git a/go.mod b/go.mod index 9457b606..27e48b2f 100644 --- a/go.mod +++ b/go.mod @@ -13,21 +13,21 @@ require ( github.com/miekg/dns v1.1.56 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/cdncheck v1.0.9 - github.com/projectdiscovery/clistats v0.0.19 + github.com/projectdiscovery/clistats v0.0.20 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 - github.com/projectdiscovery/goflags v0.1.26 - github.com/projectdiscovery/gologger v1.1.11 - github.com/projectdiscovery/hmap v0.0.24 - github.com/projectdiscovery/mapcidr v1.1.15 - github.com/projectdiscovery/rawhttp v0.1.21 - github.com/projectdiscovery/retryablehttp-go v1.0.31 + github.com/projectdiscovery/goflags v0.1.35 + github.com/projectdiscovery/gologger v1.1.12 + github.com/projectdiscovery/hmap v0.0.34 + github.com/projectdiscovery/mapcidr v1.1.16 + github.com/projectdiscovery/rawhttp v0.1.31 + github.com/projectdiscovery/retryablehttp-go v1.0.43 github.com/projectdiscovery/wappalyzergo v0.0.109 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/net v0.17.0 - golang.org/x/sys v0.15.0 // indirect + golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 ) @@ -44,11 +44,12 @@ require ( github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 github.com/mitchellh/mapstructure v1.5.0 github.com/projectdiscovery/asnmap v1.0.6 - github.com/projectdiscovery/dsl v0.0.29 - github.com/projectdiscovery/fastdialer v0.0.39 - github.com/projectdiscovery/ratelimit v0.0.14 + github.com/projectdiscovery/dsl v0.0.37 + github.com/projectdiscovery/fastdialer v0.0.51 + github.com/projectdiscovery/networkpolicy v0.0.7 + github.com/projectdiscovery/ratelimit v0.0.23 github.com/projectdiscovery/tlsx v1.1.5 - github.com/projectdiscovery/utils v0.0.62 + github.com/projectdiscovery/utils v0.0.73 github.com/stretchr/testify v1.8.4 github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 go.uber.org/multierr v1.11.0 @@ -62,7 +63,7 @@ require ( github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/alecthomas/chroma v0.10.0 // indirect - github.com/andybalholm/brotli v1.0.5 // indirect + github.com/andybalholm/brotli v1.0.6 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect @@ -76,7 +77,7 @@ require ( github.com/denisbrodbeck/machineid v1.0.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dlclark/regexp2 v1.8.1 // indirect - github.com/dsnet/compress v0.0.1 // indirect + github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect github.com/fatih/color v1.15.0 // indirect github.com/gaukas/godicttls v0.0.4 // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -88,12 +89,13 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/kataras/jwt v0.1.8 // indirect github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/pgzip v1.2.5 // indirect github.com/kljensen/snowball v0.8.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect - github.com/mholt/archiver v3.1.1+incompatible // indirect + github.com/mholt/archiver/v3 v3.5.1 // indirect github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -102,15 +104,14 @@ require ( github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect github.com/nwaples/rardecode v1.1.3 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/pierrec/lz4 v2.6.1+incompatible // indirect + github.com/pierrec/lz4/v4 v4.1.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect github.com/projectdiscovery/freeport v0.0.5 // indirect - github.com/projectdiscovery/gostruct v0.0.1 // indirect - github.com/projectdiscovery/networkpolicy v0.0.6 // indirect - github.com/projectdiscovery/retryabledns v1.0.41 // indirect + github.com/projectdiscovery/gostruct v0.0.2 // indirect + github.com/projectdiscovery/retryabledns v1.0.51 // indirect github.com/quic-go/quic-go v0.37.7 // indirect - github.com/refraction-networking/utls v1.5.3 // indirect + github.com/refraction-networking/utls v1.5.4 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/sashabaranov/go-openai v1.14.2 // indirect diff --git a/go.sum b/go.sum index 108fb51f..8735d50f 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,9 @@ github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= -github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= -github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= +github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= @@ -57,13 +58,11 @@ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/ github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.8.1 h1:6Lcdwya6GjPUNsBct8Lg/yRPwMhABj269AAzdGSiR+0= github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= -github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= +github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= -github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= @@ -86,6 +85,7 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/certificate-transparency-go v1.1.4 h1:hCyXHDbtqlr/lMXU0D4WgbalXL0Zk4dSWWMbPV8VrqY= @@ -124,9 +124,12 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/kataras/jwt v0.1.8 h1:u71baOsYD22HWeSOg32tCHbczPjdCk7V4MMeJqTtmGk= github.com/kataras/jwt v0.1.8/go.mod h1:Q5j2IkcIHnfwy+oNY3TVWuEBJNw0ADgCcXK9CaZwV4o= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= +github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kljensen/snowball v0.8.0 h1:WU4cExxK6sNW33AiGdbn4e8RvloHrhkAssu2mVJ11kg= github.com/kljensen/snowball v0.8.0/go.mod h1:OGo5gFWjaeXqCu4iIrMl5OYip9XUJHGOU5eSkPjVg2A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -152,8 +155,8 @@ github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWV github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= -github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= -github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= +github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= +github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= github.com/microcosm-cc/bluemonday v1.0.25 h1:4NEwSfiJ+Wva0VxN5B8OwMicaJvD8r9tlJWm9rtloEg= github.com/microcosm-cc/bluemonday v1.0.25/go.mod h1:ZIOjCQp1OrzBBPIJmfX4qDYFuhU02nx4bn030ixfHLE= @@ -178,6 +181,7 @@ github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc= github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= @@ -194,8 +198,8 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= -github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.2 h1:qvY3YFXRQE/XB8MlLzJH7mSzBs74eA2gg52YTk6jUPM= +github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -206,50 +210,50 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQOocUvrssFlg= github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs= -github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9ZjAnDatybhLw= -github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= -github.com/projectdiscovery/dsl v0.0.29 h1:6ne4vWL06vk0sb6lI3iTcDw2leoWa9WUkBCE+h9o2ao= -github.com/projectdiscovery/dsl v0.0.29/go.mod h1:ocF2NbK8nUQPexa5kF/1dBUG56lpsM8S/91gmqOgvmA= -github.com/projectdiscovery/fastdialer v0.0.39 h1:ng0EMzI/eT3oIYZgx1q6cCR/4dyOEdCq4/qg1YfV5GU= -github.com/projectdiscovery/fastdialer v0.0.39/go.mod h1:Bu1y41oAYPnuX09/RT42bgLZRomD8yfkHnbeRAu+zc8= +github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4= +github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= +github.com/projectdiscovery/dsl v0.0.37 h1:8DRYzYxn/nQJUqSnZ4ATUcV95X8g9uBAb8/DvWNHruM= +github.com/projectdiscovery/dsl v0.0.37/go.mod h1:PzwX11Dr1Bxt9kXDxSZnhq02jdYXPv5HOb9Mz7OMFf0= +github.com/projectdiscovery/fastdialer v0.0.51 h1:LsRry/aSzUfgSCakJve05d6Ut83w1n1NcGS5tyUqsEY= +github.com/projectdiscovery/fastdialer v0.0.51/go.mod h1:OqJbaFL/a6kX7107K6OjZ3usi2MStZ7dQop73DUOUJU= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQSS18+0QX/s= github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw= -github.com/projectdiscovery/goflags v0.1.26 h1:o3bFCN5CIYJoLUl/uyKHjXZKkITaBrGPVJ8hg2KErKE= -github.com/projectdiscovery/goflags v0.1.26/go.mod h1:SgLaQOTHspoYYCyhJEmWQZxbeAFGUfMKyultrRsAFYo= -github.com/projectdiscovery/gologger v1.1.11 h1:8vsz9oJlDT9euw6xlj7F7dZ6RWItVIqVwn4Mr6uzky8= -github.com/projectdiscovery/gologger v1.1.11/go.mod h1:UR2bgXl7zraOxYGnUwuO917hifWrwMJ0feKnVqMQkzY= -github.com/projectdiscovery/gostruct v0.0.1 h1:1KvR6Pn4mDbQqoLEQzhRfHpbreLno2R9xqRCCt5tgmU= -github.com/projectdiscovery/gostruct v0.0.1/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.24 h1:aPyHltkfOUmVu9LcT2WguHf0nV5IbGlyo8fjbU64YEk= -github.com/projectdiscovery/hmap v0.0.24/go.mod h1:NDjO7SdmdCUznX43xUvlIINOFX4B/t6yCiUGGGXBuHk= -github.com/projectdiscovery/mapcidr v1.1.15 h1:rYAgxLvMyxPU0JunE/Y3uSK1n/TcNJHK839d6YM0ms4= -github.com/projectdiscovery/mapcidr v1.1.15/go.mod h1:s9erRsoZqWcLGhJW+WT1SnbscqzhHRRnSX916xBw5ZM= -github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzVczM94W5nHiOy/1o= -github.com/projectdiscovery/networkpolicy v0.0.6/go.mod h1:8HJQ/33Pi7v3a3MRWIQGXzpj+zHw2d60TysEL4qdoQk= -github.com/projectdiscovery/ratelimit v0.0.14 h1:kzDU/nupb6yHME5qSJ4bSVb34Hixg2tDP78C2mA+kxY= -github.com/projectdiscovery/ratelimit v0.0.14/go.mod h1:MLQg9jXu4yfxtwjP7pCZWOst15M9hoOphU79i+7fh4c= -github.com/projectdiscovery/rawhttp v0.1.21 h1:fLK8+xmX70WG0dHKOFzGnA/z5asbpQZf2t93zkK1x9k= -github.com/projectdiscovery/rawhttp v0.1.21/go.mod h1:hbv4i9BMuOtoJWpA4AO1CBB5jQP7GNF9dlqd3g6/NYs= -github.com/projectdiscovery/retryabledns v1.0.41 h1:08EKCoKy+Vusxa84ZMN/KLF4CvwfHB+GYbZUWD7tJo0= -github.com/projectdiscovery/retryabledns v1.0.41/go.mod h1:AnMPvBks/ZJLcD9wIPMJ/Mu2eEv0mmShiz7oCTwuUns= -github.com/projectdiscovery/retryablehttp-go v1.0.31 h1:Z66QM4FkAyIEBje4acu3hdUPaE3NSFYNa54hR69ZlPo= -github.com/projectdiscovery/retryablehttp-go v1.0.31/go.mod h1:pFBFbxnb7fupJbl99n9M0GkoUZCtbkRZM3xfmTSWjqE= +github.com/projectdiscovery/goflags v0.1.35 h1:tSxKmJci+aoy0Eu8wgGqD5Kg8lee7Et0LI5uq51473M= +github.com/projectdiscovery/goflags v0.1.35/go.mod h1:MfK1JFmW5c4nMtcWxG6HeJCIWJVKCrVTkecuwyZyoA0= +github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= +github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= +github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= +github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= +github.com/projectdiscovery/hmap v0.0.34 h1:Xz/CjuHVw/QCrupScMFkGN0YEIkz+jkAwqbHJ9A0C9I= +github.com/projectdiscovery/hmap v0.0.34/go.mod h1:FQy913/mK7Tc2BoTpV2kZJMMuZp5FHcclGGMWtK8pBM= +github.com/projectdiscovery/mapcidr v1.1.16 h1:rjj1w5D6hbTsUQXYClLcGdfBEy9bryclgi70t0vBggo= +github.com/projectdiscovery/mapcidr v1.1.16/go.mod h1:rGqpBhStdwOQ2uS62QM9qPsybwMwIhT7CTd2bxoHs8Q= +github.com/projectdiscovery/networkpolicy v0.0.7 h1:AwHqBRXBqDQgnWzBMuoJtHBNEYBw+NFp/4qIK688x7o= +github.com/projectdiscovery/networkpolicy v0.0.7/go.mod h1:CK0CnFoLF1Nou6mY7P4WODSAxhPN8g8g7XpapgEP8tI= +github.com/projectdiscovery/ratelimit v0.0.23 h1:Fz2A57UW6GK0L0huOGVXd97EhASrJV41SC1NrGImShU= +github.com/projectdiscovery/ratelimit v0.0.23/go.mod h1:042iuvdggjUnsgAIzyxM3iLFveMaXnGTRwlCpfd03I0= +github.com/projectdiscovery/rawhttp v0.1.31 h1:ry04GKDuS8bH8UNzWdXV1uB3+5PHdzpdiM8zSwJPSMk= +github.com/projectdiscovery/rawhttp v0.1.31/go.mod h1:cjqO+O62/9MIxXLctnLNq7muOF7MrDrpWniBLo2m6tg= +github.com/projectdiscovery/retryabledns v1.0.51 h1:bX/apiRGZwhASBAT7o3qmZ0FznuBlHQlIQdCw1TAzcg= +github.com/projectdiscovery/retryabledns v1.0.51/go.mod h1:rFu1zc7HLHPEipuF91ZNMT1yGG0FKBVUnxnqLJ4OhF4= +github.com/projectdiscovery/retryablehttp-go v1.0.43 h1:Ll3QCNmVW4fRD9U0vBnTLy57DTwBKsqrPkZ8yK1zPDs= +github.com/projectdiscovery/retryablehttp-go v1.0.43/go.mod h1:3z8cxmhwXyVrvHGqOFPSNL6Uu680kEQSeiZfUlP6RrU= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v1.1.5 h1:S8KV2ckcjW3hDBa/REmDdsZfHwYJ9eKoZ7rtgETkwkM= github.com/projectdiscovery/tlsx v1.1.5/go.mod h1:0a0TdWb3fYeVpuPsJuf5AGtwZIKwkY0kxdO9lojU6S4= -github.com/projectdiscovery/utils v0.0.62 h1:1+eivihfFVGZkbQdWjQqkljg3ZYm1btTugYK55JZ44A= -github.com/projectdiscovery/utils v0.0.62/go.mod h1:vt4oY4rvRWTdkBMhLlAGPbapa/R8pa+xZBYuNZIKJgQ= +github.com/projectdiscovery/utils v0.0.73 h1:KWzxzJv9U5YKHGGOvkKHJmO7NdV5Kbzc8lPt+Frdj0o= +github.com/projectdiscovery/utils v0.0.73/go.mod h1:SEb3ZoGy1nxdnPNXAGhMZNhRcokRkoMEjC6l9H59t1s= github.com/projectdiscovery/wappalyzergo v0.0.109 h1:BERfwTRn1dvB1tbhyc5m67R8VkC9zbVuPsEq4VEm07k= github.com/projectdiscovery/wappalyzergo v0.0.109/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= github.com/quic-go/quic-go v0.37.7 h1:AgKsQLZ1+YCwZd2GYhBUsJDYZwEkA5gENtAjb+MxONU= github.com/quic-go/quic-go v0.37.7/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU= -github.com/refraction-networking/utls v1.5.3 h1:Ds5Ocg1+MC1ahNx5iBEcHe0jHeLaA/fLey61EENm7ro= -github.com/refraction-networking/utls v1.5.3/go.mod h1:SPuDbBmgLGp8s+HLNc83FuavwZCFoMmExj+ltUHiHUw= +github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= +github.com/refraction-networking/utls v1.5.4/go.mod h1:SPuDbBmgLGp8s+HLNc83FuavwZCFoMmExj+ltUHiHUw= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -307,7 +311,8 @@ github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= -github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA= @@ -429,8 +434,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/runner/banner.go b/runner/banner.go index 6cd89e63..80915ceb 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -15,7 +15,7 @@ const banner = ` ` // Version is the current version of httpx -const version = `v1.3.7` +const version = `v1.3.8` // showBanner is used to show the banner to the user func showBanner() { diff --git a/runner/headless.go b/runner/headless.go index 40b81eb6..fb76e344 100644 --- a/runner/headless.go +++ b/runner/headless.go @@ -7,6 +7,7 @@ import ( "github.com/go-rod/rod" "github.com/go-rod/rod/lib/launcher" + "github.com/go-rod/rod/lib/launcher/flags" "github.com/go-rod/rod/lib/proto" "github.com/pkg/errors" fileutil "github.com/projectdiscovery/utils/file" @@ -29,7 +30,7 @@ type Browser struct { // pids map[int32]struct{} } -func NewBrowser(proxy string, useLocal bool) (*Browser, error) { +func NewBrowser(proxy string, useLocal bool, optionalArgs map[string]string) (*Browser, error) { dataStore, err := os.MkdirTemp("", "nuclei-*") if err != nil { return nil, errors.Wrap(err, "could not create temporary directory") @@ -74,6 +75,11 @@ func NewBrowser(proxy string, useLocal bool) (*Browser, error) { if proxy != "" { chromeLauncher = chromeLauncher.Proxy(proxy) } + + for k, v := range optionalArgs { + chromeLauncher.Set(flags.Flag(k), v) + } + launcherURL, err := chromeLauncher.Launch() if err != nil { return nil, err diff --git a/runner/options.go b/runner/options.go index 36b9c43f..f3044146 100644 --- a/runner/options.go +++ b/runner/options.go @@ -78,7 +78,6 @@ type ScanOptions struct { OutputExtractRegex string extractRegexps map[string]*regexp.Regexp ExcludeCDN bool - ExcludePrivateHosts bool HostMaxErrors int ProbeAllIPS bool Favicon bool @@ -91,6 +90,7 @@ type ScanOptions struct { DisableStdin bool NoScreenshotBytes bool NoHeadlessBody bool + ScreenshotTimeout int } func (s *ScanOptions) Clone() *ScanOptions { @@ -142,6 +142,7 @@ func (s *ScanOptions) Clone() *ScanOptions { UseInstalledChrome: s.UseInstalledChrome, NoScreenshotBytes: s.NoScreenshotBytes, NoHeadlessBody: s.NoHeadlessBody, + ScreenshotTimeout: s.ScreenshotTimeout, } } @@ -243,8 +244,7 @@ type Options struct { Probe bool Resume bool resumeCfg *ResumeCfg - ExcludeCDN bool - ExcludePrivateHosts bool + Exclude goflags.StringSlice HostMaxErrors int Stream bool SkipDedupe bool @@ -289,6 +289,9 @@ type Options struct { DisableStdin bool NoScreenshotBytes bool NoHeadlessBody bool + ScreenshotTimeout int + // HeadlessOptionalArguments specifies optional arguments to pass to Chrome + HeadlessOptionalArguments goflags.StringSlice } // ParseOptions parses the command line options for application @@ -332,8 +335,10 @@ func ParseOptions() *Options { flagSet.CreateGroup("headless", "Headless", flagSet.BoolVarP(&options.Screenshot, "screenshot", "ss", false, "enable saving screenshot of the page using headless browser"), flagSet.BoolVar(&options.UseInstalledChrome, "system-chrome", false, "enable using local installed chrome for screenshot"), + flagSet.StringSliceVarP(&options.HeadlessOptionalArguments, "headless-options", "ho", nil, "start headless chrome with additional options", goflags.FileCommaSeparatedStringSliceOptions), flagSet.BoolVarP(&options.NoScreenshotBytes, "exclude-screenshot-bytes", "esb", false, "enable excluding screenshot bytes from json output"), flagSet.BoolVarP(&options.NoHeadlessBody, "exclude-headless-body", "ehb", false, "enable excluding headless header from json output"), + flagSet.IntVarP(&options.ScreenshotTimeout, "screenshot-timeout", "st", 10, "set timeout for screenshot in seconds"), ) flagSet.CreateGroup("matchers", "Matchers", @@ -454,8 +459,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.NoFallback, "no-fallback", "nf", false, "display both probed protocol (HTTPS and HTTP)"), flagSet.BoolVarP(&options.NoFallbackScheme, "no-fallback-scheme", "nfs", false, "probe with protocol scheme specified in input "), flagSet.IntVarP(&options.HostMaxErrors, "max-host-error", "maxhr", 30, "max error count per host before skipping remaining path/s"), - flagSet.BoolVarP(&options.ExcludeCDN, "exclude-cdn", "ec", false, "skip full port scans for CDN/WAF (only checks for 80,443)"), - flagSet.BoolVarP(&options.ExcludePrivateHosts, "exclude-private-hosts", "eph", false, "skip any hosts which have a private ip address"), + flagSet.StringSliceVarP(&options.Exclude, "exclude", "e", nil, "exclude host matching specified filter ('cdn', 'private-ips', cidr, ip, regex)", goflags.CommaSeparatedStringSliceOptions), flagSet.IntVar(&options.Retries, "retries", 0, "number of retries"), flagSet.IntVar(&options.Timeout, "timeout", 10, "timeout in seconds"), flagSet.DurationVar(&options.Delay, "delay", -1, "duration between each http request (eg: 200ms, 1s)"), @@ -647,6 +651,32 @@ func (options *Options) ValidateOptions() error { return nil } +// redundant with katana +func (options *Options) ParseHeadlessOptionalArguments() map[string]string { + var ( + lastKey string + optionalArguments = make(map[string]string) + ) + for _, v := range options.HeadlessOptionalArguments { + if v == "" { + continue + } + if argParts := strings.SplitN(v, "=", 2); len(argParts) >= 2 { + key := strings.TrimSpace(argParts[0]) + value := strings.TrimSpace(argParts[1]) + if key != "" && value != "" { + optionalArguments[key] = value + lastKey = key + } + } else if !strings.HasPrefix(v, "--") { + optionalArguments[lastKey] += "," + v + } else { + optionalArguments[v] = "" + } + } + return optionalArguments +} + // configureOutput configures the output on the screen func (options *Options) configureOutput() { // If the user desires verbose output, show verbose output diff --git a/runner/runner.go b/runner/runner.go index 85f570c3..090a30a1 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -34,6 +34,7 @@ import ( "github.com/projectdiscovery/httpx/common/hashes/jarm" "github.com/projectdiscovery/httpx/static" "github.com/projectdiscovery/mapcidr/asn" + "github.com/projectdiscovery/networkpolicy" errorutil "github.com/projectdiscovery/utils/errors" osutil "github.com/projectdiscovery/utils/os" @@ -77,6 +78,8 @@ type Runner struct { wappalyzer *wappalyzer.Wappalyze scanopts ScanOptions hm *hybrid.HybridMap + excludePorts map[string]struct{} + excludeCdn bool stats clistats.StatisticsClient ratelimiter ratelimit.Limiter HostErrorsCache gcache.Cache[string, int] @@ -114,6 +117,28 @@ func New(options *Options) (*Runner, error) { os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt")) } + runner.excludePorts = make(map[string]struct{}) + for _, exclude := range options.Exclude { + switch { + case exclude == "cdn": + runner.excludeCdn = true + case exclude == "private-ips": + options.Deny = append(options.Deny, networkpolicy.DefaultIPv4Denylist...) + options.Deny = append(options.Deny, networkpolicy.DefaultIPv4DenylistRanges...) + options.Deny = append(options.Deny, networkpolicy.DefaultIPv6Denylist...) + options.Deny = append(options.Deny, networkpolicy.DefaultIPv6DenylistRanges...) + case iputil.IsCIDR(exclude): + options.Deny = append(options.Deny, exclude) + case asn.IsASN(exclude): + ips := expandASNInputValue(exclude) + options.Deny = append(options.Deny, ips...) + case iputil.IsPort(exclude): + runner.excludePorts[exclude] = struct{}{} + default: + options.Deny = append(options.Deny, exclude) + } + } + httpxOptions := httpx.DefaultOptions // Enables automatically tlsgrab if tlsprobe is requested httpxOptions.TLSGrab = options.TLSGrab || options.TLSProbe @@ -127,8 +152,7 @@ func New(options *Options) (*Runner, error) { httpxOptions.Unsafe = options.Unsafe httpxOptions.UnsafeURI = options.RequestURI httpxOptions.CdnCheck = options.OutputCDN - httpxOptions.ExcludeCdn = options.ExcludeCDN - httpxOptions.ExcludePrivateHosts = options.ExcludePrivateHosts + httpxOptions.ExcludeCdn = runner.excludeCdn if options.CustomHeaders.Has("User-Agent:") { httpxOptions.RandomAgent = false } else { @@ -256,7 +280,7 @@ func New(options *Options) (*Runner, error) { scanopts.MaxResponseBodySizeToRead = options.MaxResponseBodySizeToRead scanopts.extractRegexps = make(map[string]*regexp.Regexp) if options.Screenshot { - browser, err := NewBrowser(options.HTTPProxy, options.UseInstalledChrome) + browser, err := NewBrowser(options.HTTPProxy, options.UseInstalledChrome, options.ParseHeadlessOptionalArguments()) if err != nil { return nil, err } @@ -266,6 +290,7 @@ func New(options *Options) (*Runner, error) { scanopts.NoScreenshotBytes = options.NoScreenshotBytes scanopts.NoHeadlessBody = options.NoHeadlessBody scanopts.UseInstalledChrome = options.UseInstalledChrome + scanopts.ScreenshotTimeout = options.ScreenshotTimeout if options.OutputExtractRegexs != nil { for _, regex := range options.OutputExtractRegexs { @@ -293,8 +318,7 @@ func New(options *Options) (*Runner, error) { scanopts.OutputMethod = true } - scanopts.ExcludeCDN = options.ExcludeCDN - scanopts.ExcludePrivateHosts = options.ExcludePrivateHosts + scanopts.ExcludeCDN = runner.excludeCdn scanopts.HostMaxErrors = options.HostMaxErrors scanopts.ProbeAllIPS = options.ProbeAllIPS scanopts.Favicon = options.Favicon @@ -340,6 +364,24 @@ func New(options *Options) (*Runner, error) { return runner, nil } +func expandCIDRInputValue(value string) []string { + var ips []string + ipsCh, _ := mapcidr.IPAddressesAsStream(value) + for ip := range ipsCh { + ips = append(ips, ip) + } + return ips +} + +func expandASNInputValue(value string) []string { + var ips []string + cidrs, _ := asn.GetCIDRsForASNNum(value) + for _, cidr := range cidrs { + ips = append(ips, expandCIDRInputValue(cidr.String())...) + } + return ips +} + func (r *Runner) prepareInputPaths() { // most likely, the user would provide the most simplified path to an existing file isAbsoluteOrRelativePath := filepath.Clean(r.options.RequestURIs) == r.options.RequestURIs @@ -711,7 +753,11 @@ func (r *Runner) RunEnumeration() { } if r.options.StoreResponseDir != "" { var err error - indexPath := filepath.Join(r.options.StoreResponseDir, "response", "index.txt") + responseDirPath := filepath.Join(r.options.StoreResponseDir, "response") + if err := os.MkdirAll(responseDirPath, 0755); err != nil { + gologger.Fatal().Msgf("Could not create response directory '%s': %s\n", responseDirPath, err) + } + indexPath := filepath.Join(responseDirPath, "index.txt") if r.options.Resume { indexFile, err = os.OpenFile(indexPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) } else { @@ -1147,7 +1193,6 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. result := r.analyze(hp, protocol, target, method, t, scanopts) output <- result if scanopts.TLSProbe && result.TLSData != nil { - scanopts.TLSProbe = false for _, tt := range result.TLSData.SubjectAN { if !r.testAndSet(tt) { continue @@ -1200,7 +1245,6 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. result := r.analyze(hp, protocol, target, method, t, scanopts) output <- result if scanopts.TLSProbe && result.TLSData != nil { - scanopts.TLSProbe = false for _, tt := range result.TLSData.SubjectAN { if !r.testAndSet(tt) { continue @@ -1302,15 +1346,10 @@ retry: } } - if r.skipPrivateHosts(URL.Hostname()) { - gologger.Debug().Msgf("Skipping private host %s\n", URL.Host) - return Result{URL: target.Host, Input: origInput, Err: errors.New("target has a private ip and will only connect within same local network")} - } - // check if the combination host:port should be skipped if belonging to a cdn - if r.skipCDNPort(URL.Host, URL.Port()) { - gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port()) - return Result{URL: target.Host, Input: origInput, Err: errors.New("cdn target only allows ports 80 and 443")} + skip, reason := r.skip(URL, target, origInput) + if skip { + return reason } URL.Scheme = protocol @@ -1624,7 +1663,7 @@ retry: } // web socket - isWebSocket := resp.StatusCode == 101 + isWebSocket := isWebSocket(resp) if scanopts.OutputWebSocket && isWebSocket { builder.WriteString(" [websocket]") } @@ -1942,7 +1981,7 @@ retry: var pHash uint64 if scanopts.Screenshot { var err error - screenshotBytes, headlessBody, err = r.browser.ScreenshotWithBody(fullURL, r.hp.Options.Timeout) + screenshotBytes, headlessBody, err = r.browser.ScreenshotWithBody(fullURL, time.Duration(scanopts.ScreenshotTimeout)*time.Second) if err != nil { gologger.Warning().Msgf("Could not take screenshot '%s': %s", fullURL, err) } else { @@ -2027,6 +2066,20 @@ retry: return result } +func (r *Runner) skip(URL *urlutil.URL, target httpx.Target, origInput string) (bool, Result) { + if r.skipCDNPort(URL.Host, URL.Port()) { + gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port()) + return true, Result{URL: target.Host, Input: origInput, Err: errors.New("cdn target only allows ports 80 and 443")} + } + + if _, ok := r.excludePorts[URL.Port()]; ok { + gologger.Debug().Msgf("Skipping excluded port: %s:%s\n", URL.Hostname(), URL.Port()) + return true, Result{URL: target.Host, Input: origInput, Err: errors.New("port is in the exclude list")} + } + + return false, Result{} +} + func calculatePerceptionHash(screenshotBytes []byte) (uint64, error) { reader := bytes.NewReader(screenshotBytes) img, _, err := image.Decode(reader) @@ -2191,7 +2244,7 @@ func (r Result) CSVRow(scanopts *ScanOptions) string { //nolint func (r *Runner) skipCDNPort(host string, port string) bool { // if the option is not enabled we don't skip - if !r.options.ExcludeCDN { + if !r.scanopts.ExcludeCDN { return false } // uses the dealer to pre-resolve the target @@ -2214,45 +2267,13 @@ func (r *Runner) skipCDNPort(host string, port string) bool { } // If the target is part of the CDN ips range - only ports 80 and 443 are allowed - if isCdnIP && port != "80" && port != "443" { + if isCdnIP && port != "" && port != "80" && port != "443" { return true } return false } -func (r *Runner) skipPrivateHosts(host string) bool { - // if the option is not enabled we don't skip - if !r.options.ExcludePrivateHosts { - return false - } - dnsData, err := r.hp.Dialer.GetDNSData(host) - - // if we get an error the target cannot be resolved, so we return false so that the program logic continues as usual and handles the errors accordingly - if err != nil { - return false - } - if len(dnsData.A) == 0 && len(dnsData.AAAA) == 0 { - return false - } - - var ipsToCheck []string - ipsToCheck = make([]string, 0, len(dnsData.A)+len(dnsData.AAAA)) - ipsToCheck = append(ipsToCheck, dnsData.A...) - ipsToCheck = append(ipsToCheck, dnsData.AAAA...) - - for _, ipAddr := range ipsToCheck { - ip := net.ParseIP(ipAddr) - if ip == nil { - continue //skip any bad ip addresses - } - if ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() { - return true - } - } - return false -} - // parseURL parses url based on cli option(unsafe) func (r *Runner) parseURL(url string) (*urlutil.URL, error) { urlx, err := urlutil.ParseURL(url, r.options.Unsafe) @@ -2281,3 +2302,27 @@ func normalizeHeaders(headers map[string][]string) map[string]interface{} { } return normalized } + +func isWebSocket(resp *httpx.Response) bool { + if resp.StatusCode == 101 { + return true + } + // TODO: improve this checks + // Check for specific headers that indicate WebSocket support + keyHeaders := []string{`^Sec-WebSocket-Accept:\s+.+`, `^Upgrade:\s+websocket`, `^Connection:\s+upgrade`} + for _, header := range keyHeaders { + re := regexp.MustCompile(header) + if re.MatchString(resp.RawHeaders) { + return true + } + } + // Check for specific data that indicates WebSocket support + keyData := []string{`{"socket":true,"socketUrl":"(?:wss?|ws)://.*"}`, `{"sid":"[^"]*","upgrades":\["websocket"\].*}`} + for _, data := range keyData { + re := regexp.MustCompile(data) + if re.Match(resp.RawData) { + return true + } + } + return false +}