Skip to content

Commit

Permalink
require all methods to use &http.Client{} and use a User-Agent to avo…
Browse files Browse the repository at this point in the history
…id future errors with some websites blocking requests that do not have a certain User-Agent
  • Loading branch information
ibnaleem committed Nov 20, 2024
1 parent 2d7cb40 commit f41fd7f
Showing 1 changed file with 24 additions and 2 deletions.
26 changes: 24 additions & 2 deletions gosearch.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,11 +142,22 @@ func MakeRequestWithCookies(url string, cookies [] Cookie, WebsiteErrorCode int,
func MakeRequestWithoutErrorMsg(url string, WebsiteErrorCode int, wg *sync.WaitGroup) {
defer wg.Done()

res, err := http.Get(url)
client := &http.Client{}
req, err := http.NewRequest("GET", url, nil)

if err != nil {
fmt.Printf("Error creating request in function MakeRequestWithCookies: %v\n", err)
return
}

req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:125.0) Gecko/20100101 Firefox/125.0")

res, err := client.Do(req)
if err != nil {
fmt.Printf("Error making GET request to %s: %v\n", url, err)
return
}

defer res.Body.Close()

if res.StatusCode != WebsiteErrorCode {
Expand All @@ -158,11 +169,22 @@ func MakeRequestWithoutErrorMsg(url string, WebsiteErrorCode int, wg *sync.WaitG
func MakeRequestWithErrorMsg(website Website, url string, errorMsg string, username string, wg *sync.WaitGroup) {
defer wg.Done()

res, err := http.Get(url)
client := &http.Client{}
req, err := http.NewRequest("GET", url, nil)

if err != nil {
fmt.Printf("Error creating request in function MakeRequestWithCookies: %v\n", err)
return
}

req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:125.0) Gecko/20100101 Firefox/125.0")

res, err := client.Do(req)
if err != nil {
fmt.Printf("Error making GET request to %s: %v\n", url, err)
return
}

defer res.Body.Close()

body, err := io.ReadAll(res.Body)
Expand Down

0 comments on commit f41fd7f

Please sign in to comment.