web-cache-vulnerability-scanner.go

package main

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/Hackmanit/Web-Cache-Vulnerability-Scanner/v2/pkg"
	"golang.org/x/net/http2"
)

const version = "1.0.0"

var (
	currentDate      string
	report           pkg.Report
	completedFile    *os.File
	proxyURL         *url.URL
	headerList       []string
	parameterList    []string
	noTestPreference bool
	excluded         map[string]bool
	added            map[string]bool
)

func main() {
	//pkg.ReadConfigFile()
	pkg.ParseFlags(version)
	pkg.Statistics = map[string]int{}
	/*****************************/

	/**** SET EXPORT STRUCT ****/
	report.Name = "Web_Cache_Vulnerability_Scanner"
	report.Version = version

	report.Config = &pkg.Config

	currentDate = time.Now().Format("2006-01-02_15-04-05")
	report.Date = currentDate
	report.Duration = "Not finished yet"
	/***************************/
	//Create generatePath directory
	if pkg.Config.GeneratePath != "./" {
		if !strings.HasSuffix(pkg.Config.GeneratePath, "/") {
			pkg.Config.GeneratePath += "/"
		}
		if _, err := os.Stat(pkg.Config.GeneratePath); err != nil {
			if os.IsNotExist(err) {
				err := os.Mkdir(pkg.Config.GeneratePath, 0755)
				if err != nil {
					pkg.PrintFatal(err.Error())
				}
			}
		}
	}

	// Check pkg.Config if Log shall be printed or no logs shall be made
	/* Setting Logoutput to Log file and stdout */
	f, err := os.OpenFile(pkg.Config.GeneratePath+currentDate+"_WCVS_Log", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		msg := "Log: " + err.Error() + "\n"
		pkg.PrintFatal(msg)
	}
	defer f.Close()
	//wrt := io.MultiWriter(f, os.Stdout)
	//log.SetOutput(wrt)
	log.SetOutput(f)

	report.Command = fmt.Sprint(os.Args)
	report.Command = strings.TrimPrefix(report.Command, "[")
	report.Command = strings.TrimSuffix(report.Command, "]")
	pkg.PrintVerbose(report.Command, pkg.NoColor, 2)
	/******************************************/
	if pkg.Config.Verbosity < 0 || pkg.Config.Verbosity > 2 {
		msg := fmt.Sprintf("%d is not a valid verbosity between 0 and 2!\n", pkg.Config.Verbosity)
		pkg.PrintFatal(msg)
	}

	msg := fmt.Sprintf("WCVS v%s started at %s\n", version, currentDate)
	pkg.PrintVerbose(msg, pkg.NoColor, 1)

	start := time.Now()
	// Making the random generator really random
	rand.Seed(time.Now().UnixNano())

	noTestPreference = true
	if pkg.Config.OnlyTest != "" && pkg.Config.SkipTest != "" {
		msg = "You can't set both doTest and dontTest\n"
		pkg.PrintFatal(msg)
	} else if pkg.Config.OnlyTest != "" {
		noTestPreference = false
	} else if pkg.Config.SkipTest != "" {
		noTestPreference = false
	}

	if pkg.Config.GenerateReport {
		pkg.GenerateReport(report, currentDate)
	}
	if pkg.Config.GenerateCompleted {
		completedFile = createCompletedURLs()
	}
	/***************************/

	/* Setting up proxy (e.g. burp), if wanted */
	proxyURL = setProxy()
	/*******************************************/

	// Reading header wordlist, only if it is needed
	if noTestPreference || strings.Contains(pkg.Config.OnlyTest, "header") || (pkg.Config.SkipTest != "" && !strings.Contains(pkg.Config.SkipTest, "header")) {
		headerList = pkg.ReadLocalFile(pkg.Config.HeaderWordlist)
	}

	// Reading parameter wordlist, only if it is needed
	if noTestPreference || strings.Contains(pkg.Config.OnlyTest, "parameter") || (pkg.Config.SkipTest != "" && !strings.Contains(pkg.Config.SkipTest, "parameter")) {
		parameterList = pkg.ReadLocalFile(pkg.Config.QueryWordlist)
	}

	/*******************************************************/

	excluded = make(map[string]bool)

	for _, u := range pkg.Config.RecExclude {
		u = strings.TrimSuffix(u, "\r")
		u = strings.TrimSpace(u)

		// check if empty or is a comment
		if u == "" || strings.HasPrefix(u, "//") {
			continue
		}
		if !strings.HasPrefix(u, "http://") && !strings.HasPrefix(u, "https://") {
			msg = fmt.Sprintf("URL %s doesn't begin with http:// or https:// and gets skipped\n", u)
			pkg.Print(msg, pkg.Yellow)
			continue
		}

		excluded[u] = true
	}

	added = make(map[string]bool)

	var testUrls []string
	for _, u := range pkg.Config.Urls {
		u = strings.TrimSuffix(u, "\r")
		u = strings.TrimSpace(u)

		// check if empty or is a comment
		if u == "" || strings.HasPrefix(u, "//") {
			continue
		}
		if !strings.HasPrefix(u, "http://") && !strings.HasPrefix(u, "https://") {
			prefix := "https"
			if pkg.Config.UseHTTP {
				prefix = "http"
			}
			msg = fmt.Sprintf("URL %s gets the prefix %s\n", u, prefix)
			pkg.Print(msg, pkg.Yellow)

			u = prefix + "://" + u
		}

		added[u] = true
		testUrls = append(testUrls, u)
	}

	for i, u := range testUrls {
		var recUrls []string
		var progress string

		progress = fmt.Sprintf("(%d/%d)", i+1, len(testUrls))
		runTests(0, u, progress, &recUrls, "sub_")

		for rec := 1; rec <= pkg.Config.Recursivity; rec++ {
			var urlsToAdd []string

			for ii, uu := range recUrls {
				if ii != 0 && ii == pkg.Config.RecLimit {
					msg = "RecLimit was reached. The next URL - if available - will be tested\n"
					pkg.Print(msg, pkg.NoColor)
					break
				}
				progress = fmt.Sprintf("(%d/%d):(%d/%d)", i+1, len(testUrls), ii+1, len(recUrls))
				runTests(rec, uu, progress, &urlsToAdd, "crawl_")
			}

			recUrls = urlsToAdd
		}
	}

	/* Scan finished */
	msg = "Successfully finished the scan\n"
	pkg.PrintVerbose(msg, pkg.NoColor, 1)

	duration := time.Since(start)
	msg = fmt.Sprintf("Duration: %s\n\n", duration)
	pkg.PrintVerbose(msg, pkg.NoColor, 1)
	/****************/

	if pkg.Config.GenerateReport {
		report.Duration = duration.String()
		pkg.GenerateReport(report, currentDate)
	}
}

func runTests(rec int, u string, progress string, foundUrls *[]string, stat string) {
	var repWebsite pkg.ReportWebsite
	var err error
	repWebsite.URL = u

	msg := fmt.Sprintf("Testing website%s: %s\n", progress, u)
	pkg.Print(msg, pkg.NoColor)
	fmt.Printf("-----------------------------------------------------------------------\n")

	/* Setting up client: cookies and noredirect */
	msg = "Setting up client\n"
	pkg.PrintVerbose(msg, pkg.NoColor, 2)

	// Setting cookies, specified by setcookies
	pkg.Config.Website.Cookies = []*http.Cookie{}
	for _, c := range pkg.Config.Cookies {
		c = strings.TrimSuffix(c, "\r")
		c = strings.TrimSpace(c)
		if c == "" {
			continue
		} else if !strings.Contains(c, "=") {
			msg = "Specified cookie %s doesn't contain a = and will be skipped\n"
			pkg.PrintVerbose(msg, pkg.NoColor, 2)
			continue
		} else {
			cSlice := strings.SplitAfterN(c, "=", 2)
			cSlice[0] = strings.TrimSuffix(cSlice[0], "=")

			cookie := http.Cookie{
				Name:  cSlice[0],
				Value: cSlice[1],
			}
			pkg.Config.Website.Cookies = append(pkg.Config.Website.Cookies, &cookie)
		}
	}

	timeOutDuration := time.Duration(time.Duration(pkg.Config.TimeOut) * time.Second)
	clientNoRedir := &http.Client{
		CheckRedirect: func(redirRequest *http.Request, via []*http.Request) error {
			/* Commented out, because it unnecessary bloats up logs, especially for 301/302 links
			msg := fmt.Sprintf("Redirect Request denied: %s\n", redirRequest.Header)
			pkg.PrintVerbose(msg, pkg.Yellow, 2)
			*/
			return http.ErrUseLastResponse
		},
		Timeout: timeOutDuration,
	}

	http.DefaultClient = clientNoRedir

	pkg.Statistics[stat+"total"]++

	// retrieve cookies, headers etc
	pkg.Config.Website, err = pkg.GetWebsite(u, false)
	if err != nil {
		repWebsite.HasError = true
		repWebsite.ErrorMessages = append(repWebsite.ErrorMessages, err.Error())
		report.Websites = append(report.Websites, repWebsite)

		msg := fmt.Sprintf("Couldn't test url: %s\n", err.Error())
		pkg.Print(msg, pkg.Yellow)

		pkg.Statistics[stat+"noconnection"]++
		return
	}

	if strings.HasPrefix(pkg.Config.Website.Body, "<html><head><title>Burp Suite") {
		msg := fmt.Sprintf("Couldn't connect to given url: \n%s\n", pkg.Config.Website.Body)
		pkg.Print(msg, pkg.Red)

		pkg.Statistics[stat+"noconnection"]++
		return
	}

	if !pkg.Config.DeclineCookies {
		// retrieve response with all cookies set
		pkg.Config.Website, err = pkg.GetWebsite(u, false)
		if err != nil {
			repWebsite.HasError = true
			repWebsite.ErrorMessages = append(repWebsite.ErrorMessages, err.Error())
			report.Websites = append(report.Websites, repWebsite)

			msg := fmt.Sprintf("Couldn't test url: %s\n", err.Error())
			pkg.Print(msg, pkg.Yellow)
			return
		}
	}

	// check if there's a cache and the cachebuster works
	var errSlice []error
	pkg.Config.Website.Cache, errSlice = pkg.CheckCache(stat)
	for _, err := range errSlice {
		if err != nil {
			repWebsite.HasError = true
			repWebsite.ErrorMessages = append(repWebsite.ErrorMessages, err.Error())
		}
	}
	/* dont return when there's an error. Because the crawler shall be run anyways. Also there might was a cachebuster found and an error for an other cachebuster which doesnt matter
	if len(errSlice) > 0 && !pkg.Config.Force {
		return
	}
	*/

	// retrieve response with cachebuster
	pkg.Config.Website, err = pkg.GetWebsite(u, true)
	if err != nil {
		repWebsite.HasError = true
		repWebsite.ErrorMessages = append(repWebsite.ErrorMessages, err.Error())
		report.Websites = append(report.Websites, repWebsite)

		msg := fmt.Sprintf("Couldn't test url: %s\n", err.Error())
		pkg.Print(msg, pkg.Yellow)

		pkg.Statistics[stat+"errorcachebuster"]++
		return
	}

	/*******************************************/
	if pkg.Config.Website.Cache.CBwasFound || pkg.Config.Force {
		fmt.Println()
		pkg.Statistics[stat+"tested"]++

		/* Testing for cookie poisoning */
		if noTestPreference || strings.Contains(pkg.Config.OnlyTest, "cookie") || (pkg.Config.SkipTest != "" && !strings.Contains(pkg.Config.SkipTest, "cookie")) {
			msg = "Testing for cookie poisoning\n"
			pkg.PrintVerbose(msg, pkg.NoColor, 1)

			repWebsite.Results = append(repWebsite.Results, pkg.ScanCookies())

			if len(pkg.Config.Website.Cookies) == 0 {
				msg = "There were no cookies to test!"
				pkg.Print(msg+"\n", pkg.Yellow)

				repWebsite.Results[len(repWebsite.Results)-1].ErrorMessages = append(repWebsite.Results[len(repWebsite.Results)-1].ErrorMessages, msg)
				repWebsite.Results[len(repWebsite.Results)-1].HasError = true
			}
		} else {
			msg = "Skipping to test for cookie poisoning\n"
			pkg.PrintVerbose(msg, pkg.NoColor, 1)
		}
		/*****************************/

		/* Testing for css poisoning */
		if noTestPreference || strings.Contains(pkg.Config.OnlyTest, "css") || (pkg.Config.SkipTest != "" && !strings.Contains(pkg.Config.SkipTest, "css")) {
			msg = "Testing for css poisoning\n"
			pkg.PrintVerbose(msg, pkg.NoColor, 1)

			repWebsite.Results = append(repWebsite.Results, pkg.ScanCSS())
		} else {
			msg = "Skipping to test for css poisoning\n"
			pkg.PrintVerbose(msg, pkg.NoColor, 1)
		}
		/*****************************/

		/* Testing for multiple forwarding headers for poisoning */
		if noTestPreference || strings.Contains(pkg.Config.OnlyTest, "forward") || (pkg.Config.SkipTest != "" && !strings.Contains(pkg.Config.SkipTest, "forward")) {
			msg = "Testing multiple forwarding headers for poisoning\n"
			pkg.PrintVerbose(msg, pkg.NoColor, 1)

			repWebsite.Results = append(repWebsite.Results, pkg.ScanForwardingHeaders())
		} else {
			msg = "Skipping to test multiple forwarding headers for poisoning\n"
			pkg.PrintVerbose(msg, pkg.NoColor, 1)
		}
		/*************************************************************/

		/* Testing for HTTP request smuggling poisoning */
		if noTestPreference || strings.Contains(pkg.Config.OnlyTest, "smuggl") || (pkg.Config.SkipTest != "" && !strings.Contains(pkg.Config.SkipTest, "smuggl")) {
			msg = "Testing for poisoning via HTTP request smuggling\n"
			pkg.PrintVerbose(msg, pkg.NoColor, 1)

			repWebsite.Results = append(repWebsite.Results, pkg.ScanHTTPRequestSmuggling(proxyURL))
		} else {
			msg = "Skipping to test for poisoning via HTTP request smuggling\n"
			pkg.PrintVerbose(msg, pkg.NoColor, 1)
		}
		/*************************************************************/

		/* Testing for multiple Cache Poisoned Denial Of Service Techniques */
		if noTestPreference || strings.Contains(pkg.Config.OnlyTest, "dos") || (pkg.Config.SkipTest != "" && !strings.Contains(pkg.Config.SkipTest, "dos")) {
			msg = "Testing for multiple Cache Poisoned Denial Of Service Techniques\n"
			pkg.PrintVerbose(msg, pkg.NoColor, 1)

			repWebsite.Results = append(repWebsite.Results, pkg.DOS())
		} else {
			msg = "Skipping to test for multiple Cache Poisoned Denial Of Service Techniques\n"
			pkg.PrintVerbose(msg, pkg.NoColor, 1)
		}
		/***********************************************************/

		/* Testing for header poisoning */
		if noTestPreference || strings.Contains(pkg.Config.OnlyTest, "header") || (pkg.Config.SkipTest != "" && !strings.Contains(pkg.Config.SkipTest, "header")) {
			msg = "Testing for header poisoning\n"
			pkg.PrintVerbose(msg, pkg.NoColor, 1)

			repWebsite.Results = append(repWebsite.Results, pkg.ScanHeaders(headerList))
		} else {
			msg = "Skipping to test for header poisoning\n"
			pkg.PrintVerbose(msg, pkg.NoColor, 1)
		}
		/********************************/

		/* Testing for query parameter poisoning */
		if noTestPreference || strings.Contains(pkg.Config.OnlyTest, "parameter") || (pkg.Config.SkipTest != "" && !strings.Contains(pkg.Config.SkipTest, "parameter")) {
			msg = "Testing for query parameter poisoning\n"
			pkg.PrintVerbose(msg, pkg.NoColor, 1)
			repWebsite.Results = append(repWebsite.Results, pkg.ScanParameters(parameterList))
		} else {
			msg = "Skipping to test for query parameter poisoning\n"
			pkg.PrintVerbose(msg, pkg.NoColor, 1)
		}
		/*****************************************/

		/* Testing for Fat GET */
		if pkg.Config.DoPost {
			msg = "Can't check for Fat GET, because POST was specified\n"
			pkg.PrintVerbose(msg, pkg.NoColor, 1)
		} else if noTestPreference || strings.Contains(pkg.Config.OnlyTest, "fat") || (pkg.Config.SkipTest != "" && !strings.Contains(pkg.Config.SkipTest, "fat")) {
			msg = "Testing for Fat GET\n"
			pkg.PrintVerbose(msg, pkg.NoColor, 1)

			repWebsite.Results = append(repWebsite.Results, pkg.ScanFatGET())
		} else {
			msg = "Skipping to test for Fat GET\n"
			pkg.PrintVerbose(msg, pkg.NoColor, 1)
		}
		/**********************/

		/* Testing for Parameter Cloaking */
		if pkg.Config.DoPost {
			msg = "Can't check for Parameter Cloaking, because POST was specified\n"
			pkg.PrintVerbose(msg, pkg.NoColor, 1)
		} else if noTestPreference || strings.Contains(pkg.Config.OnlyTest, "cloaking") || (pkg.Config.SkipTest != "" && !strings.Contains(pkg.Config.SkipTest, "cloaking")) {
			msg = "Testing for Parameter Cloaking\n"
			pkg.PrintVerbose(msg, pkg.NoColor, 1)

			repWebsite.Results = append(repWebsite.Results, pkg.ScanParameterCloaking())
		} else {
			msg = "Skipping to test for Parameter Cloaking\n"
			pkg.PrintVerbose(msg, pkg.NoColor, 1)
		}
		/**********************************/
	}

	/* Check for linked files */
	if pkg.Config.Recursivity > rec {
		msg = fmt.Sprintf("\nChecking recursively for urls (%d/%d)\n", rec+1, pkg.Config.Recursivity)
		pkg.Print(msg, pkg.NoColor)

		tempUrls := pkg.CrawlUrls(added, excluded)

		if len(tempUrls) > 0 {
			msg = fmt.Sprintf("Found %d urls", len(tempUrls))
			pkg.Print(msg, pkg.NoColor)

			msg = "Adding the following urls to the Queue:"
			pkg.PrintVerbose(msg, pkg.NoColor, 1)
			pkg.PrintVerbose("\n", pkg.NoColor, 0)
			for _, u := range tempUrls {
				pkg.PrintVerbose(u+"\n", pkg.NoColor, 1)
			}

			*foundUrls = append(*foundUrls, tempUrls...)
		} else {
			msg = "No urls were found to add to the queue"
			pkg.Print(msg, pkg.NoColor)
		}
	}
	/**************************/

	if pkg.Config.GenerateCompleted {
		_, err = completedFile.WriteString(u + "\n")
		if err != nil {
			pkg.Print("Writing to completed URLs file: "+err.Error(), pkg.Red)
		}
	}

	if pkg.Config.GenerateReport {
		for _, r := range repWebsite.Results {
			if r.Vulnerable {
				repWebsite.Vulnerable = true
				break
			}
		}
		report.Websites = append(report.Websites, repWebsite)

		report.Vulnerable = report.Vulnerable || repWebsite.Vulnerable
		pkg.GenerateReport(report, currentDate)
	}

	fmt.Printf("\n-----------------------------------------------------------------------\n")
}

func createCompletedURLs() *os.File {
	completedPath := pkg.Config.GeneratePath + currentDate + "_WCVS_Completed"

	_, err := os.Stat(completedPath)

	var file *os.File
	defer file.Close()

	if !os.IsNotExist(err) {
		msg := fmt.Sprintf("The file %s will be overwritten, as it already exists\n", completedPath)
		pkg.PrintVerbose(msg, pkg.Yellow, 1)
		file, err = os.OpenFile(completedPath, os.O_WRONLY, 0666)
	} else {
		file, err = os.Create(completedPath)
	}
	if err != nil {
		msg := "CompletedURLs: " + err.Error() + "\n"
		pkg.PrintFatal(msg)
	}

	return file
}

/* Setting proxy with specified proxyURL and proxyCertPath */
func setProxy() *url.URL {
	if pkg.Config.ProxyCertPath != "" {
		proxyURL, err := url.Parse(pkg.Config.ProxyURL)
		if err != nil {
			msg := "Proxy: " + err.Error() + "\n"
			pkg.PrintFatal(msg)
		}
		caCert, err := ioutil.ReadFile(pkg.Config.ProxyCertPath)
		if err != nil {
			msg := "Proxy: " + err.Error() + "\n"
			pkg.PrintFatal(msg)
		}
		//caCertPool,err := x509.SystemCertPool()
		// führt zu crypto/x509: system root pool is not available on Windows
		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(caCert)

		tlsConfig := &tls.Config{RootCAs: caCertPool,
			InsecureSkipVerify: true}
		tr := &http.Transport{
			Proxy:           http.ProxyURL(proxyURL),
			TLSClientConfig: tlsConfig}

		err = http2.ConfigureTransport(tr)
		if err != nil {
			msg := fmt.Sprintf("Proxy: Cannot switch to HTTP2: %s\n", err.Error())
			pkg.PrintFatal(msg)
		}

		http.DefaultTransport = tr
		return proxyURL
	} else {
		tlsConfig := &tls.Config{InsecureSkipVerify: true}
		tr := &http.Transport{
			TLSClientConfig: tlsConfig}

		http.DefaultTransport = tr
		return nil
	}
}