From 79218df42a5b2d72db0f0adf45506d26eaecbdb8 Mon Sep 17 00:00:00 2001 From: Sol Fisher Romanoff Date: Tue, 5 Sep 2023 18:41:31 +0300 Subject: [PATCH] cmd/scan: ignore 3xx redirects --- cmd/scan.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/scan.go b/cmd/scan.go index b7043df..4fb535a 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -37,13 +37,16 @@ func Scan(url string, timeout time.Duration, threads int, logdir string) { client := &http.Client{ Timeout: timeout, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, } resp, err := client.Get(url + "/robots.txt") if err != nil { log.Debugf("Error: %s", err) } - if resp.StatusCode != 404 { + if resp.StatusCode != 404 && resp.StatusCode != 301 && resp.StatusCode != 302 && resp.StatusCode != 307 { scanlog.Infof("file [%s] found", statusstyle.Render("robots.txt")) var robotsData []string