gitea-mirror/sif
1
0
Fork 0
mirror of https://github.com/lunchcat/sif.git synced 2025-12-05 20:40:08 -08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #25 from macdoos/main

Browse Source 
working thank you <3
This commit is contained in:
tessa
2024-06-18 22:20:41 -04:00
committed by GitHub
parent c8d12a0223 80062533c9
commit 09565b4836
1 changed files with 28 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
pkg/scan/scan.go
View File

@@ -15,8 +15,27 @@ import (
"github.com/dropalldatabases/sif/pkg/logger"
)
func Scan(url string, timeout time.Duration, threads int, logdir string) {
func fetchRobotsTXT(url string, client *http.Client) *http.Response {
resp, err := client.Get(url)
if err != nil {
log.Debugf("Error fetching robots.txt: %s", err)
return nil
}
if resp.StatusCode == http.StatusMovedPermanently {
redirectURL := resp.Header.Get("Location")
if redirectURL == "" {
log.Debugf("Redirect location is empty for %s", url)
return nil
}
resp.Body.Close()
return fetchRobotsTXT(redirectURL, client)
}
return resp
}
func Scan(url string, timeout time.Duration, threads int, logdir string) {
fmt.Println(styles.Separator.Render("🐾 Starting " + styles.Status.Render("base url scanning") + "..."))
sanitizedURL := strings.Split(url, "://")[1]
@@ -39,11 +58,12 @@ func Scan(url string, timeout time.Duration, threads int, logdir string) {
},
}
resp, err := client.Get(url + "/robots.txt")
if err != nil {
log.Debugf("Error: %s", err)
resp := fetchRobotsTXT(url+"/robots.txt", client)
if resp == nil {
return
}
defer resp.Body.Close()
if resp.StatusCode != 404 && resp.StatusCode != 301 && resp.StatusCode != 302 && resp.StatusCode != 307 {
scanlog.Infof("file [%s] found", styles.Status.Render("robots.txt"))
@@ -70,12 +90,13 @@ func Scan(url string, timeout time.Duration, threads int, logdir string) {
}
_, sanitizedRobot, _ := strings.Cut(robot, ": ")
log.Debugf("%s", robot)
scanlog.Debugf("%s", robot)
resp, err := client.Get(url + "/" + sanitizedRobot)
if err != nil {
log.Debugf("Error %s: %s", sanitizedRobot, err)
return
scanlog.Debugf("Error %s: %s", sanitizedRobot, err)
continue
}
defer resp.Body.Close()
if resp.StatusCode != 404 {
scanlog.Infof("%s from robots: [%s]", styles.Status.Render(strconv.Itoa(resp.StatusCode)), styles.Highlight.Render(sanitizedRobot))