git: implement git feature

cmd/dork.go

@@ -21,7 +21,7 @@ const (
 
 func Dork(url string, timeout time.Duration, logdir string) {
 
-	fmt.Println(separator.Render("📂 Starting " + statusstyle.Render("URL Dorking") + "..."))
+	fmt.Println(separator.Render("🤓 Starting " + statusstyle.Render("URL Dorking") + "..."))
 
 	sanitizedURL := strings.Split(url, "://")[1]
 

cmd/git.go

@@ -0,0 +1,81 @@
+package cmd
+
+import (
+	"bufio"
+	"fmt"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/charmbracelet/log"
+	// "github.com/pushfs/sif/util"
+)
+
+const (
+	gitURL  = "https://raw.githubusercontent.com/pushfs/sif-runtime/main/git/"
+	gitFile = "git.txt"
+)
+
+func Git(url string, timeout time.Duration, logdir string) {
+
+	fmt.Println(separator.Render("🌿 Starting " + statusstyle.Render("git repository scanning") + "..."))
+
+	sanitizedURL := strings.Split(url, "://")[1]
+
+	if logdir != "" {
+		f, err := os.OpenFile(logdir+"/"+sanitizedURL+".log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
+		if err != nil {
+			log.Errorf("Error creating log file: %s", err)
+			return
+		}
+		defer f.Close()
+		f.WriteString("\n\n--------------\nStarting git repository scanning\n--------------\n")
+	}
+
+	logger := log.NewWithOptions(os.Stderr, log.Options{
+		Prefix: "Git 🌿",
+	})
+	gitlog := logger.With("url", url)
+
+	gitlog.Infof("Starting repository scanning")
+
+	resp, err := http.Get(gitURL + gitFile)
+	if err != nil {
+		log.Errorf("Error downloading git list: %s", err)
+		return
+	}
+	var gitUrls []string
+	scanner := bufio.NewScanner(resp.Body)
+	scanner.Split(bufio.ScanLines)
+	for scanner.Scan() {
+		gitUrls = append(gitUrls, scanner.Text())
+	}
+
+	// util.InitProgressBar()
+	client := &http.Client{
+		Timeout: timeout,
+	}
+	for _, repourl := range gitUrls {
+		log.Debugf("%s", repourl)
+		resp, err := client.Get(url + "/" + repourl)
+		if err != nil {
+			log.Debugf("Error %s: %s", repourl, err)
+		}
+
+		if resp.StatusCode != 404 {
+			// log url, directory, and status code
+			gitlog.Infof("%s git found at [%s]", statusstyle.Render(strconv.Itoa(resp.StatusCode)), directorystyle.Render(repourl))
+			if logdir != "" {
+				f, err := os.OpenFile(logdir+"/"+sanitizedURL+".log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
+				if err != nil {
+					log.Errorf("Error creating log file: %s", err)
+					return
+				}
+				defer f.Close()
+				f.WriteString(fmt.Sprintf("%s git found at [%s]\n", strconv.Itoa(resp.StatusCode), repourl))
+			}
+		}
+	}
+}

cmd/scan.go

@@ -54,7 +54,7 @@ func Scan(url string, timeout time.Duration, logdir string) {
 
 		for _, robot := range robotsData {
 
-			if robot == "" || strings.HasPrefix(robot, "#") || strings.HasPrefix(robot, "Disallow: ") || strings.HasPrefix(robot, "User-agent: ") || strings.HasPrefix(robot, "Sitemap: ") {
+			if robot == "" || strings.HasPrefix(robot, "#") || strings.HasPrefix(robot, "User-agent: ") || strings.HasPrefix(robot, "Sitemap: ") {
 				continue
 			}
 

config.go

@@ -18,6 +18,7 @@ type Settings struct {
 	NoScan  bool
 	Ports   string
 	Dorking bool
+	Git     bool
 	Timeout time.Duration
 }
 
@@ -33,6 +34,7 @@ func parseURLs() Settings {
 	pflag.Lookup("ports").NoOptDefVal = "common"
 	var dorking = pflag.Bool("dork", false, "Enable Google dorking")
 	var noscan = pflag.Bool("noscan", false, "Do not perform base URL (robots.txt, etc) scanning")
+	var git = pflag.Bool("git", false, "Enable git repository scanning")
 	pflag.Parse()
 
 	if len(*url) > 0 {
@@ -46,6 +48,7 @@ func parseURLs() Settings {
 			Dorking: *dorking,
 			Ports:   *ports,
 			LogDir:  *logdir,
+			Git:     *git,
 		}
 	} else if *file != "" {
 		if _, err := os.Stat(*file); err != nil {
@@ -76,6 +79,7 @@ func parseURLs() Settings {
 			Ports:   *ports,
 			URLs:    urls,
 			LogDir:  *logdir,
+			Git:     *git,
 		}
 	}
 

main.go

@@ -100,6 +100,10 @@ func main() {
 			cmd.Dork(url, settings.Timeout, settings.LogDir)
 		}
 
+		if settings.Git {
+			cmd.Git(url, settings.Timeout, settings.LogDir)
+		}
+
 		// TODO: WHOIS
 
 		fmt.Println()