Merge pull request #109 from vmfunc/refactor/url-helper

refactor: dedupe url scheme stripping

internal/logger/logger.go

@@ -124,7 +124,10 @@ func (l *Logger) Close() error {
 
 // CreateFile initializes a log file for the given URL and writes the header.
 func CreateFile(logFiles *[]string, url string, dir string) error {
-	sanitizedURL := strings.Split(url, "://")[1]
+	sanitizedURL := url
+	if _, after, ok := strings.Cut(url, "://"); ok {
+		sanitizedURL = after
+	}
 	path := filepath.Join(dir, sanitizedURL+".log")
 
 	header := fmt.Sprintf("       _____________\n__________(_)__  __/\n__  ___/_  /__  /_  \n_(__  )_  / _  __/  \n/____/ /_/  /_/    \n\nsif log file for %s\nhttps://sif.sh\n\n", url)

internal/scan/builtin/frameworks_module.go

@@ -15,9 +15,10 @@ package builtin
 import (
 	"context"
 	"fmt"
+	"strings"
+
 	"github.com/dropalldatabases/sif/internal/modules"
 	"github.com/dropalldatabases/sif/internal/scan/frameworks"
-	"strings"
 )
 
 type FrameworksModule struct{}

internal/scan/builtin/nuclei_module.go

@@ -15,6 +15,7 @@ package builtin
 import (
 	"context"
 	"fmt"
+
 	"github.com/dropalldatabases/sif/internal/modules"
 	"github.com/dropalldatabases/sif/internal/scan"
 )

internal/scan/builtin/shodan_module.go

@@ -15,9 +15,10 @@ package builtin
 import (
 	"context"
 	"fmt"
+	"strings"
+
 	"github.com/dropalldatabases/sif/internal/modules"
 	"github.com/dropalldatabases/sif/internal/scan"
-	"strings"
 )
 
 type ShodanModule struct{}

internal/scan/builtin/whois_module.go

@@ -14,6 +14,7 @@ package builtin
 
 import (
 	"context"
+
 	"github.com/dropalldatabases/sif/internal/modules"
 	"github.com/dropalldatabases/sif/internal/scan"
 )

internal/scan/cloudstorage.go

@@ -33,7 +33,7 @@ type CloudStorageResult struct {
 func CloudStorage(url string, timeout time.Duration, logdir string) ([]CloudStorageResult, error) {
 	fmt.Println(styles.Separator.Render("Starting " + styles.Status.Render("Cloud Storage Misconfiguration Scan") + "..."))
 
-	sanitizedURL := strings.Split(url, "://")[1]
+	sanitizedURL := stripScheme(url)
 
 	if logdir != "" {
 		if err := logger.WriteHeader(sanitizedURL, logdir, "Cloud Storage Misconfiguration Scan"); err != nil {

internal/scan/cms.go

@@ -35,7 +35,7 @@ func CMS(url string, timeout time.Duration, logdir string) (*CMSResult, error) {
 	spin := output.NewSpinner("Detecting content management system")
 	spin.Start()
 
-	sanitizedURL := strings.Split(url, "://")[1]
+	sanitizedURL := stripScheme(url)
 
 	if logdir != "" {
 		if err := logger.WriteHeader(sanitizedURL, logdir, "CMS detection"); err != nil {

internal/scan/dirlist.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"net/http"
 	"strconv"
-	"strings"
 	"sync"
 	"time"
 
@@ -44,7 +43,7 @@ func Dirlist(size string, url string, timeout time.Duration, threads int, logdir
 	log := output.Module("DIRLIST")
 	log.Start()
 
-	sanitizedURL := strings.Split(url, "://")[1]
+	sanitizedURL := stripScheme(url)
 
 	if logdir != "" {
 		if err := logger.WriteHeader(sanitizedURL, logdir, size+" directory fuzzing"); err != nil {

internal/scan/dnslist.go

@@ -17,7 +17,6 @@ import (
 	"context"
 	"fmt"
 	"net/http"
-	"strings"
 	"sync"
 	"time"
 
@@ -67,7 +66,7 @@ func Dnslist(size string, url string, timeout time.Duration, threads int, logdir
 		dns = append(dns, scanner.Text())
 	}
 
-	sanitizedURL := strings.Split(url, "://")[1]
+	sanitizedURL := stripScheme(url)
 
 	if logdir != "" {
 		if err := logger.WriteHeader(sanitizedURL, logdir, size+" subdomain fuzzing"); err != nil {

internal/scan/dork.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"net/http"
 	"strconv"
-	"strings"
 	"sync"
 	"time"
 
@@ -60,7 +59,7 @@ func Dork(url string, timeout time.Duration, threads int, logdir string) ([]Dork
 	spin := output.NewSpinner("Running Google dorks")
 	spin.Start()
 
-	sanitizedURL := strings.Split(url, "://")[1]
+	sanitizedURL := stripScheme(url)
 
 	if logdir != "" {
 		if err := logger.WriteHeader(sanitizedURL, logdir, "URL dorking"); err != nil {

internal/scan/git.go

@@ -38,7 +38,7 @@ func Git(url string, timeout time.Duration, threads int, logdir string) ([]strin
 	spin := output.NewSpinner("Scanning for exposed git repositories")
 	spin.Start()
 
-	sanitizedURL := strings.Split(url, "://")[1]
+	sanitizedURL := stripScheme(url)
 
 	if logdir != "" {
 		if err := logger.WriteHeader(sanitizedURL, logdir, "git directory fuzzing"); err != nil {

internal/scan/headers.go

@@ -15,7 +15,6 @@ package scan
 import (
 	"context"
 	"net/http"
-	"strings"
 	"time"
 
 	"github.com/dropalldatabases/sif/internal/logger"
@@ -31,7 +30,7 @@ func Headers(url string, timeout time.Duration, logdir string) ([]HeaderResult,
 	log := output.Module("HEADERS")
 	log.Start()
 
-	sanitizedURL := strings.Split(url, "://")[1]
+	sanitizedURL := stripScheme(url)
 
 	if logdir != "" {
 		if err := logger.WriteHeader(sanitizedURL, logdir, "HTTP Header Analysis"); err != nil {

internal/scan/lfi.go

@@ -19,7 +19,6 @@ import (
 	"net/http"
 	"net/url"
 	"regexp"
-	"strings"
 	"sync"
 	"time"
 
@@ -119,7 +118,7 @@ func LFI(targetURL string, timeout time.Duration, threads int, logdir string) (*
 	spin := output.NewSpinner("Scanning for LFI vulnerabilities")
 	spin.Start()
 
-	sanitizedURL := strings.Split(targetURL, "://")[1]
+	sanitizedURL := stripScheme(targetURL)
 
 	if logdir != "" {
 		if err := logger.WriteHeader(sanitizedURL, logdir, "LFI reconnaissance"); err != nil {

internal/scan/nuclei.go

@@ -15,7 +15,6 @@ package scan
 import (
 	"context"
 	"os"
-	"strings"
 	"sync"
 	"time"
 
@@ -69,7 +68,7 @@ func Nuclei(url string, timeout time.Duration, threads int, logdir string) ([]ou
 	}
 	defer ne.Close()
 
-	sanitizedURL := strings.Split(url, "://")[1]
+	sanitizedURL := stripScheme(url)
 	ne.LoadTargets([]string{sanitizedURL}, false)
 
 	var results []output.ResultEvent

internal/scan/ports.go

@@ -19,7 +19,6 @@ import (
 	"net"
 	"net/http"
 	"strconv"
-	"strings"
 	"sync"
 	"time"
 
@@ -34,7 +33,7 @@ func Ports(ctx context.Context, scope string, url string, timeout time.Duration,
 	log := output.Module("PORTS")
 	log.Start()
 
-	sanitizedURL := strings.Split(url, "://")[1]
+	sanitizedURL := stripScheme(url)
 	if logdir != "" {
 		if err := logger.WriteHeader(sanitizedURL, logdir, scope+" port scanning"); err != nil {
 			log.Error("Error creating log file: %v", err)

internal/scan/scan.go

@@ -31,6 +31,15 @@ import (
 	"github.com/dropalldatabases/sif/internal/output"
 )
 
+// stripScheme drops the scheme:// prefix from url, or returns it unchanged when
+// there's no scheme (so a bare host doesn't panic).
+func stripScheme(url string) string {
+	if _, rest, ok := strings.Cut(url, "://"); ok {
+		return rest
+	}
+	return url
+}
+
 func fetchRobotsTXT(url string, client *http.Client) *http.Response {
 	req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, url, http.NoBody)
 	if err != nil {
@@ -67,7 +76,7 @@ func fetchRobotsTXT(url string, client *http.Client) *http.Response {
 func Scan(url string, timeout time.Duration, threads int, logdir string) {
 	output.ScanStart("base URL scanning")
 
-	sanitizedURL := strings.Split(url, "://")[1]
+	sanitizedURL := stripScheme(url)
 
 	if logdir != "" {
 		if err := logger.WriteHeader(sanitizedURL, logdir, "URL scanning"); err != nil {

internal/scan/scan_test.go

@@ -200,3 +200,24 @@ func TestHeaderResult(t *testing.T) {
 		t.Errorf("expected value 'application/json', got '%s'", result.Value)
 	}
 }
+
+func TestStripScheme(t *testing.T) {
+	tests := []struct {
+		name string
+		url  string
+		want string
+	}{
+		{"https with path", "https://example.com/path", "example.com/path"},
+		{"http", "http://example.com", "example.com"},
+		{"no scheme stays put", "example.com", "example.com"},
+		{"empty", "", ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := stripScheme(tt.url); got != tt.want {
+				t.Errorf("stripScheme(%q) = %q, want %q", tt.url, got, tt.want)
+			}
+		})
+	}
+}

internal/scan/securityheaders.go

@@ -55,7 +55,7 @@ func SecurityHeaders(url string, timeout time.Duration, logdir string) (Security
 	log := output.Module("SECHEADERS")
 	log.Start()
 
-	sanitizedURL := strings.Split(url, "://")[1]
+	sanitizedURL := stripScheme(url)
 
 	if logdir != "" {
 		if err := logger.WriteHeader(sanitizedURL, logdir, "Security Header Analysis"); err != nil {

internal/scan/securitytrails.go

@@ -100,7 +100,7 @@ func SecurityTrails(targetURL string, timeout time.Duration, logdir string) (*Se
 	spin.Stop()
 
 	if logdir != "" {
-		sanitizedURL := strings.Split(targetURL, "://")[1]
+		sanitizedURL := stripScheme(targetURL)
 		if err := logger.WriteHeader(sanitizedURL, logdir, "SecurityTrails lookup"); err != nil {
 			output.Error("error writing log header: %v", err)
 		}

internal/scan/shodan.go

@@ -134,7 +134,7 @@ func Shodan(targetURL string, timeout time.Duration, logdir string) (*ShodanResu
 
 	// log results
 	if logdir != "" {
-		sanitizedURL := strings.Split(targetURL, "://")[1]
+		sanitizedURL := stripScheme(targetURL)
 		if err := logger.WriteHeader(sanitizedURL, logdir, "Shodan lookup"); err != nil {
 			output.Error("Error writing log header: %v", err)
 		}

internal/scan/sql.go

@@ -121,7 +121,7 @@ func SQL(targetURL string, timeout time.Duration, threads int, logdir string) (*
 	spin := output.NewSpinner("Scanning for SQL exposures")
 	spin.Start()
 
-	sanitizedURL := strings.Split(targetURL, "://")[1]
+	sanitizedURL := stripScheme(targetURL)
 
 	if logdir != "" {
 		if err := logger.WriteHeader(sanitizedURL, logdir, "SQL reconnaissance"); err != nil {

internal/scan/subdomaintakeover.go

@@ -41,7 +41,7 @@ type SubdomainTakeoverResult struct {
 func SubdomainTakeover(url string, dnsResults []string, timeout time.Duration, threads int, logdir string) ([]SubdomainTakeoverResult, error) {
 	fmt.Println(styles.Separator.Render("Starting " + styles.Status.Render("Subdomain Takeover Vulnerability Check") + "..."))
 
-	sanitizedURL := strings.Split(url, "://")[1]
+	sanitizedURL := stripScheme(url)
 
 	if logdir != "" {
 		if err := logger.WriteHeader(sanitizedURL, logdir, "Subdomain Takeover Vulnerability Check"); err != nil {

internal/scan/whois.go

@@ -13,8 +13,6 @@
 package scan
 
 import (
-	"strings"
-
 	"github.com/charmbracelet/log"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
@@ -24,7 +22,7 @@ import (
 func Whois(url string, logdir string) {
 	output.ScanStart("WHOIS lookup")
 
-	sanitizedURL := strings.Split(url, "://")[1]
+	sanitizedURL := stripScheme(url)
 	if logdir != "" {
 		if err := logger.WriteHeader(sanitizedURL, logdir, " WHOIS scanning"); err != nil {
 			output.Error("Error creating log file: %v", err)