Merge pull request #118 from vmfunc/feat/httpx-client

feat: shared http client (proxy, custom headers, rate limit) + -threads clamp

README.md

@@ -172,6 +172,24 @@ sif has a modular architecture. modules are defined in yaml and can be extended
 | `-lfi` | local file inclusion |
 | `-framework` | framework detection with cve lookup |
 
+### http options
+
+these apply to every outbound request across all scanners:
+
+| flag | description |
+|------|-------------|
+| `-proxy` | route all traffic through a proxy (http/https/socks5 url) |
+| `-H`, `--header` | custom header to send (repeatable or comma-separated, `"Key: Value"`) |
+| `-cookie` | cookie header to send with every request |
+| `-rate-limit` | max requests per second (0 = unlimited, default 0) |
+
+```bash
+# scan through a socks5 proxy with a custom header, cookie and 20 req/s cap
+./sif -u https://example.com -headers -proxy socks5://127.0.0.1:1080 -H "Authorization: Bearer tok" -cookie "session=abc" -rate-limit 20
+```
+
+a scanner that sets a header explicitly (e.g. an api key) always wins over the global default.
+
 ### yaml modules
 
 list available modules:

docs/usage.md

@@ -225,7 +225,7 @@ http request timeout (default: 10s):
 
 ### --threads
 
-number of concurrent threads (default: 10):
+number of concurrent threads (default: 10). values below 1 are clamped to 1:
 
 ```bash
 ./sif -u https://example.com --threads 20
@@ -247,6 +247,42 @@ enable debug logging:
 ./sif -u https://example.com -d
 ```
 
+## http options
+
+these apply to every outbound request across all scanners (proxy, custom headers, cookie and rate limiting share one client). a scanner that sets a header explicitly still wins over the global default.
+
+### -proxy
+
+route all traffic through a proxy. supports http, https and socks5 urls:
+
+```bash
+./sif -u https://example.com -proxy socks5://127.0.0.1:1080
+```
+
+### -H, --header
+
+add a custom header to every request. repeatable or comma-separated, `"Key: Value"`:
+
+```bash
+./sif -u https://example.com -H "Authorization: Bearer tok" -H "X-Env: staging"
+```
+
+### -cookie
+
+cookie header to send with every request:
+
+```bash
+./sif -u https://example.com -cookie "session=abc; theme=dark"
+```
+
+### -rate-limit
+
+cap outbound requests per second (0 = unlimited, default 0):
+
+```bash
+./sif -u https://example.com -rate-limit 20
+```
+
 ## api options
 
 ### -api

go.mod

@@ -12,6 +12,8 @@ require (
 	github.com/projectdiscovery/nuclei/v3 v3.8.0
 	github.com/projectdiscovery/utils v0.10.1
 	github.com/rocketlaunchr/google-search v1.1.6
+	golang.org/x/net v0.53.0
+	golang.org/x/time v0.14.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -377,13 +379,11 @@ require (
 	golang.org/x/crypto v0.50.0 // indirect
 	golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect
 	golang.org/x/mod v0.35.0 // indirect
-	golang.org/x/net v0.53.0 // indirect
 	golang.org/x/oauth2 v0.34.0 // indirect
 	golang.org/x/sync v0.20.0 // indirect
 	golang.org/x/sys v0.43.0 // indirect
 	golang.org/x/term v0.42.0 // indirect
 	golang.org/x/text v0.36.0 // indirect
-	golang.org/x/time v0.14.0 // indirect
 	golang.org/x/tools v0.44.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.36.6 // indirect

internal/config/config.go

@@ -51,8 +51,17 @@ type Settings struct {
 	ModuleTags        string // Run modules matching these tags
 	AllModules        bool   // Run all loaded modules
 	ListModules       bool   // List available modules and exit
+	Proxy             string
+	Header            goflags.StringSlice // custom request headers ("Key: Value")
+	Cookie            string
+	RateLimit         int
 }
 
+// minThreads is the floor for the worker count. Threads feeds wg.Add across the
+// scanners, so 0 silently runs nothing and a negative value panics with
+// "negative WaitGroup counter"; clamp the parsed value up to this.
+const minThreads = 1
+
 const (
 	Nil goflags.EnumVariable = iota
 
@@ -109,6 +118,13 @@ func Parse() *Settings {
 		flagSet.StringVar(&settings.Template, "template", "", "Sif runtime template to use"),
 	)
 
+	flagSet.CreateGroup("http", "HTTP",
+		flagSet.StringVar(&settings.Proxy, "proxy", "", "Proxy for all requests (http/https/socks5 url)"),
+		flagSet.StringSliceVarP(&settings.Header, "header", "H", nil, "Custom header to send (repeatable or comma-separated, \"Key: Value\")", goflags.CommaSeparatedStringSliceOptions),
+		flagSet.StringVar(&settings.Cookie, "cookie", "", "Cookie header to send with every request"),
+		flagSet.IntVar(&settings.RateLimit, "rate-limit", 0, "Max requests per second (0 = unlimited)"),
+	)
+
 	flagSet.CreateGroup("api", "API",
 		flagSet.BoolVar(&settings.ApiMode, "api", false, "Enable API mode. Only useful for internal lunchcat usage"),
 	)
@@ -124,5 +140,11 @@ func Parse() *Settings {
 		log.Fatalf("Could not parse flags: %s", err)
 	}
 
+	// threads feeds wg.Add directly; floor it so 0 isn't a silent no-op and a
+	// negative value can't panic the waitgroup.
+	if settings.Threads < minThreads {
+		settings.Threads = minThreads
+	}
+
 	return settings
 }

internal/httpx/httpx.go

@@ -0,0 +1,195 @@
+/*
+·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
+:                                                                               :
+:   █▀ █ █▀▀   ·   Blazing-fast pentesting suite                                :
+:   ▄█ █ █▀    ·   BSD 3-Clause License                                         :
+:                                                                               :
+:   (c) 2022-2026 vmfunc, xyzeva,                                               :
+:                 lunchcat alumni & contributors                                :
+:                                                                               :
+·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
+*/
+
+// Package httpx is the shared http layer every scanner talks through, so a
+// single Configure call wires proxy, custom headers, cookies and rate limiting
+// into every outbound request without touching scanner signatures.
+package httpx
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/net/proxy"
+	"golang.org/x/time/rate"
+)
+
+// allowed proxy schemes
+const (
+	schemeHTTP   = "http"
+	schemeHTTPS  = "https"
+	schemeSOCKS5 = "socks5"
+)
+
+// a header is "Key: Value"; this is the separator between the two halves.
+const headerSep = ": "
+
+// burst lets the limiter absorb a small spike before pacing kicks in; a burst
+// equal to the per-second rate keeps the cap honest over any one-second window.
+const limiterBurstPerRate = 1
+
+// Options carries the runtime knobs that apply to every outbound request.
+// RateLimit is requests/sec (0 = unlimited); Headers are "Key: Value" strings.
+type Options struct {
+	Proxy     string
+	Headers   []string
+	Cookie    string
+	UserAgent string
+	RateLimit int
+}
+
+// configured holds the package-level transport built once by Configure. nil
+// means Configure was never called, so Client falls back to a plain client.
+var (
+	mu         sync.RWMutex
+	configured http.RoundTripper
+)
+
+// Configure builds the shared transport once at startup from opts. Calling it
+// again replaces the previous configuration.
+//
+//nolint:gocritic // signature is the package's stable startup api; called once.
+func Configure(opts Options) error {
+	base, err := buildTransport(opts.Proxy)
+	if err != nil {
+		return err
+	}
+
+	headers, err := parseHeaders(opts.Headers)
+	if err != nil {
+		return err
+	}
+
+	var limiter *rate.Limiter
+	if opts.RateLimit > 0 {
+		limiter = rate.NewLimiter(rate.Limit(opts.RateLimit), opts.RateLimit*limiterBurstPerRate)
+	}
+
+	rt := &roundTripper{
+		base:      base,
+		headers:   headers,
+		cookie:    opts.Cookie,
+		userAgent: opts.UserAgent,
+		limiter:   limiter,
+	}
+
+	mu.Lock()
+	configured = rt
+	mu.Unlock()
+
+	return nil
+}
+
+// Client returns an http client wired to the configured transport. It works
+// before Configure is ever called (plain transport) so existing code and tests
+// behave unchanged. A zero timeout means no timeout, matching http.Client.
+func Client(timeout time.Duration) *http.Client {
+	mu.RLock()
+	rt := configured
+	mu.RUnlock()
+
+	return &http.Client{Timeout: timeout, Transport: rt}
+}
+
+// buildTransport clones the default transport and applies the proxy. An empty
+// proxy leaves the default behavior (respects HTTP_PROXY env) intact.
+func buildTransport(proxyURL string) (*http.Transport, error) {
+	tr, ok := http.DefaultTransport.(*http.Transport)
+	if !ok {
+		// unreachable in practice, but never trust an assertion silently.
+		return nil, fmt.Errorf("default transport is not *http.Transport")
+	}
+	transport := tr.Clone()
+
+	if proxyURL == "" {
+		return transport, nil
+	}
+
+	parsed, err := url.Parse(proxyURL)
+	if err != nil {
+		return nil, fmt.Errorf("parse proxy url %q: %w", proxyURL, err)
+	}
+
+	switch parsed.Scheme {
+	case schemeHTTP, schemeHTTPS:
+		transport.Proxy = http.ProxyURL(parsed)
+	case schemeSOCKS5:
+		// socks5 needs a custom dialer; the returned dialer implements
+		// ContextDialer so cancellation/timeouts propagate.
+		dialer, err := proxy.SOCKS5("tcp", parsed.Host, nil, proxy.Direct)
+		if err != nil {
+			return nil, fmt.Errorf("socks5 proxy %q: %w", proxyURL, err)
+		}
+		ctxDialer, ok := dialer.(proxy.ContextDialer)
+		if !ok {
+			return nil, fmt.Errorf("socks5 proxy %q: dialer lacks context support", proxyURL)
+		}
+		transport.DialContext = ctxDialer.DialContext
+	default:
+		return nil, fmt.Errorf("unsupported proxy scheme %q (want http/https/socks5)", parsed.Scheme)
+	}
+
+	return transport, nil
+}
+
+// parseHeaders splits each "Key: Value" entry on the first ": ". Entries
+// without the separator are rejected so a typo fails loud instead of silently.
+// The returned map is always non-nil so callers can range it unconditionally.
+func parseHeaders(raw []string) (map[string]string, error) {
+	headers := make(map[string]string, len(raw))
+	for i := 0; i < len(raw); i++ {
+		key, value, ok := strings.Cut(raw[i], headerSep)
+		if !ok {
+			return nil, fmt.Errorf("invalid header %q (want \"Key: Value\")", raw[i])
+		}
+		headers[key] = value
+	}
+
+	return headers, nil
+}
+
+// roundTripper paces and decorates each request before delegating to base.
+type roundTripper struct {
+	base      *http.Transport
+	headers   map[string]string
+	cookie    string
+	userAgent string
+	limiter   *rate.Limiter
+}
+
+func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	if rt.limiter != nil {
+		if err := rt.limiter.Wait(req.Context()); err != nil {
+			return nil, fmt.Errorf("rate limiter: %w", err)
+		}
+	}
+
+	// only set what the caller hasn't already; a scanner that explicitly sets a
+	// header (e.g. an api key) must win over the global default.
+	for key, value := range rt.headers {
+		if req.Header.Get(key) == "" {
+			req.Header.Set(key, value)
+		}
+	}
+	if rt.cookie != "" && req.Header.Get("Cookie") == "" {
+		req.Header.Set("Cookie", rt.cookie)
+	}
+	if rt.userAgent != "" && req.Header.Get("User-Agent") == "" {
+		req.Header.Set("User-Agent", rt.userAgent)
+	}
+
+	return rt.base.RoundTrip(req)
+}

internal/httpx/httpx_test.go

@@ -0,0 +1,217 @@
+/*
+·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
+:                                                                               :
+:   █▀ █ █▀▀   ·   Blazing-fast pentesting suite                                :
+:   ▄█ █ █▀    ·   BSD 3-Clause License                                         :
+:                                                                               :
+:   (c) 2022-2026 vmfunc, xyzeva,                                               :
+:                 lunchcat alumni & contributors                                :
+:                                                                               :
+·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
+*/
+
+package httpx
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+// resetConfig clears the package-level transport so each test starts clean.
+func resetConfig(t *testing.T) {
+	t.Helper()
+	mu.Lock()
+	configured = nil
+	mu.Unlock()
+}
+
+// captureServer records the headers of the last request it served.
+func captureServer(t *testing.T, seen *http.Header) *httptest.Server {
+	t.Helper()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		*seen = r.Header.Clone()
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func get(t *testing.T, client *http.Client, url string) {
+	t.Helper()
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, http.NoBody)
+	if err != nil {
+		t.Fatalf("new request: %v", err)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		t.Fatalf("do request: %v", err)
+	}
+	resp.Body.Close()
+}
+
+func TestClientBeforeConfigure(t *testing.T) {
+	resetConfig(t)
+
+	var seen http.Header
+	srv := captureServer(t, &seen)
+
+	// a client must work with no Configure call so existing code is unaffected.
+	get(t, Client(5*time.Second), srv.URL)
+
+	if seen == nil {
+		t.Fatal("request never reached the server")
+	}
+}
+
+func TestConfigureHeadersAndCookie(t *testing.T) {
+	tests := []struct {
+		name      string
+		opts      Options
+		wantKey   string
+		wantValue string
+	}{
+		{
+			name:      "custom header injected",
+			opts:      Options{Headers: []string{"X-Test: sif"}},
+			wantKey:   "X-Test",
+			wantValue: "sif",
+		},
+		{
+			name:      "cookie injected",
+			opts:      Options{Cookie: "session=abc"},
+			wantKey:   "Cookie",
+			wantValue: "session=abc",
+		},
+		{
+			name:      "user agent injected",
+			opts:      Options{UserAgent: "sif-scanner"},
+			wantKey:   "User-Agent",
+			wantValue: "sif-scanner",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			resetConfig(t)
+
+			if err := Configure(tt.opts); err != nil {
+				t.Fatalf("Configure: %v", err)
+			}
+
+			var seen http.Header
+			srv := captureServer(t, &seen)
+			get(t, Client(5*time.Second), srv.URL)
+
+			if got := seen.Get(tt.wantKey); got != tt.wantValue {
+				t.Errorf("header %q = %q, want %q", tt.wantKey, got, tt.wantValue)
+			}
+		})
+	}
+}
+
+func TestConfigureHeaderDoesNotOverride(t *testing.T) {
+	resetConfig(t)
+
+	if err := Configure(Options{Headers: []string{"X-Test: global"}}); err != nil {
+		t.Fatalf("Configure: %v", err)
+	}
+
+	var seen http.Header
+	srv := captureServer(t, &seen)
+
+	// a caller that sets the header explicitly must win over the global default.
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, srv.URL, http.NoBody)
+	if err != nil {
+		t.Fatalf("new request: %v", err)
+	}
+	req.Header.Set("X-Test", "caller")
+	resp, err := Client(5 * time.Second).Do(req)
+	if err != nil {
+		t.Fatalf("do request: %v", err)
+	}
+	resp.Body.Close()
+
+	if got := seen.Get("X-Test"); got != "caller" {
+		t.Errorf("X-Test = %q, want caller (caller value must not be overridden)", got)
+	}
+}
+
+func TestConfigureInvalidHeader(t *testing.T) {
+	resetConfig(t)
+
+	// a header without ": " should fail loud rather than silently dropping.
+	if err := Configure(Options{Headers: []string{"missing-separator"}}); err == nil {
+		t.Fatal("expected error for malformed header, got nil")
+	}
+}
+
+func TestConfigureInvalidProxy(t *testing.T) {
+	tests := []struct {
+		name  string
+		proxy string
+	}{
+		{name: "unsupported scheme", proxy: "ftp://localhost:1080"},
+		{name: "malformed url", proxy: "://nope"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			resetConfig(t)
+			if err := Configure(Options{Proxy: tt.proxy}); err == nil {
+				t.Errorf("expected error for proxy %q, got nil", tt.proxy)
+			}
+		})
+	}
+}
+
+func TestRateLimit(t *testing.T) {
+	resetConfig(t)
+
+	const ratePerSec = 5
+	if err := Configure(Options{RateLimit: ratePerSec}); err != nil {
+		t.Fatalf("Configure: %v", err)
+	}
+
+	var seen http.Header
+	srv := captureServer(t, &seen)
+	client := Client(5 * time.Second)
+
+	// at 5 req/s the limiter starts with a full burst, so the first batch is
+	// immediate and the next request must wait roughly one tick. fire burst+1
+	// requests and assert the extra one forced a measurable delay.
+	const requests = ratePerSec + 1
+	start := time.Now()
+	for i := 0; i < requests; i++ {
+		get(t, client, srv.URL)
+	}
+	elapsed := time.Since(start)
+
+	// one request beyond the burst should cost about 1/rate; allow slack but
+	// require a non-trivial delay so an unthrottled client fails this.
+	minDelay := time.Second / ratePerSec / 2
+	if elapsed < minDelay {
+		t.Errorf("expected rate limiting to add >= %v of delay, got %v", minDelay, elapsed)
+	}
+}
+
+func TestRateLimitUnlimited(t *testing.T) {
+	resetConfig(t)
+
+	// RateLimit 0 means no limiter is installed; requests should fly through.
+	if err := Configure(Options{RateLimit: 0}); err != nil {
+		t.Fatalf("Configure: %v", err)
+	}
+
+	mu.RLock()
+	rt, ok := configured.(*roundTripper)
+	mu.RUnlock()
+	if !ok {
+		t.Fatal("configured transport is not *roundTripper")
+	}
+	if rt.limiter != nil {
+		t.Error("expected no limiter when RateLimit is 0")
+	}
+}

internal/scan/cloudstorage.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/styles"
 )
@@ -50,9 +51,7 @@ func CloudStorage(url string, timeout time.Duration, logdir string) ([]CloudStor
 		Prefix: "C3",
 	}).With("url", url)
 
-	client := &http.Client{
-		Timeout: timeout,
-	}
+	client := httpx.Client(timeout)
 
 	potentialBuckets := extractPotentialBuckets(sanitizedURL)
 

internal/scan/cms.go

@@ -19,6 +19,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -45,9 +46,7 @@ func CMS(url string, timeout time.Duration, logdir string) (*CMSResult, error) {
 		}
 	}
 
-	client := &http.Client{
-		Timeout: timeout,
-	}
+	client := httpx.Client(timeout)
 
 	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, http.NoBody)
 	if err != nil {

internal/scan/dirlist.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	charmlog "github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -64,12 +65,14 @@ func Dirlist(size string, url string, timeout time.Duration, threads int, logdir
 		list = directoryURL + bigFile
 	}
 
+	client := httpx.Client(timeout)
+
 	req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, list, http.NoBody)
 	if err != nil {
 		log.Error("Error creating directory list request: %s", err)
 		return nil, err
 	}
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := client.Do(req)
 	if err != nil {
 		log.Error("Error downloading directory list: %s", err)
 		return nil, err
@@ -83,10 +86,6 @@ func Dirlist(size string, url string, timeout time.Duration, threads int, logdir
 		directories = append(directories, scanner.Text())
 	}
 
-	client := &http.Client{
-		Timeout: timeout,
-	}
-
 	progress := output.NewProgress(len(directories), "fuzzing")
 
 	var wg sync.WaitGroup

internal/scan/dnslist.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	charmlog "github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -58,7 +59,7 @@ func Dnslist(size string, url string, timeout time.Duration, threads int, logdir
 		log.Error("Error creating request: %s", err)
 		return nil, err
 	}
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := httpx.Client(timeout).Do(req)
 	if err != nil {
 		log.Error("Error downloading DNS list: %s", err)
 		return nil, err
@@ -81,9 +82,11 @@ func Dnslist(size string, url string, timeout time.Duration, threads int, logdir
 		}
 	}
 
-	client := &http.Client{
-		Timeout:   timeout,
-		Transport: dnsTransport,
+	// per-host probe client. dnsTransport pins every dial at a fixture in
+	// integration tests; nil keeps the shared transport for real runs.
+	client := httpx.Client(timeout)
+	if dnsTransport != nil {
+		client.Transport = dnsTransport
 	}
 
 	progress := output.NewProgress(len(dns), "enumerating")

internal/scan/dork.go

@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 	googlesearch "github.com/rocketlaunchr/google-search"
@@ -76,7 +77,7 @@ func Dork(url string, timeout time.Duration, threads int, logdir string) ([]Dork
 		output.Error("Error creating dork list request: %s", err)
 		return nil, err
 	}
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := httpx.Client(timeout).Do(req)
 	if err != nil {
 		spin.Stop()
 		output.Error("Error downloading dork list: %s", err)

internal/scan/frameworks/detect.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	charmlog "github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -47,7 +48,7 @@ func DetectFramework(url string, timeout time.Duration, logdir string) (*Framewo
 	spin := output.NewSpinner("Detecting frameworks")
 	spin.Start()
 
-	client := &http.Client{Timeout: timeout}
+	client := httpx.Client(timeout)
 
 	req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, url, http.NoBody)
 	if err != nil {

internal/scan/git.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	charmlog "github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -48,13 +49,15 @@ func Git(url string, timeout time.Duration, threads int, logdir string) ([]strin
 		}
 	}
 
+	client := httpx.Client(timeout)
+
 	req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, gitURL+gitFile, http.NoBody)
 	if err != nil {
 		spin.Stop()
 		log.Error("Error creating git list request: %s", err)
 		return nil, err
 	}
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := client.Do(req)
 	if err != nil {
 		spin.Stop()
 		log.Error("Error downloading git list: %s", err)
@@ -68,11 +71,6 @@ func Git(url string, timeout time.Duration, threads int, logdir string) ([]strin
 		gitUrls = append(gitUrls, scanner.Text())
 	}
 
-	// util.InitProgressBar()
-	client := &http.Client{
-		Timeout: timeout,
-	}
-
 	var wg sync.WaitGroup
 	var mu sync.Mutex
 	wg.Add(threads)

internal/scan/headers.go

@@ -17,6 +17,7 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -39,9 +40,7 @@ func Headers(url string, timeout time.Duration, logdir string) ([]HeaderResult,
 		}
 	}
 
-	client := &http.Client{
-		Timeout: timeout,
-	}
+	client := httpx.Client(timeout)
 
 	req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, url, http.NoBody)
 	if err != nil {

internal/scan/js/frameworks/next.go

@@ -30,6 +30,7 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/dropalldatabases/sif/internal/httpx"
 	urlutil "github.com/projectdiscovery/utils/url"
 )
 
@@ -48,7 +49,9 @@ func GetPagesRouterScripts(scriptUrl string) ([]string, error) {
 		return nil, err
 	}
 
-	resp, err := http.DefaultClient.Do(req)
+	// no timeout in scope here; 0 matches the previous DefaultClient behavior
+	// while still routing through the shared transport (proxy/headers/rate-limit).
+	resp, err := httpx.Client(0).Do(req)
 	if err != nil {
 		fmt.Println(err)
 		return nil, err

internal/scan/js/scan.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/antchfx/htmlquery"
 	charmlog "github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/output"
 	"github.com/dropalldatabases/sif/internal/scan/js/frameworks"
 	urlutil "github.com/projectdiscovery/utils/url"
@@ -43,6 +44,8 @@ func JavascriptScan(url string, timeout time.Duration, threads int, logdir strin
 	spin := output.NewSpinner("Scanning JavaScript files")
 	spin.Start()
 
+	client := httpx.Client(timeout)
+
 	baseUrl, err := urlutil.Parse(url)
 	if err != nil {
 		spin.Stop()
@@ -53,7 +56,7 @@ func JavascriptScan(url string, timeout time.Duration, threads int, logdir strin
 		spin.Stop()
 		return nil, err
 	}
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := client.Do(req)
 	if err != nil {
 		spin.Stop()
 		return nil, err
@@ -120,7 +123,7 @@ func JavascriptScan(url string, timeout time.Duration, threads int, logdir strin
 			charmlog.Warnf("Failed to create request: %s", err)
 			continue
 		}
-		resp, err := http.DefaultClient.Do(req)
+		resp, err := client.Do(req)
 		if err != nil {
 			charmlog.Warnf("Failed to fetch script: %s", err)
 			continue

internal/scan/js/supabase.go

@@ -30,6 +30,7 @@ import (
 	"time"
 
 	"github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 )
 
 // jwtRegex matches JWT tokens in JavaScript content.
@@ -110,7 +111,7 @@ func getSupabaseOpenAPI(projectId, apikey string, auth *string, timeout time.Dur
 
 // doSupabaseRequest performs a GET request to the Supabase API.
 func doSupabaseRequest(projectId, path, apikey string, auth *string, timeout time.Duration) ([]byte, *http.Response, error) {
-	client := http.Client{Timeout: timeout}
+	client := httpx.Client(timeout)
 
 	req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, "https://"+projectId+".supabase.co"+path, http.NoBody)
 	if err != nil {
@@ -182,7 +183,7 @@ func ScanSupabase(jsContent string, jsUrl string, timeout time.Duration) ([]supa
 		}
 
 		supabaselog.Infof("Found valid supabase project %s with role %s", *supabaseJwt.ProjectId, *supabaseJwt.Role)
-		client := http.Client{Timeout: timeout}
+		client := httpx.Client(timeout)
 
 		req, err := http.NewRequestWithContext(context.TODO(), http.MethodPost, "https://"+*supabaseJwt.ProjectId+".supabase.co/auth/v1/signup", bytes.NewBufferString(`{"email":"automated`+strconv.Itoa(int(time.Now().Unix()))+`@sif.sh","password":"automatedacct"}`))
 		if err != nil {

internal/scan/lfi.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	charmlog "github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -136,14 +137,12 @@ func LFI(targetURL string, timeout time.Duration, threads int, logdir string) (*
 	var mu sync.Mutex
 	var wg sync.WaitGroup
 
-	client := &http.Client{
-		Timeout: timeout,
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
-			if len(via) >= 3 {
-				return http.ErrUseLastResponse
-			}
-			return nil
-		},
+	client := httpx.Client(timeout)
+	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+		if len(via) >= 3 {
+			return http.ErrUseLastResponse
+		}
+		return nil
 	}
 
 	// parse the target URL to check for existing parameters

internal/scan/ports.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	charmlog "github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -50,7 +51,7 @@ func Ports(ctx context.Context, scope string, url string, timeout time.Duration,
 			log.Error("Error creating request: %s", err)
 			return nil, err
 		}
-		resp, err := http.DefaultClient.Do(req)
+		resp, err := httpx.Client(timeout).Do(req)
 		if err != nil {
 			log.Error("Error downloading ports list: %s", err)
 			return nil, err

internal/scan/scan.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -85,11 +86,9 @@ func Scan(url string, timeout time.Duration, threads int, logdir string) {
 		}
 	}
 
-	client := &http.Client{
-		Timeout: timeout,
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
-			return http.ErrUseLastResponse
-		},
+	client := httpx.Client(timeout)
+	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+		return http.ErrUseLastResponse
 	}
 
 	resp := fetchRobotsTXT(url+"/robots.txt", client)

internal/scan/securityheaders.go

@@ -19,6 +19,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -64,9 +65,7 @@ func SecurityHeaders(url string, timeout time.Duration, logdir string) (Security
 		}
 	}
 
-	client := &http.Client{
-		Timeout: timeout,
-	}
+	client := httpx.Client(timeout)
 
 	req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, url, http.NoBody)
 	if err != nil {

internal/scan/securitytrails.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -73,7 +74,7 @@ func SecurityTrails(targetURL string, timeout time.Duration, logdir string) (*Se
 	}
 	hostname := parsedURL.Hostname()
 
-	client := &http.Client{Timeout: timeout}
+	client := httpx.Client(timeout)
 
 	result := &SecurityTrailsResult{
 		Domain: hostname,

internal/scan/shodan.go

@@ -24,6 +24,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -180,7 +181,7 @@ func resolveHostname(hostname string) (string, error) {
 }
 
 func queryShodanHost(ip string, apiKey string, timeout time.Duration) (*ShodanResult, error) {
-	client := &http.Client{Timeout: timeout}
+	client := httpx.Client(timeout)
 
 	reqURL := fmt.Sprintf("%s/shodan/host/%s?key=%s", shodanBaseURL, ip, apiKey)
 	req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, reqURL, http.NoBody)

internal/scan/sql.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	charmlog "github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -140,14 +141,12 @@ func SQL(targetURL string, timeout time.Duration, threads int, logdir string) (*
 	var mu sync.Mutex
 	var wg sync.WaitGroup
 
-	client := &http.Client{
-		Timeout: timeout,
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
-			if len(via) >= 3 {
-				return http.ErrUseLastResponse
-			}
-			return nil
-		},
+	client := httpx.Client(timeout)
+	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+		if len(via) >= 3 {
+			return http.ErrUseLastResponse
+		}
+		return nil
 	}
 
 	// check for admin panels

internal/scan/subdomaintakeover.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/styles"
 )
@@ -54,9 +55,7 @@ func SubdomainTakeover(url string, dnsResults []string, timeout time.Duration, t
 		Prefix: "Subdomain Takeover",
 	})
 
-	client := &http.Client{
-		Timeout: timeout,
-	}
+	client := httpx.Client(timeout)
 
 	var wg sync.WaitGroup
 	wg.Add(threads)

man/sif.1

@@ -103,11 +103,23 @@ per\-request timeout (default 10s).
 directory to write logs to.
 .TP
 .BR \-\-threads " \fIn\fR"
-number of concurrent workers (default 10).
+number of concurrent workers (default 10). values below 1 are clamped to 1.
 .TP
 .BR \-\-template " \fIname\fR"
 sif runtime template to use.
 .TP
+.BR \-proxy " \fIurl\fR"
+route every request through a proxy. accepts http, https or socks5 urls.
+.TP
+.BR \-H ", " \-\-header " \fIstring\fR"
+custom header to send with every request, as \fBKey: Value\fR. repeatable or comma\-separated.
+.TP
+.BR \-cookie " \fIstring\fR"
+cookie header to send with every request.
+.TP
+.BR \-rate\-limit " \fIn\fR"
+cap outbound requests per second (0 = unlimited, default 0).
+.TP
 .B \-api
 emit json results and suppress the interactive output.
 .SH MODULES

sif.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/charmbracelet/log"
 	"github.com/dropalldatabases/sif/internal/config"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/modules"
 	"github.com/dropalldatabases/sif/internal/output"
@@ -180,6 +181,18 @@ func (app *App) Run() error {
 		}()
 	}
 
+	// wire proxy/headers/cookie/rate-limit into the shared http client once,
+	// before any scanner runs. a bad proxy/header shouldn't kill the run -
+	// scanners fall back to a plain client if this fails.
+	if err := httpx.Configure(httpx.Options{
+		Proxy:     app.settings.Proxy,
+		Headers:   app.settings.Header,
+		Cookie:    app.settings.Cookie,
+		RateLimit: app.settings.RateLimit,
+	}); err != nil {
+		log.Warnf("http client config failed, continuing with defaults: %v", err)
+	}
+
 	// target expansion - securitytrails discovers new domains before scanning
 	if app.settings.SecurityTrails {
 		expanded := app.expandTargets()