feat: shared http client with proxy, custom headers and rate limiting

every scanner spun up its own &http.Client, so there was no single place
to apply a proxy, custom headers, a cookie or a rate limit. add an
internal/httpx package that builds one configured transport at startup and
hand it to every scanner via httpx.Client(timeout), keeping behavior
identical when nothing is set (plain client when Configure was never
called).

- httpx.Configure wires -proxy (http/https/socks5), -H/--header, -cookie
  and -rate-limit into a package-level RoundTripper that paces via a
  rate.Limiter and only sets headers the caller hasn't already, so a
  scanner's explicit api key still wins.
- route the scan/wordlist downloads that used http.DefaultClient through
  the shared client too; ports tcp dialing is left untouched.
- clamp -threads to a floor of 1: it feeds wg.Add across the scanners, so
  0 was a silent no-op and a negative value panicked the waitgroup.

document the new flags in the readme, usage docs and man page.

README.md

@@ -172,6 +172,24 @@ sif has a modular architecture. modules are defined in yaml and can be extended
 | `-lfi` | local file inclusion |
 | `-framework` | framework detection with cve lookup |
 
+### http options
+
+these apply to every outbound request across all scanners:
+
+| flag | description |
+|------|-------------|
+| `-proxy` | route all traffic through a proxy (http/https/socks5 url) |
+| `-H`, `--header` | custom header to send (repeatable or comma-separated, `"Key: Value"`) |
+| `-cookie` | cookie header to send with every request |
+| `-rate-limit` | max requests per second (0 = unlimited, default 0) |
+
+```bash
+# scan through a socks5 proxy with a custom header, cookie and 20 req/s cap
+./sif -u https://example.com -headers -proxy socks5://127.0.0.1:1080 -H "Authorization: Bearer tok" -cookie "session=abc" -rate-limit 20
+```
+
+a scanner that sets a header explicitly (e.g. an api key) always wins over the global default.
+
 ### yaml modules
 
 list available modules:

docs/usage.md

@@ -225,7 +225,7 @@ http request timeout (default: 10s):
 
 ### --threads
 
-number of concurrent threads (default: 10):
+number of concurrent threads (default: 10). values below 1 are clamped to 1:
 
 ```bash
 ./sif -u https://example.com --threads 20
@@ -247,6 +247,42 @@ enable debug logging:
 ./sif -u https://example.com -d
 ```
 
+## http options
+
+these apply to every outbound request across all scanners (proxy, custom headers, cookie and rate limiting share one client). a scanner that sets a header explicitly still wins over the global default.
+
+### -proxy
+
+route all traffic through a proxy. supports http, https and socks5 urls:
+
+```bash
+./sif -u https://example.com -proxy socks5://127.0.0.1:1080
+```
+
+### -H, --header
+
+add a custom header to every request. repeatable or comma-separated, `"Key: Value"`:
+
+```bash
+./sif -u https://example.com -H "Authorization: Bearer tok" -H "X-Env: staging"
+```
+
+### -cookie
+
+cookie header to send with every request:
+
+```bash
+./sif -u https://example.com -cookie "session=abc; theme=dark"
+```
+
+### -rate-limit
+
+cap outbound requests per second (0 = unlimited, default 0):
+
+```bash
+./sif -u https://example.com -rate-limit 20
+```
+
 ## api options
 
 ### -api

flake.lock

@@ -2,11 +2,11 @@
   "nodes": {
     "nixpkgs": {
       "locked": {
-        "lastModified": 1767364772,
-        "narHash": "sha256-fFUnEYMla8b7UKjijLnMe+oVFOz6HjijGGNS1l7dYaQ=",
+        "lastModified": 1780930886,
+        "narHash": "sha256-rppURzHviaQN131F+nLiLdGfcb0uCd9gGP0E5+iw9MI=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "16c7794d0a28b5a37904d55bcca36003b9109aaa",
+        "rev": "8c3cede7ddc26bd659d2d383b5610efbd2c7a16e",
         "type": "github"
       },
       "original": {

flake.nix

@@ -21,7 +21,7 @@
             version = "unstable-${self.shortRev or self.dirtyShortRev or "dev"}";
             src = ./.;
 
-            vendorHash = "sha256-ztKXnOjZS/jMxsRjtF0rIZ3lKv4YjMdZd6oQFRuAtR4=";
+            vendorHash = "sha256-fR63/dStMsZon22vancuLWIAvZiEYMLjMwY1kmRDNgM=";
 
             # Tests require network access (httptest)
             doCheck = false;

go.mod

@@ -12,6 +12,8 @@ require (
 	github.com/projectdiscovery/nuclei/v3 v3.8.0
 	github.com/projectdiscovery/utils v0.10.1
 	github.com/rocketlaunchr/google-search v1.1.6
+	golang.org/x/net v0.53.0
+	golang.org/x/time v0.14.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -377,13 +379,11 @@ require (
 	golang.org/x/crypto v0.50.0 // indirect
 	golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect
 	golang.org/x/mod v0.35.0 // indirect
-	golang.org/x/net v0.53.0 // indirect
 	golang.org/x/oauth2 v0.34.0 // indirect
 	golang.org/x/sync v0.20.0 // indirect
 	golang.org/x/sys v0.43.0 // indirect
 	golang.org/x/term v0.42.0 // indirect
 	golang.org/x/text v0.36.0 // indirect
-	golang.org/x/time v0.14.0 // indirect
 	golang.org/x/tools v0.44.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.36.6 // indirect

internal/config/config.go

@@ -51,8 +51,17 @@ type Settings struct {
 	ModuleTags        string // Run modules matching these tags
 	AllModules        bool   // Run all loaded modules
 	ListModules       bool   // List available modules and exit
+	Proxy             string
+	Header            goflags.StringSlice // custom request headers ("Key: Value")
+	Cookie            string
+	RateLimit         int
 }
 
+// minThreads is the floor for the worker count. Threads feeds wg.Add across the
+// scanners, so 0 silently runs nothing and a negative value panics with
+// "negative WaitGroup counter"; clamp the parsed value up to this.
+const minThreads = 1
+
 const (
 	Nil goflags.EnumVariable = iota
 
@@ -109,6 +118,13 @@ func Parse() *Settings {
 		flagSet.StringVar(&settings.Template, "template", "", "Sif runtime template to use"),
 	)
 
+	flagSet.CreateGroup("http", "HTTP",
+		flagSet.StringVar(&settings.Proxy, "proxy", "", "Proxy for all requests (http/https/socks5 url)"),
+		flagSet.StringSliceVarP(&settings.Header, "header", "H", nil, "Custom header to send (repeatable or comma-separated, \"Key: Value\")", goflags.CommaSeparatedStringSliceOptions),
+		flagSet.StringVar(&settings.Cookie, "cookie", "", "Cookie header to send with every request"),
+		flagSet.IntVar(&settings.RateLimit, "rate-limit", 0, "Max requests per second (0 = unlimited)"),
+	)
+
 	flagSet.CreateGroup("api", "API",
 		flagSet.BoolVar(&settings.ApiMode, "api", false, "Enable API mode. Only useful for internal lunchcat usage"),
 	)
@@ -124,5 +140,11 @@ func Parse() *Settings {
 		log.Fatalf("Could not parse flags: %s", err)
 	}
 
+	// threads feeds wg.Add directly; floor it so 0 isn't a silent no-op and a
+	// negative value can't panic the waitgroup.
+	if settings.Threads < minThreads {
+		settings.Threads = minThreads
+	}
+
 	return settings
 }

internal/httpx/httpx.go

@@ -0,0 +1,195 @@
+/*
+·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
+:                                                                               :
+:   █▀ █ █▀▀   ·   Blazing-fast pentesting suite                                :
+:   ▄█ █ █▀    ·   BSD 3-Clause License                                         :
+:                                                                               :
+:   (c) 2022-2026 vmfunc, xyzeva,                                               :
+:                 lunchcat alumni & contributors                                :
+:                                                                               :
+·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
+*/
+
+// Package httpx is the shared http layer every scanner talks through, so a
+// single Configure call wires proxy, custom headers, cookies and rate limiting
+// into every outbound request without touching scanner signatures.
+package httpx
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/net/proxy"
+	"golang.org/x/time/rate"
+)
+
+// allowed proxy schemes
+const (
+	schemeHTTP   = "http"
+	schemeHTTPS  = "https"
+	schemeSOCKS5 = "socks5"
+)
+
+// a header is "Key: Value"; this is the separator between the two halves.
+const headerSep = ": "
+
+// burst lets the limiter absorb a small spike before pacing kicks in; a burst
+// equal to the per-second rate keeps the cap honest over any one-second window.
+const limiterBurstPerRate = 1
+
+// Options carries the runtime knobs that apply to every outbound request.
+// RateLimit is requests/sec (0 = unlimited); Headers are "Key: Value" strings.
+type Options struct {
+	Proxy     string
+	Headers   []string
+	Cookie    string
+	UserAgent string
+	RateLimit int
+}
+
+// configured holds the package-level transport built once by Configure. nil
+// means Configure was never called, so Client falls back to a plain client.
+var (
+	mu         sync.RWMutex
+	configured http.RoundTripper
+)
+
+// Configure builds the shared transport once at startup from opts. Calling it
+// again replaces the previous configuration.
+//
+//nolint:gocritic // signature is the package's stable startup api; called once.
+func Configure(opts Options) error {
+	base, err := buildTransport(opts.Proxy)
+	if err != nil {
+		return err
+	}
+
+	headers, err := parseHeaders(opts.Headers)
+	if err != nil {
+		return err
+	}
+
+	var limiter *rate.Limiter
+	if opts.RateLimit > 0 {
+		limiter = rate.NewLimiter(rate.Limit(opts.RateLimit), opts.RateLimit*limiterBurstPerRate)
+	}
+
+	rt := &roundTripper{
+		base:      base,
+		headers:   headers,
+		cookie:    opts.Cookie,
+		userAgent: opts.UserAgent,
+		limiter:   limiter,
+	}
+
+	mu.Lock()
+	configured = rt
+	mu.Unlock()
+
+	return nil
+}
+
+// Client returns an http client wired to the configured transport. It works
+// before Configure is ever called (plain transport) so existing code and tests
+// behave unchanged. A zero timeout means no timeout, matching http.Client.
+func Client(timeout time.Duration) *http.Client {
+	mu.RLock()
+	rt := configured
+	mu.RUnlock()
+
+	return &http.Client{Timeout: timeout, Transport: rt}
+}
+
+// buildTransport clones the default transport and applies the proxy. An empty
+// proxy leaves the default behavior (respects HTTP_PROXY env) intact.
+func buildTransport(proxyURL string) (*http.Transport, error) {
+	tr, ok := http.DefaultTransport.(*http.Transport)
+	if !ok {
+		// unreachable in practice, but never trust an assertion silently.
+		return nil, fmt.Errorf("default transport is not *http.Transport")
+	}
+	transport := tr.Clone()
+
+	if proxyURL == "" {
+		return transport, nil
+	}
+
+	parsed, err := url.Parse(proxyURL)
+	if err != nil {
+		return nil, fmt.Errorf("parse proxy url %q: %w", proxyURL, err)
+	}
+
+	switch parsed.Scheme {
+	case schemeHTTP, schemeHTTPS:
+		transport.Proxy = http.ProxyURL(parsed)
+	case schemeSOCKS5:
+		// socks5 needs a custom dialer; the returned dialer implements
+		// ContextDialer so cancellation/timeouts propagate.
+		dialer, err := proxy.SOCKS5("tcp", parsed.Host, nil, proxy.Direct)
+		if err != nil {
+			return nil, fmt.Errorf("socks5 proxy %q: %w", proxyURL, err)
+		}
+		ctxDialer, ok := dialer.(proxy.ContextDialer)
+		if !ok {
+			return nil, fmt.Errorf("socks5 proxy %q: dialer lacks context support", proxyURL)
+		}
+		transport.DialContext = ctxDialer.DialContext
+	default:
+		return nil, fmt.Errorf("unsupported proxy scheme %q (want http/https/socks5)", parsed.Scheme)
+	}
+
+	return transport, nil
+}
+
+// parseHeaders splits each "Key: Value" entry on the first ": ". Entries
+// without the separator are rejected so a typo fails loud instead of silently.
+// The returned map is always non-nil so callers can range it unconditionally.
+func parseHeaders(raw []string) (map[string]string, error) {
+	headers := make(map[string]string, len(raw))
+	for i := 0; i < len(raw); i++ {
+		key, value, ok := strings.Cut(raw[i], headerSep)
+		if !ok {
+			return nil, fmt.Errorf("invalid header %q (want \"Key: Value\")", raw[i])
+		}
+		headers[key] = value
+	}
+
+	return headers, nil
+}
+
+// roundTripper paces and decorates each request before delegating to base.
+type roundTripper struct {
+	base      *http.Transport
+	headers   map[string]string
+	cookie    string
+	userAgent string
+	limiter   *rate.Limiter
+}
+
+func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	if rt.limiter != nil {
+		if err := rt.limiter.Wait(req.Context()); err != nil {
+			return nil, fmt.Errorf("rate limiter: %w", err)
+		}
+	}
+
+	// only set what the caller hasn't already; a scanner that explicitly sets a
+	// header (e.g. an api key) must win over the global default.
+	for key, value := range rt.headers {
+		if req.Header.Get(key) == "" {
+			req.Header.Set(key, value)
+		}
+	}
+	if rt.cookie != "" && req.Header.Get("Cookie") == "" {
+		req.Header.Set("Cookie", rt.cookie)
+	}
+	if rt.userAgent != "" && req.Header.Get("User-Agent") == "" {
+		req.Header.Set("User-Agent", rt.userAgent)
+	}
+
+	return rt.base.RoundTrip(req)
+}

internal/httpx/httpx_test.go

@@ -0,0 +1,217 @@
+/*
+·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
+:                                                                               :
+:   █▀ █ █▀▀   ·   Blazing-fast pentesting suite                                :
+:   ▄█ █ █▀    ·   BSD 3-Clause License                                         :
+:                                                                               :
+:   (c) 2022-2026 vmfunc, xyzeva,                                               :
+:                 lunchcat alumni & contributors                                :
+:                                                                               :
+·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
+*/
+
+package httpx
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+// resetConfig clears the package-level transport so each test starts clean.
+func resetConfig(t *testing.T) {
+	t.Helper()
+	mu.Lock()
+	configured = nil
+	mu.Unlock()
+}
+
+// captureServer records the headers of the last request it served.
+func captureServer(t *testing.T, seen *http.Header) *httptest.Server {
+	t.Helper()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		*seen = r.Header.Clone()
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func get(t *testing.T, client *http.Client, url string) {
+	t.Helper()
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, http.NoBody)
+	if err != nil {
+		t.Fatalf("new request: %v", err)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		t.Fatalf("do request: %v", err)
+	}
+	resp.Body.Close()
+}
+
+func TestClientBeforeConfigure(t *testing.T) {
+	resetConfig(t)
+
+	var seen http.Header
+	srv := captureServer(t, &seen)
+
+	// a client must work with no Configure call so existing code is unaffected.
+	get(t, Client(5*time.Second), srv.URL)
+
+	if seen == nil {
+		t.Fatal("request never reached the server")
+	}
+}
+
+func TestConfigureHeadersAndCookie(t *testing.T) {
+	tests := []struct {
+		name      string
+		opts      Options
+		wantKey   string
+		wantValue string
+	}{
+		{
+			name:      "custom header injected",
+			opts:      Options{Headers: []string{"X-Test: sif"}},
+			wantKey:   "X-Test",
+			wantValue: "sif",
+		},
+		{
+			name:      "cookie injected",
+			opts:      Options{Cookie: "session=abc"},
+			wantKey:   "Cookie",
+			wantValue: "session=abc",
+		},
+		{
+			name:      "user agent injected",
+			opts:      Options{UserAgent: "sif-scanner"},
+			wantKey:   "User-Agent",
+			wantValue: "sif-scanner",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			resetConfig(t)
+
+			if err := Configure(tt.opts); err != nil {
+				t.Fatalf("Configure: %v", err)
+			}
+
+			var seen http.Header
+			srv := captureServer(t, &seen)
+			get(t, Client(5*time.Second), srv.URL)
+
+			if got := seen.Get(tt.wantKey); got != tt.wantValue {
+				t.Errorf("header %q = %q, want %q", tt.wantKey, got, tt.wantValue)
+			}
+		})
+	}
+}
+
+func TestConfigureHeaderDoesNotOverride(t *testing.T) {
+	resetConfig(t)
+
+	if err := Configure(Options{Headers: []string{"X-Test: global"}}); err != nil {
+		t.Fatalf("Configure: %v", err)
+	}
+
+	var seen http.Header
+	srv := captureServer(t, &seen)
+
+	// a caller that sets the header explicitly must win over the global default.
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, srv.URL, http.NoBody)
+	if err != nil {
+		t.Fatalf("new request: %v", err)
+	}
+	req.Header.Set("X-Test", "caller")
+	resp, err := Client(5 * time.Second).Do(req)
+	if err != nil {
+		t.Fatalf("do request: %v", err)
+	}
+	resp.Body.Close()
+
+	if got := seen.Get("X-Test"); got != "caller" {
+		t.Errorf("X-Test = %q, want caller (caller value must not be overridden)", got)
+	}
+}
+
+func TestConfigureInvalidHeader(t *testing.T) {
+	resetConfig(t)
+
+	// a header without ": " should fail loud rather than silently dropping.
+	if err := Configure(Options{Headers: []string{"missing-separator"}}); err == nil {
+		t.Fatal("expected error for malformed header, got nil")
+	}
+}
+
+func TestConfigureInvalidProxy(t *testing.T) {
+	tests := []struct {
+		name  string
+		proxy string
+	}{
+		{name: "unsupported scheme", proxy: "ftp://localhost:1080"},
+		{name: "malformed url", proxy: "://nope"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			resetConfig(t)
+			if err := Configure(Options{Proxy: tt.proxy}); err == nil {
+				t.Errorf("expected error for proxy %q, got nil", tt.proxy)
+			}
+		})
+	}
+}
+
+func TestRateLimit(t *testing.T) {
+	resetConfig(t)
+
+	const ratePerSec = 5
+	if err := Configure(Options{RateLimit: ratePerSec}); err != nil {
+		t.Fatalf("Configure: %v", err)
+	}
+
+	var seen http.Header
+	srv := captureServer(t, &seen)
+	client := Client(5 * time.Second)
+
+	// at 5 req/s the limiter starts with a full burst, so the first batch is
+	// immediate and the next request must wait roughly one tick. fire burst+1
+	// requests and assert the extra one forced a measurable delay.
+	const requests = ratePerSec + 1
+	start := time.Now()
+	for i := 0; i < requests; i++ {
+		get(t, client, srv.URL)
+	}
+	elapsed := time.Since(start)
+
+	// one request beyond the burst should cost about 1/rate; allow slack but
+	// require a non-trivial delay so an unthrottled client fails this.
+	minDelay := time.Second / ratePerSec / 2
+	if elapsed < minDelay {
+		t.Errorf("expected rate limiting to add >= %v of delay, got %v", minDelay, elapsed)
+	}
+}
+
+func TestRateLimitUnlimited(t *testing.T) {
+	resetConfig(t)
+
+	// RateLimit 0 means no limiter is installed; requests should fly through.
+	if err := Configure(Options{RateLimit: 0}); err != nil {
+		t.Fatalf("Configure: %v", err)
+	}
+
+	mu.RLock()
+	rt, ok := configured.(*roundTripper)
+	mu.RUnlock()
+	if !ok {
+		t.Fatal("configured transport is not *roundTripper")
+	}
+	if rt.limiter != nil {
+		t.Error("expected no limiter when RateLimit is 0")
+	}
+}

internal/output/progress.go

@@ -28,12 +28,13 @@ const (
 
 // Progress displays a progress bar for operations with known counts
 type Progress struct {
-	total    int64
-	current  int64
-	message  string
-	lastItem string
-	mu       sync.Mutex
-	paused   bool
+	total     int64
+	current   int64
+	message   string
+	lastItem  string
+	mu        sync.Mutex
+	paused    bool
+	lastShown int // last printed milestone bucket in non-tty mode
 }
 
 // NewProgress creates a new progress bar
@@ -110,8 +111,30 @@ func (p *Progress) render() {
 		}
 		percent := int(current * 100 / total)
 
-		// Print at 0%, 25%, 50%, 75%, 100%
-		if current == 1 || percent == 25 || percent == 50 || percent == 75 || current == total {
+		// map current to a milestone bucket (0=none,1..5). concurrent workers
+		// hammer the same bucket, so only print when the bucket advances.
+		bucket := 0
+		switch {
+		case current >= total:
+			bucket = 5
+		case percent >= 75:
+			bucket = 4
+		case percent >= 50:
+			bucket = 3
+		case percent >= 25:
+			bucket = 2
+		case current >= 1:
+			bucket = 1
+		}
+
+		p.mu.Lock()
+		advanced := bucket > p.lastShown
+		if advanced {
+			p.lastShown = bucket
+		}
+		p.mu.Unlock()
+
+		if advanced {
 			fmt.Printf("    [%d%%] %d/%d\n", percent, current, total)
 		}
 		return

internal/output/progress_test.go

@@ -12,7 +12,12 @@
 
 package output
 
-import "testing"
+import (
+	"os"
+	"strings"
+	"sync"
+	"testing"
+)
 
 // the non-tty milestone path divides current*100/total, so a zero-total bar
 // used to panic with integer divide-by-zero when piped or redirected.
@@ -32,3 +37,60 @@ func TestProgressCounts(t *testing.T) {
 		t.Errorf("current = %d, want 4", p.current)
 	}
 }
+
+// many concurrent workers used to spam the same milestone bucket (e.g. ten
+// "[25%] .../1000" lines). each bucket must now print at most once.
+func TestProgressNonTTYDedupesMilestones(t *testing.T) {
+	savedTTY, savedAPI := IsTTY, apiMode
+	IsTTY, apiMode = false, false
+	defer func() { IsTTY, apiMode = savedTTY, savedAPI }()
+
+	out := captureStdout(t, func() {
+		p := NewProgress(1000, "scanning")
+		var wg sync.WaitGroup
+		for i := 0; i < 40; i++ {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				for j := 0; j < 25; j++ {
+					p.Increment("x")
+				}
+			}()
+		}
+		wg.Wait()
+	})
+
+	lines := strings.Count(out, "\n")
+	if lines > 5 {
+		t.Errorf("printed %d milestone lines, want <=5:\n%s", lines, out)
+	}
+}
+
+func captureStdout(t *testing.T, fn func()) string {
+	t.Helper()
+	r, w, err := os.Pipe()
+	if err != nil {
+		t.Fatalf("pipe: %v", err)
+	}
+	saved := os.Stdout
+	os.Stdout = w
+
+	done := make(chan string, 1)
+	go func() {
+		buf := make([]byte, 0, 4096)
+		tmp := make([]byte, 1024)
+		for {
+			n, rerr := r.Read(tmp)
+			buf = append(buf, tmp[:n]...)
+			if rerr != nil {
+				break
+			}
+		}
+		done <- string(buf)
+	}()
+
+	fn()
+	os.Stdout = saved
+	w.Close()
+	return <-done
+}

internal/scan/cloudstorage.go

@@ -21,10 +21,15 @@ import (
 	"time"
 
 	"github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/styles"
 )
 
+// s3EndpointFmt is a var so integration tests can repoint it at a fixture; the
+// %s is the bucket name.
+var s3EndpointFmt = "https://%s.s3.amazonaws.com"
+
 type CloudStorageResult struct {
 	BucketName string `json:"bucket_name"`
 	IsPublic   bool   `json:"is_public"`
@@ -46,9 +51,7 @@ func CloudStorage(url string, timeout time.Duration, logdir string) ([]CloudStor
 		Prefix: "C3",
 	}).With("url", url)
 
-	client := &http.Client{
-		Timeout: timeout,
-	}
+	client := httpx.Client(timeout)
 
 	potentialBuckets := extractPotentialBuckets(sanitizedURL)
 
@@ -96,7 +99,7 @@ func extractPotentialBuckets(url string) []string {
 }
 
 func checkS3Bucket(ctx context.Context, bucket string, client *http.Client) (bool, error) {
-	url := fmt.Sprintf("https://%s.s3.amazonaws.com", bucket)
+	url := fmt.Sprintf(s3EndpointFmt, bucket)
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody)
 	if err != nil {
 		return false, err

internal/scan/cms.go

@@ -19,6 +19,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -45,9 +46,7 @@ func CMS(url string, timeout time.Duration, logdir string) (*CMSResult, error) {
 		}
 	}
 
-	client := &http.Client{
-		Timeout: timeout,
-	}
+	client := httpx.Client(timeout)
 
 	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, http.NoBody)
 	if err != nil {

internal/scan/dirlist.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	charmlog "github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -64,12 +65,14 @@ func Dirlist(size string, url string, timeout time.Duration, threads int, logdir
 		list = directoryURL + bigFile
 	}
 
+	client := httpx.Client(timeout)
+
 	req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, list, http.NoBody)
 	if err != nil {
 		log.Error("Error creating directory list request: %s", err)
 		return nil, err
 	}
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := client.Do(req)
 	if err != nil {
 		log.Error("Error downloading directory list: %s", err)
 		return nil, err
@@ -83,10 +86,6 @@ func Dirlist(size string, url string, timeout time.Duration, threads int, logdir
 		directories = append(directories, scanner.Text())
 	}
 
-	client := &http.Client{
-		Timeout: timeout,
-	}
-
 	progress := output.NewProgress(len(directories), "fuzzing")
 
 	var wg sync.WaitGroup

internal/scan/dnslist.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	charmlog "github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -28,6 +29,10 @@ import (
 // dnsURL is a var so integration tests can repoint it at a fixture.
 var dnsURL = "https://raw.githubusercontent.com/dropalldatabases/sif-runtime/main/dnslist/"
 
+// dnsTransport is a var so integration tests can route the per-host probes at a
+// local server instead of resolving real DNS. nil keeps http.DefaultTransport.
+var dnsTransport http.RoundTripper
+
 const (
 	dnsSmallFile  = "subdomains-100.txt"
 	dnsMediumFile = "subdomains-1000.txt"
@@ -54,7 +59,7 @@ func Dnslist(size string, url string, timeout time.Duration, threads int, logdir
 		log.Error("Error creating request: %s", err)
 		return nil, err
 	}
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := httpx.Client(timeout).Do(req)
 	if err != nil {
 		log.Error("Error downloading DNS list: %s", err)
 		return nil, err
@@ -77,8 +82,11 @@ func Dnslist(size string, url string, timeout time.Duration, threads int, logdir
 		}
 	}
 
-	client := &http.Client{
-		Timeout: timeout,
+	// per-host probe client. dnsTransport pins every dial at a fixture in
+	// integration tests; nil keeps the shared transport for real runs.
+	client := httpx.Client(timeout)
+	if dnsTransport != nil {
+		client.Transport = dnsTransport
 	}
 
 	progress := output.NewProgress(len(dns), "enumerating")

internal/scan/dork.go

@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 	googlesearch "github.com/rocketlaunchr/google-search"
@@ -76,7 +77,7 @@ func Dork(url string, timeout time.Duration, threads int, logdir string) ([]Dork
 		output.Error("Error creating dork list request: %s", err)
 		return nil, err
 	}
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := httpx.Client(timeout).Do(req)
 	if err != nil {
 		spin.Stop()
 		output.Error("Error downloading dork list: %s", err)

internal/scan/frameworks/detect.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	charmlog "github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -47,7 +48,7 @@ func DetectFramework(url string, timeout time.Duration, logdir string) (*Framewo
 	spin := output.NewSpinner("Detecting frameworks")
 	spin.Start()
 
-	client := &http.Client{Timeout: timeout}
+	client := httpx.Client(timeout)
 
 	req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, url, http.NoBody)
 	if err != nil {

internal/scan/git.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	charmlog "github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -48,13 +49,15 @@ func Git(url string, timeout time.Duration, threads int, logdir string) ([]strin
 		}
 	}
 
+	client := httpx.Client(timeout)
+
 	req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, gitURL+gitFile, http.NoBody)
 	if err != nil {
 		spin.Stop()
 		log.Error("Error creating git list request: %s", err)
 		return nil, err
 	}
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := client.Do(req)
 	if err != nil {
 		spin.Stop()
 		log.Error("Error downloading git list: %s", err)
@@ -68,11 +71,6 @@ func Git(url string, timeout time.Duration, threads int, logdir string) ([]strin
 		gitUrls = append(gitUrls, scanner.Text())
 	}
 
-	// util.InitProgressBar()
-	client := &http.Client{
-		Timeout: timeout,
-	}
-
 	var wg sync.WaitGroup
 	var mu sync.Mutex
 	wg.Add(threads)

internal/scan/headers.go

@@ -17,6 +17,7 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -39,9 +40,7 @@ func Headers(url string, timeout time.Duration, logdir string) ([]HeaderResult,
 		}
 	}
 
-	client := &http.Client{
-		Timeout: timeout,
-	}
+	client := httpx.Client(timeout)
 
 	req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, url, http.NoBody)
 	if err != nil {

internal/scan/integration_test.go

@@ -20,6 +20,7 @@ package scan
 
 import (
 	"context"
+	"encoding/json"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -41,6 +42,9 @@ func newVulnApp() *httptest.Server {
 	mux.HandleFunc("/directory-list-2.3-small.txt", func(w http.ResponseWriter, r *http.Request) {
 		w.Write([]byte("admin\nlogin\nnope\n"))
 	})
+	mux.HandleFunc("/subdomains-100.txt", func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("dev\nstaging\n"))
+	})
 
 	// an exposed git repo: HEAD is a real find, config is html so it's excluded
 	mux.HandleFunc("/.git/HEAD", func(w http.ResponseWriter, r *http.Request) {
@@ -209,6 +213,164 @@ func TestIntegrationPorts(t *testing.T) {
 	}
 }
 
+func TestIntegrationShodan(t *testing.T) {
+	// a local server stands in for api.shodan.io; example.com resolves to a real
+	// IP but the lookup never leaves the box.
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Query().Get("key") != "test-key" {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		json.NewEncoder(w).Encode(shodanHostResponse{
+			IP:        "93.184.216.34",
+			Hostnames: []string{"example.com"},
+			Org:       "EDGECAST",
+			Ports:     []int{80, 443},
+			Data: []shodanData{
+				{Port: 80, Transport: "tcp", Product: "nginx", Version: "1.18.0"},
+			},
+		})
+	}))
+	defer srv.Close()
+	orig := shodanBaseURL
+	shodanBaseURL = srv.URL
+	defer func() { shodanBaseURL = orig }()
+
+	t.Setenv("SHODAN_API_KEY", "test-key")
+
+	result, err := Shodan("https://example.com", 5*time.Second, "")
+	if err != nil {
+		t.Fatalf("Shodan: %v", err)
+	}
+	if result == nil || result.IP != "93.184.216.34" {
+		t.Fatalf("expected parsed shodan result, got %+v", result)
+	}
+	if len(result.Services) != 1 || result.Services[0].Product != "nginx" {
+		t.Errorf("expected one nginx service, got %+v", result.Services)
+	}
+}
+
+func TestIntegrationSecurityTrails(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("APIKEY") != "test-key" {
+			w.WriteHeader(http.StatusForbidden)
+			return
+		}
+		switch {
+		case strings.HasSuffix(r.URL.Path, "/subdomains"):
+			json.NewEncoder(w).Encode(stSubdomainsResponse{Subdomains: []string{"www", "api"}})
+		case strings.HasSuffix(r.URL.Path, "/associated"):
+			json.NewEncoder(w).Encode(stAssociatedResponse{Records: []stAssociatedRecord{{Hostname: "example.org"}}})
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+	orig := securityTrailsBaseURL
+	securityTrailsBaseURL = srv.URL
+	defer func() { securityTrailsBaseURL = orig }()
+
+	t.Setenv("SECURITYTRAILS_API_KEY", "test-key")
+
+	result, err := SecurityTrails("https://example.com", 5*time.Second, "")
+	if err != nil {
+		t.Fatalf("SecurityTrails: %v", err)
+	}
+	if len(result.Subdomains) != 2 {
+		t.Errorf("expected 2 subdomains, got %v", result.Subdomains)
+	}
+	if len(result.AssociatedDomains) != 1 || result.AssociatedDomains[0] != "example.org" {
+		t.Errorf("expected example.org associated, got %v", result.AssociatedDomains)
+	}
+
+	urls := result.DiscoveredURLs()
+	if !contains(urls, "https://www.example.com") || !contains(urls, "https://example.org") {
+		t.Errorf("expected discovered urls to expand subs and associated, got %v", urls)
+	}
+}
+
+func TestIntegrationCloudStorage(t *testing.T) {
+	// the fixture returns 200 only for the planted bucket, so any candidate that
+	// matches it is reported public.
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/example" {
+			w.WriteHeader(http.StatusOK)
+			return
+		}
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer srv.Close()
+	orig := s3EndpointFmt
+	s3EndpointFmt = srv.URL + "/%s"
+	defer func() { s3EndpointFmt = orig }()
+
+	results, err := CloudStorage("https://example.com", 5*time.Second, "")
+	if err != nil {
+		t.Fatalf("CloudStorage: %v", err)
+	}
+
+	var public bool
+	for _, r := range results {
+		if r.BucketName == "example" && r.IsPublic {
+			public = true
+		}
+	}
+	if !public {
+		t.Errorf("expected the example bucket to be flagged public, got %+v", results)
+	}
+}
+
+func TestIntegrationDnslist(t *testing.T) {
+	// the probe server answers any host routed to it; dnsTransport pins every
+	// dial here so no real DNS is touched.
+	probe := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer probe.Close()
+	probeAddr := strings.TrimPrefix(probe.URL, "http://")
+
+	list := newVulnApp()
+	defer list.Close()
+	origURL := dnsURL
+	dnsURL = list.URL + "/"
+	defer func() { dnsURL = origURL }()
+
+	origTr := dnsTransport
+	dnsTransport = &http.Transport{
+		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			return (&net.Dialer{}).DialContext(ctx, network, probeAddr)
+		},
+	}
+	defer func() { dnsTransport = origTr }()
+
+	found, err := Dnslist("small", "http://example.com", 5*time.Second, 2, "")
+	if err != nil {
+		t.Fatalf("Dnslist: %v", err)
+	}
+	// http probes land on the plain-http probe server; https fails the tls
+	// handshake and is dropped, which is fine - the planted sub still shows up.
+	if !hasSuffixIn(sliceSet(found), "dev.example.com") {
+		t.Errorf("expected dev.example.com among findings, got %v", found)
+	}
+}
+
+func contains(s []string, v string) bool {
+	for i := 0; i < len(s); i++ {
+		if s[i] == v {
+			return true
+		}
+	}
+	return false
+}
+
+func sliceSet(s []string) map[string]bool {
+	set := make(map[string]bool, len(s))
+	for i := 0; i < len(s); i++ {
+		set[s[i]] = true
+	}
+	return set
+}
+
 func hasSuffixIn(set map[string]bool, suffix string) bool {
 	for k := range set {
 		if strings.HasSuffix(k, suffix) {

internal/scan/js/frameworks/next.go

@@ -30,6 +30,7 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/dropalldatabases/sif/internal/httpx"
 	urlutil "github.com/projectdiscovery/utils/url"
 )
 
@@ -48,7 +49,9 @@ func GetPagesRouterScripts(scriptUrl string) ([]string, error) {
 		return nil, err
 	}
 
-	resp, err := http.DefaultClient.Do(req)
+	// no timeout in scope here; 0 matches the previous DefaultClient behavior
+	// while still routing through the shared transport (proxy/headers/rate-limit).
+	resp, err := httpx.Client(0).Do(req)
 	if err != nil {
 		fmt.Println(err)
 		return nil, err

internal/scan/js/scan.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/antchfx/htmlquery"
 	charmlog "github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/output"
 	"github.com/dropalldatabases/sif/internal/scan/js/frameworks"
 	urlutil "github.com/projectdiscovery/utils/url"
@@ -43,6 +44,8 @@ func JavascriptScan(url string, timeout time.Duration, threads int, logdir strin
 	spin := output.NewSpinner("Scanning JavaScript files")
 	spin.Start()
 
+	client := httpx.Client(timeout)
+
 	baseUrl, err := urlutil.Parse(url)
 	if err != nil {
 		spin.Stop()
@@ -53,7 +56,7 @@ func JavascriptScan(url string, timeout time.Duration, threads int, logdir strin
 		spin.Stop()
 		return nil, err
 	}
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := client.Do(req)
 	if err != nil {
 		spin.Stop()
 		return nil, err
@@ -120,7 +123,7 @@ func JavascriptScan(url string, timeout time.Duration, threads int, logdir strin
 			charmlog.Warnf("Failed to create request: %s", err)
 			continue
 		}
-		resp, err := http.DefaultClient.Do(req)
+		resp, err := client.Do(req)
 		if err != nil {
 			charmlog.Warnf("Failed to fetch script: %s", err)
 			continue

internal/scan/js/supabase.go

@@ -30,6 +30,7 @@ import (
 	"time"
 
 	"github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 )
 
 // jwtRegex matches JWT tokens in JavaScript content.
@@ -110,7 +111,7 @@ func getSupabaseOpenAPI(projectId, apikey string, auth *string, timeout time.Dur
 
 // doSupabaseRequest performs a GET request to the Supabase API.
 func doSupabaseRequest(projectId, path, apikey string, auth *string, timeout time.Duration) ([]byte, *http.Response, error) {
-	client := http.Client{Timeout: timeout}
+	client := httpx.Client(timeout)
 
 	req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, "https://"+projectId+".supabase.co"+path, http.NoBody)
 	if err != nil {
@@ -182,7 +183,7 @@ func ScanSupabase(jsContent string, jsUrl string, timeout time.Duration) ([]supa
 		}
 
 		supabaselog.Infof("Found valid supabase project %s with role %s", *supabaseJwt.ProjectId, *supabaseJwt.Role)
-		client := http.Client{Timeout: timeout}
+		client := httpx.Client(timeout)
 
 		req, err := http.NewRequestWithContext(context.TODO(), http.MethodPost, "https://"+*supabaseJwt.ProjectId+".supabase.co/auth/v1/signup", bytes.NewBufferString(`{"email":"automated`+strconv.Itoa(int(time.Now().Unix()))+`@sif.sh","password":"automatedacct"}`))
 		if err != nil {

internal/scan/lfi.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	charmlog "github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -136,14 +137,12 @@ func LFI(targetURL string, timeout time.Duration, threads int, logdir string) (*
 	var mu sync.Mutex
 	var wg sync.WaitGroup
 
-	client := &http.Client{
-		Timeout: timeout,
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
-			if len(via) >= 3 {
-				return http.ErrUseLastResponse
-			}
-			return nil
-		},
+	client := httpx.Client(timeout)
+	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+		if len(via) >= 3 {
+			return http.ErrUseLastResponse
+		}
+		return nil
 	}
 
 	// parse the target URL to check for existing parameters

internal/scan/ports.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	charmlog "github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -50,7 +51,7 @@ func Ports(ctx context.Context, scope string, url string, timeout time.Duration,
 			log.Error("Error creating request: %s", err)
 			return nil, err
 		}
-		resp, err := http.DefaultClient.Do(req)
+		resp, err := httpx.Client(timeout).Do(req)
 		if err != nil {
 			log.Error("Error downloading ports list: %s", err)
 			return nil, err

internal/scan/scan.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -85,11 +86,9 @@ func Scan(url string, timeout time.Duration, threads int, logdir string) {
 		}
 	}
 
-	client := &http.Client{
-		Timeout: timeout,
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
-			return http.ErrUseLastResponse
-		},
+	client := httpx.Client(timeout)
+	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+		return http.ErrUseLastResponse
 	}
 
 	resp := fetchRobotsTXT(url+"/robots.txt", client)

internal/scan/securityheaders.go

@@ -19,6 +19,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -64,9 +65,7 @@ func SecurityHeaders(url string, timeout time.Duration, logdir string) (Security
 		}
 	}
 
-	client := &http.Client{
-		Timeout: timeout,
-	}
+	client := httpx.Client(timeout)
 
 	req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, url, http.NoBody)
 	if err != nil {

internal/scan/securitytrails.go

@@ -23,11 +23,13 @@ import (
 	"strings"
 	"time"
 
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
 
-const securityTrailsBaseURL = "https://api.securitytrails.com/v1"
+// securityTrailsBaseURL is a var so integration tests can repoint it at a fixture.
+var securityTrailsBaseURL = "https://api.securitytrails.com/v1"
 
 // SecurityTrailsResult holds discovered domains from SecurityTrails API
 type SecurityTrailsResult struct {
@@ -72,7 +74,7 @@ func SecurityTrails(targetURL string, timeout time.Duration, logdir string) (*Se
 	}
 	hostname := parsedURL.Hostname()
 
-	client := &http.Client{Timeout: timeout}
+	client := httpx.Client(timeout)
 
 	result := &SecurityTrailsResult{
 		Domain: hostname,

internal/scan/shodan.go

@@ -24,11 +24,13 @@ import (
 	"strings"
 	"time"
 
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
 
-const shodanBaseURL = "https://api.shodan.io"
+// shodanBaseURL is a var so integration tests can repoint it at a fixture.
+var shodanBaseURL = "https://api.shodan.io"
 
 // ShodanResult represents the results from a Shodan host lookup
 type ShodanResult struct {
@@ -179,7 +181,7 @@ func resolveHostname(hostname string) (string, error) {
 }
 
 func queryShodanHost(ip string, apiKey string, timeout time.Duration) (*ShodanResult, error) {
-	client := &http.Client{Timeout: timeout}
+	client := httpx.Client(timeout)
 
 	reqURL := fmt.Sprintf("%s/shodan/host/%s?key=%s", shodanBaseURL, ip, apiKey)
 	req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, reqURL, http.NoBody)

internal/scan/sql.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	charmlog "github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/output"
 )
@@ -140,14 +141,12 @@ func SQL(targetURL string, timeout time.Duration, threads int, logdir string) (*
 	var mu sync.Mutex
 	var wg sync.WaitGroup
 
-	client := &http.Client{
-		Timeout: timeout,
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
-			if len(via) >= 3 {
-				return http.ErrUseLastResponse
-			}
-			return nil
-		},
+	client := httpx.Client(timeout)
+	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+		if len(via) >= 3 {
+			return http.ErrUseLastResponse
+		}
+		return nil
 	}
 
 	// check for admin panels

internal/scan/subdomaintakeover.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/charmbracelet/log"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/styles"
 )
@@ -54,9 +55,7 @@ func SubdomainTakeover(url string, dnsResults []string, timeout time.Duration, t
 		Prefix: "Subdomain Takeover",
 	})
 
-	client := &http.Client{
-		Timeout: timeout,
-	}
+	client := httpx.Client(timeout)
 
 	var wg sync.WaitGroup
 	wg.Add(threads)

man/sif.1

@@ -103,11 +103,23 @@ per\-request timeout (default 10s).
 directory to write logs to.
 .TP
 .BR \-\-threads " \fIn\fR"
-number of concurrent workers (default 10).
+number of concurrent workers (default 10). values below 1 are clamped to 1.
 .TP
 .BR \-\-template " \fIname\fR"
 sif runtime template to use.
 .TP
+.BR \-proxy " \fIurl\fR"
+route every request through a proxy. accepts http, https or socks5 urls.
+.TP
+.BR \-H ", " \-\-header " \fIstring\fR"
+custom header to send with every request, as \fBKey: Value\fR. repeatable or comma\-separated.
+.TP
+.BR \-cookie " \fIstring\fR"
+cookie header to send with every request.
+.TP
+.BR \-rate\-limit " \fIn\fR"
+cap outbound requests per second (0 = unlimited, default 0).
+.TP
 .B \-api
 emit json results and suppress the interactive output.
 .SH MODULES

sif.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/charmbracelet/log"
 	"github.com/dropalldatabases/sif/internal/config"
+	"github.com/dropalldatabases/sif/internal/httpx"
 	"github.com/dropalldatabases/sif/internal/logger"
 	"github.com/dropalldatabases/sif/internal/modules"
 	"github.com/dropalldatabases/sif/internal/output"
@@ -180,6 +181,18 @@ func (app *App) Run() error {
 		}()
 	}
 
+	// wire proxy/headers/cookie/rate-limit into the shared http client once,
+	// before any scanner runs. a bad proxy/header shouldn't kill the run -
+	// scanners fall back to a plain client if this fails.
+	if err := httpx.Configure(httpx.Options{
+		Proxy:     app.settings.Proxy,
+		Headers:   app.settings.Header,
+		Cookie:    app.settings.Cookie,
+		RateLimit: app.settings.RateLimit,
+	}); err != nil {
+		log.Warnf("http client config failed, continuing with defaults: %v", err)
+	}
+
 	// target expansion - securitytrails discovers new domains before scanning
 	if app.settings.SecurityTrails {
 		expanded := app.expandTargets()