From d0bdcf16908a721f7774708b192fe17e511e0191 Mon Sep 17 00:00:00 2001 From: vmfunc Date: Tue, 9 Jun 2026 17:28:14 -0700 Subject: [PATCH] feat: shared http client with proxy, custom headers and rate limiting every scanner spun up its own &http.Client, so there was no single place to apply a proxy, custom headers, a cookie or a rate limit. add an internal/httpx package that builds one configured transport at startup and hand it to every scanner via httpx.Client(timeout), keeping behavior identical when nothing is set (plain client when Configure was never called). - httpx.Configure wires -proxy (http/https/socks5), -H/--header, -cookie and -rate-limit into a package-level RoundTripper that paces via a rate.Limiter and only sets headers the caller hasn't already, so a scanner's explicit api key still wins. - route the scan/wordlist downloads that used http.DefaultClient through the shared client too; ports tcp dialing is left untouched. - clamp -threads to a floor of 1: it feeds wg.Add across the scanners, so 0 was a silent no-op and a negative value panicked the waitgroup. document the new flags in the readme, usage docs and man page. --- README.md | 18 +++ docs/usage.md | 38 ++++- go.mod | 4 +- internal/config/config.go | 22 +++ internal/httpx/httpx.go | 195 +++++++++++++++++++++++++ internal/httpx/httpx_test.go | 217 ++++++++++++++++++++++++++++ internal/scan/cloudstorage.go | 5 +- internal/scan/cms.go | 5 +- internal/scan/dirlist.go | 9 +- internal/scan/dnslist.go | 11 +- internal/scan/dork.go | 3 +- internal/scan/frameworks/detect.go | 3 +- internal/scan/git.go | 10 +- internal/scan/headers.go | 5 +- internal/scan/js/frameworks/next.go | 5 +- internal/scan/js/scan.go | 7 +- internal/scan/js/supabase.go | 5 +- internal/scan/lfi.go | 15 +- internal/scan/ports.go | 3 +- internal/scan/scan.go | 9 +- internal/scan/securityheaders.go | 5 +- internal/scan/securitytrails.go | 3 +- internal/scan/shodan.go | 3 +- internal/scan/sql.go | 15 +- internal/scan/subdomaintakeover.go | 5 +- man/sif.1 | 14 +- sif.go | 13 ++ 27 files changed, 582 insertions(+), 65 deletions(-) create mode 100644 internal/httpx/httpx.go create mode 100644 internal/httpx/httpx_test.go diff --git a/README.md b/README.md index 3bb3efe..7f9911b 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,24 @@ sif has a modular architecture. modules are defined in yaml and can be extended | `-lfi` | local file inclusion | | `-framework` | framework detection with cve lookup | +### http options + +these apply to every outbound request across all scanners: + +| flag | description | +|------|-------------| +| `-proxy` | route all traffic through a proxy (http/https/socks5 url) | +| `-H`, `--header` | custom header to send (repeatable or comma-separated, `"Key: Value"`) | +| `-cookie` | cookie header to send with every request | +| `-rate-limit` | max requests per second (0 = unlimited, default 0) | + +```bash +# scan through a socks5 proxy with a custom header, cookie and 20 req/s cap +./sif -u https://example.com -headers -proxy socks5://127.0.0.1:1080 -H "Authorization: Bearer tok" -cookie "session=abc" -rate-limit 20 +``` + +a scanner that sets a header explicitly (e.g. an api key) always wins over the global default. + ### yaml modules list available modules: diff --git a/docs/usage.md b/docs/usage.md index 2d788f5..5a1db9e 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -225,7 +225,7 @@ http request timeout (default: 10s): ### --threads -number of concurrent threads (default: 10): +number of concurrent threads (default: 10). values below 1 are clamped to 1: ```bash ./sif -u https://example.com --threads 20 @@ -247,6 +247,42 @@ enable debug logging: ./sif -u https://example.com -d ``` +## http options + +these apply to every outbound request across all scanners (proxy, custom headers, cookie and rate limiting share one client). a scanner that sets a header explicitly still wins over the global default. + +### -proxy + +route all traffic through a proxy. supports http, https and socks5 urls: + +```bash +./sif -u https://example.com -proxy socks5://127.0.0.1:1080 +``` + +### -H, --header + +add a custom header to every request. repeatable or comma-separated, `"Key: Value"`: + +```bash +./sif -u https://example.com -H "Authorization: Bearer tok" -H "X-Env: staging" +``` + +### -cookie + +cookie header to send with every request: + +```bash +./sif -u https://example.com -cookie "session=abc; theme=dark" +``` + +### -rate-limit + +cap outbound requests per second (0 = unlimited, default 0): + +```bash +./sif -u https://example.com -rate-limit 20 +``` + ## api options ### -api diff --git a/go.mod b/go.mod index 39bccf0..4382257 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,8 @@ require ( github.com/projectdiscovery/nuclei/v3 v3.8.0 github.com/projectdiscovery/utils v0.10.1 github.com/rocketlaunchr/google-search v1.1.6 + golang.org/x/net v0.53.0 + golang.org/x/time v0.14.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -377,13 +379,11 @@ require ( golang.org/x/crypto v0.50.0 // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/mod v0.35.0 // indirect - golang.org/x/net v0.53.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.43.0 // indirect golang.org/x/term v0.42.0 // indirect golang.org/x/text v0.36.0 // indirect - golang.org/x/time v0.14.0 // indirect golang.org/x/tools v0.44.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.36.6 // indirect diff --git a/internal/config/config.go b/internal/config/config.go index 13a2a3b..7e6755d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -51,8 +51,17 @@ type Settings struct { ModuleTags string // Run modules matching these tags AllModules bool // Run all loaded modules ListModules bool // List available modules and exit + Proxy string + Header goflags.StringSlice // custom request headers ("Key: Value") + Cookie string + RateLimit int } +// minThreads is the floor for the worker count. Threads feeds wg.Add across the +// scanners, so 0 silently runs nothing and a negative value panics with +// "negative WaitGroup counter"; clamp the parsed value up to this. +const minThreads = 1 + const ( Nil goflags.EnumVariable = iota @@ -109,6 +118,13 @@ func Parse() *Settings { flagSet.StringVar(&settings.Template, "template", "", "Sif runtime template to use"), ) + flagSet.CreateGroup("http", "HTTP", + flagSet.StringVar(&settings.Proxy, "proxy", "", "Proxy for all requests (http/https/socks5 url)"), + flagSet.StringSliceVarP(&settings.Header, "header", "H", nil, "Custom header to send (repeatable or comma-separated, \"Key: Value\")", goflags.CommaSeparatedStringSliceOptions), + flagSet.StringVar(&settings.Cookie, "cookie", "", "Cookie header to send with every request"), + flagSet.IntVar(&settings.RateLimit, "rate-limit", 0, "Max requests per second (0 = unlimited)"), + ) + flagSet.CreateGroup("api", "API", flagSet.BoolVar(&settings.ApiMode, "api", false, "Enable API mode. Only useful for internal lunchcat usage"), ) @@ -124,5 +140,11 @@ func Parse() *Settings { log.Fatalf("Could not parse flags: %s", err) } + // threads feeds wg.Add directly; floor it so 0 isn't a silent no-op and a + // negative value can't panic the waitgroup. + if settings.Threads < minThreads { + settings.Threads = minThreads + } + return settings } diff --git a/internal/httpx/httpx.go b/internal/httpx/httpx.go new file mode 100644 index 0000000..d0f32dd --- /dev/null +++ b/internal/httpx/httpx.go @@ -0,0 +1,195 @@ +/* +·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━· +: : +: █▀ █ █▀▀ · Blazing-fast pentesting suite : +: ▄█ █ █▀ · BSD 3-Clause License : +: : +: (c) 2022-2026 vmfunc, xyzeva, : +: lunchcat alumni & contributors : +: : +·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━· +*/ + +// Package httpx is the shared http layer every scanner talks through, so a +// single Configure call wires proxy, custom headers, cookies and rate limiting +// into every outbound request without touching scanner signatures. +package httpx + +import ( + "fmt" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "golang.org/x/net/proxy" + "golang.org/x/time/rate" +) + +// allowed proxy schemes +const ( + schemeHTTP = "http" + schemeHTTPS = "https" + schemeSOCKS5 = "socks5" +) + +// a header is "Key: Value"; this is the separator between the two halves. +const headerSep = ": " + +// burst lets the limiter absorb a small spike before pacing kicks in; a burst +// equal to the per-second rate keeps the cap honest over any one-second window. +const limiterBurstPerRate = 1 + +// Options carries the runtime knobs that apply to every outbound request. +// RateLimit is requests/sec (0 = unlimited); Headers are "Key: Value" strings. +type Options struct { + Proxy string + Headers []string + Cookie string + UserAgent string + RateLimit int +} + +// configured holds the package-level transport built once by Configure. nil +// means Configure was never called, so Client falls back to a plain client. +var ( + mu sync.RWMutex + configured http.RoundTripper +) + +// Configure builds the shared transport once at startup from opts. Calling it +// again replaces the previous configuration. +// +//nolint:gocritic // signature is the package's stable startup api; called once. +func Configure(opts Options) error { + base, err := buildTransport(opts.Proxy) + if err != nil { + return err + } + + headers, err := parseHeaders(opts.Headers) + if err != nil { + return err + } + + var limiter *rate.Limiter + if opts.RateLimit > 0 { + limiter = rate.NewLimiter(rate.Limit(opts.RateLimit), opts.RateLimit*limiterBurstPerRate) + } + + rt := &roundTripper{ + base: base, + headers: headers, + cookie: opts.Cookie, + userAgent: opts.UserAgent, + limiter: limiter, + } + + mu.Lock() + configured = rt + mu.Unlock() + + return nil +} + +// Client returns an http client wired to the configured transport. It works +// before Configure is ever called (plain transport) so existing code and tests +// behave unchanged. A zero timeout means no timeout, matching http.Client. +func Client(timeout time.Duration) *http.Client { + mu.RLock() + rt := configured + mu.RUnlock() + + return &http.Client{Timeout: timeout, Transport: rt} +} + +// buildTransport clones the default transport and applies the proxy. An empty +// proxy leaves the default behavior (respects HTTP_PROXY env) intact. +func buildTransport(proxyURL string) (*http.Transport, error) { + tr, ok := http.DefaultTransport.(*http.Transport) + if !ok { + // unreachable in practice, but never trust an assertion silently. + return nil, fmt.Errorf("default transport is not *http.Transport") + } + transport := tr.Clone() + + if proxyURL == "" { + return transport, nil + } + + parsed, err := url.Parse(proxyURL) + if err != nil { + return nil, fmt.Errorf("parse proxy url %q: %w", proxyURL, err) + } + + switch parsed.Scheme { + case schemeHTTP, schemeHTTPS: + transport.Proxy = http.ProxyURL(parsed) + case schemeSOCKS5: + // socks5 needs a custom dialer; the returned dialer implements + // ContextDialer so cancellation/timeouts propagate. + dialer, err := proxy.SOCKS5("tcp", parsed.Host, nil, proxy.Direct) + if err != nil { + return nil, fmt.Errorf("socks5 proxy %q: %w", proxyURL, err) + } + ctxDialer, ok := dialer.(proxy.ContextDialer) + if !ok { + return nil, fmt.Errorf("socks5 proxy %q: dialer lacks context support", proxyURL) + } + transport.DialContext = ctxDialer.DialContext + default: + return nil, fmt.Errorf("unsupported proxy scheme %q (want http/https/socks5)", parsed.Scheme) + } + + return transport, nil +} + +// parseHeaders splits each "Key: Value" entry on the first ": ". Entries +// without the separator are rejected so a typo fails loud instead of silently. +// The returned map is always non-nil so callers can range it unconditionally. +func parseHeaders(raw []string) (map[string]string, error) { + headers := make(map[string]string, len(raw)) + for i := 0; i < len(raw); i++ { + key, value, ok := strings.Cut(raw[i], headerSep) + if !ok { + return nil, fmt.Errorf("invalid header %q (want \"Key: Value\")", raw[i]) + } + headers[key] = value + } + + return headers, nil +} + +// roundTripper paces and decorates each request before delegating to base. +type roundTripper struct { + base *http.Transport + headers map[string]string + cookie string + userAgent string + limiter *rate.Limiter +} + +func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + if rt.limiter != nil { + if err := rt.limiter.Wait(req.Context()); err != nil { + return nil, fmt.Errorf("rate limiter: %w", err) + } + } + + // only set what the caller hasn't already; a scanner that explicitly sets a + // header (e.g. an api key) must win over the global default. + for key, value := range rt.headers { + if req.Header.Get(key) == "" { + req.Header.Set(key, value) + } + } + if rt.cookie != "" && req.Header.Get("Cookie") == "" { + req.Header.Set("Cookie", rt.cookie) + } + if rt.userAgent != "" && req.Header.Get("User-Agent") == "" { + req.Header.Set("User-Agent", rt.userAgent) + } + + return rt.base.RoundTrip(req) +} diff --git a/internal/httpx/httpx_test.go b/internal/httpx/httpx_test.go new file mode 100644 index 0000000..4b37548 --- /dev/null +++ b/internal/httpx/httpx_test.go @@ -0,0 +1,217 @@ +/* +·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━· +: : +: █▀ █ █▀▀ · Blazing-fast pentesting suite : +: ▄█ █ █▀ · BSD 3-Clause License : +: : +: (c) 2022-2026 vmfunc, xyzeva, : +: lunchcat alumni & contributors : +: : +·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━· +*/ + +package httpx + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +// resetConfig clears the package-level transport so each test starts clean. +func resetConfig(t *testing.T) { + t.Helper() + mu.Lock() + configured = nil + mu.Unlock() +} + +// captureServer records the headers of the last request it served. +func captureServer(t *testing.T, seen *http.Header) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + *seen = r.Header.Clone() + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(srv.Close) + return srv +} + +func get(t *testing.T, client *http.Client, url string) { + t.Helper() + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, http.NoBody) + if err != nil { + t.Fatalf("new request: %v", err) + } + resp, err := client.Do(req) + if err != nil { + t.Fatalf("do request: %v", err) + } + resp.Body.Close() +} + +func TestClientBeforeConfigure(t *testing.T) { + resetConfig(t) + + var seen http.Header + srv := captureServer(t, &seen) + + // a client must work with no Configure call so existing code is unaffected. + get(t, Client(5*time.Second), srv.URL) + + if seen == nil { + t.Fatal("request never reached the server") + } +} + +func TestConfigureHeadersAndCookie(t *testing.T) { + tests := []struct { + name string + opts Options + wantKey string + wantValue string + }{ + { + name: "custom header injected", + opts: Options{Headers: []string{"X-Test: sif"}}, + wantKey: "X-Test", + wantValue: "sif", + }, + { + name: "cookie injected", + opts: Options{Cookie: "session=abc"}, + wantKey: "Cookie", + wantValue: "session=abc", + }, + { + name: "user agent injected", + opts: Options{UserAgent: "sif-scanner"}, + wantKey: "User-Agent", + wantValue: "sif-scanner", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resetConfig(t) + + if err := Configure(tt.opts); err != nil { + t.Fatalf("Configure: %v", err) + } + + var seen http.Header + srv := captureServer(t, &seen) + get(t, Client(5*time.Second), srv.URL) + + if got := seen.Get(tt.wantKey); got != tt.wantValue { + t.Errorf("header %q = %q, want %q", tt.wantKey, got, tt.wantValue) + } + }) + } +} + +func TestConfigureHeaderDoesNotOverride(t *testing.T) { + resetConfig(t) + + if err := Configure(Options{Headers: []string{"X-Test: global"}}); err != nil { + t.Fatalf("Configure: %v", err) + } + + var seen http.Header + srv := captureServer(t, &seen) + + // a caller that sets the header explicitly must win over the global default. + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, srv.URL, http.NoBody) + if err != nil { + t.Fatalf("new request: %v", err) + } + req.Header.Set("X-Test", "caller") + resp, err := Client(5 * time.Second).Do(req) + if err != nil { + t.Fatalf("do request: %v", err) + } + resp.Body.Close() + + if got := seen.Get("X-Test"); got != "caller" { + t.Errorf("X-Test = %q, want caller (caller value must not be overridden)", got) + } +} + +func TestConfigureInvalidHeader(t *testing.T) { + resetConfig(t) + + // a header without ": " should fail loud rather than silently dropping. + if err := Configure(Options{Headers: []string{"missing-separator"}}); err == nil { + t.Fatal("expected error for malformed header, got nil") + } +} + +func TestConfigureInvalidProxy(t *testing.T) { + tests := []struct { + name string + proxy string + }{ + {name: "unsupported scheme", proxy: "ftp://localhost:1080"}, + {name: "malformed url", proxy: "://nope"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resetConfig(t) + if err := Configure(Options{Proxy: tt.proxy}); err == nil { + t.Errorf("expected error for proxy %q, got nil", tt.proxy) + } + }) + } +} + +func TestRateLimit(t *testing.T) { + resetConfig(t) + + const ratePerSec = 5 + if err := Configure(Options{RateLimit: ratePerSec}); err != nil { + t.Fatalf("Configure: %v", err) + } + + var seen http.Header + srv := captureServer(t, &seen) + client := Client(5 * time.Second) + + // at 5 req/s the limiter starts with a full burst, so the first batch is + // immediate and the next request must wait roughly one tick. fire burst+1 + // requests and assert the extra one forced a measurable delay. + const requests = ratePerSec + 1 + start := time.Now() + for i := 0; i < requests; i++ { + get(t, client, srv.URL) + } + elapsed := time.Since(start) + + // one request beyond the burst should cost about 1/rate; allow slack but + // require a non-trivial delay so an unthrottled client fails this. + minDelay := time.Second / ratePerSec / 2 + if elapsed < minDelay { + t.Errorf("expected rate limiting to add >= %v of delay, got %v", minDelay, elapsed) + } +} + +func TestRateLimitUnlimited(t *testing.T) { + resetConfig(t) + + // RateLimit 0 means no limiter is installed; requests should fly through. + if err := Configure(Options{RateLimit: 0}); err != nil { + t.Fatalf("Configure: %v", err) + } + + mu.RLock() + rt, ok := configured.(*roundTripper) + mu.RUnlock() + if !ok { + t.Fatal("configured transport is not *roundTripper") + } + if rt.limiter != nil { + t.Error("expected no limiter when RateLimit is 0") + } +} diff --git a/internal/scan/cloudstorage.go b/internal/scan/cloudstorage.go index 417a37e..d1d0f90 100644 --- a/internal/scan/cloudstorage.go +++ b/internal/scan/cloudstorage.go @@ -21,6 +21,7 @@ import ( "time" "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/styles" ) @@ -50,9 +51,7 @@ func CloudStorage(url string, timeout time.Duration, logdir string) ([]CloudStor Prefix: "C3", }).With("url", url) - client := &http.Client{ - Timeout: timeout, - } + client := httpx.Client(timeout) potentialBuckets := extractPotentialBuckets(sanitizedURL) diff --git a/internal/scan/cms.go b/internal/scan/cms.go index 22600eb..d07cc95 100644 --- a/internal/scan/cms.go +++ b/internal/scan/cms.go @@ -19,6 +19,7 @@ import ( "strings" "time" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -45,9 +46,7 @@ func CMS(url string, timeout time.Duration, logdir string) (*CMSResult, error) { } } - client := &http.Client{ - Timeout: timeout, - } + client := httpx.Client(timeout) req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, http.NoBody) if err != nil { diff --git a/internal/scan/dirlist.go b/internal/scan/dirlist.go index 102c523..f56824b 100644 --- a/internal/scan/dirlist.go +++ b/internal/scan/dirlist.go @@ -22,6 +22,7 @@ import ( "time" charmlog "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -64,12 +65,14 @@ func Dirlist(size string, url string, timeout time.Duration, threads int, logdir list = directoryURL + bigFile } + client := httpx.Client(timeout) + req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, list, http.NoBody) if err != nil { log.Error("Error creating directory list request: %s", err) return nil, err } - resp, err := http.DefaultClient.Do(req) + resp, err := client.Do(req) if err != nil { log.Error("Error downloading directory list: %s", err) return nil, err @@ -83,10 +86,6 @@ func Dirlist(size string, url string, timeout time.Duration, threads int, logdir directories = append(directories, scanner.Text()) } - client := &http.Client{ - Timeout: timeout, - } - progress := output.NewProgress(len(directories), "fuzzing") var wg sync.WaitGroup diff --git a/internal/scan/dnslist.go b/internal/scan/dnslist.go index b490a55..7f7c592 100644 --- a/internal/scan/dnslist.go +++ b/internal/scan/dnslist.go @@ -21,6 +21,7 @@ import ( "time" charmlog "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -58,7 +59,7 @@ func Dnslist(size string, url string, timeout time.Duration, threads int, logdir log.Error("Error creating request: %s", err) return nil, err } - resp, err := http.DefaultClient.Do(req) + resp, err := httpx.Client(timeout).Do(req) if err != nil { log.Error("Error downloading DNS list: %s", err) return nil, err @@ -81,9 +82,11 @@ func Dnslist(size string, url string, timeout time.Duration, threads int, logdir } } - client := &http.Client{ - Timeout: timeout, - Transport: dnsTransport, + // per-host probe client. dnsTransport pins every dial at a fixture in + // integration tests; nil keeps the shared transport for real runs. + client := httpx.Client(timeout) + if dnsTransport != nil { + client.Transport = dnsTransport } progress := output.NewProgress(len(dns), "enumerating") diff --git a/internal/scan/dork.go b/internal/scan/dork.go index 43ec6a3..c685285 100644 --- a/internal/scan/dork.go +++ b/internal/scan/dork.go @@ -25,6 +25,7 @@ import ( "time" "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" googlesearch "github.com/rocketlaunchr/google-search" @@ -76,7 +77,7 @@ func Dork(url string, timeout time.Duration, threads int, logdir string) ([]Dork output.Error("Error creating dork list request: %s", err) return nil, err } - resp, err := http.DefaultClient.Do(req) + resp, err := httpx.Client(timeout).Do(req) if err != nil { spin.Stop() output.Error("Error downloading dork list: %s", err) diff --git a/internal/scan/frameworks/detect.go b/internal/scan/frameworks/detect.go index 2ae683a..487b244 100644 --- a/internal/scan/frameworks/detect.go +++ b/internal/scan/frameworks/detect.go @@ -22,6 +22,7 @@ import ( "time" charmlog "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -47,7 +48,7 @@ func DetectFramework(url string, timeout time.Duration, logdir string) (*Framewo spin := output.NewSpinner("Detecting frameworks") spin.Start() - client := &http.Client{Timeout: timeout} + client := httpx.Client(timeout) req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, url, http.NoBody) if err != nil { diff --git a/internal/scan/git.go b/internal/scan/git.go index 0a4d3a7..b026ac5 100644 --- a/internal/scan/git.go +++ b/internal/scan/git.go @@ -22,6 +22,7 @@ import ( "time" charmlog "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -48,13 +49,15 @@ func Git(url string, timeout time.Duration, threads int, logdir string) ([]strin } } + client := httpx.Client(timeout) + req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, gitURL+gitFile, http.NoBody) if err != nil { spin.Stop() log.Error("Error creating git list request: %s", err) return nil, err } - resp, err := http.DefaultClient.Do(req) + resp, err := client.Do(req) if err != nil { spin.Stop() log.Error("Error downloading git list: %s", err) @@ -68,11 +71,6 @@ func Git(url string, timeout time.Duration, threads int, logdir string) ([]strin gitUrls = append(gitUrls, scanner.Text()) } - // util.InitProgressBar() - client := &http.Client{ - Timeout: timeout, - } - var wg sync.WaitGroup var mu sync.Mutex wg.Add(threads) diff --git a/internal/scan/headers.go b/internal/scan/headers.go index 6badd5f..35ce693 100644 --- a/internal/scan/headers.go +++ b/internal/scan/headers.go @@ -17,6 +17,7 @@ import ( "net/http" "time" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -39,9 +40,7 @@ func Headers(url string, timeout time.Duration, logdir string) ([]HeaderResult, } } - client := &http.Client{ - Timeout: timeout, - } + client := httpx.Client(timeout) req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, url, http.NoBody) if err != nil { diff --git a/internal/scan/js/frameworks/next.go b/internal/scan/js/frameworks/next.go index 62b5be9..593ade3 100644 --- a/internal/scan/js/frameworks/next.go +++ b/internal/scan/js/frameworks/next.go @@ -30,6 +30,7 @@ import ( "regexp" "strings" + "github.com/dropalldatabases/sif/internal/httpx" urlutil "github.com/projectdiscovery/utils/url" ) @@ -48,7 +49,9 @@ func GetPagesRouterScripts(scriptUrl string) ([]string, error) { return nil, err } - resp, err := http.DefaultClient.Do(req) + // no timeout in scope here; 0 matches the previous DefaultClient behavior + // while still routing through the shared transport (proxy/headers/rate-limit). + resp, err := httpx.Client(0).Do(req) if err != nil { fmt.Println(err) return nil, err diff --git a/internal/scan/js/scan.go b/internal/scan/js/scan.go index 3e4ae11..519b11e 100644 --- a/internal/scan/js/scan.go +++ b/internal/scan/js/scan.go @@ -23,6 +23,7 @@ import ( "github.com/antchfx/htmlquery" charmlog "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/output" "github.com/dropalldatabases/sif/internal/scan/js/frameworks" urlutil "github.com/projectdiscovery/utils/url" @@ -43,6 +44,8 @@ func JavascriptScan(url string, timeout time.Duration, threads int, logdir strin spin := output.NewSpinner("Scanning JavaScript files") spin.Start() + client := httpx.Client(timeout) + baseUrl, err := urlutil.Parse(url) if err != nil { spin.Stop() @@ -53,7 +56,7 @@ func JavascriptScan(url string, timeout time.Duration, threads int, logdir strin spin.Stop() return nil, err } - resp, err := http.DefaultClient.Do(req) + resp, err := client.Do(req) if err != nil { spin.Stop() return nil, err @@ -120,7 +123,7 @@ func JavascriptScan(url string, timeout time.Duration, threads int, logdir strin charmlog.Warnf("Failed to create request: %s", err) continue } - resp, err := http.DefaultClient.Do(req) + resp, err := client.Do(req) if err != nil { charmlog.Warnf("Failed to fetch script: %s", err) continue diff --git a/internal/scan/js/supabase.go b/internal/scan/js/supabase.go index be853d1..6e86180 100644 --- a/internal/scan/js/supabase.go +++ b/internal/scan/js/supabase.go @@ -30,6 +30,7 @@ import ( "time" "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" ) // jwtRegex matches JWT tokens in JavaScript content. @@ -110,7 +111,7 @@ func getSupabaseOpenAPI(projectId, apikey string, auth *string, timeout time.Dur // doSupabaseRequest performs a GET request to the Supabase API. func doSupabaseRequest(projectId, path, apikey string, auth *string, timeout time.Duration) ([]byte, *http.Response, error) { - client := http.Client{Timeout: timeout} + client := httpx.Client(timeout) req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, "https://"+projectId+".supabase.co"+path, http.NoBody) if err != nil { @@ -182,7 +183,7 @@ func ScanSupabase(jsContent string, jsUrl string, timeout time.Duration) ([]supa } supabaselog.Infof("Found valid supabase project %s with role %s", *supabaseJwt.ProjectId, *supabaseJwt.Role) - client := http.Client{Timeout: timeout} + client := httpx.Client(timeout) req, err := http.NewRequestWithContext(context.TODO(), http.MethodPost, "https://"+*supabaseJwt.ProjectId+".supabase.co/auth/v1/signup", bytes.NewBufferString(`{"email":"automated`+strconv.Itoa(int(time.Now().Unix()))+`@sif.sh","password":"automatedacct"}`)) if err != nil { diff --git a/internal/scan/lfi.go b/internal/scan/lfi.go index f93f33a..583d1a8 100644 --- a/internal/scan/lfi.go +++ b/internal/scan/lfi.go @@ -23,6 +23,7 @@ import ( "time" charmlog "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -136,14 +137,12 @@ func LFI(targetURL string, timeout time.Duration, threads int, logdir string) (* var mu sync.Mutex var wg sync.WaitGroup - client := &http.Client{ - Timeout: timeout, - CheckRedirect: func(req *http.Request, via []*http.Request) error { - if len(via) >= 3 { - return http.ErrUseLastResponse - } - return nil - }, + client := httpx.Client(timeout) + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if len(via) >= 3 { + return http.ErrUseLastResponse + } + return nil } // parse the target URL to check for existing parameters diff --git a/internal/scan/ports.go b/internal/scan/ports.go index 87110d7..a0566fa 100644 --- a/internal/scan/ports.go +++ b/internal/scan/ports.go @@ -23,6 +23,7 @@ import ( "time" charmlog "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -50,7 +51,7 @@ func Ports(ctx context.Context, scope string, url string, timeout time.Duration, log.Error("Error creating request: %s", err) return nil, err } - resp, err := http.DefaultClient.Do(req) + resp, err := httpx.Client(timeout).Do(req) if err != nil { log.Error("Error downloading ports list: %s", err) return nil, err diff --git a/internal/scan/scan.go b/internal/scan/scan.go index 95330e0..91e3aab 100644 --- a/internal/scan/scan.go +++ b/internal/scan/scan.go @@ -27,6 +27,7 @@ import ( "time" "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -85,11 +86,9 @@ func Scan(url string, timeout time.Duration, threads int, logdir string) { } } - client := &http.Client{ - Timeout: timeout, - CheckRedirect: func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - }, + client := httpx.Client(timeout) + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse } resp := fetchRobotsTXT(url+"/robots.txt", client) diff --git a/internal/scan/securityheaders.go b/internal/scan/securityheaders.go index 2444fa7..dc64fc8 100644 --- a/internal/scan/securityheaders.go +++ b/internal/scan/securityheaders.go @@ -19,6 +19,7 @@ import ( "strings" "time" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -64,9 +65,7 @@ func SecurityHeaders(url string, timeout time.Duration, logdir string) (Security } } - client := &http.Client{ - Timeout: timeout, - } + client := httpx.Client(timeout) req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, url, http.NoBody) if err != nil { diff --git a/internal/scan/securitytrails.go b/internal/scan/securitytrails.go index aeccdd9..42aaf62 100644 --- a/internal/scan/securitytrails.go +++ b/internal/scan/securitytrails.go @@ -23,6 +23,7 @@ import ( "strings" "time" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -73,7 +74,7 @@ func SecurityTrails(targetURL string, timeout time.Duration, logdir string) (*Se } hostname := parsedURL.Hostname() - client := &http.Client{Timeout: timeout} + client := httpx.Client(timeout) result := &SecurityTrailsResult{ Domain: hostname, diff --git a/internal/scan/shodan.go b/internal/scan/shodan.go index 238e3e9..f288b03 100644 --- a/internal/scan/shodan.go +++ b/internal/scan/shodan.go @@ -24,6 +24,7 @@ import ( "strings" "time" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -180,7 +181,7 @@ func resolveHostname(hostname string) (string, error) { } func queryShodanHost(ip string, apiKey string, timeout time.Duration) (*ShodanResult, error) { - client := &http.Client{Timeout: timeout} + client := httpx.Client(timeout) reqURL := fmt.Sprintf("%s/shodan/host/%s?key=%s", shodanBaseURL, ip, apiKey) req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, reqURL, http.NoBody) diff --git a/internal/scan/sql.go b/internal/scan/sql.go index d0f7e73..6f0e821 100644 --- a/internal/scan/sql.go +++ b/internal/scan/sql.go @@ -23,6 +23,7 @@ import ( "time" charmlog "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -140,14 +141,12 @@ func SQL(targetURL string, timeout time.Duration, threads int, logdir string) (* var mu sync.Mutex var wg sync.WaitGroup - client := &http.Client{ - Timeout: timeout, - CheckRedirect: func(req *http.Request, via []*http.Request) error { - if len(via) >= 3 { - return http.ErrUseLastResponse - } - return nil - }, + client := httpx.Client(timeout) + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if len(via) >= 3 { + return http.ErrUseLastResponse + } + return nil } // check for admin panels diff --git a/internal/scan/subdomaintakeover.go b/internal/scan/subdomaintakeover.go index d77b857..1d0cfbd 100644 --- a/internal/scan/subdomaintakeover.go +++ b/internal/scan/subdomaintakeover.go @@ -24,6 +24,7 @@ import ( "time" "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/styles" ) @@ -54,9 +55,7 @@ func SubdomainTakeover(url string, dnsResults []string, timeout time.Duration, t Prefix: "Subdomain Takeover", }) - client := &http.Client{ - Timeout: timeout, - } + client := httpx.Client(timeout) var wg sync.WaitGroup wg.Add(threads) diff --git a/man/sif.1 b/man/sif.1 index 47ebce7..f70e3cd 100644 --- a/man/sif.1 +++ b/man/sif.1 @@ -103,11 +103,23 @@ per\-request timeout (default 10s). directory to write logs to. .TP .BR \-\-threads " \fIn\fR" -number of concurrent workers (default 10). +number of concurrent workers (default 10). values below 1 are clamped to 1. .TP .BR \-\-template " \fIname\fR" sif runtime template to use. .TP +.BR \-proxy " \fIurl\fR" +route every request through a proxy. accepts http, https or socks5 urls. +.TP +.BR \-H ", " \-\-header " \fIstring\fR" +custom header to send with every request, as \fBKey: Value\fR. repeatable or comma\-separated. +.TP +.BR \-cookie " \fIstring\fR" +cookie header to send with every request. +.TP +.BR \-rate\-limit " \fIn\fR" +cap outbound requests per second (0 = unlimited, default 0). +.TP .B \-api emit json results and suppress the interactive output. .SH MODULES diff --git a/sif.go b/sif.go index 0932571..cee06fd 100644 --- a/sif.go +++ b/sif.go @@ -25,6 +25,7 @@ import ( "github.com/charmbracelet/log" "github.com/dropalldatabases/sif/internal/config" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/modules" "github.com/dropalldatabases/sif/internal/output" @@ -180,6 +181,18 @@ func (app *App) Run() error { }() } + // wire proxy/headers/cookie/rate-limit into the shared http client once, + // before any scanner runs. a bad proxy/header shouldn't kill the run - + // scanners fall back to a plain client if this fails. + if err := httpx.Configure(httpx.Options{ + Proxy: app.settings.Proxy, + Headers: app.settings.Header, + Cookie: app.settings.Cookie, + RateLimit: app.settings.RateLimit, + }); err != nil { + log.Warnf("http client config failed, continuing with defaults: %v", err) + } + // target expansion - securitytrails discovers new domains before scanning if app.settings.SecurityTrails { expanded := app.expandTargets()