diff --git a/README.md b/README.md index 3bb3efe..7f9911b 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,24 @@ sif has a modular architecture. modules are defined in yaml and can be extended | `-lfi` | local file inclusion | | `-framework` | framework detection with cve lookup | +### http options + +these apply to every outbound request across all scanners: + +| flag | description | +|------|-------------| +| `-proxy` | route all traffic through a proxy (http/https/socks5 url) | +| `-H`, `--header` | custom header to send (repeatable or comma-separated, `"Key: Value"`) | +| `-cookie` | cookie header to send with every request | +| `-rate-limit` | max requests per second (0 = unlimited, default 0) | + +```bash +# scan through a socks5 proxy with a custom header, cookie and 20 req/s cap +./sif -u https://example.com -headers -proxy socks5://127.0.0.1:1080 -H "Authorization: Bearer tok" -cookie "session=abc" -rate-limit 20 +``` + +a scanner that sets a header explicitly (e.g. an api key) always wins over the global default. + ### yaml modules list available modules: diff --git a/docs/usage.md b/docs/usage.md index 2d788f5..5a1db9e 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -225,7 +225,7 @@ http request timeout (default: 10s): ### --threads -number of concurrent threads (default: 10): +number of concurrent threads (default: 10). values below 1 are clamped to 1: ```bash ./sif -u https://example.com --threads 20 @@ -247,6 +247,42 @@ enable debug logging: ./sif -u https://example.com -d ``` +## http options + +these apply to every outbound request across all scanners (proxy, custom headers, cookie and rate limiting share one client). a scanner that sets a header explicitly still wins over the global default. + +### -proxy + +route all traffic through a proxy. supports http, https and socks5 urls: + +```bash +./sif -u https://example.com -proxy socks5://127.0.0.1:1080 +``` + +### -H, --header + +add a custom header to every request. repeatable or comma-separated, `"Key: Value"`: + +```bash +./sif -u https://example.com -H "Authorization: Bearer tok" -H "X-Env: staging" +``` + +### -cookie + +cookie header to send with every request: + +```bash +./sif -u https://example.com -cookie "session=abc; theme=dark" +``` + +### -rate-limit + +cap outbound requests per second (0 = unlimited, default 0): + +```bash +./sif -u https://example.com -rate-limit 20 +``` + ## api options ### -api diff --git a/go.mod b/go.mod index 39bccf0..4382257 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,8 @@ require ( github.com/projectdiscovery/nuclei/v3 v3.8.0 github.com/projectdiscovery/utils v0.10.1 github.com/rocketlaunchr/google-search v1.1.6 + golang.org/x/net v0.53.0 + golang.org/x/time v0.14.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -377,13 +379,11 @@ require ( golang.org/x/crypto v0.50.0 // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/mod v0.35.0 // indirect - golang.org/x/net v0.53.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.43.0 // indirect golang.org/x/term v0.42.0 // indirect golang.org/x/text v0.36.0 // indirect - golang.org/x/time v0.14.0 // indirect golang.org/x/tools v0.44.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.36.6 // indirect diff --git a/internal/config/config.go b/internal/config/config.go index 13a2a3b..7e6755d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -51,8 +51,17 @@ type Settings struct { ModuleTags string // Run modules matching these tags AllModules bool // Run all loaded modules ListModules bool // List available modules and exit + Proxy string + Header goflags.StringSlice // custom request headers ("Key: Value") + Cookie string + RateLimit int } +// minThreads is the floor for the worker count. Threads feeds wg.Add across the +// scanners, so 0 silently runs nothing and a negative value panics with +// "negative WaitGroup counter"; clamp the parsed value up to this. +const minThreads = 1 + const ( Nil goflags.EnumVariable = iota @@ -109,6 +118,13 @@ func Parse() *Settings { flagSet.StringVar(&settings.Template, "template", "", "Sif runtime template to use"), ) + flagSet.CreateGroup("http", "HTTP", + flagSet.StringVar(&settings.Proxy, "proxy", "", "Proxy for all requests (http/https/socks5 url)"), + flagSet.StringSliceVarP(&settings.Header, "header", "H", nil, "Custom header to send (repeatable or comma-separated, \"Key: Value\")", goflags.CommaSeparatedStringSliceOptions), + flagSet.StringVar(&settings.Cookie, "cookie", "", "Cookie header to send with every request"), + flagSet.IntVar(&settings.RateLimit, "rate-limit", 0, "Max requests per second (0 = unlimited)"), + ) + flagSet.CreateGroup("api", "API", flagSet.BoolVar(&settings.ApiMode, "api", false, "Enable API mode. Only useful for internal lunchcat usage"), ) @@ -124,5 +140,11 @@ func Parse() *Settings { log.Fatalf("Could not parse flags: %s", err) } + // threads feeds wg.Add directly; floor it so 0 isn't a silent no-op and a + // negative value can't panic the waitgroup. + if settings.Threads < minThreads { + settings.Threads = minThreads + } + return settings } diff --git a/internal/httpx/httpx.go b/internal/httpx/httpx.go new file mode 100644 index 0000000..d0f32dd --- /dev/null +++ b/internal/httpx/httpx.go @@ -0,0 +1,195 @@ +/* +·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━· +: : +: █▀ █ █▀▀ · Blazing-fast pentesting suite : +: ▄█ █ █▀ · BSD 3-Clause License : +: : +: (c) 2022-2026 vmfunc, xyzeva, : +: lunchcat alumni & contributors : +: : +·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━· +*/ + +// Package httpx is the shared http layer every scanner talks through, so a +// single Configure call wires proxy, custom headers, cookies and rate limiting +// into every outbound request without touching scanner signatures. +package httpx + +import ( + "fmt" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "golang.org/x/net/proxy" + "golang.org/x/time/rate" +) + +// allowed proxy schemes +const ( + schemeHTTP = "http" + schemeHTTPS = "https" + schemeSOCKS5 = "socks5" +) + +// a header is "Key: Value"; this is the separator between the two halves. +const headerSep = ": " + +// burst lets the limiter absorb a small spike before pacing kicks in; a burst +// equal to the per-second rate keeps the cap honest over any one-second window. +const limiterBurstPerRate = 1 + +// Options carries the runtime knobs that apply to every outbound request. +// RateLimit is requests/sec (0 = unlimited); Headers are "Key: Value" strings. +type Options struct { + Proxy string + Headers []string + Cookie string + UserAgent string + RateLimit int +} + +// configured holds the package-level transport built once by Configure. nil +// means Configure was never called, so Client falls back to a plain client. +var ( + mu sync.RWMutex + configured http.RoundTripper +) + +// Configure builds the shared transport once at startup from opts. Calling it +// again replaces the previous configuration. +// +//nolint:gocritic // signature is the package's stable startup api; called once. +func Configure(opts Options) error { + base, err := buildTransport(opts.Proxy) + if err != nil { + return err + } + + headers, err := parseHeaders(opts.Headers) + if err != nil { + return err + } + + var limiter *rate.Limiter + if opts.RateLimit > 0 { + limiter = rate.NewLimiter(rate.Limit(opts.RateLimit), opts.RateLimit*limiterBurstPerRate) + } + + rt := &roundTripper{ + base: base, + headers: headers, + cookie: opts.Cookie, + userAgent: opts.UserAgent, + limiter: limiter, + } + + mu.Lock() + configured = rt + mu.Unlock() + + return nil +} + +// Client returns an http client wired to the configured transport. It works +// before Configure is ever called (plain transport) so existing code and tests +// behave unchanged. A zero timeout means no timeout, matching http.Client. +func Client(timeout time.Duration) *http.Client { + mu.RLock() + rt := configured + mu.RUnlock() + + return &http.Client{Timeout: timeout, Transport: rt} +} + +// buildTransport clones the default transport and applies the proxy. An empty +// proxy leaves the default behavior (respects HTTP_PROXY env) intact. +func buildTransport(proxyURL string) (*http.Transport, error) { + tr, ok := http.DefaultTransport.(*http.Transport) + if !ok { + // unreachable in practice, but never trust an assertion silently. + return nil, fmt.Errorf("default transport is not *http.Transport") + } + transport := tr.Clone() + + if proxyURL == "" { + return transport, nil + } + + parsed, err := url.Parse(proxyURL) + if err != nil { + return nil, fmt.Errorf("parse proxy url %q: %w", proxyURL, err) + } + + switch parsed.Scheme { + case schemeHTTP, schemeHTTPS: + transport.Proxy = http.ProxyURL(parsed) + case schemeSOCKS5: + // socks5 needs a custom dialer; the returned dialer implements + // ContextDialer so cancellation/timeouts propagate. + dialer, err := proxy.SOCKS5("tcp", parsed.Host, nil, proxy.Direct) + if err != nil { + return nil, fmt.Errorf("socks5 proxy %q: %w", proxyURL, err) + } + ctxDialer, ok := dialer.(proxy.ContextDialer) + if !ok { + return nil, fmt.Errorf("socks5 proxy %q: dialer lacks context support", proxyURL) + } + transport.DialContext = ctxDialer.DialContext + default: + return nil, fmt.Errorf("unsupported proxy scheme %q (want http/https/socks5)", parsed.Scheme) + } + + return transport, nil +} + +// parseHeaders splits each "Key: Value" entry on the first ": ". Entries +// without the separator are rejected so a typo fails loud instead of silently. +// The returned map is always non-nil so callers can range it unconditionally. +func parseHeaders(raw []string) (map[string]string, error) { + headers := make(map[string]string, len(raw)) + for i := 0; i < len(raw); i++ { + key, value, ok := strings.Cut(raw[i], headerSep) + if !ok { + return nil, fmt.Errorf("invalid header %q (want \"Key: Value\")", raw[i]) + } + headers[key] = value + } + + return headers, nil +} + +// roundTripper paces and decorates each request before delegating to base. +type roundTripper struct { + base *http.Transport + headers map[string]string + cookie string + userAgent string + limiter *rate.Limiter +} + +func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + if rt.limiter != nil { + if err := rt.limiter.Wait(req.Context()); err != nil { + return nil, fmt.Errorf("rate limiter: %w", err) + } + } + + // only set what the caller hasn't already; a scanner that explicitly sets a + // header (e.g. an api key) must win over the global default. + for key, value := range rt.headers { + if req.Header.Get(key) == "" { + req.Header.Set(key, value) + } + } + if rt.cookie != "" && req.Header.Get("Cookie") == "" { + req.Header.Set("Cookie", rt.cookie) + } + if rt.userAgent != "" && req.Header.Get("User-Agent") == "" { + req.Header.Set("User-Agent", rt.userAgent) + } + + return rt.base.RoundTrip(req) +} diff --git a/internal/httpx/httpx_test.go b/internal/httpx/httpx_test.go new file mode 100644 index 0000000..4b37548 --- /dev/null +++ b/internal/httpx/httpx_test.go @@ -0,0 +1,217 @@ +/* +·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━· +: : +: █▀ █ █▀▀ · Blazing-fast pentesting suite : +: ▄█ █ █▀ · BSD 3-Clause License : +: : +: (c) 2022-2026 vmfunc, xyzeva, : +: lunchcat alumni & contributors : +: : +·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━· +*/ + +package httpx + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +// resetConfig clears the package-level transport so each test starts clean. +func resetConfig(t *testing.T) { + t.Helper() + mu.Lock() + configured = nil + mu.Unlock() +} + +// captureServer records the headers of the last request it served. +func captureServer(t *testing.T, seen *http.Header) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + *seen = r.Header.Clone() + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(srv.Close) + return srv +} + +func get(t *testing.T, client *http.Client, url string) { + t.Helper() + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, http.NoBody) + if err != nil { + t.Fatalf("new request: %v", err) + } + resp, err := client.Do(req) + if err != nil { + t.Fatalf("do request: %v", err) + } + resp.Body.Close() +} + +func TestClientBeforeConfigure(t *testing.T) { + resetConfig(t) + + var seen http.Header + srv := captureServer(t, &seen) + + // a client must work with no Configure call so existing code is unaffected. + get(t, Client(5*time.Second), srv.URL) + + if seen == nil { + t.Fatal("request never reached the server") + } +} + +func TestConfigureHeadersAndCookie(t *testing.T) { + tests := []struct { + name string + opts Options + wantKey string + wantValue string + }{ + { + name: "custom header injected", + opts: Options{Headers: []string{"X-Test: sif"}}, + wantKey: "X-Test", + wantValue: "sif", + }, + { + name: "cookie injected", + opts: Options{Cookie: "session=abc"}, + wantKey: "Cookie", + wantValue: "session=abc", + }, + { + name: "user agent injected", + opts: Options{UserAgent: "sif-scanner"}, + wantKey: "User-Agent", + wantValue: "sif-scanner", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resetConfig(t) + + if err := Configure(tt.opts); err != nil { + t.Fatalf("Configure: %v", err) + } + + var seen http.Header + srv := captureServer(t, &seen) + get(t, Client(5*time.Second), srv.URL) + + if got := seen.Get(tt.wantKey); got != tt.wantValue { + t.Errorf("header %q = %q, want %q", tt.wantKey, got, tt.wantValue) + } + }) + } +} + +func TestConfigureHeaderDoesNotOverride(t *testing.T) { + resetConfig(t) + + if err := Configure(Options{Headers: []string{"X-Test: global"}}); err != nil { + t.Fatalf("Configure: %v", err) + } + + var seen http.Header + srv := captureServer(t, &seen) + + // a caller that sets the header explicitly must win over the global default. + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, srv.URL, http.NoBody) + if err != nil { + t.Fatalf("new request: %v", err) + } + req.Header.Set("X-Test", "caller") + resp, err := Client(5 * time.Second).Do(req) + if err != nil { + t.Fatalf("do request: %v", err) + } + resp.Body.Close() + + if got := seen.Get("X-Test"); got != "caller" { + t.Errorf("X-Test = %q, want caller (caller value must not be overridden)", got) + } +} + +func TestConfigureInvalidHeader(t *testing.T) { + resetConfig(t) + + // a header without ": " should fail loud rather than silently dropping. + if err := Configure(Options{Headers: []string{"missing-separator"}}); err == nil { + t.Fatal("expected error for malformed header, got nil") + } +} + +func TestConfigureInvalidProxy(t *testing.T) { + tests := []struct { + name string + proxy string + }{ + {name: "unsupported scheme", proxy: "ftp://localhost:1080"}, + {name: "malformed url", proxy: "://nope"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resetConfig(t) + if err := Configure(Options{Proxy: tt.proxy}); err == nil { + t.Errorf("expected error for proxy %q, got nil", tt.proxy) + } + }) + } +} + +func TestRateLimit(t *testing.T) { + resetConfig(t) + + const ratePerSec = 5 + if err := Configure(Options{RateLimit: ratePerSec}); err != nil { + t.Fatalf("Configure: %v", err) + } + + var seen http.Header + srv := captureServer(t, &seen) + client := Client(5 * time.Second) + + // at 5 req/s the limiter starts with a full burst, so the first batch is + // immediate and the next request must wait roughly one tick. fire burst+1 + // requests and assert the extra one forced a measurable delay. + const requests = ratePerSec + 1 + start := time.Now() + for i := 0; i < requests; i++ { + get(t, client, srv.URL) + } + elapsed := time.Since(start) + + // one request beyond the burst should cost about 1/rate; allow slack but + // require a non-trivial delay so an unthrottled client fails this. + minDelay := time.Second / ratePerSec / 2 + if elapsed < minDelay { + t.Errorf("expected rate limiting to add >= %v of delay, got %v", minDelay, elapsed) + } +} + +func TestRateLimitUnlimited(t *testing.T) { + resetConfig(t) + + // RateLimit 0 means no limiter is installed; requests should fly through. + if err := Configure(Options{RateLimit: 0}); err != nil { + t.Fatalf("Configure: %v", err) + } + + mu.RLock() + rt, ok := configured.(*roundTripper) + mu.RUnlock() + if !ok { + t.Fatal("configured transport is not *roundTripper") + } + if rt.limiter != nil { + t.Error("expected no limiter when RateLimit is 0") + } +} diff --git a/internal/scan/cloudstorage.go b/internal/scan/cloudstorage.go index 417a37e..d1d0f90 100644 --- a/internal/scan/cloudstorage.go +++ b/internal/scan/cloudstorage.go @@ -21,6 +21,7 @@ import ( "time" "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/styles" ) @@ -50,9 +51,7 @@ func CloudStorage(url string, timeout time.Duration, logdir string) ([]CloudStor Prefix: "C3", }).With("url", url) - client := &http.Client{ - Timeout: timeout, - } + client := httpx.Client(timeout) potentialBuckets := extractPotentialBuckets(sanitizedURL) diff --git a/internal/scan/cms.go b/internal/scan/cms.go index 22600eb..d07cc95 100644 --- a/internal/scan/cms.go +++ b/internal/scan/cms.go @@ -19,6 +19,7 @@ import ( "strings" "time" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -45,9 +46,7 @@ func CMS(url string, timeout time.Duration, logdir string) (*CMSResult, error) { } } - client := &http.Client{ - Timeout: timeout, - } + client := httpx.Client(timeout) req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, http.NoBody) if err != nil { diff --git a/internal/scan/dirlist.go b/internal/scan/dirlist.go index 102c523..f56824b 100644 --- a/internal/scan/dirlist.go +++ b/internal/scan/dirlist.go @@ -22,6 +22,7 @@ import ( "time" charmlog "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -64,12 +65,14 @@ func Dirlist(size string, url string, timeout time.Duration, threads int, logdir list = directoryURL + bigFile } + client := httpx.Client(timeout) + req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, list, http.NoBody) if err != nil { log.Error("Error creating directory list request: %s", err) return nil, err } - resp, err := http.DefaultClient.Do(req) + resp, err := client.Do(req) if err != nil { log.Error("Error downloading directory list: %s", err) return nil, err @@ -83,10 +86,6 @@ func Dirlist(size string, url string, timeout time.Duration, threads int, logdir directories = append(directories, scanner.Text()) } - client := &http.Client{ - Timeout: timeout, - } - progress := output.NewProgress(len(directories), "fuzzing") var wg sync.WaitGroup diff --git a/internal/scan/dnslist.go b/internal/scan/dnslist.go index b490a55..7f7c592 100644 --- a/internal/scan/dnslist.go +++ b/internal/scan/dnslist.go @@ -21,6 +21,7 @@ import ( "time" charmlog "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -58,7 +59,7 @@ func Dnslist(size string, url string, timeout time.Duration, threads int, logdir log.Error("Error creating request: %s", err) return nil, err } - resp, err := http.DefaultClient.Do(req) + resp, err := httpx.Client(timeout).Do(req) if err != nil { log.Error("Error downloading DNS list: %s", err) return nil, err @@ -81,9 +82,11 @@ func Dnslist(size string, url string, timeout time.Duration, threads int, logdir } } - client := &http.Client{ - Timeout: timeout, - Transport: dnsTransport, + // per-host probe client. dnsTransport pins every dial at a fixture in + // integration tests; nil keeps the shared transport for real runs. + client := httpx.Client(timeout) + if dnsTransport != nil { + client.Transport = dnsTransport } progress := output.NewProgress(len(dns), "enumerating") diff --git a/internal/scan/dork.go b/internal/scan/dork.go index 43ec6a3..c685285 100644 --- a/internal/scan/dork.go +++ b/internal/scan/dork.go @@ -25,6 +25,7 @@ import ( "time" "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" googlesearch "github.com/rocketlaunchr/google-search" @@ -76,7 +77,7 @@ func Dork(url string, timeout time.Duration, threads int, logdir string) ([]Dork output.Error("Error creating dork list request: %s", err) return nil, err } - resp, err := http.DefaultClient.Do(req) + resp, err := httpx.Client(timeout).Do(req) if err != nil { spin.Stop() output.Error("Error downloading dork list: %s", err) diff --git a/internal/scan/frameworks/detect.go b/internal/scan/frameworks/detect.go index 2ae683a..487b244 100644 --- a/internal/scan/frameworks/detect.go +++ b/internal/scan/frameworks/detect.go @@ -22,6 +22,7 @@ import ( "time" charmlog "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -47,7 +48,7 @@ func DetectFramework(url string, timeout time.Duration, logdir string) (*Framewo spin := output.NewSpinner("Detecting frameworks") spin.Start() - client := &http.Client{Timeout: timeout} + client := httpx.Client(timeout) req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, url, http.NoBody) if err != nil { diff --git a/internal/scan/git.go b/internal/scan/git.go index 0a4d3a7..b026ac5 100644 --- a/internal/scan/git.go +++ b/internal/scan/git.go @@ -22,6 +22,7 @@ import ( "time" charmlog "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -48,13 +49,15 @@ func Git(url string, timeout time.Duration, threads int, logdir string) ([]strin } } + client := httpx.Client(timeout) + req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, gitURL+gitFile, http.NoBody) if err != nil { spin.Stop() log.Error("Error creating git list request: %s", err) return nil, err } - resp, err := http.DefaultClient.Do(req) + resp, err := client.Do(req) if err != nil { spin.Stop() log.Error("Error downloading git list: %s", err) @@ -68,11 +71,6 @@ func Git(url string, timeout time.Duration, threads int, logdir string) ([]strin gitUrls = append(gitUrls, scanner.Text()) } - // util.InitProgressBar() - client := &http.Client{ - Timeout: timeout, - } - var wg sync.WaitGroup var mu sync.Mutex wg.Add(threads) diff --git a/internal/scan/headers.go b/internal/scan/headers.go index 6badd5f..35ce693 100644 --- a/internal/scan/headers.go +++ b/internal/scan/headers.go @@ -17,6 +17,7 @@ import ( "net/http" "time" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -39,9 +40,7 @@ func Headers(url string, timeout time.Duration, logdir string) ([]HeaderResult, } } - client := &http.Client{ - Timeout: timeout, - } + client := httpx.Client(timeout) req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, url, http.NoBody) if err != nil { diff --git a/internal/scan/js/frameworks/next.go b/internal/scan/js/frameworks/next.go index 62b5be9..593ade3 100644 --- a/internal/scan/js/frameworks/next.go +++ b/internal/scan/js/frameworks/next.go @@ -30,6 +30,7 @@ import ( "regexp" "strings" + "github.com/dropalldatabases/sif/internal/httpx" urlutil "github.com/projectdiscovery/utils/url" ) @@ -48,7 +49,9 @@ func GetPagesRouterScripts(scriptUrl string) ([]string, error) { return nil, err } - resp, err := http.DefaultClient.Do(req) + // no timeout in scope here; 0 matches the previous DefaultClient behavior + // while still routing through the shared transport (proxy/headers/rate-limit). + resp, err := httpx.Client(0).Do(req) if err != nil { fmt.Println(err) return nil, err diff --git a/internal/scan/js/scan.go b/internal/scan/js/scan.go index 3e4ae11..519b11e 100644 --- a/internal/scan/js/scan.go +++ b/internal/scan/js/scan.go @@ -23,6 +23,7 @@ import ( "github.com/antchfx/htmlquery" charmlog "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/output" "github.com/dropalldatabases/sif/internal/scan/js/frameworks" urlutil "github.com/projectdiscovery/utils/url" @@ -43,6 +44,8 @@ func JavascriptScan(url string, timeout time.Duration, threads int, logdir strin spin := output.NewSpinner("Scanning JavaScript files") spin.Start() + client := httpx.Client(timeout) + baseUrl, err := urlutil.Parse(url) if err != nil { spin.Stop() @@ -53,7 +56,7 @@ func JavascriptScan(url string, timeout time.Duration, threads int, logdir strin spin.Stop() return nil, err } - resp, err := http.DefaultClient.Do(req) + resp, err := client.Do(req) if err != nil { spin.Stop() return nil, err @@ -120,7 +123,7 @@ func JavascriptScan(url string, timeout time.Duration, threads int, logdir strin charmlog.Warnf("Failed to create request: %s", err) continue } - resp, err := http.DefaultClient.Do(req) + resp, err := client.Do(req) if err != nil { charmlog.Warnf("Failed to fetch script: %s", err) continue diff --git a/internal/scan/js/supabase.go b/internal/scan/js/supabase.go index be853d1..6e86180 100644 --- a/internal/scan/js/supabase.go +++ b/internal/scan/js/supabase.go @@ -30,6 +30,7 @@ import ( "time" "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" ) // jwtRegex matches JWT tokens in JavaScript content. @@ -110,7 +111,7 @@ func getSupabaseOpenAPI(projectId, apikey string, auth *string, timeout time.Dur // doSupabaseRequest performs a GET request to the Supabase API. func doSupabaseRequest(projectId, path, apikey string, auth *string, timeout time.Duration) ([]byte, *http.Response, error) { - client := http.Client{Timeout: timeout} + client := httpx.Client(timeout) req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, "https://"+projectId+".supabase.co"+path, http.NoBody) if err != nil { @@ -182,7 +183,7 @@ func ScanSupabase(jsContent string, jsUrl string, timeout time.Duration) ([]supa } supabaselog.Infof("Found valid supabase project %s with role %s", *supabaseJwt.ProjectId, *supabaseJwt.Role) - client := http.Client{Timeout: timeout} + client := httpx.Client(timeout) req, err := http.NewRequestWithContext(context.TODO(), http.MethodPost, "https://"+*supabaseJwt.ProjectId+".supabase.co/auth/v1/signup", bytes.NewBufferString(`{"email":"automated`+strconv.Itoa(int(time.Now().Unix()))+`@sif.sh","password":"automatedacct"}`)) if err != nil { diff --git a/internal/scan/lfi.go b/internal/scan/lfi.go index f93f33a..583d1a8 100644 --- a/internal/scan/lfi.go +++ b/internal/scan/lfi.go @@ -23,6 +23,7 @@ import ( "time" charmlog "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -136,14 +137,12 @@ func LFI(targetURL string, timeout time.Duration, threads int, logdir string) (* var mu sync.Mutex var wg sync.WaitGroup - client := &http.Client{ - Timeout: timeout, - CheckRedirect: func(req *http.Request, via []*http.Request) error { - if len(via) >= 3 { - return http.ErrUseLastResponse - } - return nil - }, + client := httpx.Client(timeout) + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if len(via) >= 3 { + return http.ErrUseLastResponse + } + return nil } // parse the target URL to check for existing parameters diff --git a/internal/scan/ports.go b/internal/scan/ports.go index 87110d7..a0566fa 100644 --- a/internal/scan/ports.go +++ b/internal/scan/ports.go @@ -23,6 +23,7 @@ import ( "time" charmlog "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -50,7 +51,7 @@ func Ports(ctx context.Context, scope string, url string, timeout time.Duration, log.Error("Error creating request: %s", err) return nil, err } - resp, err := http.DefaultClient.Do(req) + resp, err := httpx.Client(timeout).Do(req) if err != nil { log.Error("Error downloading ports list: %s", err) return nil, err diff --git a/internal/scan/scan.go b/internal/scan/scan.go index 95330e0..91e3aab 100644 --- a/internal/scan/scan.go +++ b/internal/scan/scan.go @@ -27,6 +27,7 @@ import ( "time" "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -85,11 +86,9 @@ func Scan(url string, timeout time.Duration, threads int, logdir string) { } } - client := &http.Client{ - Timeout: timeout, - CheckRedirect: func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - }, + client := httpx.Client(timeout) + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse } resp := fetchRobotsTXT(url+"/robots.txt", client) diff --git a/internal/scan/securityheaders.go b/internal/scan/securityheaders.go index 2444fa7..dc64fc8 100644 --- a/internal/scan/securityheaders.go +++ b/internal/scan/securityheaders.go @@ -19,6 +19,7 @@ import ( "strings" "time" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -64,9 +65,7 @@ func SecurityHeaders(url string, timeout time.Duration, logdir string) (Security } } - client := &http.Client{ - Timeout: timeout, - } + client := httpx.Client(timeout) req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, url, http.NoBody) if err != nil { diff --git a/internal/scan/securitytrails.go b/internal/scan/securitytrails.go index aeccdd9..42aaf62 100644 --- a/internal/scan/securitytrails.go +++ b/internal/scan/securitytrails.go @@ -23,6 +23,7 @@ import ( "strings" "time" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -73,7 +74,7 @@ func SecurityTrails(targetURL string, timeout time.Duration, logdir string) (*Se } hostname := parsedURL.Hostname() - client := &http.Client{Timeout: timeout} + client := httpx.Client(timeout) result := &SecurityTrailsResult{ Domain: hostname, diff --git a/internal/scan/shodan.go b/internal/scan/shodan.go index 238e3e9..f288b03 100644 --- a/internal/scan/shodan.go +++ b/internal/scan/shodan.go @@ -24,6 +24,7 @@ import ( "strings" "time" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -180,7 +181,7 @@ func resolveHostname(hostname string) (string, error) { } func queryShodanHost(ip string, apiKey string, timeout time.Duration) (*ShodanResult, error) { - client := &http.Client{Timeout: timeout} + client := httpx.Client(timeout) reqURL := fmt.Sprintf("%s/shodan/host/%s?key=%s", shodanBaseURL, ip, apiKey) req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, reqURL, http.NoBody) diff --git a/internal/scan/sql.go b/internal/scan/sql.go index d0f7e73..6f0e821 100644 --- a/internal/scan/sql.go +++ b/internal/scan/sql.go @@ -23,6 +23,7 @@ import ( "time" charmlog "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/output" ) @@ -140,14 +141,12 @@ func SQL(targetURL string, timeout time.Duration, threads int, logdir string) (* var mu sync.Mutex var wg sync.WaitGroup - client := &http.Client{ - Timeout: timeout, - CheckRedirect: func(req *http.Request, via []*http.Request) error { - if len(via) >= 3 { - return http.ErrUseLastResponse - } - return nil - }, + client := httpx.Client(timeout) + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if len(via) >= 3 { + return http.ErrUseLastResponse + } + return nil } // check for admin panels diff --git a/internal/scan/subdomaintakeover.go b/internal/scan/subdomaintakeover.go index d77b857..1d0cfbd 100644 --- a/internal/scan/subdomaintakeover.go +++ b/internal/scan/subdomaintakeover.go @@ -24,6 +24,7 @@ import ( "time" "github.com/charmbracelet/log" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/styles" ) @@ -54,9 +55,7 @@ func SubdomainTakeover(url string, dnsResults []string, timeout time.Duration, t Prefix: "Subdomain Takeover", }) - client := &http.Client{ - Timeout: timeout, - } + client := httpx.Client(timeout) var wg sync.WaitGroup wg.Add(threads) diff --git a/man/sif.1 b/man/sif.1 index 47ebce7..f70e3cd 100644 --- a/man/sif.1 +++ b/man/sif.1 @@ -103,11 +103,23 @@ per\-request timeout (default 10s). directory to write logs to. .TP .BR \-\-threads " \fIn\fR" -number of concurrent workers (default 10). +number of concurrent workers (default 10). values below 1 are clamped to 1. .TP .BR \-\-template " \fIname\fR" sif runtime template to use. .TP +.BR \-proxy " \fIurl\fR" +route every request through a proxy. accepts http, https or socks5 urls. +.TP +.BR \-H ", " \-\-header " \fIstring\fR" +custom header to send with every request, as \fBKey: Value\fR. repeatable or comma\-separated. +.TP +.BR \-cookie " \fIstring\fR" +cookie header to send with every request. +.TP +.BR \-rate\-limit " \fIn\fR" +cap outbound requests per second (0 = unlimited, default 0). +.TP .B \-api emit json results and suppress the interactive output. .SH MODULES diff --git a/sif.go b/sif.go index 0932571..cee06fd 100644 --- a/sif.go +++ b/sif.go @@ -25,6 +25,7 @@ import ( "github.com/charmbracelet/log" "github.com/dropalldatabases/sif/internal/config" + "github.com/dropalldatabases/sif/internal/httpx" "github.com/dropalldatabases/sif/internal/logger" "github.com/dropalldatabases/sif/internal/modules" "github.com/dropalldatabases/sif/internal/output" @@ -180,6 +181,18 @@ func (app *App) Run() error { }() } + // wire proxy/headers/cookie/rate-limit into the shared http client once, + // before any scanner runs. a bad proxy/header shouldn't kill the run - + // scanners fall back to a plain client if this fails. + if err := httpx.Configure(httpx.Options{ + Proxy: app.settings.Proxy, + Headers: app.settings.Header, + Cookie: app.settings.Cookie, + RateLimit: app.settings.RateLimit, + }); err != nil { + log.Warnf("http client config failed, continuing with defaults: %v", err) + } + // target expansion - securitytrails discovers new domains before scanning if app.settings.SecurityTrails { expanded := app.expandTargets()