From 0383a7bcd2ade285fb089f0abf9be0dd98804f42 Mon Sep 17 00:00:00 2001 From: vmfunc Date: Wed, 10 Jun 2026 15:43:32 -0700 Subject: [PATCH] perf(scan): drain response bodies so pooled connections get reused go only returns a conn to the idle pool when the body is read to EOF before Close. the header-only and early-return scan paths closed an unread body, leaking the conn and forcing a fresh dial each request. route those close sites through httpx.DrainClose so the tuned pool from phase 1 actually gets reused. body-read paths (scanner/io.ReadAll) are left untouched. --- internal/scan/cloudstorage.go | 5 +++-- internal/scan/cms.go | 5 +++-- internal/scan/cors.go | 6 +++--- internal/scan/dnslist.go | 5 +++-- internal/scan/git.go | 5 +++-- internal/scan/headers.go | 5 +++-- internal/scan/js/supabase.go | 7 +++++-- internal/scan/passive.go | 6 ++++-- internal/scan/redirect.go | 6 ++++-- internal/scan/scan.go | 14 +++++++++----- internal/scan/securityheaders.go | 5 +++-- internal/scan/securitytrails.go | 6 ++++-- internal/scan/shodan.go | 6 ++++-- internal/scan/sql.go | 3 ++- 14 files changed, 53 insertions(+), 31 deletions(-) diff --git a/internal/scan/cloudstorage.go b/internal/scan/cloudstorage.go index d1d0f90..d8b4811 100644 --- a/internal/scan/cloudstorage.go +++ b/internal/scan/cloudstorage.go @@ -104,11 +104,12 @@ func checkS3Bucket(ctx context.Context, bucket string, client *http.Client) (boo if err != nil { return false, err } - resp, err := client.Do(req) + resp, err := client.Do(req) //nolint:bodyclose // drained and closed via httpx.DrainClose if err != nil { return false, err } - defer resp.Body.Close() + // status only; drain on close so the conn returns to the pool. + defer httpx.DrainClose(resp) // If we can access the bucket listing, it's public return resp.StatusCode == http.StatusOK, nil diff --git a/internal/scan/cms.go b/internal/scan/cms.go index d07cc95..dd3247a 100644 --- a/internal/scan/cms.go +++ b/internal/scan/cms.go @@ -128,10 +128,11 @@ func detectWordPress(url string, client *http.Client, bodyString string) bool { if err != nil { continue } - resp, err := client.Do(req) + resp, err := client.Do(req) //nolint:bodyclose // drained and closed via httpx.DrainClose if err == nil { found := resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusFound - resp.Body.Close() + // status only; drain so the conn returns to the pool. + httpx.DrainClose(resp) if found { return true } diff --git a/internal/scan/cors.go b/internal/scan/cors.go index 3828628..c974376 100644 --- a/internal/scan/cors.go +++ b/internal/scan/cors.go @@ -175,13 +175,13 @@ func probeCORS(client *http.Client, targetURL, origin, note string) (CORSFinding } req.Header.Set("Origin", origin) - resp, err := client.Do(req) + resp, err := client.Do(req) //nolint:bodyclose // drained and closed via httpx.DrainClose if err != nil { charmlog.Debugf("cors: request %s with origin %s: %v", targetURL, origin, err) return CORSFinding{}, false } - // headers are all we need; drain nothing, just close. - resp.Body.Close() + // headers are all we need; drain the body so the conn returns to the pool. + httpx.DrainClose(resp) allowOrigin := resp.Header.Get("Access-Control-Allow-Origin") if allowOrigin == "" { diff --git a/internal/scan/dnslist.go b/internal/scan/dnslist.go index ba1e1ab..9f2ef72 100644 --- a/internal/scan/dnslist.go +++ b/internal/scan/dnslist.go @@ -229,14 +229,15 @@ func probeSubdomain(client *http.Client, host string) (string, dnsScheme) { charmlog.Debugf("Error %s: %s", host, err) continue } - resp, err := client.Do(req) + resp, err := client.Do(req) //nolint:bodyclose // drained and closed via httpx.DrainClose if err != nil { charmlog.Debugf("Error %s: %s", host, err) continue } code := resp.StatusCode resolved := resp.Request.URL.String() - resp.Body.Close() + // status/url only; drain so the conn returns to the pool. + httpx.DrainClose(resp) if meaningfulStatus(code) { return resolved, schemes[i].label diff --git a/internal/scan/git.go b/internal/scan/git.go index b026ac5..9e8b425 100644 --- a/internal/scan/git.go +++ b/internal/scan/git.go @@ -91,7 +91,7 @@ func Git(url string, timeout time.Duration, threads int, logdir string) ([]strin charmlog.Debugf("Error creating request for %s: %s", repourl, err) continue } - resp, err := client.Do(gitReq) + resp, err := client.Do(gitReq) //nolint:bodyclose // drained and closed via httpx.DrainClose if err != nil { charmlog.Debugf("Error %s: %s", repourl, err) continue @@ -109,7 +109,8 @@ func Git(url string, timeout time.Duration, threads int, logdir string) ([]strin foundUrls = append(foundUrls, resp.Request.URL.String()) mu.Unlock() } - resp.Body.Close() + // status/headers only; drain so the conn returns to the pool. + httpx.DrainClose(resp) } }(thread) } diff --git a/internal/scan/headers.go b/internal/scan/headers.go index 35ce693..a481c85 100644 --- a/internal/scan/headers.go +++ b/internal/scan/headers.go @@ -46,11 +46,12 @@ func Headers(url string, timeout time.Duration, logdir string) ([]HeaderResult, if err != nil { return nil, err } - resp, err := client.Do(req) + resp, err := client.Do(req) //nolint:bodyclose // drained and closed via httpx.DrainClose if err != nil { return nil, err } - defer resp.Body.Close() + // header-only scan: drain on close so the conn is returned to the pool. + defer httpx.DrainClose(resp) var results []HeaderResult diff --git a/internal/scan/js/supabase.go b/internal/scan/js/supabase.go index 6e86180..4923865 100644 --- a/internal/scan/js/supabase.go +++ b/internal/scan/js/supabase.go @@ -129,7 +129,9 @@ func doSupabaseRequest(projectId, path, apikey string, auth *string, timeout tim if err != nil { return nil, nil, err } - defer resp.Body.Close() + // the non-200 branch returns before reading the body, so drain on close to + // keep the conn reusable instead of leaking it. + defer httpx.DrainClose(resp) if resp.StatusCode != 200 { return nil, nil, errors.New("request to " + resp.Request.URL.String() + " failed with status code " + strconv.Itoa(resp.StatusCode)) @@ -215,7 +217,8 @@ func ScanSupabase(jsContent string, jsUrl string, timeout time.Duration) ([]supa auth = authResp.AccessToken supabaselog.Infof("Created account with JWT %s", auth) } else { - resp.Body.Close() + // non-200 signup: body never read, so drain to reuse the conn. + httpx.DrainClose(resp) } var collections = []supabaseCollection{} diff --git a/internal/scan/passive.go b/internal/scan/passive.go index 8d02ced..dac7ebf 100644 --- a/internal/scan/passive.go +++ b/internal/scan/passive.go @@ -205,11 +205,13 @@ func passiveGET(ctx context.Context, client *http.Client, reqURL string) ([]byte } req.Header.Set("Accept", "application/json") - resp, err := client.Do(req) + resp, err := client.Do(req) //nolint:bodyclose // drained and closed via httpx.DrainClose if err != nil { return nil, fmt.Errorf("request failed: %w", err) } - defer resp.Body.Close() + // the non-200 branch returns before reading the body, so drain on close to + // keep the conn reusable instead of leaking it. + defer httpx.DrainClose(resp) if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("unexpected status %d", resp.StatusCode) diff --git a/internal/scan/redirect.go b/internal/scan/redirect.go index 7597c31..cff3596 100644 --- a/internal/scan/redirect.go +++ b/internal/scan/redirect.go @@ -229,12 +229,14 @@ func probeRedirect(client *http.Client, testURL string) (location, via string, o charmlog.Debugf("redirect: build request for %s: %v", testURL, err) return "", "", false } - resp, err := client.Do(req) + resp, err := client.Do(req) //nolint:bodyclose // drained and closed via httpx.DrainClose if err != nil { charmlog.Debugf("redirect: request %s: %v", testURL, err) return "", "", false } - defer resp.Body.Close() + // the header-redirect branch returns before reading the body, so drain on + // close to keep that conn reusable instead of leaking it. + defer httpx.DrainClose(resp) // header redirect: a 30x whose Location resolves to the sentinel host if resp.StatusCode >= http.StatusMultipleChoices && resp.StatusCode < http.StatusBadRequest { diff --git a/internal/scan/scan.go b/internal/scan/scan.go index 487dcaa..4145f2b 100644 --- a/internal/scan/scan.go +++ b/internal/scan/scan.go @@ -74,7 +74,8 @@ func fetchRobotsTXT(url string, client *http.Client) *http.Response { } redirectURL := resp.Header.Get("Location") - resp.Body.Close() + // only the Location header is used here; drain so the conn is reusable. + httpx.DrainClose(resp) if redirectURL == "" { log.Debugf("Redirect location is empty for %s", url) return nil @@ -111,11 +112,13 @@ func Scan(url string, timeout time.Duration, threads int, logdir string) { return http.ErrUseLastResponse } - resp := fetchRobotsTXT(url+"/robots.txt", client) + resp := fetchRobotsTXT(url+"/robots.txt", client) //nolint:bodyclose // drained and closed via httpx.DrainClose if resp == nil { return } - defer resp.Body.Close() + // drain on close: the non-success branch never reads the body, so a bare + // close would leak the conn instead of returning it to the pool. + defer httpx.DrainClose(resp) if resp.StatusCode != 404 && resp.StatusCode != 301 && resp.StatusCode != 302 && resp.StatusCode != 307 { output.Success("File %s found", output.Status.Render("robots.txt")) @@ -149,7 +152,7 @@ func Scan(url string, timeout time.Duration, threads int, logdir string) { log.Debugf("Error creating request for %s: %s", sanitizedRobot, err) continue } - resp, err := client.Do(robotReq) + resp, err := client.Do(robotReq) //nolint:bodyclose // drained and closed via httpx.DrainClose if err != nil { log.Debugf("Error %s: %s", sanitizedRobot, err) continue @@ -161,7 +164,8 @@ func Scan(url string, timeout time.Duration, threads int, logdir string) { logger.Write(sanitizedURL, logdir, strconv.Itoa(resp.StatusCode)+" from robots: ["+sanitizedRobot+"]\n") } } - resp.Body.Close() + // status only; drain so the conn returns to the pool. + httpx.DrainClose(resp) } }(thread) diff --git a/internal/scan/securityheaders.go b/internal/scan/securityheaders.go index dc64fc8..4222180 100644 --- a/internal/scan/securityheaders.go +++ b/internal/scan/securityheaders.go @@ -71,11 +71,12 @@ func SecurityHeaders(url string, timeout time.Duration, logdir string) (Security if err != nil { return nil, err } - resp, err := client.Do(req) + resp, err := client.Do(req) //nolint:bodyclose // drained and closed via httpx.DrainClose if err != nil { return nil, err } - defer resp.Body.Close() + // header-only scan: drain on close so the conn is returned to the pool. + defer httpx.DrainClose(resp) results := gradeSecurityHeaders(resp.Header, strings.HasPrefix(url, "https://")) diff --git a/internal/scan/securitytrails.go b/internal/scan/securitytrails.go index 42aaf62..e132473 100644 --- a/internal/scan/securitytrails.go +++ b/internal/scan/securitytrails.go @@ -187,11 +187,13 @@ func doSTRequest(client *http.Client, reqURL, apiKey string) ([]byte, error) { req.Header.Set("APIKEY", apiKey) req.Header.Set("Accept", "application/json") - resp, err := client.Do(req) + resp, err := client.Do(req) //nolint:bodyclose // drained and closed via httpx.DrainClose if err != nil { return nil, fmt.Errorf("SecurityTrails request failed: %w", err) } - defer resp.Body.Close() + // the auth/rate-limit branches return before reading the body, so drain on + // close to keep the conn reusable instead of leaking it. + defer httpx.DrainClose(resp) if resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusUnauthorized { return nil, fmt.Errorf("invalid SecurityTrails API key (status %d)", resp.StatusCode) diff --git a/internal/scan/shodan.go b/internal/scan/shodan.go index f288b03..fbe943b 100644 --- a/internal/scan/shodan.go +++ b/internal/scan/shodan.go @@ -188,11 +188,13 @@ func queryShodanHost(ip string, apiKey string, timeout time.Duration) (*ShodanRe if err != nil { return nil, fmt.Errorf("failed to create Shodan request: %w", err) } - resp, err := client.Do(req) + resp, err := client.Do(req) //nolint:bodyclose // drained and closed via httpx.DrainClose if err != nil { return nil, fmt.Errorf("failed to query Shodan: %w", err) } - defer resp.Body.Close() + // the unauthorized/not-found branches return before reading the body, so + // drain on close to keep the conn reusable instead of leaking it. + defer httpx.DrainClose(resp) if resp.StatusCode == http.StatusUnauthorized { return nil, fmt.Errorf("invalid Shodan API key") diff --git a/internal/scan/sql.go b/internal/scan/sql.go index 6f0e821..600c28c 100644 --- a/internal/scan/sql.go +++ b/internal/scan/sql.go @@ -208,7 +208,8 @@ func SQL(targetURL string, timeout time.Duration, threads int, logdir string) (* } } } else { - resp.Body.Close() + // uninteresting status; body never read, so drain to reuse the conn. + httpx.DrainClose(resp) } } }()