feat(scan): add -concurrency flag to scan targets in parallel (#367)

* refactor(cli): extract per-target scan into scanTarget

pull the target loop body into scanTarget, which returns a targetScan
holding that target's findings, report rows, scan labels and log files
instead of appending onto run-wide slices. the run loop merges those in
target order and finishRun handles the post-loop notify/silent/report/
summary steps. behavior is identical for the sequential run today; the
isolation is the seam a bounded worker pool needs next.

* feat(cli): scan targets in parallel with -concurrency

add -concurrency N (default 1) to run the per-target scan pool over
several targets at once. scanTarget already returns isolated
accumulators, so workers share only the console: output.SetConcurrent
serializes sink writes and de-animates spinners/progress so parallel
lines stay whole. results merge in input order, and concurrency 1 keeps
the sequential path unchanged.
This commit is contained in:
Tigah
2026-07-12 14:23:35 -07:00
committed by GitHub
parent 24f3481fb1
commit d13622dc1c
7 changed files with 717 additions and 404 deletions
+8
View File
@@ -40,6 +40,7 @@ type Settings struct {
Git bool
Whois bool
Threads int
Concurrency int
Nuclei bool
JavaScript bool
Timeout time.Duration
@@ -171,6 +172,7 @@ func registerFlags(settings *Settings) *goflags.FlagSet {
flagSet.DurationVarP(&settings.Timeout, "timeout", "t", 10*time.Second, "HTTP request timeout"),
flagSet.StringVarP(&settings.LogDir, "log", "l", "", "Directory to store logs in"),
flagSet.IntVar(&settings.Threads, "threads", 10, "Number of threads to run scans on"),
flagSet.IntVar(&settings.Concurrency, "concurrency", 1, "Number of targets to scan in parallel (>1 interleaves console output)"),
flagSet.StringVar(&settings.Template, "template", "", "Load scan settings from a template (preset minimal/recon/full, or a local yaml file)"),
)
@@ -241,5 +243,11 @@ func Parse() *Settings {
settings.Threads = minThreads
}
// concurrency bounds the target worker pool; floor it so 0/negative means
// sequential rather than a stalled or panicking pool.
if settings.Concurrency < 1 {
settings.Concurrency = 1
}
return settings
}
+38 -1
View File
@@ -17,6 +17,7 @@ import (
"io"
"os"
"strings"
"sync"
"github.com/charmbracelet/lipgloss"
)
@@ -154,6 +155,40 @@ func Silent() bool {
return silent
}
// concurrent is set when multiple targets are scanned in parallel. it serializes
// sink writes so lines from different targets never garble, and disables live
// widgets, which cannot share one terminal across goroutines.
var concurrent bool
// SetConcurrent switches the sink into parallel-safe mode: writes go through a
// mutex and interactive widgets (spinners, live progress) are gated off. call it
// once before launching target workers; it is not meant to be toggled back.
func SetConcurrent(enabled bool) {
concurrent = enabled
if enabled {
sink = &lockingWriter{w: sink}
}
}
// Concurrent reports whether parallel-target mode is active; widget code gates on
// it so spinners and progress bars stay silent when targets interleave.
func Concurrent() bool {
return concurrent
}
// lockingWriter serializes concurrent writes to the wrapped sink so one target's
// line is never interleaved mid-write with another's.
type lockingWriter struct {
mu sync.Mutex
w io.Writer
}
func (l *lockingWriter) Write(p []byte) (int, error) {
l.mu.Lock()
defer l.mu.Unlock()
return l.w.Write(p)
}
// Writer is the current chrome sink (stdout normally, stderr under -silent).
// callers that render their own chrome (the startup banner) write here so it
// follows the same routing as everything else.
@@ -286,7 +321,9 @@ func (m *ModuleLogger) Complete(resultCount int, resultType string) {
// ClearLine clears the current line (for progress bar updates). silent mode is
// non-interactive, so there's no live line to clear and stdout stays untouched.
func ClearLine() {
if !IsTTY || silent {
// under -concurrency there is no single live line to clear, and emitting the
// escape would wipe whatever another target just wrote.
if !IsTTY || silent || concurrent {
return
}
fmt.Fprint(sink, "\033[2K\r")
+6 -3
View File
@@ -91,7 +91,9 @@ func (p *Progress) Resume() {
// Done clears the progress bar line
func (p *Progress) Done() {
if apiMode || !IsTTY {
// under -concurrency there is no live line to clear (render printed milestones),
// and clearing would wipe another target's output.
if apiMode || !IsTTY || concurrent {
return
}
ClearLine()
@@ -102,8 +104,9 @@ func (p *Progress) render() {
return
}
// In non-TTY mode, print progress at milestones only
if !IsTTY {
// In non-TTY mode, or under -concurrency, print progress at milestones only
// rather than redrawing a live line that parallel targets would corrupt.
if !IsTTY || concurrent {
current := atomic.LoadInt64(&p.current)
total := p.total
if total <= 0 {
+3 -2
View File
@@ -54,8 +54,9 @@ func (s *Spinner) Start() {
s.done = make(chan struct{})
s.mu.Unlock()
// In non-TTY mode, just print the message once
if !IsTTY {
// In non-TTY mode, or when targets scan in parallel, just print the message
// once: a shared animated line can't be driven from concurrent goroutines.
if !IsTTY || concurrent {
fmt.Fprintf(sink, " %s...\n", s.message)
return
}
+113 -7
View File
@@ -23,6 +23,7 @@ import (
"io"
"os"
"strings"
"sync"
"github.com/charmbracelet/log"
"github.com/vmfunc/sif/internal/config"
@@ -322,14 +323,108 @@ func (app *App) Run() error {
}
}
for _, url := range app.targets {
results, err := app.scanAllTargets(storeDir, wantReport)
if err != nil {
return err
}
// merge per-target results in input order so the run-wide view is identical
// regardless of the order workers finished under -concurrency.
for _, ts := range results {
scansRun = append(scansRun, ts.scansRun...)
app.logFiles = append(app.logFiles, ts.logFiles...)
allFindings = append(allFindings, ts.findings...)
if wantReport {
reportResults = append(reportResults, ts.reportResults...)
}
}
return app.finishRun(scansRun, allFindings, reportResults, wantReport)
}
// targetScan holds one target's isolated scan output: its findings, report rows,
// scan labels and log files, which the run loop merges in target order.
type targetScan struct {
findings []finding.Finding
reportResults []report.Result
scansRun []string
logFiles []string
}
// scanAllTargets scans every target and returns the per-target results in input
// order. With concurrency 1 it runs sequentially, behaviour-identical to a plain
// loop. Above 1 it runs a bounded worker pool: scanTarget is self-contained
// (isolated accumulators, no run-wide writes), so the only shared surface is the
// console, which output.SetConcurrent serializes and de-animates. Results are
// indexed by target position, so the caller merges them in a stable order no
// matter which worker finished first.
func (app *App) scanAllTargets(storeDir string, wantReport bool) ([]targetScan, error) {
results := make([]targetScan, len(app.targets))
concurrency := app.settings.Concurrency
if concurrency < 1 {
concurrency = 1
}
if concurrency > len(app.targets) {
concurrency = len(app.targets)
}
if concurrency <= 1 {
for i, url := range app.targets {
ts, err := app.scanTarget(url, storeDir, wantReport)
if err != nil {
return nil, err
}
results[i] = ts
}
return results, nil
}
output.SetConcurrent(true)
errs := make([]error, len(app.targets))
sem := make(chan struct{}, concurrency)
var wg sync.WaitGroup
for i, url := range app.targets {
wg.Add(1)
sem <- struct{}{}
go func(i int, url string) {
defer wg.Done()
defer func() { <-sem }()
// a panic in one target's scan (a module bug, a nil deref deep in a
// third-party client) must not take down every other worker's
// in-flight scan; convert it into that target's error instead.
defer func() {
if r := recover(); r != nil {
errs[i] = fmt.Errorf("panic scanning %s: %v", url, r)
}
}()
results[i], errs[i] = app.scanTarget(url, storeDir, wantReport)
}(i, url)
}
wg.Wait()
for _, err := range errs {
if err != nil {
return nil, err
}
}
return results, nil
}
// scanTarget runs the full scanner set for one target and returns its isolated
// accumulators without mutating run-wide state.
func (app *App) scanTarget(url, storeDir string, wantReport bool) (targetScan, error) {
var scansRun []string
var logFiles []string
output.Info("Starting scan on %s", output.Highlight.Render(url))
moduleResults := make([]ModuleResult, 0, 16)
if app.settings.LogDir != "" {
if err := logger.CreateFile(&app.logFiles, url, app.settings.LogDir); err != nil {
return err
if err := logger.CreateFile(&logFiles, url, app.settings.LogDir); err != nil {
return targetScan{}, err
}
}
@@ -702,13 +797,12 @@ func (app *App) Run() error {
marshalled, err := json.Marshal(result)
if err != nil {
log.Errorf("failed to marshal result: %s", err)
continue
return targetScan{scansRun: scansRun, logFiles: logFiles}, nil
}
fmt.Println(string(marshalled))
}
targetFindings := collectFindings(url, moduleResults)
allFindings = append(allFindings, targetFindings...)
// diff mode is per-target: load this target's last snapshot, surface only
// the delta, then overwrite the snapshot so the next run diffs against now.
@@ -720,11 +814,23 @@ func (app *App) Run() error {
// the report carries raw blobs and is only built when an export flag is
// set, so the common path skips the marshalling entirely.
var reportResults []report.Result
if wantReport {
reportResults = append(reportResults, collectReportResults(url, moduleResults)...)
}
reportResults = collectReportResults(url, moduleResults)
}
return targetScan{
findings: targetFindings,
reportResults: reportResults,
scansRun: scansRun,
logFiles: logFiles,
}, nil
}
// finishRun performs the run-wide steps after every target has been scanned:
// notify, the silent findings stream, report files and the summary. It consumes
// the merged accumulators so per-target scanning stays isolated in scanTarget.
func (app *App) finishRun(scansRun []string, allFindings []finding.Finding, reportResults []report.Result, wantReport bool) error {
// the normalized findings are the handoff point for notify/diff; surface the
// count now so the path is live and observable without changing output.
log.Debugf("normalized %d findings across %d targets", len(allFindings), len(app.targets))
+80
View File
@@ -0,0 +1,80 @@
/*
·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
: :
: █▀ █ █▀▀ · Blazing-fast pentesting suite :
: ▄█ █ █▀ · BSD 3-Clause License :
: :
: (c) 2022-2026 vmfunc, xyzeva, :
: lunchcat alumni & contributors :
: :
·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
*/
package sif
import (
"testing"
"github.com/vmfunc/sif/internal/output"
)
// TestScanAllTargetsConcurrentIsolation runs the target pool at concurrency 3
// against three separate servers and asserts each target's result is isolated:
// exactly one scan recorded per target, results in input order. Run under -race
// it also backstops scanTarget's concurrency safety and the serialized sink.
func TestScanAllTargetsConcurrentIsolation(t *testing.T) {
defer output.SetConcurrent(false)
srvA := okServer()
defer srvA.Close()
srvB := okServer()
defer srvB.Close()
srvC := okServer()
defer srvC.Close()
app := headersOnlyApp()
app.targets = []string{srvA.URL, srvB.URL, srvC.URL}
app.settings.Concurrency = 3
results, err := app.scanAllTargets("", false)
if err != nil {
t.Fatalf("scanAllTargets: %v", err)
}
if len(results) != len(app.targets) {
t.Fatalf("got %d results, want %d", len(results), len(app.targets))
}
for i, ts := range results {
if len(ts.scansRun) != 1 || ts.scansRun[0] != "HTTP Headers" {
t.Errorf("target %d scansRun = %v, want exactly [HTTP Headers] (accumulator leaked across targets)", i, ts.scansRun)
}
}
}
// TestScanAllTargetsSequentialMatchesInputOrder pins that concurrency 1 keeps the
// plain sequential path: one result per target, in order, no pool engaged.
func TestScanAllTargetsSequentialMatchesInputOrder(t *testing.T) {
srvA := okServer()
defer srvA.Close()
srvB := okServer()
defer srvB.Close()
app := headersOnlyApp()
app.targets = []string{srvA.URL, srvB.URL}
app.settings.Concurrency = 1
results, err := app.scanAllTargets("", false)
if err != nil {
t.Fatalf("scanAllTargets: %v", err)
}
if len(results) != 2 {
t.Fatalf("got %d results, want 2", len(results))
}
for i, ts := range results {
if len(ts.scansRun) != 1 || ts.scansRun[0] != "HTTP Headers" {
t.Errorf("target %d scansRun = %v, want exactly [HTTP Headers]", i, ts.scansRun)
}
}
if output.Concurrent() {
t.Error("concurrency 1 must not switch output into concurrent mode")
}
}
+78
View File
@@ -0,0 +1,78 @@
/*
·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
: :
: █▀ █ █▀▀ · Blazing-fast pentesting suite :
: ▄█ █ █▀ · BSD 3-Clause License :
: :
: (c) 2022-2026 vmfunc, xyzeva, :
: lunchcat alumni & contributors :
: :
·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
*/
package sif
import (
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/vmfunc/sif/internal/config"
)
// headersOnlyApp builds an App that runs only the HTTP headers scan against
// live servers, with every network-touching scanner gated off. it isolates the
// scanTarget path to a single deterministic scan.
func headersOnlyApp() *App {
return &App{settings: &config.Settings{
Headers: true,
NoScan: true,
Dirlist: "none",
Dnslist: "none",
Ports: "none",
Timeout: 5 * time.Second,
}}
}
func okServer() *httptest.Server {
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("X-Sif-Test", "1")
w.WriteHeader(http.StatusOK)
}))
}
// TestScanTargetIsolatesPerTargetState is the accumulator seam the worker pool
// depends on: each scanTarget call must own its scansRun rather than share a
// run-wide slice. two sequential scans of different targets must each report a
// single scan, not a growing total; a shared accumulator would make the second
// call report two.
func TestScanTargetIsolatesPerTargetState(t *testing.T) {
srvA := okServer()
defer srvA.Close()
srvB := okServer()
defer srvB.Close()
app := headersOnlyApp()
tsA, err := app.scanTarget(srvA.URL, "", false)
if err != nil {
t.Fatalf("scanTarget(A): %v", err)
}
if len(tsA.scansRun) != 1 || tsA.scansRun[0] != "HTTP Headers" {
t.Fatalf("target A scansRun = %v, want exactly [HTTP Headers]", tsA.scansRun)
}
tsB, err := app.scanTarget(srvB.URL, "", false)
if err != nil {
t.Fatalf("scanTarget(B): %v", err)
}
if len(tsB.scansRun) != 1 || tsB.scansRun[0] != "HTTP Headers" {
t.Fatalf("target B scansRun = %v, want exactly [HTTP Headers] (accumulator leaked across targets)", tsB.scansRun)
}
// no log dir configured, so no per-target log file should be recorded.
if len(tsA.logFiles) != 0 || len(tsB.logFiles) != 0 {
t.Errorf("logFiles should be empty without a log dir, got A=%v B=%v", tsA.logFiles, tsB.logFiles)
}
}