mirror of
https://github.com/lunchcat/sif.git
synced 2026-07-28 14:37:01 -07:00
feat(scan): add -concurrency flag to scan targets in parallel (#367)
* refactor(cli): extract per-target scan into scanTarget pull the target loop body into scanTarget, which returns a targetScan holding that target's findings, report rows, scan labels and log files instead of appending onto run-wide slices. the run loop merges those in target order and finishRun handles the post-loop notify/silent/report/ summary steps. behavior is identical for the sequential run today; the isolation is the seam a bounded worker pool needs next. * feat(cli): scan targets in parallel with -concurrency add -concurrency N (default 1) to run the per-target scan pool over several targets at once. scanTarget already returns isolated accumulators, so workers share only the console: output.SetConcurrent serializes sink writes and de-animates spinners/progress so parallel lines stay whole. results merge in input order, and concurrency 1 keeps the sequential path unchanged.
This commit is contained in:
@@ -40,6 +40,7 @@ type Settings struct {
|
||||
Git bool
|
||||
Whois bool
|
||||
Threads int
|
||||
Concurrency int
|
||||
Nuclei bool
|
||||
JavaScript bool
|
||||
Timeout time.Duration
|
||||
@@ -171,6 +172,7 @@ func registerFlags(settings *Settings) *goflags.FlagSet {
|
||||
flagSet.DurationVarP(&settings.Timeout, "timeout", "t", 10*time.Second, "HTTP request timeout"),
|
||||
flagSet.StringVarP(&settings.LogDir, "log", "l", "", "Directory to store logs in"),
|
||||
flagSet.IntVar(&settings.Threads, "threads", 10, "Number of threads to run scans on"),
|
||||
flagSet.IntVar(&settings.Concurrency, "concurrency", 1, "Number of targets to scan in parallel (>1 interleaves console output)"),
|
||||
flagSet.StringVar(&settings.Template, "template", "", "Load scan settings from a template (preset minimal/recon/full, or a local yaml file)"),
|
||||
)
|
||||
|
||||
@@ -241,5 +243,11 @@ func Parse() *Settings {
|
||||
settings.Threads = minThreads
|
||||
}
|
||||
|
||||
// concurrency bounds the target worker pool; floor it so 0/negative means
|
||||
// sequential rather than a stalled or panicking pool.
|
||||
if settings.Concurrency < 1 {
|
||||
settings.Concurrency = 1
|
||||
}
|
||||
|
||||
return settings
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
)
|
||||
@@ -154,6 +155,40 @@ func Silent() bool {
|
||||
return silent
|
||||
}
|
||||
|
||||
// concurrent is set when multiple targets are scanned in parallel. it serializes
|
||||
// sink writes so lines from different targets never garble, and disables live
|
||||
// widgets, which cannot share one terminal across goroutines.
|
||||
var concurrent bool
|
||||
|
||||
// SetConcurrent switches the sink into parallel-safe mode: writes go through a
|
||||
// mutex and interactive widgets (spinners, live progress) are gated off. call it
|
||||
// once before launching target workers; it is not meant to be toggled back.
|
||||
func SetConcurrent(enabled bool) {
|
||||
concurrent = enabled
|
||||
if enabled {
|
||||
sink = &lockingWriter{w: sink}
|
||||
}
|
||||
}
|
||||
|
||||
// Concurrent reports whether parallel-target mode is active; widget code gates on
|
||||
// it so spinners and progress bars stay silent when targets interleave.
|
||||
func Concurrent() bool {
|
||||
return concurrent
|
||||
}
|
||||
|
||||
// lockingWriter serializes concurrent writes to the wrapped sink so one target's
|
||||
// line is never interleaved mid-write with another's.
|
||||
type lockingWriter struct {
|
||||
mu sync.Mutex
|
||||
w io.Writer
|
||||
}
|
||||
|
||||
func (l *lockingWriter) Write(p []byte) (int, error) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.w.Write(p)
|
||||
}
|
||||
|
||||
// Writer is the current chrome sink (stdout normally, stderr under -silent).
|
||||
// callers that render their own chrome (the startup banner) write here so it
|
||||
// follows the same routing as everything else.
|
||||
@@ -286,7 +321,9 @@ func (m *ModuleLogger) Complete(resultCount int, resultType string) {
|
||||
// ClearLine clears the current line (for progress bar updates). silent mode is
|
||||
// non-interactive, so there's no live line to clear and stdout stays untouched.
|
||||
func ClearLine() {
|
||||
if !IsTTY || silent {
|
||||
// under -concurrency there is no single live line to clear, and emitting the
|
||||
// escape would wipe whatever another target just wrote.
|
||||
if !IsTTY || silent || concurrent {
|
||||
return
|
||||
}
|
||||
fmt.Fprint(sink, "\033[2K\r")
|
||||
|
||||
@@ -91,7 +91,9 @@ func (p *Progress) Resume() {
|
||||
|
||||
// Done clears the progress bar line
|
||||
func (p *Progress) Done() {
|
||||
if apiMode || !IsTTY {
|
||||
// under -concurrency there is no live line to clear (render printed milestones),
|
||||
// and clearing would wipe another target's output.
|
||||
if apiMode || !IsTTY || concurrent {
|
||||
return
|
||||
}
|
||||
ClearLine()
|
||||
@@ -102,8 +104,9 @@ func (p *Progress) render() {
|
||||
return
|
||||
}
|
||||
|
||||
// In non-TTY mode, print progress at milestones only
|
||||
if !IsTTY {
|
||||
// In non-TTY mode, or under -concurrency, print progress at milestones only
|
||||
// rather than redrawing a live line that parallel targets would corrupt.
|
||||
if !IsTTY || concurrent {
|
||||
current := atomic.LoadInt64(&p.current)
|
||||
total := p.total
|
||||
if total <= 0 {
|
||||
|
||||
@@ -54,8 +54,9 @@ func (s *Spinner) Start() {
|
||||
s.done = make(chan struct{})
|
||||
s.mu.Unlock()
|
||||
|
||||
// In non-TTY mode, just print the message once
|
||||
if !IsTTY {
|
||||
// In non-TTY mode, or when targets scan in parallel, just print the message
|
||||
// once: a shared animated line can't be driven from concurrent goroutines.
|
||||
if !IsTTY || concurrent {
|
||||
fmt.Fprintf(sink, " %s...\n", s.message)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/charmbracelet/log"
|
||||
"github.com/vmfunc/sif/internal/config"
|
||||
@@ -322,14 +323,108 @@ func (app *App) Run() error {
|
||||
}
|
||||
}
|
||||
|
||||
for _, url := range app.targets {
|
||||
results, err := app.scanAllTargets(storeDir, wantReport)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// merge per-target results in input order so the run-wide view is identical
|
||||
// regardless of the order workers finished under -concurrency.
|
||||
for _, ts := range results {
|
||||
scansRun = append(scansRun, ts.scansRun...)
|
||||
app.logFiles = append(app.logFiles, ts.logFiles...)
|
||||
allFindings = append(allFindings, ts.findings...)
|
||||
if wantReport {
|
||||
reportResults = append(reportResults, ts.reportResults...)
|
||||
}
|
||||
}
|
||||
|
||||
return app.finishRun(scansRun, allFindings, reportResults, wantReport)
|
||||
}
|
||||
|
||||
// targetScan holds one target's isolated scan output: its findings, report rows,
|
||||
// scan labels and log files, which the run loop merges in target order.
|
||||
type targetScan struct {
|
||||
findings []finding.Finding
|
||||
reportResults []report.Result
|
||||
scansRun []string
|
||||
logFiles []string
|
||||
}
|
||||
|
||||
// scanAllTargets scans every target and returns the per-target results in input
|
||||
// order. With concurrency 1 it runs sequentially, behaviour-identical to a plain
|
||||
// loop. Above 1 it runs a bounded worker pool: scanTarget is self-contained
|
||||
// (isolated accumulators, no run-wide writes), so the only shared surface is the
|
||||
// console, which output.SetConcurrent serializes and de-animates. Results are
|
||||
// indexed by target position, so the caller merges them in a stable order no
|
||||
// matter which worker finished first.
|
||||
func (app *App) scanAllTargets(storeDir string, wantReport bool) ([]targetScan, error) {
|
||||
results := make([]targetScan, len(app.targets))
|
||||
|
||||
concurrency := app.settings.Concurrency
|
||||
if concurrency < 1 {
|
||||
concurrency = 1
|
||||
}
|
||||
if concurrency > len(app.targets) {
|
||||
concurrency = len(app.targets)
|
||||
}
|
||||
|
||||
if concurrency <= 1 {
|
||||
for i, url := range app.targets {
|
||||
ts, err := app.scanTarget(url, storeDir, wantReport)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
results[i] = ts
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
output.SetConcurrent(true)
|
||||
|
||||
errs := make([]error, len(app.targets))
|
||||
sem := make(chan struct{}, concurrency)
|
||||
var wg sync.WaitGroup
|
||||
for i, url := range app.targets {
|
||||
wg.Add(1)
|
||||
sem <- struct{}{}
|
||||
go func(i int, url string) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
// a panic in one target's scan (a module bug, a nil deref deep in a
|
||||
// third-party client) must not take down every other worker's
|
||||
// in-flight scan; convert it into that target's error instead.
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
errs[i] = fmt.Errorf("panic scanning %s: %v", url, r)
|
||||
}
|
||||
}()
|
||||
results[i], errs[i] = app.scanTarget(url, storeDir, wantReport)
|
||||
}(i, url)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
for _, err := range errs {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// scanTarget runs the full scanner set for one target and returns its isolated
|
||||
// accumulators without mutating run-wide state.
|
||||
func (app *App) scanTarget(url, storeDir string, wantReport bool) (targetScan, error) {
|
||||
var scansRun []string
|
||||
var logFiles []string
|
||||
|
||||
output.Info("Starting scan on %s", output.Highlight.Render(url))
|
||||
|
||||
moduleResults := make([]ModuleResult, 0, 16)
|
||||
|
||||
if app.settings.LogDir != "" {
|
||||
if err := logger.CreateFile(&app.logFiles, url, app.settings.LogDir); err != nil {
|
||||
return err
|
||||
if err := logger.CreateFile(&logFiles, url, app.settings.LogDir); err != nil {
|
||||
return targetScan{}, err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -702,13 +797,12 @@ func (app *App) Run() error {
|
||||
marshalled, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
log.Errorf("failed to marshal result: %s", err)
|
||||
continue
|
||||
return targetScan{scansRun: scansRun, logFiles: logFiles}, nil
|
||||
}
|
||||
fmt.Println(string(marshalled))
|
||||
}
|
||||
|
||||
targetFindings := collectFindings(url, moduleResults)
|
||||
allFindings = append(allFindings, targetFindings...)
|
||||
|
||||
// diff mode is per-target: load this target's last snapshot, surface only
|
||||
// the delta, then overwrite the snapshot so the next run diffs against now.
|
||||
@@ -720,11 +814,23 @@ func (app *App) Run() error {
|
||||
|
||||
// the report carries raw blobs and is only built when an export flag is
|
||||
// set, so the common path skips the marshalling entirely.
|
||||
var reportResults []report.Result
|
||||
if wantReport {
|
||||
reportResults = append(reportResults, collectReportResults(url, moduleResults)...)
|
||||
}
|
||||
reportResults = collectReportResults(url, moduleResults)
|
||||
}
|
||||
|
||||
return targetScan{
|
||||
findings: targetFindings,
|
||||
reportResults: reportResults,
|
||||
scansRun: scansRun,
|
||||
logFiles: logFiles,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// finishRun performs the run-wide steps after every target has been scanned:
|
||||
// notify, the silent findings stream, report files and the summary. It consumes
|
||||
// the merged accumulators so per-target scanning stays isolated in scanTarget.
|
||||
func (app *App) finishRun(scansRun []string, allFindings []finding.Finding, reportResults []report.Result, wantReport bool) error {
|
||||
// the normalized findings are the handoff point for notify/diff; surface the
|
||||
// count now so the path is live and observable without changing output.
|
||||
log.Debugf("normalized %d findings across %d targets", len(allFindings), len(app.targets))
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
/*
|
||||
·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
|
||||
: :
|
||||
: █▀ █ █▀▀ · Blazing-fast pentesting suite :
|
||||
: ▄█ █ █▀ · BSD 3-Clause License :
|
||||
: :
|
||||
: (c) 2022-2026 vmfunc, xyzeva, :
|
||||
: lunchcat alumni & contributors :
|
||||
: :
|
||||
·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
|
||||
*/
|
||||
|
||||
package sif
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/vmfunc/sif/internal/output"
|
||||
)
|
||||
|
||||
// TestScanAllTargetsConcurrentIsolation runs the target pool at concurrency 3
|
||||
// against three separate servers and asserts each target's result is isolated:
|
||||
// exactly one scan recorded per target, results in input order. Run under -race
|
||||
// it also backstops scanTarget's concurrency safety and the serialized sink.
|
||||
func TestScanAllTargetsConcurrentIsolation(t *testing.T) {
|
||||
defer output.SetConcurrent(false)
|
||||
|
||||
srvA := okServer()
|
||||
defer srvA.Close()
|
||||
srvB := okServer()
|
||||
defer srvB.Close()
|
||||
srvC := okServer()
|
||||
defer srvC.Close()
|
||||
|
||||
app := headersOnlyApp()
|
||||
app.targets = []string{srvA.URL, srvB.URL, srvC.URL}
|
||||
app.settings.Concurrency = 3
|
||||
|
||||
results, err := app.scanAllTargets("", false)
|
||||
if err != nil {
|
||||
t.Fatalf("scanAllTargets: %v", err)
|
||||
}
|
||||
if len(results) != len(app.targets) {
|
||||
t.Fatalf("got %d results, want %d", len(results), len(app.targets))
|
||||
}
|
||||
for i, ts := range results {
|
||||
if len(ts.scansRun) != 1 || ts.scansRun[0] != "HTTP Headers" {
|
||||
t.Errorf("target %d scansRun = %v, want exactly [HTTP Headers] (accumulator leaked across targets)", i, ts.scansRun)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestScanAllTargetsSequentialMatchesInputOrder pins that concurrency 1 keeps the
|
||||
// plain sequential path: one result per target, in order, no pool engaged.
|
||||
func TestScanAllTargetsSequentialMatchesInputOrder(t *testing.T) {
|
||||
srvA := okServer()
|
||||
defer srvA.Close()
|
||||
srvB := okServer()
|
||||
defer srvB.Close()
|
||||
|
||||
app := headersOnlyApp()
|
||||
app.targets = []string{srvA.URL, srvB.URL}
|
||||
app.settings.Concurrency = 1
|
||||
|
||||
results, err := app.scanAllTargets("", false)
|
||||
if err != nil {
|
||||
t.Fatalf("scanAllTargets: %v", err)
|
||||
}
|
||||
if len(results) != 2 {
|
||||
t.Fatalf("got %d results, want 2", len(results))
|
||||
}
|
||||
for i, ts := range results {
|
||||
if len(ts.scansRun) != 1 || ts.scansRun[0] != "HTTP Headers" {
|
||||
t.Errorf("target %d scansRun = %v, want exactly [HTTP Headers]", i, ts.scansRun)
|
||||
}
|
||||
}
|
||||
if output.Concurrent() {
|
||||
t.Error("concurrency 1 must not switch output into concurrent mode")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
/*
|
||||
·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
|
||||
: :
|
||||
: █▀ █ █▀▀ · Blazing-fast pentesting suite :
|
||||
: ▄█ █ █▀ · BSD 3-Clause License :
|
||||
: :
|
||||
: (c) 2022-2026 vmfunc, xyzeva, :
|
||||
: lunchcat alumni & contributors :
|
||||
: :
|
||||
·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
|
||||
*/
|
||||
|
||||
package sif
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/vmfunc/sif/internal/config"
|
||||
)
|
||||
|
||||
// headersOnlyApp builds an App that runs only the HTTP headers scan against
|
||||
// live servers, with every network-touching scanner gated off. it isolates the
|
||||
// scanTarget path to a single deterministic scan.
|
||||
func headersOnlyApp() *App {
|
||||
return &App{settings: &config.Settings{
|
||||
Headers: true,
|
||||
NoScan: true,
|
||||
Dirlist: "none",
|
||||
Dnslist: "none",
|
||||
Ports: "none",
|
||||
Timeout: 5 * time.Second,
|
||||
}}
|
||||
}
|
||||
|
||||
func okServer() *httptest.Server {
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("X-Sif-Test", "1")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
}
|
||||
|
||||
// TestScanTargetIsolatesPerTargetState is the accumulator seam the worker pool
|
||||
// depends on: each scanTarget call must own its scansRun rather than share a
|
||||
// run-wide slice. two sequential scans of different targets must each report a
|
||||
// single scan, not a growing total; a shared accumulator would make the second
|
||||
// call report two.
|
||||
func TestScanTargetIsolatesPerTargetState(t *testing.T) {
|
||||
srvA := okServer()
|
||||
defer srvA.Close()
|
||||
srvB := okServer()
|
||||
defer srvB.Close()
|
||||
|
||||
app := headersOnlyApp()
|
||||
|
||||
tsA, err := app.scanTarget(srvA.URL, "", false)
|
||||
if err != nil {
|
||||
t.Fatalf("scanTarget(A): %v", err)
|
||||
}
|
||||
if len(tsA.scansRun) != 1 || tsA.scansRun[0] != "HTTP Headers" {
|
||||
t.Fatalf("target A scansRun = %v, want exactly [HTTP Headers]", tsA.scansRun)
|
||||
}
|
||||
|
||||
tsB, err := app.scanTarget(srvB.URL, "", false)
|
||||
if err != nil {
|
||||
t.Fatalf("scanTarget(B): %v", err)
|
||||
}
|
||||
if len(tsB.scansRun) != 1 || tsB.scansRun[0] != "HTTP Headers" {
|
||||
t.Fatalf("target B scansRun = %v, want exactly [HTTP Headers] (accumulator leaked across targets)", tsB.scansRun)
|
||||
}
|
||||
|
||||
// no log dir configured, so no per-target log file should be recorded.
|
||||
if len(tsA.logFiles) != 0 || len(tsB.logFiles) != 0 {
|
||||
t.Errorf("logFiles should be empty without a log dir, got A=%v B=%v", tsA.logFiles, tsB.logFiles)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user