feat(modules): add dynamic-variable request chaining (#317)

http modules could only fire independent one-shot requests: an extractor
recorded a value for reporting, but nothing could feed it back into a
later request. add a requests: chain where steps run in order sharing a
variable map, so a step's extractors populate {{name}} references in the
path, headers and body of later steps.

a step with matchers records a finding on a match and halts the chain on
a miss, so a login or setup step can gate an authenticated follow-up. the
single-request form is unchanged and stays the concurrent default when no
chain is defined.

Co-authored-by: vmfunc <vmfunc.lc@gmail.com>

internal/modules/chain_test.go

@@ -0,0 +1,182 @@
+/*
+·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
+:                                                                               :
+:   █▀ █ █▀▀   ·   Blazing-fast pentesting suite                                :
+:   ▄█ █ █▀    ·   BSD 3-Clause License                                         :
+:                                                                               :
+:   (c) 2022-2026 vmfunc, xyzeva,                                               :
+:                 lunchcat alumni & contributors                                :
+:                                                                               :
+·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
+*/
+
+package modules
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/vmfunc/sif/internal/httpx"
+)
+
+// TestParseYAMLChainModule pins that the requests: chain schema round-trips from
+// yaml into HTTPStep, so a module file can express a chain end to end.
+func TestParseYAMLChainModule(t *testing.T) {
+	const src = `
+id: chain-example
+type: http
+info:
+  name: chain example
+  severity: high
+http:
+  requests:
+    - name: login
+      path: "{{BaseURL}}/login"
+      extractors:
+        - type: json
+          name: token
+          json: ["token"]
+    - name: authed
+      path: "{{BaseURL}}/secret"
+      headers:
+        Authorization: "Bearer {{token}}"
+      matchers:
+        - type: word
+          part: body
+          words: ["TOP SECRET"]
+`
+	path := filepath.Join(t.TempDir(), "chain-example.yaml")
+	if err := os.WriteFile(path, []byte(src), 0o600); err != nil {
+		t.Fatalf("write module: %v", err)
+	}
+	def, err := ParseYAMLModule(path)
+	if err != nil {
+		t.Fatalf("ParseYAMLModule: %v", err)
+	}
+	if def.HTTP == nil || len(def.HTTP.Requests) != 2 {
+		t.Fatalf("want 2 chain steps, got %+v", def.HTTP)
+	}
+	if def.HTTP.Requests[0].Extractors[0].Name != "token" {
+		t.Errorf("step 1 extractor name = %q, want token", def.HTTP.Requests[0].Extractors[0].Name)
+	}
+	if def.HTTP.Requests[1].Headers["Authorization"] != "Bearer {{token}}" {
+		t.Errorf("step 2 auth header = %q, want templated bearer", def.HTTP.Requests[1].Headers["Authorization"])
+	}
+}
+
+// chainServer is a two-endpoint app: /login hands back a token, /secret only
+// serves its body when that token rides in the Authorization header. it's the
+// canonical fetch-token-then-use-it flow the request chain exists to express.
+func chainServer() *httptest.Server {
+	const token = "tok-abc123"
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/login":
+			w.WriteHeader(http.StatusOK)
+			_, _ = w.Write([]byte(`{"token":"` + token + `"}`))
+		case "/secret":
+			if r.Header.Get("Authorization") == "Bearer "+token {
+				w.WriteHeader(http.StatusOK)
+				_, _ = w.Write([]byte("TOP SECRET DATA"))
+				return
+			}
+			w.WriteHeader(http.StatusUnauthorized)
+			_, _ = w.Write([]byte("denied"))
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+}
+
+func chainModule() *YAMLModule {
+	return &YAMLModule{
+		ID:   "test-chain",
+		Type: TypeHTTP,
+		Info: YAMLModuleInfo{Severity: "high"},
+		HTTP: &HTTPConfig{
+			Requests: []HTTPStep{
+				{
+					Name:       "login",
+					Path:       "{{BaseURL}}/login",
+					Extractors: []Extractor{{Type: "json", Name: "token", Part: "body", JSON: []string{"token"}}},
+				},
+				{
+					Name:     "authed",
+					Path:     "{{BaseURL}}/secret",
+					Headers:  map[string]string{"Authorization": "Bearer {{token}}"},
+					Matchers: []Matcher{{Type: "word", Part: "body", Words: []string{"TOP SECRET"}}},
+				},
+			},
+		},
+	}
+}
+
+// TestExecuteHTTPChainPropagatesVariable checks the core of chaining: step 1
+// extracts a token, step 2 injects it into a header and only then can match the
+// protected body. it fails unless the value actually crossed between requests.
+func TestExecuteHTTPChainPropagatesVariable(t *testing.T) {
+	srv := chainServer()
+	defer srv.Close()
+
+	opts := Options{Timeout: testTimeout, Client: httpx.Client(testTimeout)}
+	result, err := ExecuteHTTPModule(context.Background(), srv.URL, chainModule(), opts)
+	if err != nil {
+		t.Fatalf("ExecuteHTTPModule: %v", err)
+	}
+
+	if len(result.Findings) != 1 {
+		t.Fatalf("got %d findings, want 1 (chain should reach the protected step)", len(result.Findings))
+	}
+	f := result.Findings[0]
+	if f.URL != srv.URL+"/secret" {
+		t.Errorf("finding url = %q, want the protected step %q", f.URL, srv.URL+"/secret")
+	}
+	if f.Extracted["token"] != "tok-abc123" {
+		t.Errorf("extracted token = %q, want tok-abc123", f.Extracted["token"])
+	}
+}
+
+// TestExecuteHTTPChainHaltsWithoutToken proves the value really gates access:
+// drop the extractor so {{token}} never resolves, and the authed step must miss
+// its matcher, leaving no finding.
+func TestExecuteHTTPChainHaltsWithoutToken(t *testing.T) {
+	srv := chainServer()
+	defer srv.Close()
+
+	def := chainModule()
+	def.HTTP.Requests[0].Extractors = nil // never learn the token
+
+	opts := Options{Timeout: testTimeout, Client: httpx.Client(testTimeout)}
+	result, err := ExecuteHTTPModule(context.Background(), srv.URL, def, opts)
+	if err != nil {
+		t.Fatalf("ExecuteHTTPModule: %v", err)
+	}
+	if len(result.Findings) != 0 {
+		t.Fatalf("got %d findings, want 0 (unauthed step must not match)", len(result.Findings))
+	}
+}
+
+// TestExecuteHTTPChainStepMatcherHalts confirms a failed matcher on an early
+// step stops the chain before the later step records a finding.
+func TestExecuteHTTPChainStepMatcherHalts(t *testing.T) {
+	srv := chainServer()
+	defer srv.Close()
+
+	def := chainModule()
+	// gate the login step on a body word it never returns; the chain must stop
+	// there and never reach /secret.
+	def.HTTP.Requests[0].Matchers = []Matcher{{Type: "word", Part: "body", Words: []string{"nonexistent-marker"}}}
+
+	opts := Options{Timeout: testTimeout, Client: httpx.Client(testTimeout)}
+	result, err := ExecuteHTTPModule(context.Background(), srv.URL, def, opts)
+	if err != nil {
+		t.Fatalf("ExecuteHTTPModule: %v", err)
+	}
+	if len(result.Findings) != 0 {
+		t.Fatalf("got %d findings, want 0 (chain should halt on the failed login matcher)", len(result.Findings))
+	}
+}

internal/modules/executor.go

@@ -84,6 +84,13 @@ func ExecuteHTTPModule(ctx context.Context, target string, def *YAMLModule, opts
 		client = &scoped
 	}
 
+	// a module with an explicit request chain runs its steps in order, threading
+	// extracted variables between them; the concurrent single-request path below
+	// stays the default when no chain is defined.
+	if len(cfg.Requests) > 0 {
+		return executeHTTPChain(ctx, client, target, def)
+	}
+
 	// Generate requests based on paths and payloads
 	requests, err := generateHTTPRequests(target, cfg)
 	if err != nil {
@@ -141,6 +148,98 @@ func ExecuteHTTPModule(ctx context.Context, target string, def *YAMLModule, opts
 	return result, nil
 }
 
+// executeHTTPChain runs a module's ordered request chain. steps run in sequence
+// sharing one variable map: each step's extractors feed {{name}} references in
+// later steps' path, headers and body. a step with matchers records a finding
+// on a match and halts the chain on a miss, so a login/setup step can gate an
+// authenticated follow-up. steps are sequential by construction (a later step
+// depends on an earlier one), so this path is not concurrent.
+func executeHTTPChain(ctx context.Context, client *http.Client, target string, def *YAMLModule) (*Result, error) {
+	result := &Result{
+		ModuleID: def.ID,
+		Target:   target,
+		Findings: make([]Finding, 0),
+	}
+	base := strings.TrimSuffix(target, "/")
+	vars := make(map[string]string)
+
+	for i := range def.HTTP.Requests {
+		if err := ctx.Err(); err != nil {
+			return result, err
+		}
+		step := &def.HTTP.Requests[i]
+
+		method := step.Method
+		if method == "" {
+			method = "GET"
+		}
+		url := substituteVariablesWithVars(step.Path, base, "", vars)
+
+		var bodyReader io.Reader
+		if bodyStr := substituteVariablesWithVars(step.Body, base, "", vars); bodyStr != "" {
+			bodyReader = strings.NewReader(bodyStr)
+		}
+
+		req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
+		if err != nil {
+			return result, nil
+		}
+		for k, v := range step.Headers {
+			req.Header.Set(k, substituteVariablesWithVars(v, base, "", vars))
+		}
+		if req.Header.Get("User-Agent") == "" {
+			req.Header.Set("User-Agent", defaultUserAgent)
+		}
+
+		resp, err := client.Do(req)
+		if err != nil {
+			// a transport error breaks the chain; return whatever matched earlier.
+			return result, nil
+		}
+		respBody, err := io.ReadAll(io.LimitReader(resp.Body, MaxBodySize))
+		resp.Body.Close()
+		if err != nil {
+			return result, nil
+		}
+		respStr := string(respBody)
+
+		// feed this step's extractions into the shared vars for later steps,
+		// regardless of whether the step also carries matchers.
+		for k, v := range runExtractors(step.Extractors, resp, respStr) {
+			vars[k] = v
+		}
+
+		// a step with matchers gates the chain: a match records a finding, a miss
+		// means the precondition failed so the chain stops here.
+		if len(step.Matchers) > 0 {
+			if !checkMatchers(step.Matchers, step.MatchersCondition, resp, respStr) {
+				break
+			}
+			result.Findings = append(result.Findings, Finding{
+				URL:       url,
+				Severity:  def.Info.Severity,
+				Evidence:  truncateEvidence(respStr),
+				Extracted: snapshotVars(vars),
+			})
+		}
+	}
+
+	return result, nil
+}
+
+// snapshotVars copies the running chain variables so each finding carries an
+// independent view rather than aliasing the map that later steps keep mutating.
+func snapshotVars(vars map[string]string) map[string]string {
+	if len(vars) == 0 {
+		return nil
+	}
+	out := make(map[string]string, len(vars))
+	for k, v := range vars {
+		out[k] = v
+	}
+	return out
+}
+
 // generateHTTPRequests creates all requests based on paths and payloads.
 func generateHTTPRequests(target string, cfg *HTTPConfig) ([]*httpRequest, error) {
 	var requests []*httpRequest
@@ -282,6 +381,20 @@ func substituteVariables(template, baseURL, payload string) string {
 	return result
 }
 
+// substituteVariablesWithVars extends substituteVariables with the named values
+// a request chain accumulates: after {{BaseURL}}/{{payload}} it replaces every
+// {{name}} with the value an earlier step extracted under that name.
+func substituteVariablesWithVars(template, baseURL, payload string, vars map[string]string) string {
+	result := substituteVariables(template, baseURL, payload)
+	for name, value := range vars {
+		result = strings.ReplaceAll(result, "{{"+name+"}}", value)
+	}
+	return result
+}
+
+// defaultUserAgent is sent when a module doesn't set its own User-Agent header.
+const defaultUserAgent = "Mozilla/5.0 (compatible; sif/1.0)"
+
 // executeHTTPRequest executes a single HTTP request and checks matchers.
 func executeHTTPRequest(ctx context.Context, client *http.Client, r *httpRequest, cfg *HTTPConfig, severity string) (Finding, bool) {
 	var body io.Reader
@@ -299,7 +412,7 @@ func executeHTTPRequest(ctx context.Context, client *http.Client, r *httpRequest
 		req.Header.Set(k, v)
 	}
 	if req.Header.Get("User-Agent") == "" {
-		req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; sif/1.0)")
+		req.Header.Set("User-Agent", defaultUserAgent)
 	}
 
 	resp, err := client.Do(req)

internal/modules/yaml.go

@@ -65,6 +65,24 @@ type HTTPConfig struct {
 	Matchers          []Matcher         `yaml:"matchers"`
 	MatchersCondition string            `yaml:"matchers-condition,omitempty"` // and (default), or
 	Extractors        []Extractor       `yaml:"extractors,omitempty"`
+	Requests          []HTTPStep        `yaml:"requests,omitempty"` // ordered request chain; see HTTPStep
+}
+
+// HTTPStep is one request in a chain. steps run in order and share a variable
+// map: each step's extractors populate {{name}} references usable in the path,
+// headers and body of later steps. a step whose matchers don't match halts the
+// chain, so a login step can gate an authenticated follow-up request. when
+// HTTPConfig.Requests is empty the single-request fields above drive the module
+// unchanged.
+type HTTPStep struct {
+	Name              string            `yaml:"name,omitempty"`
+	Method            string            `yaml:"method,omitempty"`
+	Path              string            `yaml:"path"`
+	Headers           map[string]string `yaml:"headers,omitempty"`
+	Body              string            `yaml:"body,omitempty"`
+	Matchers          []Matcher         `yaml:"matchers,omitempty"`
+	MatchersCondition string            `yaml:"matchers-condition,omitempty"`
+	Extractors        []Extractor       `yaml:"extractors,omitempty"`
 }
 
 // DNSConfig defines DNS module settings
@@ -116,6 +134,18 @@ func ParseYAMLModuleBytes(data []byte) (*YAMLModule, error) {
 		if err := validateMatchersCondition(ym.HTTP.MatchersCondition); err != nil {
 			return nil, fmt.Errorf("module %q: %w", ym.ID, err)
 		}
+		for i := range ym.HTTP.Requests {
+			step := &ym.HTTP.Requests[i]
+			if step.Path == "" {
+				return nil, fmt.Errorf("module %q: request %d missing required field: path", ym.ID, i)
+			}
+			if err := validateMatchersCondition(step.MatchersCondition); err != nil {
+				return nil, fmt.Errorf("module %q request %d: %w", ym.ID, i, err)
+			}
+			if err := validateMatchers(step.Matchers); err != nil {
+				return nil, fmt.Errorf("module %q request %d: %w", ym.ID, i, err)
+			}
+		}
 	}
 	if ym.TCP != nil {
 		if err := validateTCP(ym.TCP); err != nil {