mirror of
https://github.com/lunchcat/sif.git
synced 2026-07-28 14:37:01 -07:00
feat(frameworks): report every detected technology (#314)
framework detection collapsed to a single argmax winner, so a page built from several technologies (react behind a next.js server, say) reported only one and dropped the rest. add DetectFrameworks, which keeps every detector hit above the threshold ranked most-confident first, and wire the -framework path to emit one finding per detected framework. the single-winner DetectFramework stays for callers that want just the top match; both share one fetch through the new gatherDetections helper, so the extra coverage costs no additional request. Co-authored-by: vmfunc <vmfunc.lc@gmail.com>
This commit is contained in:
@@ -17,6 +17,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
@@ -44,43 +45,36 @@ type detectionResult struct {
|
|||||||
// so config-defined detectors join the registry without a per-target re-read.
|
// so config-defined detectors join the registry without a per-target re-read.
|
||||||
var loadCustomOnce sync.Once
|
var loadCustomOnce sync.Once
|
||||||
|
|
||||||
// DetectFramework runs all registered detectors against the target URL.
|
// gatherDetections fetches the target once and runs every registered detector
|
||||||
func DetectFramework(url string, timeout time.Duration, logdir string) (*FrameworkResult, error) {
|
// against the response, returning each detector's raw confidence. it owns the
|
||||||
loadCustomOnce.Do(loadCustomDetectors)
|
// spinner so the single- and multi-result entry points share one fetch. an
|
||||||
|
// empty result slice means no detectors were registered.
|
||||||
log := output.Module("FRAMEWORK")
|
func gatherDetections(url string, timeout time.Duration) ([]detectionResult, string, error) {
|
||||||
log.Start()
|
|
||||||
|
|
||||||
spin := output.NewSpinner("Detecting frameworks")
|
spin := output.NewSpinner("Detecting frameworks")
|
||||||
spin.Start()
|
spin.Start()
|
||||||
|
defer spin.Stop()
|
||||||
|
|
||||||
client := httpx.Client(timeout)
|
client := httpx.Client(timeout)
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, url, http.NoBody)
|
req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, url, http.NoBody)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
spin.Stop()
|
return nil, "", err
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
resp, err := client.Do(req) //nolint:bodyclose // closed via defer below
|
resp, err := client.Do(req) //nolint:bodyclose // closed via defer below
|
||||||
if err != nil {
|
if err != nil {
|
||||||
spin.Stop()
|
return nil, "", err
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodySize))
|
body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodySize))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
spin.Stop()
|
return nil, "", err
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
bodyStr := string(body)
|
bodyStr := string(body)
|
||||||
|
|
||||||
// Get all registered detectors
|
|
||||||
detectors := GetDetectors()
|
detectors := GetDetectors()
|
||||||
if len(detectors) == 0 {
|
if len(detectors) == 0 {
|
||||||
spin.Stop()
|
return nil, bodyStr, nil
|
||||||
log.Warn("No framework detectors registered")
|
|
||||||
return nil, nil //nolint:nilnil // no detectors registered is not an error
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run all detectors concurrently
|
// Run all detectors concurrently
|
||||||
@@ -106,40 +100,118 @@ func DetectFramework(url string, timeout time.Duration, logdir string) (*Framewo
|
|||||||
close(results)
|
close(results)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
// Find the best match
|
out := make([]detectionResult, 0, len(detectors))
|
||||||
// results arrive in goroutine-completion order; tie-break on name so the
|
|
||||||
// winner is deterministic when two detectors land on the same confidence.
|
|
||||||
var best detectionResult
|
|
||||||
for r := range results {
|
for r := range results {
|
||||||
|
out = append(out, r)
|
||||||
|
}
|
||||||
|
return out, bodyStr, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DetectFramework runs all registered detectors against the target URL and
|
||||||
|
// returns the single highest-confidence match. see DetectFrameworks for the
|
||||||
|
// full set of detected technologies.
|
||||||
|
func DetectFramework(url string, timeout time.Duration, logdir string) (*FrameworkResult, error) {
|
||||||
|
loadCustomOnce.Do(loadCustomDetectors)
|
||||||
|
|
||||||
|
log := output.Module("FRAMEWORK")
|
||||||
|
log.Start()
|
||||||
|
|
||||||
|
results, bodyStr, err := gatherDetections(url, timeout)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(results) == 0 {
|
||||||
|
log.Warn("No framework detectors registered")
|
||||||
|
return nil, nil //nolint:nilnil // no detectors registered is not an error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find the best match. tie-break on name so the winner is deterministic when
|
||||||
|
// two detectors land on the same confidence.
|
||||||
|
var best detectionResult
|
||||||
|
for _, r := range results {
|
||||||
if r.confidence > best.confidence || (r.confidence == best.confidence && r.name < best.name) {
|
if r.confidence > best.confidence || (r.confidence == best.confidence && r.name < best.name) {
|
||||||
best = r
|
best = r
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
spin.Stop()
|
|
||||||
|
|
||||||
if best.confidence <= detectionThreshold {
|
if best.confidence <= detectionThreshold {
|
||||||
log.Info("No framework detected with sufficient confidence")
|
log.Info("No framework detected with sufficient confidence")
|
||||||
log.Complete(0, "detected")
|
log.Complete(0, "detected")
|
||||||
return nil, nil //nolint:nilnil // no framework detected is not an error
|
return nil, nil //nolint:nilnil // no framework detected is not an error
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get version match details. the detector's own best.version is often
|
result := assembleResult(best, bodyStr, url, logdir, log)
|
||||||
// "unknown" (it only fingerprints the framework, not always the version),
|
log.Complete(1, "detected")
|
||||||
// while ExtractVersionOptimized digs the real version out of the body. prefer
|
|
||||||
// that for both the reported version and the cve lookup, otherwise CVEs that
|
|
||||||
// only match a concrete version are silently missed.
|
|
||||||
versionMatch := ExtractVersionOptimized(bodyStr, best.name)
|
|
||||||
version := resolveVersion(best.version, versionMatch.Version)
|
|
||||||
cves, suggestions := getVulnerabilities(best.name, version)
|
|
||||||
|
|
||||||
result := NewFrameworkResult(best.name, version, best.confidence, versionMatch.Confidence)
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DetectFrameworks runs all registered detectors and returns every framework
|
||||||
|
// that clears the detection threshold, ranked most-confident first. unlike
|
||||||
|
// DetectFramework it does not collapse to a single winner, so a page built from
|
||||||
|
// several technologies (say react behind a next.js server) reports each.
|
||||||
|
func DetectFrameworks(url string, timeout time.Duration, logdir string) ([]*FrameworkResult, error) {
|
||||||
|
loadCustomOnce.Do(loadCustomDetectors)
|
||||||
|
|
||||||
|
log := output.Module("FRAMEWORK")
|
||||||
|
log.Start()
|
||||||
|
|
||||||
|
results, bodyStr, err := gatherDetections(url, timeout)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(results) == 0 {
|
||||||
|
log.Warn("No framework detectors registered")
|
||||||
|
return nil, nil //nolint:nilnil // no detectors registered is not an error
|
||||||
|
}
|
||||||
|
|
||||||
|
// keep everything above the threshold, most-confident first with a name
|
||||||
|
// tie-break so the order is stable across runs.
|
||||||
|
detected := make([]detectionResult, 0, len(results))
|
||||||
|
for _, r := range results {
|
||||||
|
if r.confidence > detectionThreshold {
|
||||||
|
detected = append(detected, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Slice(detected, func(i, j int) bool {
|
||||||
|
if detected[i].confidence != detected[j].confidence {
|
||||||
|
return detected[i].confidence > detected[j].confidence
|
||||||
|
}
|
||||||
|
return detected[i].name < detected[j].name
|
||||||
|
})
|
||||||
|
|
||||||
|
if len(detected) == 0 {
|
||||||
|
log.Info("No framework detected with sufficient confidence")
|
||||||
|
log.Complete(0, "detected")
|
||||||
|
return nil, nil //nolint:nilnil // no framework detected is not an error
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make([]*FrameworkResult, 0, len(detected))
|
||||||
|
for i := range detected {
|
||||||
|
out = append(out, assembleResult(detected[i], bodyStr, url, logdir, log))
|
||||||
|
}
|
||||||
|
log.Complete(len(out), "detected")
|
||||||
|
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// assembleResult turns one detector hit into a full FrameworkResult: it digs out
|
||||||
|
// the concrete version, looks up CVEs for it, and logs the detection. the
|
||||||
|
// detector's own version is often "unknown" (it fingerprints the framework, not
|
||||||
|
// always the version), while ExtractVersionOptimized reads the real version from
|
||||||
|
// the body; prefer that for both the reported version and the cve lookup, else
|
||||||
|
// CVEs that only match a concrete version are silently missed.
|
||||||
|
func assembleResult(d detectionResult, bodyStr, url, logdir string, log *output.ModuleLogger) *FrameworkResult {
|
||||||
|
versionMatch := ExtractVersionOptimized(bodyStr, d.name)
|
||||||
|
version := resolveVersion(d.version, versionMatch.Version)
|
||||||
|
cves, suggestions := getVulnerabilities(d.name, version)
|
||||||
|
|
||||||
|
result := NewFrameworkResult(d.name, version, d.confidence, versionMatch.Confidence)
|
||||||
result.WithVulnerabilities(cves, suggestions)
|
result.WithVulnerabilities(cves, suggestions)
|
||||||
|
|
||||||
// Log results
|
|
||||||
if logdir != "" {
|
if logdir != "" {
|
||||||
logEntry := fmt.Sprintf("Detected framework: %s (version: %s, confidence: %.2f, version_confidence: %.2f)\n",
|
logEntry := fmt.Sprintf("Detected framework: %s (version: %s, confidence: %.2f, version_confidence: %.2f)\n",
|
||||||
best.name, version, best.confidence, versionMatch.Confidence)
|
d.name, version, d.confidence, versionMatch.Confidence)
|
||||||
if len(cves) > 0 {
|
if len(cves) > 0 {
|
||||||
logEntry += fmt.Sprintf(" Risk Level: %s\n", result.RiskLevel)
|
logEntry += fmt.Sprintf(" Risk Level: %s\n", result.RiskLevel)
|
||||||
logEntry += fmt.Sprintf(" CVEs: %v\n", cves)
|
logEntry += fmt.Sprintf(" CVEs: %v\n", cves)
|
||||||
@@ -149,7 +221,7 @@ func DetectFramework(url string, timeout time.Duration, logdir string) (*Framewo
|
|||||||
}
|
}
|
||||||
|
|
||||||
log.Success("Detected %s framework (version: %s, confidence: %.2f)",
|
log.Success("Detected %s framework (version: %s, confidence: %.2f)",
|
||||||
output.Highlight.Render(best.name), version, best.confidence)
|
output.Highlight.Render(d.name), version, d.confidence)
|
||||||
|
|
||||||
if versionMatch.Confidence > 0 {
|
if versionMatch.Confidence > 0 {
|
||||||
charmlog.Debugf("Version detected from: %s (confidence: %.2f)",
|
charmlog.Debugf("Version detected from: %s (confidence: %.2f)",
|
||||||
@@ -166,9 +238,7 @@ func DetectFramework(url string, timeout time.Duration, logdir string) (*Framewo
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Complete(1, "detected")
|
return result
|
||||||
|
|
||||||
return result, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// unknownVersion is the sentinel both detectors and the version extractor emit
|
// unknownVersion is the sentinel both detectors and the version extractor emit
|
||||||
|
|||||||
@@ -110,6 +110,66 @@ func TestDetectFramework_NextJS(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDetectFrameworks_ReportsMultiple(t *testing.T) {
|
||||||
|
// a next.js app served behind an express-tagged server carries markers for
|
||||||
|
// both; the multi-report path must surface each rather than one winner.
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "text/html")
|
||||||
|
w.Header().Set("X-Powered-By", "Express")
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
w.Write([]byte(`
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html><head><title>Test</title></head>
|
||||||
|
<body>
|
||||||
|
<script id="__NEXT_DATA__" type="application/json">{"props":{}}</script>
|
||||||
|
<script src="/_next/static/chunks/main.js"></script>
|
||||||
|
</body></html>
|
||||||
|
`))
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
results, err := frameworks.DetectFrameworks(server.URL, 5*time.Second, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(results) < 2 {
|
||||||
|
t.Fatalf("expected at least 2 frameworks, got %d: %+v", len(results), results)
|
||||||
|
}
|
||||||
|
|
||||||
|
names := make(map[string]bool)
|
||||||
|
for i := range results {
|
||||||
|
names[results[i].Name] = true
|
||||||
|
}
|
||||||
|
for _, want := range []string{"Next.js", "Express.js"} {
|
||||||
|
if !names[want] {
|
||||||
|
t.Errorf("expected %q in multi-report, got %v", want, names)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// results must be ranked most-confident first.
|
||||||
|
for i := 1; i < len(results); i++ {
|
||||||
|
if results[i-1].Confidence < results[i].Confidence {
|
||||||
|
t.Errorf("results not sorted by confidence: %v then %v", results[i-1].Confidence, results[i].Confidence)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDetectFrameworks_NoMatch(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
w.Write([]byte(`<!DOCTYPE html><html><body>plain</body></html>`))
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
results, err := frameworks.DetectFrameworks(server.URL, 5*time.Second, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(results) != 0 {
|
||||||
|
t.Errorf("expected no frameworks on a blank page, got %+v", results)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDetectFramework_Express(t *testing.T) {
|
func TestDetectFramework_Express(t *testing.T) {
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
w.Header().Set("X-Powered-By", "Express")
|
w.Header().Set("X-Powered-By", "Express")
|
||||||
|
|||||||
@@ -434,11 +434,13 @@ func (app *App) scanTarget(url, storeDir string, wantReport bool) (targetScan, e
|
|||||||
}
|
}
|
||||||
|
|
||||||
if app.settings.Framework {
|
if app.settings.Framework {
|
||||||
result, err := frameworks.DetectFramework(url, app.settings.Timeout, app.settings.LogDir)
|
results, err := frameworks.DetectFrameworks(url, app.settings.Timeout, app.settings.LogDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("Error while running framework detection: %s", err)
|
log.Errorf("Error while running framework detection: %s", err)
|
||||||
} else if result != nil {
|
} else if len(results) > 0 {
|
||||||
moduleResults = append(moduleResults, NewModuleResult(result))
|
for _, result := range results {
|
||||||
|
moduleResults = append(moduleResults, NewModuleResult(result))
|
||||||
|
}
|
||||||
scansRun = append(scansRun, "Framework Detection")
|
scansRun = append(scansRun, "Framework Detection")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user