fix(scan): report the found url in dork results and dedupe by url (#283)

internal/finding/finding.go

@@ -562,7 +562,7 @@ func flattenDork(target string, rs []scan.DorkResult) []Finding {
 			Module:   "dork",
 			Severity: sevRecon,
 			Key:      key("dork", d.Url),
-			Title:    "dork hit",
+			Title:    fmt.Sprintf("dork hit [%s]", d.Dork),
 			Raw:      d.Url,
 		})
 	}

internal/finding/finding_test.go

@@ -174,7 +174,7 @@ func coverageCases() []coverageCase {
 			wantItems: 1,
 		},
 		{
-			value:     scan.DorkResults{{Url: "http://x/leak", Count: 1}},
+			value:     scan.DorkResults{{Url: "http://x/leak", Dork: "site:x filetype:pdf", Count: 1}},
 			typed:     scan.DorkResults{},
 			module:    "dork",
 			wantItems: 1,

internal/scan/dork.go

@@ -37,10 +37,11 @@ const (
 	dorkFile = "dork.txt"
 )
 
-// DorkResult represents the result of a Google dork search.
+// DorkResult represents a single URL found by a Google dork search.
 type DorkResult struct {
 	Url   string `json:"url"`   // The URL found by the dork
-	Count int    `json:"count"` // The number of times this URL was found
+	Dork  string `json:"dork"`  // The dork query that surfaced this URL
+	Count int    `json:"count"` // The number of times this URL appeared in the dork's results
 }
 
 // Dork performs Google dorking operations on the target URL.
@@ -110,13 +111,8 @@ func Dork(url string, timeout time.Duration, threads int, logdir string) ([]Dork
 				_ = logger.Write(sanitizedURL, logdir, strconv.Itoa(len(results))+" dork results found for dork ["+dork+"]\n")
 			}
 
-			result := DorkResult{
-				Url:   dork,
-				Count: len(results),
-			}
-
 			mu.Lock()
-			dorkResults = append(dorkResults, result)
+			dorkResults = append(dorkResults, groupDorkResults(dork, results)...)
 			mu.Unlock()
 		}
 	})
@@ -125,3 +121,31 @@ func Dork(url string, timeout time.Duration, threads int, logdir string) ([]Dork
 	output.ScanComplete("URL dorking", len(dorkResults), "found")
 	return dorkResults, nil
 }
+
+// groupDorkResults turns the raw search hits for a single dork query into
+// DorkResults, one per unique URL found, with Count tracking how many times
+// that URL showed up in the query's results. Results with an empty URL are
+// dropped since there is nothing to report.
+func groupDorkResults(dork string, results []googlesearch.Result) []DorkResult {
+	counts := make(map[string]int, len(results))
+	order := make([]string, 0, len(results))
+	for _, r := range results {
+		if r.URL == "" {
+			continue
+		}
+		if counts[r.URL] == 0 {
+			order = append(order, r.URL)
+		}
+		counts[r.URL]++
+	}
+
+	out := make([]DorkResult, 0, len(order))
+	for _, foundURL := range order {
+		out = append(out, DorkResult{
+			Url:   foundURL,
+			Dork:  dork,
+			Count: counts[foundURL],
+		})
+	}
+	return out
+}

internal/scan/scan_test.go

@@ -8,6 +8,8 @@ import (
 	"sync/atomic"
 	"testing"
 	"time"
+
+	googlesearch "github.com/rocketlaunchr/google-search"
 )
 
 func TestCheckSubdomainTakeover_GitHubPages(t *testing.T) {
@@ -273,16 +275,22 @@ func TestSubdomainTakeoverResult(t *testing.T) {
 }
 
 func TestDorkResult(t *testing.T) {
+	// the Url field must hold the URL the dork actually found, not the dork
+	// query that produced it. the query belongs in Dork.
 	result := DorkResult{
-		Url:   "site:example.com filetype:pdf",
-		Count: 42,
+		Url:   "https://example.com/leaked.pdf",
+		Dork:  "site:example.com filetype:pdf",
+		Count: 2,
 	}
 
-	if result.Url != "site:example.com filetype:pdf" {
-		t.Errorf("expected url 'site:example.com filetype:pdf', got '%s'", result.Url)
+	if result.Url != "https://example.com/leaked.pdf" {
+		t.Errorf("expected url 'https://example.com/leaked.pdf', got '%s'", result.Url)
 	}
-	if result.Count != 42 {
-		t.Errorf("expected count 42, got %d", result.Count)
+	if result.Dork != "site:example.com filetype:pdf" {
+		t.Errorf("expected dork 'site:example.com filetype:pdf', got '%s'", result.Dork)
+	}
+	if result.Count != 2 {
+		t.Errorf("expected count 2, got %d", result.Count)
 	}
 }
 
@@ -300,6 +308,39 @@ func TestHeaderResult(t *testing.T) {
 	}
 }
 
+func TestGroupDorkResults(t *testing.T) {
+	dork := "site:example.com filetype:pdf"
+	results := []googlesearch.Result{
+		{URL: "https://example.com/a.pdf"},
+		{URL: "https://example.com/b.pdf"},
+		{URL: "https://example.com/a.pdf"}, // duplicate, must be counted not re-listed
+		{URL: ""},                          // must be dropped, nothing to report
+	}
+
+	got := groupDorkResults(dork, results)
+
+	if len(got) != 2 {
+		t.Fatalf("expected 2 unique urls, got %d: %+v", len(got), got)
+	}
+
+	// guard against Url regressing back to holding the dork query itself.
+	for _, r := range got {
+		if r.Url == dork {
+			t.Errorf("Url must not be the dork query, got %q", r.Url)
+		}
+		if r.Dork != dork {
+			t.Errorf("expected Dork %q, got %q", dork, r.Dork)
+		}
+	}
+
+	if got[0].Url != "https://example.com/a.pdf" || got[0].Count != 2 {
+		t.Errorf("expected a.pdf with count 2, got %+v", got[0])
+	}
+	if got[1].Url != "https://example.com/b.pdf" || got[1].Count != 1 {
+		t.Errorf("expected b.pdf with count 1, got %+v", got[1])
+	}
+}
+
 func TestStripScheme(t *testing.T) {
 	tests := []struct {
 		name string