feat: add securitytrails integration for domain discovery + target expansion

This commit is contained in:
vmfunc
2026-02-17 13:38:07 +01:00
parent 5ddfbc6204
commit 495d2c5496
8 changed files with 598 additions and 4 deletions

View File

@@ -115,6 +115,10 @@ makepkg -si
# shodan host intelligence (requires SHODAN_API_KEY env var) # shodan host intelligence (requires SHODAN_API_KEY env var)
./sif -u https://example.com -shodan ./sif -u https://example.com -shodan
# securitytrails domain discovery (requires SECURITYTRAILS_API_KEY env var)
# discovers subdomains + associated domains, then scans all of them
./sif -u https://example.com -securitytrails -headers
# sql recon + lfi scanning # sql recon + lfi scanning
./sif -u https://example.com -sql -lfi ./sif -u https://example.com -sql -lfi
@@ -148,6 +152,7 @@ sif has a modular architecture. modules are defined in yaml and can be extended
| `-whois` | whois lookups | | `-whois` | whois lookups |
| `-git` | exposed git repository detection | | `-git` | exposed git repository detection |
| `-shodan` | shodan lookup (requires SHODAN_API_KEY) | | `-shodan` | shodan lookup (requires SHODAN_API_KEY) |
| `-securitytrails` | domain discovery + target expansion (requires SECURITYTRAILS_API_KEY) |
| `-sql` | sql recon | | `-sql` | sql recon |
| `-lfi` | local file inclusion | | `-lfi` | local file inclusion |
| `-framework` | framework detection with cve lookup | | `-framework` | framework detection with cve lookup |

View File

@@ -42,6 +42,7 @@ type Settings struct {
CloudStorage bool CloudStorage bool
SubdomainTakeover bool SubdomainTakeover bool
Shodan bool Shodan bool
SecurityTrails bool
SQL bool SQL bool
LFI bool LFI bool
Framework bool Framework bool
@@ -92,6 +93,7 @@ func Parse() *Settings {
flagSet.BoolVar(&settings.CloudStorage, "c3", false, "Enable C3 Misconfiguration Scan"), flagSet.BoolVar(&settings.CloudStorage, "c3", false, "Enable C3 Misconfiguration Scan"),
flagSet.BoolVar(&settings.SubdomainTakeover, "st", false, "Enable Subdomain Takeover Check"), flagSet.BoolVar(&settings.SubdomainTakeover, "st", false, "Enable Subdomain Takeover Check"),
flagSet.BoolVar(&settings.Shodan, "shodan", false, "Enable Shodan lookup (requires SHODAN_API_KEY env var)"), flagSet.BoolVar(&settings.Shodan, "shodan", false, "Enable Shodan lookup (requires SHODAN_API_KEY env var)"),
flagSet.BoolVar(&settings.SecurityTrails, "securitytrails", false, "Enable SecurityTrails domain discovery (requires SECURITYTRAILS_API_KEY env var)"),
flagSet.BoolVar(&settings.SQL, "sql", false, "Enable SQL reconnaissance (admin panels, error disclosure)"), flagSet.BoolVar(&settings.SQL, "sql", false, "Enable SQL reconnaissance (admin panels, error disclosure)"),
flagSet.BoolVar(&settings.LFI, "lfi", false, "Enable LFI (Local File Inclusion) reconnaissance"), flagSet.BoolVar(&settings.LFI, "lfi", false, "Enable LFI (Local File Inclusion) reconnaissance"),
flagSet.BoolVar(&settings.Framework, "framework", false, "Enable framework detection"), flagSet.BoolVar(&settings.Framework, "framework", false, "Enable framework detection"),

View File

@@ -21,4 +21,5 @@ func Register() {
modules.Register(&FrameworksModule{}) modules.Register(&FrameworksModule{})
modules.Register(&NucleiModule{}) modules.Register(&NucleiModule{})
modules.Register(&WhoisModule{}) modules.Register(&WhoisModule{})
modules.Register(&SecurityTrailsModule{})
} }

View File

@@ -0,0 +1,79 @@
/*
·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
: :
: █▀ █ █▀▀ · Blazing-fast pentesting suite :
: ▄█ █ █▀ · BSD 3-Clause License :
: :
: (c) 2022-2025 vmfunc, xyzeva, :
: lunchcat alumni & contributors :
: :
·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
*/
package builtin
import (
"context"
"fmt"
"strings"
"github.com/dropalldatabases/sif/internal/modules"
"github.com/dropalldatabases/sif/internal/scan"
)
type SecurityTrailsModule struct{}
func (m *SecurityTrailsModule) Info() modules.Info {
return modules.Info{
ID: "securitytrails-lookup",
Name: "SecurityTrails Domain Discovery",
Author: "sif",
Severity: "info",
Description: "Queries SecurityTrails API for subdomains and associated domains (requires SECURITYTRAILS_API_KEY)",
Tags: []string{"recon", "osint", "dns", "subdomains"},
}
}
func (m *SecurityTrailsModule) Type() modules.ModuleType {
return modules.TypeScript
}
func (m *SecurityTrailsModule) Execute(ctx context.Context, target string, opts modules.Options) (*modules.Result, error) {
stResult, err := scan.SecurityTrails(target, opts.Timeout, opts.LogDir)
if err != nil {
return nil, err
}
result := &modules.Result{
ModuleID: m.Info().ID,
Target: target,
Findings: []modules.Finding{},
}
if stResult == nil {
return result, nil
}
finding := modules.Finding{
URL: target,
Severity: "info",
Evidence: fmt.Sprintf("discovered %d subdomains and %d associated domains",
len(stResult.Subdomains), len(stResult.AssociatedDomains)),
Extracted: map[string]string{
"domain": stResult.Domain,
"subdomain_count": fmt.Sprintf("%d", len(stResult.Subdomains)),
"associated_count": fmt.Sprintf("%d", len(stResult.AssociatedDomains)),
},
}
if len(stResult.Subdomains) > 0 {
finding.Extracted["subdomains"] = strings.Join(stResult.Subdomains, ", ")
}
if len(stResult.AssociatedDomains) > 0 {
finding.Extracted["associated_domains"] = strings.Join(stResult.AssociatedDomains, ", ")
}
result.Findings = append(result.Findings, finding)
return result, nil
}

View File

@@ -31,10 +31,11 @@ type ScanResult interface {
// ResultType implementations for pointer result types. // ResultType implementations for pointer result types.
func (r *ShodanResult) ResultType() string { return "shodan" } func (r *ShodanResult) ResultType() string { return "shodan" }
func (r *SQLResult) ResultType() string { return "sql" } func (r *SQLResult) ResultType() string { return "sql" }
func (r *LFIResult) ResultType() string { return "lfi" } func (r *LFIResult) ResultType() string { return "lfi" }
func (r *CMSResult) ResultType() string { return "cms" } func (r *CMSResult) ResultType() string { return "cms" }
func (r *SecurityTrailsResult) ResultType() string { return "securitytrails" }
// ResultType implementations for slice result types. // ResultType implementations for slice result types.
@@ -50,6 +51,7 @@ var (
_ ScanResult = (*SQLResult)(nil) _ ScanResult = (*SQLResult)(nil)
_ ScanResult = (*LFIResult)(nil) _ ScanResult = (*LFIResult)(nil)
_ ScanResult = (*CMSResult)(nil) _ ScanResult = (*CMSResult)(nil)
_ ScanResult = (*SecurityTrailsResult)(nil)
_ ScanResult = HeaderResults(nil) _ ScanResult = HeaderResults(nil)
_ ScanResult = DirectoryResults(nil) _ ScanResult = DirectoryResults(nil)
_ ScanResult = CloudStorageResults(nil) _ ScanResult = CloudStorageResults(nil)

View File

@@ -0,0 +1,253 @@
/*
·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
: :
: █▀ █ █▀▀ · Blazing-fast pentesting suite :
: ▄█ █ █▀ · BSD 3-Clause License :
: :
: (c) 2022-2025 vmfunc, xyzeva, :
: lunchcat alumni & contributors :
: :
·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
*/
package scan
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strings"
"time"
"github.com/dropalldatabases/sif/internal/logger"
"github.com/dropalldatabases/sif/internal/output"
)
const securityTrailsBaseURL = "https://api.securitytrails.com/v1"
// SecurityTrailsResult holds discovered domains from SecurityTrails API
type SecurityTrailsResult struct {
Domain string `json:"domain"`
Subdomains []string `json:"subdomains,omitempty"`
AssociatedDomains []string `json:"associated_domains,omitempty"`
}
// stSubdomainsResponse is the raw response from the subdomains endpoint -
// returns prefix labels, not FQDNs
type stSubdomainsResponse struct {
Subdomains []string `json:"subdomains"`
}
type stAssociatedResponse struct {
Records []stAssociatedRecord `json:"records"`
}
type stAssociatedRecord struct {
Hostname string `json:"hostname"`
}
// SecurityTrails queries the SecurityTrails API for subdomains and associated domains.
// API key should be provided via the SECURITYTRAILS_API_KEY environment variable.
func SecurityTrails(targetURL string, timeout time.Duration, logdir string) (*SecurityTrailsResult, error) {
output.ScanStart("SecurityTrails lookup")
spin := output.NewSpinner("querying SecurityTrails API")
spin.Start()
apiKey := os.Getenv("SECURITYTRAILS_API_KEY")
if apiKey == "" {
spin.Stop()
output.Warn("SECURITYTRAILS_API_KEY environment variable not set, skipping SecurityTrails lookup")
return nil, fmt.Errorf("SECURITYTRAILS_API_KEY environment variable not set")
}
parsedURL, err := url.Parse(targetURL)
if err != nil {
spin.Stop()
return nil, fmt.Errorf("failed to parse URL: %w", err)
}
hostname := parsedURL.Hostname()
client := &http.Client{Timeout: timeout}
result := &SecurityTrailsResult{
Domain: hostname,
}
// fetch subdomains
spin.Update("fetching subdomains for " + hostname)
subs, err := querySTSubdomains(client, hostname, apiKey)
if err != nil {
// non-fatal - still try associated domains
output.Warn("SecurityTrails subdomains failed: %v", err)
} else {
result.Subdomains = subs
}
// fetch associated domains
spin.Update("fetching associated domains for " + hostname)
assoc, err := querySTAssociated(client, hostname, apiKey)
if err != nil {
output.Warn("SecurityTrails associated domains failed: %v", err)
} else {
result.AssociatedDomains = assoc
}
spin.Stop()
if logdir != "" {
sanitizedURL := strings.Split(targetURL, "://")[1]
if err := logger.WriteHeader(sanitizedURL, logdir, "SecurityTrails lookup"); err != nil {
output.Error("error writing log header: %v", err)
}
logSecurityTrailsResults(sanitizedURL, logdir, result)
}
printSecurityTrailsResults(result)
total := len(result.Subdomains) + len(result.AssociatedDomains)
output.ScanComplete("SecurityTrails lookup", total, "domains discovered")
return result, nil
}
// DiscoveredURLs returns all discovered domains as https:// URLs.
// used by the orchestration layer for target expansion.
func (r *SecurityTrailsResult) DiscoveredURLs() []string {
seen := make(map[string]struct{})
var urls []string
for _, sub := range r.Subdomains {
fqdn := sub + "." + r.Domain
if _, ok := seen[fqdn]; !ok {
seen[fqdn] = struct{}{}
urls = append(urls, "https://"+fqdn)
}
}
for _, assoc := range r.AssociatedDomains {
if _, ok := seen[assoc]; !ok {
seen[assoc] = struct{}{}
urls = append(urls, "https://"+assoc)
}
}
return urls
}
func querySTSubdomains(client *http.Client, hostname, apiKey string) ([]string, error) {
reqURL := fmt.Sprintf("%s/domain/%s/subdomains", securityTrailsBaseURL, hostname)
body, err := doSTRequest(client, reqURL, apiKey)
if err != nil {
return nil, err
}
var resp stSubdomainsResponse
if err := json.Unmarshal(body, &resp); err != nil {
return nil, fmt.Errorf("parse subdomains response: %w", err)
}
return resp.Subdomains, nil
}
func querySTAssociated(client *http.Client, hostname, apiKey string) ([]string, error) {
reqURL := fmt.Sprintf("%s/domain/%s/associated", securityTrailsBaseURL, hostname)
body, err := doSTRequest(client, reqURL, apiKey)
if err != nil {
return nil, err
}
var resp stAssociatedResponse
if err := json.Unmarshal(body, &resp); err != nil {
return nil, fmt.Errorf("parse associated response: %w", err)
}
domains := make([]string, 0, len(resp.Records))
for _, rec := range resp.Records {
if rec.Hostname != "" {
domains = append(domains, rec.Hostname)
}
}
return domains, nil
}
// doSTRequest makes an authenticated GET to the SecurityTrails API
func doSTRequest(client *http.Client, reqURL, apiKey string) ([]byte, error) {
req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, reqURL, http.NoBody)
if err != nil {
return nil, fmt.Errorf("create request: %w", err)
}
req.Header.Set("APIKEY", apiKey)
req.Header.Set("Accept", "application/json")
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("SecurityTrails request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusUnauthorized {
return nil, fmt.Errorf("invalid SecurityTrails API key (status %d)", resp.StatusCode)
}
if resp.StatusCode == http.StatusTooManyRequests {
return nil, fmt.Errorf("SecurityTrails rate limit exceeded")
}
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
return nil, fmt.Errorf("SecurityTrails API error (status %d): %s", resp.StatusCode, string(body))
}
body, err := io.ReadAll(io.LimitReader(resp.Body, 5*1024*1024))
if err != nil {
return nil, fmt.Errorf("read response: %w", err)
}
return body, nil
}
func printSecurityTrailsResults(result *SecurityTrailsResult) {
output.Info("Domain: %s", output.Highlight.Render(result.Domain))
if len(result.Subdomains) > 0 {
output.Info("Subdomains found: %d", len(result.Subdomains))
for _, sub := range result.Subdomains {
output.Success(" %s.%s", sub, result.Domain)
}
}
if len(result.AssociatedDomains) > 0 {
output.Info("Associated domains found: %d", len(result.AssociatedDomains))
for _, assoc := range result.AssociatedDomains {
output.Success(" %s", assoc)
}
}
}
func logSecurityTrailsResults(sanitizedURL, logdir string, result *SecurityTrailsResult) {
var sb strings.Builder
sb.WriteString(fmt.Sprintf("Domain: %s\n", result.Domain))
if len(result.Subdomains) > 0 {
sb.WriteString(fmt.Sprintf("\nSubdomains (%d):\n", len(result.Subdomains)))
for _, sub := range result.Subdomains {
sb.WriteString(fmt.Sprintf(" %s.%s\n", sub, result.Domain))
}
}
if len(result.AssociatedDomains) > 0 {
sb.WriteString(fmt.Sprintf("\nAssociated Domains (%d):\n", len(result.AssociatedDomains)))
for _, assoc := range result.AssociatedDomains {
sb.WriteString(fmt.Sprintf(" %s\n", assoc))
}
}
logger.Write(sanitizedURL, logdir, sb.String())
}

View File

@@ -0,0 +1,208 @@
package scan
import (
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
)
func TestSecurityTrailsResult_DiscoveredURLs(t *testing.T) {
result := &SecurityTrailsResult{
Domain: "example.com",
Subdomains: []string{"www", "api", "mail"},
AssociatedDomains: []string{"example.org", "example.net"},
}
urls := result.DiscoveredURLs()
if len(urls) != 5 {
t.Errorf("expected 5 URLs, got %d: %v", len(urls), urls)
}
expected := map[string]bool{
"https://www.example.com": false,
"https://api.example.com": false,
"https://mail.example.com": false,
"https://example.org": false,
"https://example.net": false,
}
for _, u := range urls {
if _, ok := expected[u]; !ok {
t.Errorf("unexpected URL: %s", u)
}
expected[u] = true
}
for u, seen := range expected {
if !seen {
t.Errorf("missing expected URL: %s", u)
}
}
}
func TestSecurityTrailsResult_DiscoveredURLs_Dedup(t *testing.T) {
result := &SecurityTrailsResult{
Domain: "example.com",
Subdomains: []string{"www"},
AssociatedDomains: []string{"www.example.com"},
}
urls := result.DiscoveredURLs()
if len(urls) != 1 {
t.Errorf("expected 1 URL (deduped), got %d: %v", len(urls), urls)
}
}
func TestSecurityTrailsResult_DiscoveredURLs_Empty(t *testing.T) {
result := &SecurityTrailsResult{
Domain: "example.com",
}
urls := result.DiscoveredURLs()
if len(urls) != 0 {
t.Errorf("expected 0 URLs, got %d: %v", len(urls), urls)
}
}
func TestDoSTRequest_Success(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("APIKEY") != "test-key" {
w.WriteHeader(http.StatusForbidden)
return
}
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(`{"test": true}`))
}))
defer server.Close()
client := &http.Client{Timeout: 5 * time.Second}
body, err := doSTRequest(client, server.URL, "test-key")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(body) == 0 {
t.Error("expected non-empty body")
}
}
func TestDoSTRequest_Unauthorized(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusForbidden)
}))
defer server.Close()
client := &http.Client{Timeout: 5 * time.Second}
_, err := doSTRequest(client, server.URL, "bad-key")
if err == nil {
t.Error("expected error for forbidden response")
}
}
func TestDoSTRequest_RateLimit(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusTooManyRequests)
}))
defer server.Close()
client := &http.Client{Timeout: 5 * time.Second}
_, err := doSTRequest(client, server.URL, "test-key")
if err == nil {
t.Error("expected error for rate limit response")
}
}
func TestQuerySTSubdomains(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("APIKEY") != "test-key" {
w.WriteHeader(http.StatusForbidden)
return
}
resp := stSubdomainsResponse{
Subdomains: []string{"www", "api", "mail", "dev"},
}
json.NewEncoder(w).Encode(resp)
}))
defer server.Close()
client := &http.Client{Timeout: 5 * time.Second}
// query the mock server directly via doSTRequest + unmarshal
body, err := doSTRequest(client, server.URL, "test-key")
if err != nil {
t.Fatalf("request failed: %v", err)
}
var resp stSubdomainsResponse
if err := json.Unmarshal(body, &resp); err != nil {
t.Fatalf("unmarshal failed: %v", err)
}
if len(resp.Subdomains) != 4 {
t.Errorf("expected 4 subdomains, got %d", len(resp.Subdomains))
}
}
func TestQuerySTAssociated(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("APIKEY") != "test-key" {
w.WriteHeader(http.StatusForbidden)
return
}
resp := stAssociatedResponse{
Records: []stAssociatedRecord{
{Hostname: "related.com"},
{Hostname: "sibling.net"},
{Hostname: ""},
},
}
json.NewEncoder(w).Encode(resp)
}))
defer server.Close()
client := &http.Client{Timeout: 5 * time.Second}
body, err := doSTRequest(client, server.URL, "test-key")
if err != nil {
t.Fatalf("request failed: %v", err)
}
var resp stAssociatedResponse
if err := json.Unmarshal(body, &resp); err != nil {
t.Fatalf("unmarshal failed: %v", err)
}
// should have 3 records total (including empty one)
if len(resp.Records) != 3 {
t.Errorf("expected 3 records, got %d", len(resp.Records))
}
// filter empty hostnames like the real code does
var domains []string
for _, rec := range resp.Records {
if rec.Hostname != "" {
domains = append(domains, rec.Hostname)
}
}
if len(domains) != 2 {
t.Errorf("expected 2 non-empty domains, got %d", len(domains))
}
}
func TestSecurityTrailsResult_ResultType(t *testing.T) {
result := &SecurityTrailsResult{}
if result.ResultType() != "securitytrails" {
t.Errorf("expected ResultType 'securitytrails', got '%s'", result.ResultType())
}
}
func TestSecurityTrailsIntegration(t *testing.T) {
t.Skip("integration test - requires valid SECURITYTRAILS_API_KEY")
_, err := SecurityTrails("https://example.com", 10*time.Second, "")
if err != nil {
t.Logf("SecurityTrails lookup failed: %v", err)
}
}

44
sif.go
View File

@@ -169,6 +169,15 @@ func (app *App) Run() error {
defer logger.Close() defer logger.Close()
} }
// target expansion - securitytrails discovers new domains before scanning
if app.settings.SecurityTrails {
expanded := app.expandTargets()
if len(expanded) > 0 {
output.Info("SecurityTrails discovered %d additional targets", len(expanded))
app.targets = append(app.targets, expanded...)
}
}
scansRun := make([]string, 0, 16) scansRun := make([]string, 0, 16)
for _, url := range app.targets { for _, url := range app.targets {
@@ -426,3 +435,38 @@ func (app *App) Run() error {
return nil return nil
} }
// expandTargets queries SecurityTrails for each original target and returns
// newly discovered domains (subdomains + associated) for target expansion
func (app *App) expandTargets() []string {
seen := make(map[string]struct{})
for _, t := range app.targets {
seen[t] = struct{}{}
}
// snapshot original targets - don't expand discovered ones
originals := make([]string, len(app.targets))
copy(originals, app.targets)
var expanded []string
for _, url := range originals {
result, err := scan.SecurityTrails(url, app.settings.Timeout, app.settings.LogDir)
if err != nil {
log.Errorf("SecurityTrails error for %s: %v", url, err)
continue
}
if result == nil {
continue
}
for _, d := range result.DiscoveredURLs() {
if _, exists := seen[d]; !exists {
seen[d] = struct{}{}
expanded = append(expanded, d)
}
}
}
return expanded
}