From 912f6e8e0e8efaeb6ddba9fc1cd5c2022f506fc7 Mon Sep 17 00:00:00 2001 From: vmfunc Date: Tue, 9 Jun 2026 16:07:20 -0700 Subject: [PATCH] test(scan): seam shodan/securitytrails/cloudstorage/dnslist for hermetic integration tests the remaining hardcoded base urls had no test seam, so their drivers could only be exercised against the live apis. promote them to package vars (matching the dirlist/git/ports pattern from #112) and route dnslist's per-host probes through an injectable transport, then add integration tests that pin each at a local httptest fixture. defaults equal the old const values so behavior is unchanged. --- internal/scan/cloudstorage.go | 6 +- internal/scan/dnslist.go | 7 +- internal/scan/integration_test.go | 162 ++++++++++++++++++++++++++++++ internal/scan/securitytrails.go | 3 +- internal/scan/shodan.go | 3 +- 5 files changed, 177 insertions(+), 4 deletions(-) diff --git a/internal/scan/cloudstorage.go b/internal/scan/cloudstorage.go index 809e191..417a37e 100644 --- a/internal/scan/cloudstorage.go +++ b/internal/scan/cloudstorage.go @@ -25,6 +25,10 @@ import ( "github.com/dropalldatabases/sif/internal/styles" ) +// s3EndpointFmt is a var so integration tests can repoint it at a fixture; the +// %s is the bucket name. +var s3EndpointFmt = "https://%s.s3.amazonaws.com" + type CloudStorageResult struct { BucketName string `json:"bucket_name"` IsPublic bool `json:"is_public"` @@ -96,7 +100,7 @@ func extractPotentialBuckets(url string) []string { } func checkS3Bucket(ctx context.Context, bucket string, client *http.Client) (bool, error) { - url := fmt.Sprintf("https://%s.s3.amazonaws.com", bucket) + url := fmt.Sprintf(s3EndpointFmt, bucket) req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) if err != nil { return false, err diff --git a/internal/scan/dnslist.go b/internal/scan/dnslist.go index c23f3e3..b490a55 100644 --- a/internal/scan/dnslist.go +++ b/internal/scan/dnslist.go @@ -28,6 +28,10 @@ import ( // dnsURL is a var so integration tests can repoint it at a fixture. var dnsURL = "https://raw.githubusercontent.com/dropalldatabases/sif-runtime/main/dnslist/" +// dnsTransport is a var so integration tests can route the per-host probes at a +// local server instead of resolving real DNS. nil keeps http.DefaultTransport. +var dnsTransport http.RoundTripper + const ( dnsSmallFile = "subdomains-100.txt" dnsMediumFile = "subdomains-1000.txt" @@ -78,7 +82,8 @@ func Dnslist(size string, url string, timeout time.Duration, threads int, logdir } client := &http.Client{ - Timeout: timeout, + Timeout: timeout, + Transport: dnsTransport, } progress := output.NewProgress(len(dns), "enumerating") diff --git a/internal/scan/integration_test.go b/internal/scan/integration_test.go index 5e09ca4..7894a7f 100644 --- a/internal/scan/integration_test.go +++ b/internal/scan/integration_test.go @@ -20,6 +20,7 @@ package scan import ( "context" + "encoding/json" "net" "net/http" "net/http/httptest" @@ -41,6 +42,9 @@ func newVulnApp() *httptest.Server { mux.HandleFunc("/directory-list-2.3-small.txt", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("admin\nlogin\nnope\n")) }) + mux.HandleFunc("/subdomains-100.txt", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("dev\nstaging\n")) + }) // an exposed git repo: HEAD is a real find, config is html so it's excluded mux.HandleFunc("/.git/HEAD", func(w http.ResponseWriter, r *http.Request) { @@ -209,6 +213,164 @@ func TestIntegrationPorts(t *testing.T) { } } +func TestIntegrationShodan(t *testing.T) { + // a local server stands in for api.shodan.io; example.com resolves to a real + // IP but the lookup never leaves the box. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("key") != "test-key" { + w.WriteHeader(http.StatusUnauthorized) + return + } + json.NewEncoder(w).Encode(shodanHostResponse{ + IP: "93.184.216.34", + Hostnames: []string{"example.com"}, + Org: "EDGECAST", + Ports: []int{80, 443}, + Data: []shodanData{ + {Port: 80, Transport: "tcp", Product: "nginx", Version: "1.18.0"}, + }, + }) + })) + defer srv.Close() + orig := shodanBaseURL + shodanBaseURL = srv.URL + defer func() { shodanBaseURL = orig }() + + t.Setenv("SHODAN_API_KEY", "test-key") + + result, err := Shodan("https://example.com", 5*time.Second, "") + if err != nil { + t.Fatalf("Shodan: %v", err) + } + if result == nil || result.IP != "93.184.216.34" { + t.Fatalf("expected parsed shodan result, got %+v", result) + } + if len(result.Services) != 1 || result.Services[0].Product != "nginx" { + t.Errorf("expected one nginx service, got %+v", result.Services) + } +} + +func TestIntegrationSecurityTrails(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("APIKEY") != "test-key" { + w.WriteHeader(http.StatusForbidden) + return + } + switch { + case strings.HasSuffix(r.URL.Path, "/subdomains"): + json.NewEncoder(w).Encode(stSubdomainsResponse{Subdomains: []string{"www", "api"}}) + case strings.HasSuffix(r.URL.Path, "/associated"): + json.NewEncoder(w).Encode(stAssociatedResponse{Records: []stAssociatedRecord{{Hostname: "example.org"}}}) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + orig := securityTrailsBaseURL + securityTrailsBaseURL = srv.URL + defer func() { securityTrailsBaseURL = orig }() + + t.Setenv("SECURITYTRAILS_API_KEY", "test-key") + + result, err := SecurityTrails("https://example.com", 5*time.Second, "") + if err != nil { + t.Fatalf("SecurityTrails: %v", err) + } + if len(result.Subdomains) != 2 { + t.Errorf("expected 2 subdomains, got %v", result.Subdomains) + } + if len(result.AssociatedDomains) != 1 || result.AssociatedDomains[0] != "example.org" { + t.Errorf("expected example.org associated, got %v", result.AssociatedDomains) + } + + urls := result.DiscoveredURLs() + if !contains(urls, "https://www.example.com") || !contains(urls, "https://example.org") { + t.Errorf("expected discovered urls to expand subs and associated, got %v", urls) + } +} + +func TestIntegrationCloudStorage(t *testing.T) { + // the fixture returns 200 only for the planted bucket, so any candidate that + // matches it is reported public. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/example" { + w.WriteHeader(http.StatusOK) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + orig := s3EndpointFmt + s3EndpointFmt = srv.URL + "/%s" + defer func() { s3EndpointFmt = orig }() + + results, err := CloudStorage("https://example.com", 5*time.Second, "") + if err != nil { + t.Fatalf("CloudStorage: %v", err) + } + + var public bool + for _, r := range results { + if r.BucketName == "example" && r.IsPublic { + public = true + } + } + if !public { + t.Errorf("expected the example bucket to be flagged public, got %+v", results) + } +} + +func TestIntegrationDnslist(t *testing.T) { + // the probe server answers any host routed to it; dnsTransport pins every + // dial here so no real DNS is touched. + probe := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer probe.Close() + probeAddr := strings.TrimPrefix(probe.URL, "http://") + + list := newVulnApp() + defer list.Close() + origURL := dnsURL + dnsURL = list.URL + "/" + defer func() { dnsURL = origURL }() + + origTr := dnsTransport + dnsTransport = &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, probeAddr) + }, + } + defer func() { dnsTransport = origTr }() + + found, err := Dnslist("small", "http://example.com", 5*time.Second, 2, "") + if err != nil { + t.Fatalf("Dnslist: %v", err) + } + // http probes land on the plain-http probe server; https fails the tls + // handshake and is dropped, which is fine - the planted sub still shows up. + if !hasSuffixIn(sliceSet(found), "dev.example.com") { + t.Errorf("expected dev.example.com among findings, got %v", found) + } +} + +func contains(s []string, v string) bool { + for i := 0; i < len(s); i++ { + if s[i] == v { + return true + } + } + return false +} + +func sliceSet(s []string) map[string]bool { + set := make(map[string]bool, len(s)) + for i := 0; i < len(s); i++ { + set[s[i]] = true + } + return set +} + func hasSuffixIn(set map[string]bool, suffix string) bool { for k := range set { if strings.HasSuffix(k, suffix) { diff --git a/internal/scan/securitytrails.go b/internal/scan/securitytrails.go index 1004cdd..aeccdd9 100644 --- a/internal/scan/securitytrails.go +++ b/internal/scan/securitytrails.go @@ -27,7 +27,8 @@ import ( "github.com/dropalldatabases/sif/internal/output" ) -const securityTrailsBaseURL = "https://api.securitytrails.com/v1" +// securityTrailsBaseURL is a var so integration tests can repoint it at a fixture. +var securityTrailsBaseURL = "https://api.securitytrails.com/v1" // SecurityTrailsResult holds discovered domains from SecurityTrails API type SecurityTrailsResult struct { diff --git a/internal/scan/shodan.go b/internal/scan/shodan.go index e47afee..238e3e9 100644 --- a/internal/scan/shodan.go +++ b/internal/scan/shodan.go @@ -28,7 +28,8 @@ import ( "github.com/dropalldatabases/sif/internal/output" ) -const shodanBaseURL = "https://api.shodan.io" +// shodanBaseURL is a var so integration tests can repoint it at a fixture. +var shodanBaseURL = "https://api.shodan.io" // ShodanResult represents the results from a Shodan host lookup type ShodanResult struct {