test(scan): seam shodan/securitytrails/cloudstorage/dnslist for hermetic integration tests

the remaining hardcoded base urls had no test seam, so their drivers could
only be exercised against the live apis. promote them to package vars (matching
the dirlist/git/ports pattern from #112) and route dnslist's per-host probes
through an injectable transport, then add integration tests that pin each at a
local httptest fixture. defaults equal the old const values so behavior is
unchanged.

internal/scan/cloudstorage.go

@@ -25,6 +25,10 @@ import (
 	"github.com/dropalldatabases/sif/internal/styles"
 )
 
+// s3EndpointFmt is a var so integration tests can repoint it at a fixture; the
+// %s is the bucket name.
+var s3EndpointFmt = "https://%s.s3.amazonaws.com"
+
 type CloudStorageResult struct {
 	BucketName string `json:"bucket_name"`
 	IsPublic   bool   `json:"is_public"`
@@ -96,7 +100,7 @@ func extractPotentialBuckets(url string) []string {
 }
 
 func checkS3Bucket(ctx context.Context, bucket string, client *http.Client) (bool, error) {
-	url := fmt.Sprintf("https://%s.s3.amazonaws.com", bucket)
+	url := fmt.Sprintf(s3EndpointFmt, bucket)
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody)
 	if err != nil {
 		return false, err

internal/scan/dnslist.go

@@ -28,6 +28,10 @@ import (
 // dnsURL is a var so integration tests can repoint it at a fixture.
 var dnsURL = "https://raw.githubusercontent.com/dropalldatabases/sif-runtime/main/dnslist/"
 
+// dnsTransport is a var so integration tests can route the per-host probes at a
+// local server instead of resolving real DNS. nil keeps http.DefaultTransport.
+var dnsTransport http.RoundTripper
+
 const (
 	dnsSmallFile  = "subdomains-100.txt"
 	dnsMediumFile = "subdomains-1000.txt"
@@ -78,7 +82,8 @@ func Dnslist(size string, url string, timeout time.Duration, threads int, logdir
 	}
 
 	client := &http.Client{
-		Timeout: timeout,
+		Timeout:   timeout,
+		Transport: dnsTransport,
 	}
 
 	progress := output.NewProgress(len(dns), "enumerating")

internal/scan/integration_test.go

@@ -20,6 +20,7 @@ package scan
 
 import (
 	"context"
+	"encoding/json"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -41,6 +42,9 @@ func newVulnApp() *httptest.Server {
 	mux.HandleFunc("/directory-list-2.3-small.txt", func(w http.ResponseWriter, r *http.Request) {
 		w.Write([]byte("admin\nlogin\nnope\n"))
 	})
+	mux.HandleFunc("/subdomains-100.txt", func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("dev\nstaging\n"))
+	})
 
 	// an exposed git repo: HEAD is a real find, config is html so it's excluded
 	mux.HandleFunc("/.git/HEAD", func(w http.ResponseWriter, r *http.Request) {
@@ -209,6 +213,164 @@ func TestIntegrationPorts(t *testing.T) {
 	}
 }
 
+func TestIntegrationShodan(t *testing.T) {
+	// a local server stands in for api.shodan.io; example.com resolves to a real
+	// IP but the lookup never leaves the box.
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Query().Get("key") != "test-key" {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		json.NewEncoder(w).Encode(shodanHostResponse{
+			IP:        "93.184.216.34",
+			Hostnames: []string{"example.com"},
+			Org:       "EDGECAST",
+			Ports:     []int{80, 443},
+			Data: []shodanData{
+				{Port: 80, Transport: "tcp", Product: "nginx", Version: "1.18.0"},
+			},
+		})
+	}))
+	defer srv.Close()
+	orig := shodanBaseURL
+	shodanBaseURL = srv.URL
+	defer func() { shodanBaseURL = orig }()
+
+	t.Setenv("SHODAN_API_KEY", "test-key")
+
+	result, err := Shodan("https://example.com", 5*time.Second, "")
+	if err != nil {
+		t.Fatalf("Shodan: %v", err)
+	}
+	if result == nil || result.IP != "93.184.216.34" {
+		t.Fatalf("expected parsed shodan result, got %+v", result)
+	}
+	if len(result.Services) != 1 || result.Services[0].Product != "nginx" {
+		t.Errorf("expected one nginx service, got %+v", result.Services)
+	}
+}
+
+func TestIntegrationSecurityTrails(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("APIKEY") != "test-key" {
+			w.WriteHeader(http.StatusForbidden)
+			return
+		}
+		switch {
+		case strings.HasSuffix(r.URL.Path, "/subdomains"):
+			json.NewEncoder(w).Encode(stSubdomainsResponse{Subdomains: []string{"www", "api"}})
+		case strings.HasSuffix(r.URL.Path, "/associated"):
+			json.NewEncoder(w).Encode(stAssociatedResponse{Records: []stAssociatedRecord{{Hostname: "example.org"}}})
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+	orig := securityTrailsBaseURL
+	securityTrailsBaseURL = srv.URL
+	defer func() { securityTrailsBaseURL = orig }()
+
+	t.Setenv("SECURITYTRAILS_API_KEY", "test-key")
+
+	result, err := SecurityTrails("https://example.com", 5*time.Second, "")
+	if err != nil {
+		t.Fatalf("SecurityTrails: %v", err)
+	}
+	if len(result.Subdomains) != 2 {
+		t.Errorf("expected 2 subdomains, got %v", result.Subdomains)
+	}
+	if len(result.AssociatedDomains) != 1 || result.AssociatedDomains[0] != "example.org" {
+		t.Errorf("expected example.org associated, got %v", result.AssociatedDomains)
+	}
+
+	urls := result.DiscoveredURLs()
+	if !contains(urls, "https://www.example.com") || !contains(urls, "https://example.org") {
+		t.Errorf("expected discovered urls to expand subs and associated, got %v", urls)
+	}
+}
+
+func TestIntegrationCloudStorage(t *testing.T) {
+	// the fixture returns 200 only for the planted bucket, so any candidate that
+	// matches it is reported public.
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/example" {
+			w.WriteHeader(http.StatusOK)
+			return
+		}
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer srv.Close()
+	orig := s3EndpointFmt
+	s3EndpointFmt = srv.URL + "/%s"
+	defer func() { s3EndpointFmt = orig }()
+
+	results, err := CloudStorage("https://example.com", 5*time.Second, "")
+	if err != nil {
+		t.Fatalf("CloudStorage: %v", err)
+	}
+
+	var public bool
+	for _, r := range results {
+		if r.BucketName == "example" && r.IsPublic {
+			public = true
+		}
+	}
+	if !public {
+		t.Errorf("expected the example bucket to be flagged public, got %+v", results)
+	}
+}
+
+func TestIntegrationDnslist(t *testing.T) {
+	// the probe server answers any host routed to it; dnsTransport pins every
+	// dial here so no real DNS is touched.
+	probe := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer probe.Close()
+	probeAddr := strings.TrimPrefix(probe.URL, "http://")
+
+	list := newVulnApp()
+	defer list.Close()
+	origURL := dnsURL
+	dnsURL = list.URL + "/"
+	defer func() { dnsURL = origURL }()
+
+	origTr := dnsTransport
+	dnsTransport = &http.Transport{
+		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			return (&net.Dialer{}).DialContext(ctx, network, probeAddr)
+		},
+	}
+	defer func() { dnsTransport = origTr }()
+
+	found, err := Dnslist("small", "http://example.com", 5*time.Second, 2, "")
+	if err != nil {
+		t.Fatalf("Dnslist: %v", err)
+	}
+	// http probes land on the plain-http probe server; https fails the tls
+	// handshake and is dropped, which is fine - the planted sub still shows up.
+	if !hasSuffixIn(sliceSet(found), "dev.example.com") {
+		t.Errorf("expected dev.example.com among findings, got %v", found)
+	}
+}
+
+func contains(s []string, v string) bool {
+	for i := 0; i < len(s); i++ {
+		if s[i] == v {
+			return true
+		}
+	}
+	return false
+}
+
+func sliceSet(s []string) map[string]bool {
+	set := make(map[string]bool, len(s))
+	for i := 0; i < len(s); i++ {
+		set[s[i]] = true
+	}
+	return set
+}
+
 func hasSuffixIn(set map[string]bool, suffix string) bool {
 	for k := range set {
 		if strings.HasSuffix(k, suffix) {

internal/scan/securitytrails.go

@@ -27,7 +27,8 @@ import (
 	"github.com/dropalldatabases/sif/internal/output"
 )
 
-const securityTrailsBaseURL = "https://api.securitytrails.com/v1"
+// securityTrailsBaseURL is a var so integration tests can repoint it at a fixture.
+var securityTrailsBaseURL = "https://api.securitytrails.com/v1"
 
 // SecurityTrailsResult holds discovered domains from SecurityTrails API
 type SecurityTrailsResult struct {

internal/scan/shodan.go

@@ -28,7 +28,8 @@ import (
 	"github.com/dropalldatabases/sif/internal/output"
 )
 
-const shodanBaseURL = "https://api.shodan.io"
+// shodanBaseURL is a var so integration tests can repoint it at a fixture.
+var shodanBaseURL = "https://api.shodan.io"
 
 // ShodanResult represents the results from a Shodan host lookup
 type ShodanResult struct {