diff --git a/internal/scan/cloudstorage.go b/internal/scan/cloudstorage.go index 809e191..417a37e 100644 --- a/internal/scan/cloudstorage.go +++ b/internal/scan/cloudstorage.go @@ -25,6 +25,10 @@ import ( "github.com/dropalldatabases/sif/internal/styles" ) +// s3EndpointFmt is a var so integration tests can repoint it at a fixture; the +// %s is the bucket name. +var s3EndpointFmt = "https://%s.s3.amazonaws.com" + type CloudStorageResult struct { BucketName string `json:"bucket_name"` IsPublic bool `json:"is_public"` @@ -96,7 +100,7 @@ func extractPotentialBuckets(url string) []string { } func checkS3Bucket(ctx context.Context, bucket string, client *http.Client) (bool, error) { - url := fmt.Sprintf("https://%s.s3.amazonaws.com", bucket) + url := fmt.Sprintf(s3EndpointFmt, bucket) req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) if err != nil { return false, err diff --git a/internal/scan/dnslist.go b/internal/scan/dnslist.go index c23f3e3..b490a55 100644 --- a/internal/scan/dnslist.go +++ b/internal/scan/dnslist.go @@ -28,6 +28,10 @@ import ( // dnsURL is a var so integration tests can repoint it at a fixture. var dnsURL = "https://raw.githubusercontent.com/dropalldatabases/sif-runtime/main/dnslist/" +// dnsTransport is a var so integration tests can route the per-host probes at a +// local server instead of resolving real DNS. nil keeps http.DefaultTransport. +var dnsTransport http.RoundTripper + const ( dnsSmallFile = "subdomains-100.txt" dnsMediumFile = "subdomains-1000.txt" @@ -78,7 +82,8 @@ func Dnslist(size string, url string, timeout time.Duration, threads int, logdir } client := &http.Client{ - Timeout: timeout, + Timeout: timeout, + Transport: dnsTransport, } progress := output.NewProgress(len(dns), "enumerating") diff --git a/internal/scan/integration_test.go b/internal/scan/integration_test.go index 5e09ca4..7894a7f 100644 --- a/internal/scan/integration_test.go +++ b/internal/scan/integration_test.go @@ -20,6 +20,7 @@ package scan import ( "context" + "encoding/json" "net" "net/http" "net/http/httptest" @@ -41,6 +42,9 @@ func newVulnApp() *httptest.Server { mux.HandleFunc("/directory-list-2.3-small.txt", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("admin\nlogin\nnope\n")) }) + mux.HandleFunc("/subdomains-100.txt", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("dev\nstaging\n")) + }) // an exposed git repo: HEAD is a real find, config is html so it's excluded mux.HandleFunc("/.git/HEAD", func(w http.ResponseWriter, r *http.Request) { @@ -209,6 +213,164 @@ func TestIntegrationPorts(t *testing.T) { } } +func TestIntegrationShodan(t *testing.T) { + // a local server stands in for api.shodan.io; example.com resolves to a real + // IP but the lookup never leaves the box. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("key") != "test-key" { + w.WriteHeader(http.StatusUnauthorized) + return + } + json.NewEncoder(w).Encode(shodanHostResponse{ + IP: "93.184.216.34", + Hostnames: []string{"example.com"}, + Org: "EDGECAST", + Ports: []int{80, 443}, + Data: []shodanData{ + {Port: 80, Transport: "tcp", Product: "nginx", Version: "1.18.0"}, + }, + }) + })) + defer srv.Close() + orig := shodanBaseURL + shodanBaseURL = srv.URL + defer func() { shodanBaseURL = orig }() + + t.Setenv("SHODAN_API_KEY", "test-key") + + result, err := Shodan("https://example.com", 5*time.Second, "") + if err != nil { + t.Fatalf("Shodan: %v", err) + } + if result == nil || result.IP != "93.184.216.34" { + t.Fatalf("expected parsed shodan result, got %+v", result) + } + if len(result.Services) != 1 || result.Services[0].Product != "nginx" { + t.Errorf("expected one nginx service, got %+v", result.Services) + } +} + +func TestIntegrationSecurityTrails(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("APIKEY") != "test-key" { + w.WriteHeader(http.StatusForbidden) + return + } + switch { + case strings.HasSuffix(r.URL.Path, "/subdomains"): + json.NewEncoder(w).Encode(stSubdomainsResponse{Subdomains: []string{"www", "api"}}) + case strings.HasSuffix(r.URL.Path, "/associated"): + json.NewEncoder(w).Encode(stAssociatedResponse{Records: []stAssociatedRecord{{Hostname: "example.org"}}}) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + orig := securityTrailsBaseURL + securityTrailsBaseURL = srv.URL + defer func() { securityTrailsBaseURL = orig }() + + t.Setenv("SECURITYTRAILS_API_KEY", "test-key") + + result, err := SecurityTrails("https://example.com", 5*time.Second, "") + if err != nil { + t.Fatalf("SecurityTrails: %v", err) + } + if len(result.Subdomains) != 2 { + t.Errorf("expected 2 subdomains, got %v", result.Subdomains) + } + if len(result.AssociatedDomains) != 1 || result.AssociatedDomains[0] != "example.org" { + t.Errorf("expected example.org associated, got %v", result.AssociatedDomains) + } + + urls := result.DiscoveredURLs() + if !contains(urls, "https://www.example.com") || !contains(urls, "https://example.org") { + t.Errorf("expected discovered urls to expand subs and associated, got %v", urls) + } +} + +func TestIntegrationCloudStorage(t *testing.T) { + // the fixture returns 200 only for the planted bucket, so any candidate that + // matches it is reported public. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/example" { + w.WriteHeader(http.StatusOK) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + orig := s3EndpointFmt + s3EndpointFmt = srv.URL + "/%s" + defer func() { s3EndpointFmt = orig }() + + results, err := CloudStorage("https://example.com", 5*time.Second, "") + if err != nil { + t.Fatalf("CloudStorage: %v", err) + } + + var public bool + for _, r := range results { + if r.BucketName == "example" && r.IsPublic { + public = true + } + } + if !public { + t.Errorf("expected the example bucket to be flagged public, got %+v", results) + } +} + +func TestIntegrationDnslist(t *testing.T) { + // the probe server answers any host routed to it; dnsTransport pins every + // dial here so no real DNS is touched. + probe := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer probe.Close() + probeAddr := strings.TrimPrefix(probe.URL, "http://") + + list := newVulnApp() + defer list.Close() + origURL := dnsURL + dnsURL = list.URL + "/" + defer func() { dnsURL = origURL }() + + origTr := dnsTransport + dnsTransport = &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, probeAddr) + }, + } + defer func() { dnsTransport = origTr }() + + found, err := Dnslist("small", "http://example.com", 5*time.Second, 2, "") + if err != nil { + t.Fatalf("Dnslist: %v", err) + } + // http probes land on the plain-http probe server; https fails the tls + // handshake and is dropped, which is fine - the planted sub still shows up. + if !hasSuffixIn(sliceSet(found), "dev.example.com") { + t.Errorf("expected dev.example.com among findings, got %v", found) + } +} + +func contains(s []string, v string) bool { + for i := 0; i < len(s); i++ { + if s[i] == v { + return true + } + } + return false +} + +func sliceSet(s []string) map[string]bool { + set := make(map[string]bool, len(s)) + for i := 0; i < len(s); i++ { + set[s[i]] = true + } + return set +} + func hasSuffixIn(set map[string]bool, suffix string) bool { for k := range set { if strings.HasSuffix(k, suffix) { diff --git a/internal/scan/securitytrails.go b/internal/scan/securitytrails.go index 1004cdd..aeccdd9 100644 --- a/internal/scan/securitytrails.go +++ b/internal/scan/securitytrails.go @@ -27,7 +27,8 @@ import ( "github.com/dropalldatabases/sif/internal/output" ) -const securityTrailsBaseURL = "https://api.securitytrails.com/v1" +// securityTrailsBaseURL is a var so integration tests can repoint it at a fixture. +var securityTrailsBaseURL = "https://api.securitytrails.com/v1" // SecurityTrailsResult holds discovered domains from SecurityTrails API type SecurityTrailsResult struct { diff --git a/internal/scan/shodan.go b/internal/scan/shodan.go index e47afee..238e3e9 100644 --- a/internal/scan/shodan.go +++ b/internal/scan/shodan.go @@ -28,7 +28,8 @@ import ( "github.com/dropalldatabases/sif/internal/output" ) -const shodanBaseURL = "https://api.shodan.io" +// shodanBaseURL is a var so integration tests can repoint it at a fixture. +var shodanBaseURL = "https://api.shodan.io" // ShodanResult represents the results from a Shodan host lookup type ShodanResult struct {