diff --git a/internal/scan/git.go b/internal/scan/git.go index 7fc8b4a..781a29f 100644 --- a/internal/scan/git.go +++ b/internal/scan/git.go @@ -15,7 +15,10 @@ package scan import ( "bufio" "context" + "encoding/json" + "io" "net/http" + "regexp" "strconv" "strings" "sync" @@ -33,6 +36,64 @@ var gitURL = "https://raw.githubusercontent.com/vmfunc/sif-runtime/main/git/" const gitFile = "git.txt" +// a 200 alone proves nothing: a server that answers every path with the same +// catch-all page reports the whole git list as exposed. gitValidateCap bounds +// how much of the body we read to confirm the response is the requested git +// artifact and not that shell. +const gitValidateCap = 8 << 10 + +var ( + // a git object name is a sha1 (40 hex) or, on sha256 repos, 64 hex. refs and + // a detached HEAD hold exactly this and nothing else. + gitObjectName = regexp.MustCompile(`^[0-9a-f]{40}([0-9a-f]{24})?$`) + // a reflog line opens with " "; the zero sha is valid for + // the first entry, so match the shape rather than reject all-zero hashes. + gitReflogLine = regexp.MustCompile(`^[0-9a-f]{40}([0-9a-f]{24})? [0-9a-f]{40}`) +) + +// looksLikeGit reports whether body is plausibly the git artifact at repoPath +// rather than a catch-all 200 page. paths with a recognizable shape (HEAD, the +// index magic, the config header, refs and reflogs) are matched positively; the +// rest (.gitignore and anything new in the list) fall back to rejecting the +// html/json bodies a soft-404 or spa shell hands back. +func looksLikeGit(repoPath string, body []byte) bool { + text := strings.TrimSpace(string(body)) + switch { + case strings.Contains(repoPath, "/logs/"): + // reflog lives under .git/logs/, including .git/logs/refs/..., so this + // must be checked before the /refs/ case below. + return gitReflogLine.MatchString(text) + case strings.HasSuffix(repoPath, "/index"): + return strings.HasPrefix(string(body), "DIRC") + case strings.HasSuffix(repoPath, "/config"): + return strings.Contains(text, "[core]") + case strings.HasSuffix(repoPath, "/HEAD"): + return strings.HasPrefix(text, "ref:") || gitObjectName.MatchString(text) + case strings.Contains(repoPath, "/refs/"): + return gitObjectName.MatchString(text) + default: + return shapelessGitArtifact(text, body) + } +} + +// shapelessGitArtifact is the fallback for paths with no fixed shape (.gitignore +// and anything new in the list). a real artifact is neither html nor json; the +// leading-byte rejects also catch a shell whose body is truncated past the read +// cap. a '[' is ambiguous - a json array shell, but also a valid .gitignore +// character class like "[Bb]in/" - so json validity, not the bare byte, decides. +func shapelessGitArtifact(text string, body []byte) bool { + switch { + case text == "": + return false + case strings.HasPrefix(text, "<"), strings.HasPrefix(text, "{"): + return false + case strings.HasPrefix(text, "["): + return !json.Valid(body) + default: + return true + } +} + func Git(url string, timeout time.Duration, threads int, logdir string) ([]string, error) { log := output.Module("GIT") log.Start() @@ -87,21 +148,32 @@ func Git(url string, timeout time.Duration, threads int, logdir string) ([]strin charmlog.Debugf("Error %s: %s", repourl, err) return } - - if resp.StatusCode == 200 && !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") { - spin.Stop() - log.Success("Git found at %s [%s]", output.Highlight.Render(repourl), output.Status.Render(strconv.Itoa(resp.StatusCode))) - spin.Start() - if logdir != "" { - logger.Write(sanitizedURL, logdir, strconv.Itoa(resp.StatusCode)+" git found at ["+repourl+"]\n") - } - - mu.Lock() - foundUrls = append(foundUrls, resp.Request.URL.String()) - mu.Unlock() + if resp.StatusCode != 200 { + httpx.DrainClose(resp) + return } - // status/headers only; drain so the conn returns to the pool. + + body, err := io.ReadAll(io.LimitReader(resp.Body, gitValidateCap)) + foundURL := resp.Request.URL.String() httpx.DrainClose(resp) + if err != nil { + charmlog.Debugf("Error reading %s: %s", repourl, err) + return + } + if !looksLikeGit(repourl, body) { + return + } + + spin.Stop() + log.Success("Git found at %s [%s]", output.Highlight.Render(repourl), output.Status.Render(strconv.Itoa(resp.StatusCode))) + spin.Start() + if logdir != "" { + logger.Write(sanitizedURL, logdir, strconv.Itoa(resp.StatusCode)+" git found at ["+repourl+"]\n") + } + + mu.Lock() + foundUrls = append(foundUrls, foundURL) + mu.Unlock() }) spin.Stop() diff --git a/internal/scan/git_test.go b/internal/scan/git_test.go new file mode 100644 index 0000000..a7a33b4 --- /dev/null +++ b/internal/scan/git_test.go @@ -0,0 +1,118 @@ +/* +·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━· +: : +: █▀ █ █▀▀ · Blazing-fast pentesting suite : +: ▄█ █ █▀ · BSD 3-Clause License : +: : +: (c) 2022-2026 vmfunc, xyzeva, : +: lunchcat alumni & contributors : +: : +·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━· +*/ + +package scan + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +const ( + // reused across the artifact fixtures below. + fakeSHA = "1d69b25b0d808e0e78ce366cd2ccf897e43783be" + // the all-zero sha git writes as the previous value of a ref's first reflog. + zeroSHA = "0000000000000000000000000000000000000000" +) + +func TestLooksLikeGit(t *testing.T) { + tests := []struct { + name string + path string + body string + want bool + }{ + {"head ref", ".git/HEAD", "ref: refs/heads/main\n", true}, + {"head detached", ".git/HEAD", fakeSHA + "\n", true}, + {"config core", ".git/config", "[core]\n\trepositoryformatversion = 0\n", true}, + {"index magic", ".git/index", "DIRC\x00\x00\x00\x02", true}, + {"ref sha", ".git/refs/remotes/origin/main", fakeSHA + "\n", true}, + {"ref sha256", ".git/refs/remotes/origin/main", strings.Repeat("a", 64) + "\n", true}, + {"reflog line", ".git/logs/refs/remotes/origin/main", zeroSHA + " " + fakeSHA + " a 1 -0700\tx\n", true}, + {"gitignore text", ".gitignore", "node_modules/\n", true}, + {"gitignore comment lead", ".gitignore", "# build output\n*.log\n", true}, + // a .gitignore can legitimately open with a '[' character-class glob + // (common in .net repos), so the fallback must not treat '[' as json. + {"gitignore bracket glob", ".gitignore", "[Bb]in/\n[Oo]bj/\n", true}, + + // a server that answers every path with the same shell is the dominant + // false positive; none of these are the requested artifact. + {"head json shell", ".git/HEAD", `{"error":"not found"}`, false}, + {"head text shell", ".git/HEAD", "Not Found", false}, + {"head html shell", ".git/HEAD", "", false}, + {"config json shell", ".git/config", `{"error":"not found"}`, false}, + {"index json shell", ".git/index", `{"error":"nope"}`, false}, + {"ref json shell", ".git/refs/remotes/origin/main", `{"ok":true}`, false}, + {"reflog text shell", ".git/logs/refs/remotes/origin/main", "Page not found", false}, + {"gitignore json shell", ".gitignore", `{"error":"not found"}`, false}, + {"gitignore json array shell", ".gitignore", `[{"error":"not found"}]`, false}, + {"gitignore html shell", ".gitignore", "", false}, + // known limit: .gitignore has no fixed shape, so a plain-text soft-404 is + // indistinguishable from a real ignore file and is accepted. the structured + // artifacts (HEAD/config/index/refs) carry the detection weight. + {"gitignore plaintext soft404 accepted", ".gitignore", "Not Found", true}, + {"empty body", ".git/HEAD", "", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := looksLikeGit(tt.path, []byte(tt.body)); got != tt.want { + t.Errorf("looksLikeGit(%q, %q) = %v, want %v", tt.path, tt.body, got, tt.want) + } + }) + } +} + +// gitFixtureApp backs TestGit_SuppressesCatchAll: real .git/HEAD plus a catch-all 200 json shell everywhere else. +func gitFixtureApp() *httptest.Server { + mux := http.NewServeMux() + mux.HandleFunc("/.git/HEAD", func(w http.ResponseWriter, _ *http.Request) { + w.Write([]byte("ref: refs/heads/main\n")) + }) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.git/HEAD" { + return + } + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"error":"not found"}`)) + }) + return httptest.NewServer(mux) +} + +func TestGit_SuppressesCatchAll(t *testing.T) { + target := gitFixtureApp() + defer target.Close() + + list := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Write([]byte(".git/HEAD\n.git/config\n.git/index\n")) + })) + defer list.Close() + + orig := gitURL + gitURL = list.URL + "/" + t.Cleanup(func() { gitURL = orig }) + + found, err := Git(target.URL, 5*time.Second, 3, "") + if err != nil { + t.Fatalf("Git: %v", err) + } + + if len(found) != 1 { + t.Fatalf("expected only the real .git/HEAD to surface, got %d: %v", len(found), found) + } + if !strings.HasSuffix(found[0], "/.git/HEAD") { + t.Errorf("expected .git/HEAD, got %q", found[0]) + } +}