fix(scan): fetch wayback source over https (#351)

the wayback passive source used http while the crt.sh and certspotter
sources use https. passive results flatten straight into findings, so
an on-path attacker could tamper with the plaintext response to inject
or strip historical urls. use https to match the other two sources.

Co-authored-by: vmfunc <vmfunc.lc@gmail.com>
This commit is contained in:
Tigah
2026-07-22 21:01:33 +00:00
committed by GitHub
co-authored by vmfunc
parent a664f36934
commit d52cd842a9
2 changed files with 17 additions and 1 deletions
+1 -1
View File
@@ -33,7 +33,7 @@ import (
var (
crtshBaseURL = "https://crt.sh/?q=%%25.%s&output=json"
certspotterBaseURL = "https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names"
waybackBaseURL = "http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=text&fl=original&collapse=urlkey"
waybackBaseURL = "https://web.archive.org/cdx/search/cdx?url=*.%s/*&output=text&fl=original&collapse=urlkey"
)
// cap the response we read from any one source so a hostile/huge feed can't
+16
View File
@@ -169,6 +169,22 @@ func TestPassive_ScopesSubdomainsToTarget(t *testing.T) {
}
}
func TestPassive_SourcesUseTLS(t *testing.T) {
// all three passive feeds must be fetched over https: an on-path attacker
// able to tamper with a plain-http response could inject or drop
// subdomains/urls in the reported results without detection.
sources := map[string]string{
"crtsh": crtshBaseURL,
"certspotter": certspotterBaseURL,
"wayback": waybackBaseURL,
}
for name, base := range sources {
if !strings.HasPrefix(base, "https://") {
t.Errorf("%s base url is not https: %q", name, base)
}
}
}
func TestPassive_WaybackLongLineKeepsFeed(t *testing.T) {
// a single archived url with a huge query string (data:/base64 blobs do
// occur) must not discard every other harvested url.