From 22935328b5ed4386db30c9bb0905cf010a17929c Mon Sep 17 00:00:00 2001
From: Tigah <88289044+TBX3D@users.noreply.github.com>
Date: Wed, 22 Jul 2026 12:51:57 -0700
Subject: [PATCH] fix(scan): stop one over-long url from dropping the wayback
 feed (#321)

the wayback line scanner capped each line at 1mb, so a single archived
url with a huge query string made scanner.Err return ErrTooLong and the
whole feed was discarded. read lines without a per-line cap, still bounded
by the passiveMaxBytes read limit.
---
 internal/scan/passive.go      | 24 ++++++++++++++----------
 internal/scan/passive_test.go | 25 +++++++++++++++++++++++++
 2 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/internal/scan/passive.go b/internal/scan/passive.go
index 1aaec43..2fc1469 100644
--- a/internal/scan/passive.go
+++ b/internal/scan/passive.go
@@ -181,17 +181,21 @@ func fetchWayback(ctx context.Context, client *http.Client, domain string) ([]st
 	}
 
 	var urls []string
-	scanner := bufio.NewScanner(strings.NewReader(string(body)))
-	// historical urls can be long; give the scanner a generous line buffer.
-	scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
-	for scanner.Scan() {
-		line := strings.TrimSpace(scanner.Text())
-		if line != "" {
-			urls = append(urls, line)
+	// read line by line without a per-line cap: a single archived url with a
+	// huge query string must not abort the whole feed. overall size is already
+	// bounded by the passiveMaxBytes read limit.
+	reader := bufio.NewReader(strings.NewReader(string(body)))
+	for {
+		line, err := reader.ReadString('\n')
+		if trimmed := strings.TrimSpace(line); trimmed != "" {
+			urls = append(urls, trimmed)
+		}
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, fmt.Errorf("read wayback lines: %w", err)
 		}
-	}
-	if err := scanner.Err(); err != nil {
-		return nil, fmt.Errorf("read wayback lines: %w", err)
 	}
 	return urls, nil
 }
diff --git a/internal/scan/passive_test.go b/internal/scan/passive_test.go
index 547a389..977619b 100644
--- a/internal/scan/passive_test.go
+++ b/internal/scan/passive_test.go
@@ -15,6 +15,7 @@ package scan
 import (
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 	"time"
 )
@@ -168,6 +169,30 @@ func TestPassive_ScopesSubdomainsToTarget(t *testing.T) {
 	}
 }
 
+func TestPassive_WaybackLongLineKeepsFeed(t *testing.T) {
+	// a single archived url with a huge query string (data:/base64 blobs do
+	// occur) must not discard every other harvested url.
+	longURL := "http://example.com/?blob=" + strings.Repeat("a", 2*1024*1024)
+	wayback := longURL + "\n" +
+		"http://example.com/keep-one\n" +
+		"http://example.com/keep-two\n"
+	fixtureServer(t, "[]", "[]", wayback)
+
+	result, err := Passive("https://example.com", 5*time.Second, "")
+	if err != nil {
+		t.Fatalf("Passive: %v", err)
+	}
+
+	for _, want := range []string{"http://example.com/keep-one", "http://example.com/keep-two"} {
+		if !urlsContain(result.URLs, want) {
+			t.Errorf("over-long wayback line dropped the feed; missing %q, got %d urls", want, len(result.URLs))
+		}
+	}
+	if !urlsContain(result.URLs, longURL) {
+		t.Errorf("the over-long url itself was dropped, got %d urls", len(result.URLs))
+	}
+}
+
 func TestNormalizeHost(t *testing.T) {
 	tests := []struct {
 		in   string