test: add end-to-end testing framework with image scan and proxy tests (#9231)

Co-authored-by: knqyf263 <knqyf263@users.noreply.github.com>

.github/workflows/test.yaml

@@ -187,6 +187,25 @@ jobs:
         run: |
           mage test:vm
 
+  e2e-test:
+    name: E2E Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.1.6
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: false
+
+      - name: Install Go tools
+        run: go install tool # GOBIN is added to the PATH by the setup-go action
+
+      - name: Run E2E tests
+        run: mage test:e2e
+
   build-test:
     name: Build Test
     runs-on: ${{ matrix.operating-system }}

e2e/README.md

@@ -0,0 +1,70 @@
+# End-to-End (E2E) Tests
+
+## Testing Philosophy
+
+The E2E tests in this directory are designed to test Trivy's functionality in realistic environments with **external dependencies and network connectivity**. These tests complement unit tests and integration tests by focusing on scenarios that require real external resources.
+
+### What E2E Tests Should Cover
+
+E2E tests should focus on functionality that involves:
+- **External network connections** (downloading container images, vulnerability databases)
+- **External service dependencies** (Docker daemon, registry access, proxy servers)
+- **Real-world scenarios** that cannot be easily mocked or simulated
+- **Cross-component integration** involving external systems
+
+### What E2E Tests Should NOT Cover
+
+E2E tests should **avoid** detailed assertions and comprehensive validation:
+- **Detailed JSON output validation** - this should be covered by integration tests
+- **Comprehensive vulnerability detection** - this should be covered by unit tests
+- **Complex result comparison** - basic functionality verification is sufficient
+- **Edge cases and error conditions** - these should be covered by unit tests
+
+### Testing Approach
+
+- **Minimal assertions**: Focus on basic functionality rather than detailed output validation
+- **External dependencies**: Use real registries, databases, and services where practical
+- **Environment isolation**: Each test should use isolated cache and working directories
+- **Golden files**: Use -update flag for maintainable output comparison
+- **Conditional execution**: Tests should validate required dependencies during setup
+
+### Dependencies
+
+- **Docker**: Required for local image scanning tests
+- **Internet access**: Required for downloading images and databases
+
+### Test Execution
+
+The E2E tests build and execute trivy in isolated temporary directories. When you run `mage test:e2e`, it automatically:
+1. Builds trivy in a test-specific temporary directory (via `t.TempDir()`)
+2. Adds the temporary directory to the PATH for test execution
+3. Runs the E2E tests using the isolated binary
+
+This approach ensures:
+- No pollution of the global environment
+- Each test run uses a freshly built binary
+- Test isolation between different test runs
+- Clean test environment without side effects
+
+### Running Tests
+
+```bash
+# Run all E2E tests
+mage test:e2e
+
+# Run specific test
+go test -v -tags=e2e ./e2e/ -run TestE2E/image_scan
+
+# Update golden files when output changes
+go test -v -tags=e2e ./e2e/ -update
+```
+
+### Adding New Tests
+
+When adding new E2E tests:
+1. Focus on external dependencies and real-world scenarios
+2. Use minimal assertions - verify functionality, not detailed output
+3. Use golden files with -update flag for output comparison
+4. Validate required dependencies in test setup
+5. Use fixed/pinned versions for reproducible results
+6. Include clear test documentation explaining the scenario being tested

e2e/e2e_test.go

@@ -0,0 +1,94 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/rogpeppe/go-internal/testscript"
+)
+
+var update = flag.Bool("update", false, "update golden files")
+
+func TestE2E(t *testing.T) {
+	testscript.Run(t, testscript.Params{
+		Dir: "testdata",
+		Setup: func(env *testscript.Env) error {
+			return setupTestEnvironment(t, env)
+		},
+		UpdateScripts: *update,
+	})
+}
+
+func buildTrivy(t *testing.T) string {
+	t.Helper()
+
+	tmp := t.TempDir() // Test-specific directory
+	exe := filepath.Join(tmp, "trivy")
+	if runtime.GOOS == "windows" {
+		exe += ".exe"
+	}
+
+	cmd := exec.Command("go", "build",
+		"-o", exe,
+		"../cmd/trivy",
+	)
+	// Prevent environment pollution
+	cmd.Env = append(os.Environ(), "CGO_ENABLED=0")
+
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("Trivy build failed: %v\n%s", err, out)
+	}
+	return exe
+}
+
+func setupTestEnvironment(t *testing.T, env *testscript.Env) error {
+	// Validate Docker availability - fail if not available
+	if err := validateDockerAvailability(); err != nil {
+		return fmt.Errorf("Docker validation failed: %v", err)
+	}
+
+	// Build Trivy once and cache it
+	trivyExe := buildTrivy(t)
+
+	// Add directory containing trivy to PATH
+	env.Setenv("PATH", filepath.Dir(trivyExe)+string(os.PathListSeparator)+env.Getenv("PATH"))
+
+	// Set environment variables for test scripts
+	env.Setenv("TRIVY_DB_DIGEST", "sha256:b4d3718a89a78d4a6b02250953e92fcd87776de4774e64e818c1d0e01c928025")
+	// Disable VEX notice in test environment
+	env.Setenv("TRIVY_DISABLE_VEX_NOTICE", "true")
+	
+	// Define test image
+	testImage := "alpine@sha256:c5b1261d6d3e43071626931fc004f70149baeba2c8ec672bd4f27761f8e1ad6b"
+	env.Setenv("TEST_IMAGE", testImage)
+
+	// Pre-pull the test image to Docker daemon
+	t.Logf("Pre-pulling test image: %s", testImage)
+	cmd := exec.Command("docker", "pull", testImage)
+	if output, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("failed to pull test image: %v\nOutput: %s", err, output)
+	}
+
+	// Pass through DOCKER_HOST if set
+	if dockerHost := os.Getenv("DOCKER_HOST"); dockerHost != "" {
+		env.Setenv("DOCKER_HOST", dockerHost)
+	}
+
+	return nil
+}
+
+func validateDockerAvailability() error {
+	cmd := exec.Command("docker", "version")
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("Docker is not available or not running: %v", err)
+	}
+	return nil
+}

e2e/testdata/image_scan.txtar

@@ -0,0 +1,58 @@
+# Image scan test
+# This test verifies that Trivy can scan images from both local Docker daemon and remote registry
+# and that both methods produce equivalent results
+#
+# Image is already pre-pulled in setupTestEnvironment
+
+# Test 1: Remote image scanning
+exec trivy image --image-src remote --cache-dir $WORK/.cache --format table --output remote_result.txt --db-repository mirror.gcr.io/aquasec/trivy-db@sha256:b4d3718a89a78d4a6b02250953e92fcd87776de4774e64e818c1d0e01c928025 --severity HIGH,CRITICAL --no-progress $TEST_IMAGE
+
+# Verify DB download message appears for remote scan
+stderr 'Downloading vulnerability DB...'
+
+# Test 2: Local Docker daemon scanning  
+exec trivy image --image-src docker --cache-dir $WORK/.cache --format table --output local_result.txt --db-repository mirror.gcr.io/aquasec/trivy-db@sha256:b4d3718a89a78d4a6b02250953e92fcd87776de4774e64e818c1d0e01c928025 --severity HIGH,CRITICAL --no-progress $TEST_IMAGE
+
+# Test 3: Exit code testing - scan with exit code for vulnerabilities
+! exec trivy image --exit-code 1 --cache-dir $WORK/.cache --format table --db-repository mirror.gcr.io/aquasec/trivy-db@sha256:b4d3718a89a78d4a6b02250953e92fcd87776de4774e64e818c1d0e01c928025 --severity HIGH,CRITICAL --no-progress $TEST_IMAGE
+
+# Verify all scans completed successfully
+! stderr 'FATAL'
+exists remote_result.txt
+exists local_result.txt
+
+
+# Verify both scans produce equivalent results
+cmp remote_result.txt local_result.txt
+
+# Compare with golden file to ensure expected output format
+cmp remote_result.txt image_scan_golden.txt
+
+-- image_scan_golden.txt --
+
+Report Summary
+
+┌────────────────────────────────────────────────────────────────────────────────┬────────┬─────────────────┬─────────┐
+│                                     Target                                     │  Type  │ Vulnerabilities │ Secrets │
+├────────────────────────────────────────────────────────────────────────────────┼────────┼─────────────────┼─────────┤
+│ alpine@sha256:c5b1261d6d3e43071626931fc004f70149baeba2c8ec672bd4f27761f8e1ad6b │ alpine │        2        │    -    │
+│ (alpine 3.19.1)                                                                │        │                 │         │
+└────────────────────────────────────────────────────────────────────────────────┴────────┴─────────────────┴─────────┘
+Legend:
+- '-': Not scanned
+- '0': Clean (no security findings detected)
+
+
+alpine@sha256:c5b1261d6d3e43071626931fc004f70149baeba2c8ec672bd4f27761f8e1ad6b (alpine 3.19.1)
+==============================================================================================
+Total: 2 (HIGH: 2, CRITICAL: 0)
+
+┌────────────┬───────────────┬──────────┬────────┬───────────────────┬───────────────┬──────────────────────────────────────────────────────────┐
+│  Library   │ Vulnerability │ Severity │ Status │ Installed Version │ Fixed Version │                          Title                           │
+├────────────┼───────────────┼──────────┼────────┼───────────────────┼───────────────┼──────────────────────────────────────────────────────────┤
+│ libcrypto3 │ CVE-2024-6119 │ HIGH     │ fixed  │ 3.1.4-r5          │ 3.1.7-r0      │ openssl: Possible denial of service in X.509 name checks │
+│            │               │          │        │                   │               │ https://avd.aquasec.com/nvd/cve-2024-6119                │
+├────────────┤               │          │        │                   │               │                                                          │
+│ libssl3    │               │          │        │                   │               │                                                          │
+│            │               │          │        │                   │               │                                                          │
+└────────────┴───────────────┴──────────┴────────┴───────────────────┴───────────────┴──────────────────────────────────────────────────────────┘

e2e/testdata/proxy_scan.txtar

@@ -0,0 +1,27 @@
+# Proxy scan test
+# This test verifies that Trivy can work through HTTP/HTTPS proxy with self-signed certificates
+# Tests both --insecure flag behavior: failure without it, success with it
+
+# Start mitmdump proxy container on port 18080
+exec docker rm -f mitmproxy-e2e-test
+exec docker run -d --name mitmproxy-e2e-test -p 18080:8080 mitmproxy/mitmproxy:latest mitmdump
+exec sleep 3
+
+# Set proxy environment variables
+env HTTPS_PROXY=http://localhost:18080
+env HTTP_PROXY=http://localhost:18080
+
+# Test 1: Scan without --insecure flag (should fail due to certificate issues)
+! exec trivy image --cache-dir $WORK/.cache --format table --db-repository mirror.gcr.io/aquasec/trivy-db@sha256:b4d3718a89a78d4a6b02250953e92fcd87776de4774e64e818c1d0e01c928025 --severity HIGH,CRITICAL --no-progress alpine@sha256:c5b1261d6d3e43071626931fc004f70149baeba2c8ec672bd4f27761f8e1ad6b
+
+# Verify certificate error occurred
+stderr 'certificate'
+
+# Test 2: Scan with --insecure flag (should succeed despite certificate issues)
+exec trivy image --insecure --cache-dir $WORK/.cache --format table --db-repository mirror.gcr.io/aquasec/trivy-db@sha256:b4d3718a89a78d4a6b02250953e92fcd87776de4774e64e818c1d0e01c928025 --severity HIGH,CRITICAL --no-progress alpine@sha256:c5b1261d6d3e43071626931fc004f70149baeba2c8ec672bd4f27761f8e1ad6b
+
+# Verify scan completed successfully with --insecure
+! stderr 'FATAL'
+
+# Stop proxy container
+exec docker rm -f mitmproxy-e2e-test

go.mod

@@ -132,6 +132,8 @@ require (
 	modernc.org/sqlite v1.38.0
 )
 
+require github.com/rogpeppe/go-internal v1.14.1
+
 require (
 	buf.build/gen/go/bufbuild/bufplugin/protocolbuffers/go v1.36.6-20250121211742-6d880cc6cc8d.1 // indirect
 	buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250613105001-9f2d3c737feb.1 // indirect

magefiles/magefile.go

@@ -331,6 +331,11 @@ func (t Test) UpdateVMGolden() error {
 	return sh.RunWithV(ENV, "go", "test", "-v", "-tags=vm_integration", "./integration/...", "-update")
 }
 
+// E2e runs E2E tests using testscript framework
+func (t Test) E2e() error {
+	return sh.RunWithV(ENV, "go", "test", "-v", "-tags=e2e", "./e2e/...")
+}
+
 type Lint mg.Namespace
 
 // Run runs linters