From 4bd7512e90d551bf9f309c6c7461a62b1cd0afbc Mon Sep 17 00:00:00 2001 From: Teppei Fukuda Date: Thu, 24 Jul 2025 11:06:01 +0400 Subject: [PATCH] test: add end-to-end testing framework with image scan and proxy tests (#9231) Co-authored-by: knqyf263 --- .github/workflows/test.yaml | 19 +++++++ e2e/README.md | 70 ++++++++++++++++++++++++++ e2e/e2e_test.go | 94 +++++++++++++++++++++++++++++++++++ e2e/testdata/image_scan.txtar | 58 +++++++++++++++++++++ e2e/testdata/proxy_scan.txtar | 27 ++++++++++ go.mod | 2 + magefiles/magefile.go | 5 ++ 7 files changed, 275 insertions(+) create mode 100644 e2e/README.md create mode 100644 e2e/e2e_test.go create mode 100644 e2e/testdata/image_scan.txtar create mode 100644 e2e/testdata/proxy_scan.txtar diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2be82302e8..fba0ac420b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -187,6 +187,25 @@ jobs: run: | mage test:vm + e2e-test: + name: E2E Test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4.1.6 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: false + + - name: Install Go tools + run: go install tool # GOBIN is added to the PATH by the setup-go action + + - name: Run E2E tests + run: mage test:e2e + build-test: name: Build Test runs-on: ${{ matrix.operating-system }} diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 0000000000..1f44a8b869 --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,70 @@ +# End-to-End (E2E) Tests + +## Testing Philosophy + +The E2E tests in this directory are designed to test Trivy's functionality in realistic environments with **external dependencies and network connectivity**. These tests complement unit tests and integration tests by focusing on scenarios that require real external resources. + +### What E2E Tests Should Cover + +E2E tests should focus on functionality that involves: +- **External network connections** (downloading container images, vulnerability databases) +- **External service dependencies** (Docker daemon, registry access, proxy servers) +- **Real-world scenarios** that cannot be easily mocked or simulated +- **Cross-component integration** involving external systems + +### What E2E Tests Should NOT Cover + +E2E tests should **avoid** detailed assertions and comprehensive validation: +- **Detailed JSON output validation** - this should be covered by integration tests +- **Comprehensive vulnerability detection** - this should be covered by unit tests +- **Complex result comparison** - basic functionality verification is sufficient +- **Edge cases and error conditions** - these should be covered by unit tests + +### Testing Approach + +- **Minimal assertions**: Focus on basic functionality rather than detailed output validation +- **External dependencies**: Use real registries, databases, and services where practical +- **Environment isolation**: Each test should use isolated cache and working directories +- **Golden files**: Use -update flag for maintainable output comparison +- **Conditional execution**: Tests should validate required dependencies during setup + +### Dependencies + +- **Docker**: Required for local image scanning tests +- **Internet access**: Required for downloading images and databases + +### Test Execution + +The E2E tests build and execute trivy in isolated temporary directories. When you run `mage test:e2e`, it automatically: +1. Builds trivy in a test-specific temporary directory (via `t.TempDir()`) +2. Adds the temporary directory to the PATH for test execution +3. Runs the E2E tests using the isolated binary + +This approach ensures: +- No pollution of the global environment +- Each test run uses a freshly built binary +- Test isolation between different test runs +- Clean test environment without side effects + +### Running Tests + +```bash +# Run all E2E tests +mage test:e2e + +# Run specific test +go test -v -tags=e2e ./e2e/ -run TestE2E/image_scan + +# Update golden files when output changes +go test -v -tags=e2e ./e2e/ -update +``` + +### Adding New Tests + +When adding new E2E tests: +1. Focus on external dependencies and real-world scenarios +2. Use minimal assertions - verify functionality, not detailed output +3. Use golden files with -update flag for output comparison +4. Validate required dependencies in test setup +5. Use fixed/pinned versions for reproducible results +6. Include clear test documentation explaining the scenario being tested \ No newline at end of file diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go new file mode 100644 index 0000000000..850fdb02b0 --- /dev/null +++ b/e2e/e2e_test.go @@ -0,0 +1,94 @@ +//go:build e2e + +package e2e + +import ( + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" + + "github.com/rogpeppe/go-internal/testscript" +) + +var update = flag.Bool("update", false, "update golden files") + +func TestE2E(t *testing.T) { + testscript.Run(t, testscript.Params{ + Dir: "testdata", + Setup: func(env *testscript.Env) error { + return setupTestEnvironment(t, env) + }, + UpdateScripts: *update, + }) +} + +func buildTrivy(t *testing.T) string { + t.Helper() + + tmp := t.TempDir() // Test-specific directory + exe := filepath.Join(tmp, "trivy") + if runtime.GOOS == "windows" { + exe += ".exe" + } + + cmd := exec.Command("go", "build", + "-o", exe, + "../cmd/trivy", + ) + // Prevent environment pollution + cmd.Env = append(os.Environ(), "CGO_ENABLED=0") + + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("Trivy build failed: %v\n%s", err, out) + } + return exe +} + +func setupTestEnvironment(t *testing.T, env *testscript.Env) error { + // Validate Docker availability - fail if not available + if err := validateDockerAvailability(); err != nil { + return fmt.Errorf("Docker validation failed: %v", err) + } + + // Build Trivy once and cache it + trivyExe := buildTrivy(t) + + // Add directory containing trivy to PATH + env.Setenv("PATH", filepath.Dir(trivyExe)+string(os.PathListSeparator)+env.Getenv("PATH")) + + // Set environment variables for test scripts + env.Setenv("TRIVY_DB_DIGEST", "sha256:b4d3718a89a78d4a6b02250953e92fcd87776de4774e64e818c1d0e01c928025") + // Disable VEX notice in test environment + env.Setenv("TRIVY_DISABLE_VEX_NOTICE", "true") + + // Define test image + testImage := "alpine@sha256:c5b1261d6d3e43071626931fc004f70149baeba2c8ec672bd4f27761f8e1ad6b" + env.Setenv("TEST_IMAGE", testImage) + + // Pre-pull the test image to Docker daemon + t.Logf("Pre-pulling test image: %s", testImage) + cmd := exec.Command("docker", "pull", testImage) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to pull test image: %v\nOutput: %s", err, output) + } + + // Pass through DOCKER_HOST if set + if dockerHost := os.Getenv("DOCKER_HOST"); dockerHost != "" { + env.Setenv("DOCKER_HOST", dockerHost) + } + + return nil +} + +func validateDockerAvailability() error { + cmd := exec.Command("docker", "version") + if err := cmd.Run(); err != nil { + return fmt.Errorf("Docker is not available or not running: %v", err) + } + return nil +} \ No newline at end of file diff --git a/e2e/testdata/image_scan.txtar b/e2e/testdata/image_scan.txtar new file mode 100644 index 0000000000..53963e9256 --- /dev/null +++ b/e2e/testdata/image_scan.txtar @@ -0,0 +1,58 @@ +# Image scan test +# This test verifies that Trivy can scan images from both local Docker daemon and remote registry +# and that both methods produce equivalent results +# +# Image is already pre-pulled in setupTestEnvironment + +# Test 1: Remote image scanning +exec trivy image --image-src remote --cache-dir $WORK/.cache --format table --output remote_result.txt --db-repository mirror.gcr.io/aquasec/trivy-db@sha256:b4d3718a89a78d4a6b02250953e92fcd87776de4774e64e818c1d0e01c928025 --severity HIGH,CRITICAL --no-progress $TEST_IMAGE + +# Verify DB download message appears for remote scan +stderr 'Downloading vulnerability DB...' + +# Test 2: Local Docker daemon scanning +exec trivy image --image-src docker --cache-dir $WORK/.cache --format table --output local_result.txt --db-repository mirror.gcr.io/aquasec/trivy-db@sha256:b4d3718a89a78d4a6b02250953e92fcd87776de4774e64e818c1d0e01c928025 --severity HIGH,CRITICAL --no-progress $TEST_IMAGE + +# Test 3: Exit code testing - scan with exit code for vulnerabilities +! exec trivy image --exit-code 1 --cache-dir $WORK/.cache --format table --db-repository mirror.gcr.io/aquasec/trivy-db@sha256:b4d3718a89a78d4a6b02250953e92fcd87776de4774e64e818c1d0e01c928025 --severity HIGH,CRITICAL --no-progress $TEST_IMAGE + +# Verify all scans completed successfully +! stderr 'FATAL' +exists remote_result.txt +exists local_result.txt + + +# Verify both scans produce equivalent results +cmp remote_result.txt local_result.txt + +# Compare with golden file to ensure expected output format +cmp remote_result.txt image_scan_golden.txt + +-- image_scan_golden.txt -- + +Report Summary + +┌────────────────────────────────────────────────────────────────────────────────┬────────┬─────────────────┬─────────┐ +│ Target │ Type │ Vulnerabilities │ Secrets │ +├────────────────────────────────────────────────────────────────────────────────┼────────┼─────────────────┼─────────┤ +│ alpine@sha256:c5b1261d6d3e43071626931fc004f70149baeba2c8ec672bd4f27761f8e1ad6b │ alpine │ 2 │ - │ +│ (alpine 3.19.1) │ │ │ │ +└────────────────────────────────────────────────────────────────────────────────┴────────┴─────────────────┴─────────┘ +Legend: +- '-': Not scanned +- '0': Clean (no security findings detected) + + +alpine@sha256:c5b1261d6d3e43071626931fc004f70149baeba2c8ec672bd4f27761f8e1ad6b (alpine 3.19.1) +============================================================================================== +Total: 2 (HIGH: 2, CRITICAL: 0) + +┌────────────┬───────────────┬──────────┬────────┬───────────────────┬───────────────┬──────────────────────────────────────────────────────────┐ +│ Library │ Vulnerability │ Severity │ Status │ Installed Version │ Fixed Version │ Title │ +├────────────┼───────────────┼──────────┼────────┼───────────────────┼───────────────┼──────────────────────────────────────────────────────────┤ +│ libcrypto3 │ CVE-2024-6119 │ HIGH │ fixed │ 3.1.4-r5 │ 3.1.7-r0 │ openssl: Possible denial of service in X.509 name checks │ +│ │ │ │ │ │ │ https://avd.aquasec.com/nvd/cve-2024-6119 │ +├────────────┤ │ │ │ │ │ │ +│ libssl3 │ │ │ │ │ │ │ +│ │ │ │ │ │ │ │ +└────────────┴───────────────┴──────────┴────────┴───────────────────┴───────────────┴──────────────────────────────────────────────────────────┘ diff --git a/e2e/testdata/proxy_scan.txtar b/e2e/testdata/proxy_scan.txtar new file mode 100644 index 0000000000..d8ced2410e --- /dev/null +++ b/e2e/testdata/proxy_scan.txtar @@ -0,0 +1,27 @@ +# Proxy scan test +# This test verifies that Trivy can work through HTTP/HTTPS proxy with self-signed certificates +# Tests both --insecure flag behavior: failure without it, success with it + +# Start mitmdump proxy container on port 18080 +exec docker rm -f mitmproxy-e2e-test +exec docker run -d --name mitmproxy-e2e-test -p 18080:8080 mitmproxy/mitmproxy:latest mitmdump +exec sleep 3 + +# Set proxy environment variables +env HTTPS_PROXY=http://localhost:18080 +env HTTP_PROXY=http://localhost:18080 + +# Test 1: Scan without --insecure flag (should fail due to certificate issues) +! exec trivy image --cache-dir $WORK/.cache --format table --db-repository mirror.gcr.io/aquasec/trivy-db@sha256:b4d3718a89a78d4a6b02250953e92fcd87776de4774e64e818c1d0e01c928025 --severity HIGH,CRITICAL --no-progress alpine@sha256:c5b1261d6d3e43071626931fc004f70149baeba2c8ec672bd4f27761f8e1ad6b + +# Verify certificate error occurred +stderr 'certificate' + +# Test 2: Scan with --insecure flag (should succeed despite certificate issues) +exec trivy image --insecure --cache-dir $WORK/.cache --format table --db-repository mirror.gcr.io/aquasec/trivy-db@sha256:b4d3718a89a78d4a6b02250953e92fcd87776de4774e64e818c1d0e01c928025 --severity HIGH,CRITICAL --no-progress alpine@sha256:c5b1261d6d3e43071626931fc004f70149baeba2c8ec672bd4f27761f8e1ad6b + +# Verify scan completed successfully with --insecure +! stderr 'FATAL' + +# Stop proxy container +exec docker rm -f mitmproxy-e2e-test \ No newline at end of file diff --git a/go.mod b/go.mod index 69c57e1ee4..115681f53a 100644 --- a/go.mod +++ b/go.mod @@ -132,6 +132,8 @@ require ( modernc.org/sqlite v1.38.0 ) +require github.com/rogpeppe/go-internal v1.14.1 + require ( buf.build/gen/go/bufbuild/bufplugin/protocolbuffers/go v1.36.6-20250121211742-6d880cc6cc8d.1 // indirect buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250613105001-9f2d3c737feb.1 // indirect diff --git a/magefiles/magefile.go b/magefiles/magefile.go index a612a4a154..93c546d6d9 100644 --- a/magefiles/magefile.go +++ b/magefiles/magefile.go @@ -331,6 +331,11 @@ func (t Test) UpdateVMGolden() error { return sh.RunWithV(ENV, "go", "test", "-v", "-tags=vm_integration", "./integration/...", "-update") } +// E2e runs E2E tests using testscript framework +func (t Test) E2e() error { + return sh.RunWithV(ENV, "go", "test", "-v", "-tags=e2e", "./e2e/...") +} + type Lint mg.Namespace // Run runs linters