diff --git a/README.md b/README.md index 3e088b96b7..cd8f5428fc 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ A Simple and Comprehensive Vulnerability Scanner for Containers and other Artifa + [Specify exit code](#specify-exit-code) + [Ignore the specified vulnerabilities](#ignore-the-specified-vulnerabilities) + [Specify cache directory](#specify-cache-directory) + + [Specify cache backend](#specify-cache-backend) + [Clear caches](#clear-caches) + [Reset](#reset) + [Use lightweight DB](#use-lightweight-db) @@ -1331,6 +1332,21 @@ Total: 0 (UNKNOWN: 0, LOW: 0, MEDIUM: 0, HIGH: 0, CRITICAL: 0) $ trivy --cache-dir /tmp/trivy/ image python:3.4-alpine3.9 ``` +### Specify cache backend +[EXPERIMENTAL] This feature might change without preserving backwards compatibility. + +Trivy supports local filesystem and Redis as the cache backend. This option is useful especially for client/server mode. + +Two options: +- `fs` + - the cache path can be specified by `--cache-dir` +- `redis://` + - `redis://[HOST]:[PORT]` + +``` +$ trivy server --cache-backend redis://localhost:6379 +``` + ### Clear caches The `--clear-cache` option removes caches. This option is useful if the image which has the same tag is updated (such as when using `latest` tag). diff --git a/go.mod b/go.mod index b36e47258a..c9b80fc3bb 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/cheggaaa/pb/v3 v3.0.3 github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7 github.com/docker/go-connections v0.4.0 + github.com/go-redis/redis/v8 v8.4.0 github.com/golang/protobuf v1.4.2 github.com/google/go-containerregistry v0.0.0-20200331213917-3d03ed9b1ca2 github.com/google/go-github/v28 v28.1.1 diff --git a/go.sum b/go.sum index 81a3621ac0..37a40fb1f0 100644 --- a/go.sum +++ b/go.sum @@ -477,14 +477,12 @@ github.com/olekukonko/tablewriter v0.0.2-0.20190607075207-195002e6e56a/go.mod h1 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M= github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= diff --git a/integration/client_server_test.go b/integration/client_server_test.go index 51385d9ba4..3468906972 100644 --- a/integration/client_server_test.go +++ b/integration/client_server_test.go @@ -12,8 +12,10 @@ import ( "testing" "time" + "github.com/docker/go-connections/nat" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + testcontainers "github.com/testcontainers/testcontainers-go" "github.com/urfave/cli/v2" "github.com/aquasecurity/trivy/internal" @@ -332,7 +334,7 @@ func TestClientServer(t *testing.T) { }, } - app, addr, cacheDir := setup(t, "", "") + app, addr, cacheDir := setup(t, setupOptions{}) for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -394,7 +396,10 @@ func TestClientServerWithToken(t *testing.T) { serverToken := "token" serverTokenHeader := "Trivy-Token" - app, addr, cacheDir := setup(t, serverToken, serverTokenHeader) + app, addr, cacheDir := setup(t, setupOptions{ + token: serverToken, + tokenHeader: serverTokenHeader, + }) defer os.RemoveAll(cacheDir) for _, c := range cases { @@ -418,7 +423,54 @@ func TestClientServerWithToken(t *testing.T) { } } -func setup(t *testing.T, token, tokenHeader string) (*cli.App, string, string) { +func TestClientServerWithRedis(t *testing.T) { + // Set up a Redis container + ctx := context.Background() + redisC, addr := setupRedis(t, ctx) + + // Set up Trivy server + app, addr, cacheDir := setup(t, setupOptions{cacheBackend: addr}) + defer os.RemoveAll(cacheDir) + + // Test parameters + testArgs := args{ + Version: "dev", + Input: "testdata/fixtures/centos-7.tar.gz", + } + golden := "testdata/centos-7.json.golden" + + t.Run("centos 7", func(t *testing.T) { + osArgs, outputFile, cleanup := setupClient(t, testArgs, addr, cacheDir, golden) + defer cleanup() + + // Run Trivy client + err := app.Run(osArgs) + require.NoError(t, err) + + compare(t, golden, outputFile) + }) + + // Terminate the Redis container + require.NoError(t, redisC.Terminate(ctx)) + + t.Run("sad path", func(t *testing.T) { + osArgs, _, cleanup := setupClient(t, testArgs, addr, cacheDir, golden) + defer cleanup() + + // Run Trivy client + err := app.Run(osArgs) + require.NotNil(t, err) + assert.Contains(t, err.Error(), "connect: connection refused") + }) +} + +type setupOptions struct { + token string + tokenHeader string + cacheBackend string +} + +func setup(t *testing.T, options setupOptions) (*cli.App, string, string) { t.Helper() version := "dev" @@ -434,7 +486,7 @@ func setup(t *testing.T, token, tokenHeader string) (*cli.App, string, string) { // Setup CLI App app := internal.NewApp(version) app.Writer = ioutil.Discard - osArgs := setupServer(addr, token, tokenHeader, cacheDir) + osArgs := setupServer(addr, options.token, options.tokenHeader, cacheDir, options.cacheBackend) // Run Trivy server app.Run(osArgs) @@ -451,11 +503,14 @@ func setup(t *testing.T, token, tokenHeader string) (*cli.App, string, string) { return app, addr, cacheDir } -func setupServer(addr, token, tokenHeader, cacheDir string) []string { +func setupServer(addr, token, tokenHeader, cacheDir, cacheBackend string) []string { osArgs := []string{"trivy", "server", "--skip-update", "--cache-dir", cacheDir, "--listen", addr} if token != "" { osArgs = append(osArgs, []string{"--token", token, "--token-header", tokenHeader}...) } + if cacheBackend != "" { + osArgs = append(osArgs, "--cache-backend", cacheBackend) + } return osArgs } @@ -519,6 +574,32 @@ func setupClient(t *testing.T, c args, addr string, cacheDir string, golden stri return osArgs, outputFile, cleanup } +func setupRedis(t *testing.T, ctx context.Context) (testcontainers.Container, string) { + t.Helper() + imageName := "redis:5.0" + port := "6379/tcp" + req := testcontainers.ContainerRequest{ + Name: "redis", + Image: imageName, + ExposedPorts: []string{port}, + } + + redis, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + require.NoError(t, err) + + ip, err := redis.Host(ctx) + require.NoError(t, err) + + p, err := redis.MappedPort(ctx, nat.Port(port)) + require.NoError(t, err) + + addr := fmt.Sprintf("redis://%s:%s", ip, p.Port()) + return redis, addr +} + func compare(t *testing.T, wantFile, gotFile string) { t.Helper() // Compare want and got diff --git a/internal/app.go b/internal/app.go index a0f6461dad..359628915b 100644 --- a/internal/app.go +++ b/internal/app.go @@ -144,6 +144,13 @@ var ( EnvVars: []string{"TRIVY_CACHE_DIR"}, } + cacheBackendFlag = cli.StringFlag{ + Name: "cache-backend", + Value: "fs", + Usage: "cache backend (e.g. redis://localhost:6379)", + EnvVars: []string{"TRIVY_CACHE_BACKEND"}, + } + ignoreFileFlag = cli.StringFlag{ Name: "ignorefile", Value: vulnerability.DefaultIgnoreFile, @@ -229,6 +236,7 @@ var ( &listAllPackages, &skipFiles, &skipDirectories, + &cacheBackendFlag, } // deprecated options @@ -385,6 +393,7 @@ func NewFilesystemCommand() *cli.Command { &vulnTypeFlag, &ignoreFileFlag, &cacheDirFlag, + &cacheBackendFlag, &timeoutFlag, &noProgressFlag, &ignorePolicy, @@ -419,6 +428,7 @@ func NewRepositoryCommand() *cli.Command { &vulnTypeFlag, &ignoreFileFlag, &cacheDirFlag, + &cacheBackendFlag, &timeoutFlag, &noProgressFlag, &ignorePolicy, @@ -487,6 +497,7 @@ func NewServerCommand() *cli.Command { &quietFlag, &debugFlag, &cacheDirFlag, + &cacheBackendFlag, // original flags &token, diff --git a/internal/artifact/config/config.go b/internal/artifact/config/config.go index 577c79a5c1..a1e17300a6 100644 --- a/internal/artifact/config/config.go +++ b/internal/artifact/config/config.go @@ -14,6 +14,7 @@ type Config struct { config.DBConfig config.ImageConfig config.ReportConfig + config.CacheConfig // deprecated onlyUpdate string @@ -36,6 +37,7 @@ func New(c *cli.Context) (Config, error) { DBConfig: config.NewDBConfig(c), ImageConfig: config.NewImageConfig(c), ReportConfig: config.NewReportConfig(c), + CacheConfig: config.NewCacheConfig(c), onlyUpdate: c.String("only-update"), refresh: c.Bool("refresh"), @@ -45,13 +47,11 @@ func New(c *cli.Context) (Config, error) { // Init initializes the artifact config func (c *Config) Init(image bool) error { - if err := c.ReportConfig.Init(c.Logger); err != nil { - return err - } if c.onlyUpdate != "" || c.refresh || c.autoRefresh { c.Logger.Warn("--only-update, --refresh and --auto-refresh are unnecessary and ignored now. These commands will be removed in the next version.") } - if err := c.DBConfig.Init(); err != nil { + + if err := c.initPreScanConfigs(); err != nil { return err } @@ -73,6 +73,19 @@ func (c *Config) Init(image bool) error { return nil } +func (c *Config) initPreScanConfigs() error { + if err := c.ReportConfig.Init(c.Logger); err != nil { + return err + } + if err := c.DBConfig.Init(); err != nil { + return err + } + if err := c.CacheConfig.Init(); err != nil { + return err + } + return nil +} + func (c *Config) skipScan() bool { if c.ClearCache || c.DownloadDBOnly || c.Reset { return true diff --git a/internal/artifact/run.go b/internal/artifact/run.go index b92f1830b3..dbcf23f01e 100644 --- a/internal/artifact/run.go +++ b/internal/artifact/run.go @@ -32,20 +32,18 @@ func run(c config.Config, initializeScanner InitializeScanner) error { // configure cache dir utils.SetCacheDir(c.CacheDir) - cacheClient, err := cache.NewFSCache(c.CacheDir) + cache, err := operation.NewCache(c.CacheBackend) if err != nil { return xerrors.Errorf("unable to initialize the cache: %w", err) } - defer cacheClient.Close() - - cacheOperation := operation.NewCache(cacheClient) + defer cache.Close() log.Logger.Debugf("cache dir: %s", utils.CacheDir()) if c.Reset { - return cacheOperation.Reset() + return cache.Reset() } if c.ClearCache { - return cacheOperation.ClearImages() + return cache.ClearImages() } // download the database file @@ -70,7 +68,7 @@ func run(c config.Config, initializeScanner InitializeScanner) error { ctx, cancel := context.WithTimeout(context.Background(), c.Timeout) defer cancel() - scanner, cleanup, err := initializeScanner(ctx, target, cacheClient, cacheClient, c.Timeout) + scanner, cleanup, err := initializeScanner(ctx, target, cache, cache, c.Timeout) if err != nil { return xerrors.Errorf("unable to initialize a scanner: %w", err) } diff --git a/internal/config/cache.go b/internal/config/cache.go new file mode 100644 index 0000000000..1b7e827714 --- /dev/null +++ b/internal/config/cache.go @@ -0,0 +1,31 @@ +package config + +import ( + "strings" + + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" +) + +// CacheConfig holds the config for cache +type CacheConfig struct { + CacheBackend string +} + +// NewCacheConfig returns an instance of CacheConfig +func NewCacheConfig(c *cli.Context) CacheConfig { + return CacheConfig{ + CacheBackend: c.String("cache-backend"), + } +} + +// Init initialize the CacheConfig +func (c *CacheConfig) Init() error { + // "redis://" or "fs" are allowed for now + // An empty value is also allowed for testability + if !strings.HasPrefix(c.CacheBackend, "redis://") && + c.CacheBackend != "fs" && c.CacheBackend != "" { + return xerrors.Errorf("unsupported cache backend: %s", c.CacheBackend) + } + return nil +} diff --git a/internal/config/cache_test.go b/internal/config/cache_test.go new file mode 100644 index 0000000000..b16ee7dea1 --- /dev/null +++ b/internal/config/cache_test.go @@ -0,0 +1,92 @@ +package config_test + +import ( + "flag" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/urfave/cli/v2" + + "github.com/aquasecurity/trivy/internal/config" +) + +func TestNewCacheConfig(t *testing.T) { + tests := []struct { + name string + args []string + want config.CacheConfig + }{ + { + name: "happy path", + args: []string{"--cache-backend", "redis://localhost:6379"}, + want: config.CacheConfig{ + CacheBackend: "redis://localhost:6379", + }, + }, + { + name: "default", + args: []string{}, + want: config.CacheConfig{ + CacheBackend: "fs", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + app := &cli.App{} + set := flag.NewFlagSet("test", 0) + set.String("cache-backend", "fs", "") + + c := cli.NewContext(app, set, nil) + _ = set.Parse(tt.args) + + got := config.NewCacheConfig(c) + assert.Equal(t, tt.want, got, tt.name) + }) + } +} + +func TestCacheConfig_Init(t *testing.T) { + type fields struct { + backend string + } + tests := []struct { + name string + fields fields + wantErr string + }{ + { + name: "fs", + fields: fields{ + backend: "fs", + }, + }, + { + name: "redis", + fields: fields{ + backend: "redis://localhost:6379", + }, + }, + { + name: "sad path", + fields: fields{ + backend: "unknown://", + }, + wantErr: "unsupported cache backend: unknown://", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &config.CacheConfig{ + CacheBackend: tt.fields.backend, + } + + err := c.Init() + if tt.wantErr != "" { + assert.EqualError(t, err, tt.wantErr, err) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/internal/operation/operation.go b/internal/operation/operation.go index 1c635f58ed..a64be6c041 100644 --- a/internal/operation/operation.go +++ b/internal/operation/operation.go @@ -3,10 +3,11 @@ package operation import ( "context" "os" + "strings" - "github.com/spf13/afero" - + "github.com/go-redis/redis/v8" "github.com/google/wire" + "github.com/spf13/afero" "golang.org/x/xerrors" "github.com/aquasecurity/fanal/cache" @@ -24,12 +25,23 @@ var SuperSet = wire.NewSet( // Cache implements the local cache type Cache struct { - client cache.LocalArtifactCache + cache.Cache } // NewCache is the factory method for Cache -func NewCache(client cache.LocalArtifactCache) Cache { - return Cache{client: client} +func NewCache(backend string) (Cache, error) { + if strings.HasPrefix(backend, "redis://") { + log.Logger.Info("Redis cache: %s", backend) + redisCache := cache.NewRedisCache(&redis.Options{ + Addr: strings.TrimPrefix(backend, "redis://"), + }) + return Cache{Cache: redisCache}, nil + } + fsCache, err := cache.NewFSCache(utils.CacheDir()) + if err != nil { + return Cache{}, xerrors.Errorf("unable to initialize fs cache: %w", err) + } + return Cache{Cache: fsCache}, nil } // Reset resets the cache @@ -55,7 +67,7 @@ func (c Cache) ClearDB() (err error) { // ClearImages clears the cache images func (c Cache) ClearImages() error { log.Logger.Info("Removing image caches...") - if err := c.client.Clear(); err != nil { + if err := c.Clear(); err != nil { return xerrors.Errorf("failed to remove the cache: %w", err) } return nil diff --git a/internal/server/config/config.go b/internal/server/config/config.go index 3288668242..074e3e1944 100644 --- a/internal/server/config/config.go +++ b/internal/server/config/config.go @@ -6,23 +6,25 @@ import ( "github.com/aquasecurity/trivy/internal/config" ) -// Config holds the Trivy config +// Config holds the Trivy config type Config struct { config.GlobalConfig config.DBConfig + config.CacheConfig Listen string Token string TokenHeader string } -// New is the factory method to return cofig +// New is the factory method to return config func New(c *cli.Context) Config { // the error is ignored because logger is unnecessary gc, _ := config.NewGlobalConfig(c) // nolint: errcheck return Config{ GlobalConfig: gc, DBConfig: config.NewDBConfig(c), + CacheConfig: config.NewCacheConfig(c), Listen: c.String("listen"), Token: c.String("token"), @@ -30,11 +32,14 @@ func New(c *cli.Context) Config { } } -// Init initializes the DB config +// Init initializes the config func (c *Config) Init() (err error) { if err := c.DBConfig.Init(); err != nil { return err } + if err := c.CacheConfig.Init(); err != nil { + return err + } return nil } diff --git a/internal/server/run.go b/internal/server/run.go index 77baa43d8a..c76afb6b32 100644 --- a/internal/server/run.go +++ b/internal/server/run.go @@ -4,7 +4,6 @@ import ( "github.com/urfave/cli/v2" "golang.org/x/xerrors" - "github.com/aquasecurity/fanal/cache" "github.com/aquasecurity/trivy-db/pkg/db" "github.com/aquasecurity/trivy/internal/operation" "github.com/aquasecurity/trivy/internal/server/config" @@ -30,17 +29,15 @@ func run(c config.Config) (err error) { // configure cache dir utils.SetCacheDir(c.CacheDir) + cache, err := operation.NewCache(c.CacheBackend) + if err != nil { + return xerrors.Errorf("server cache error: %w", err) + } + defer cache.Close() log.Logger.Debugf("cache dir: %s", utils.CacheDir()) - fsCache, err := cache.NewFSCache(utils.CacheDir()) - if err != nil { - return xerrors.Errorf("unable to initialize cache: %w", err) - } - - // server doesn't have image cache - cacheOperation := operation.NewCache(fsCache) if c.Reset { - return cacheOperation.ClearDB() + return cache.ClearDB() } // download the database file @@ -56,5 +53,5 @@ func run(c config.Config) (err error) { return xerrors.Errorf("error in vulnerability DB initialize: %w", err) } - return server.ListenAndServe(c, fsCache) + return server.ListenAndServe(c, cache) } diff --git a/pkg/rpc/server/listen.go b/pkg/rpc/server/listen.go index 0c64ead1ce..3d29df12c3 100644 --- a/pkg/rpc/server/listen.go +++ b/pkg/rpc/server/listen.go @@ -31,7 +31,7 @@ var DBWorkerSuperSet = wire.NewSet( ) // ListenAndServe starts Trivy server -func ListenAndServe(c config.Config, fsCache cache.FSCache) error { +func ListenAndServe(c config.Config, serverCache cache.Cache) error { requestWg := &sync.WaitGroup{} dbUpdateWg := &sync.WaitGroup{} @@ -46,13 +46,13 @@ func ListenAndServe(c config.Config, fsCache cache.FSCache) error { } }() - mux := newServeMux(fsCache, dbUpdateWg, requestWg, c.Token, c.TokenHeader) + mux := newServeMux(serverCache, dbUpdateWg, requestWg, c.Token, c.TokenHeader) log.Logger.Infof("Listening %s...", c.Listen) return http.ListenAndServe(c.Listen, mux) } -func newServeMux(fsCache cache.FSCache, dbUpdateWg, requestWg *sync.WaitGroup, token, tokenHeader string) *http.ServeMux { +func newServeMux(serverCache cache.Cache, dbUpdateWg, requestWg *sync.WaitGroup, token, tokenHeader string) *http.ServeMux { withWaitGroup := func(base http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Stop processing requests during DB update @@ -69,10 +69,10 @@ func newServeMux(fsCache cache.FSCache, dbUpdateWg, requestWg *sync.WaitGroup, t mux := http.NewServeMux() - scanHandler := rpcScanner.NewScannerServer(initializeScanServer(fsCache), nil) + scanHandler := rpcScanner.NewScannerServer(initializeScanServer(serverCache), nil) mux.Handle(rpcScanner.ScannerPathPrefix, withToken(withWaitGroup(scanHandler), token, tokenHeader)) - layerHandler := rpcCache.NewCacheServer(NewCacheServer(fsCache), nil) + layerHandler := rpcCache.NewCacheServer(NewCacheServer(serverCache), nil) mux.Handle(rpcCache.CachePathPrefix, withToken(withWaitGroup(layerHandler), token, tokenHeader)) // osHandler is for backward compatibility