feat(cache): support Redis (#770)

* feat(config): add --cache-backend

* feat(operation): embed cache.Cache into operation.Cache

* feat(cache): support redis://

* test(integration): add redis test

* chore(README): add --cache-backend

* chore(mod): update

* chore: add disclaimer

README.md

@@ -55,6 +55,7 @@ A Simple and Comprehensive Vulnerability Scanner for Containers and other Artifa
     + [Specify exit code](#specify-exit-code)
     + [Ignore the specified vulnerabilities](#ignore-the-specified-vulnerabilities)
     + [Specify cache directory](#specify-cache-directory)
+    + [Specify cache backend](#specify-cache-backend)
     + [Clear caches](#clear-caches)
     + [Reset](#reset)
     + [Use lightweight DB](#use-lightweight-db)
@@ -1331,6 +1332,21 @@ Total: 0 (UNKNOWN: 0, LOW: 0, MEDIUM: 0, HIGH: 0, CRITICAL: 0)
 $ trivy --cache-dir /tmp/trivy/ image python:3.4-alpine3.9
 ```
 
+### Specify cache backend
+[EXPERIMENTAL] This feature might change without preserving backwards compatibility.
+
+Trivy supports local filesystem and Redis as the cache backend. This option is useful especially for client/server mode.
+
+Two options:
+- `fs` 
+  - the cache path can be specified by `--cache-dir`
+- `redis://`
+  - `redis://[HOST]:[PORT]`
+
+```
+$ trivy server --cache-backend redis://localhost:6379
+```
+
 ### Clear caches
 
 The `--clear-cache` option removes caches. This option is useful if the image which has the same tag is updated (such as when using `latest` tag).

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/cheggaaa/pb/v3 v3.0.3
 	github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7
 	github.com/docker/go-connections v0.4.0
+	github.com/go-redis/redis/v8 v8.4.0
 	github.com/golang/protobuf v1.4.2
 	github.com/google/go-containerregistry v0.0.0-20200331213917-3d03ed9b1ca2
 	github.com/google/go-github/v28 v28.1.1

go.sum

@@ -477,14 +477,12 @@ github.com/olekukonko/tablewriter v0.0.2-0.20190607075207-195002e6e56a/go.mod h1
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M=
 github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=

integration/client_server_test.go

@@ -12,8 +12,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/docker/go-connections/nat"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	testcontainers "github.com/testcontainers/testcontainers-go"
 	"github.com/urfave/cli/v2"
 
 	"github.com/aquasecurity/trivy/internal"
@@ -332,7 +334,7 @@ func TestClientServer(t *testing.T) {
 		},
 	}
 
-	app, addr, cacheDir := setup(t, "", "")
+	app, addr, cacheDir := setup(t, setupOptions{})
 
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
@@ -394,7 +396,10 @@ func TestClientServerWithToken(t *testing.T) {
 
 	serverToken := "token"
 	serverTokenHeader := "Trivy-Token"
-	app, addr, cacheDir := setup(t, serverToken, serverTokenHeader)
+	app, addr, cacheDir := setup(t, setupOptions{
+		token:       serverToken,
+		tokenHeader: serverTokenHeader,
+	})
 	defer os.RemoveAll(cacheDir)
 
 	for _, c := range cases {
@@ -418,7 +423,54 @@ func TestClientServerWithToken(t *testing.T) {
 	}
 }
 
-func setup(t *testing.T, token, tokenHeader string) (*cli.App, string, string) {
+func TestClientServerWithRedis(t *testing.T) {
+	// Set up a Redis container
+	ctx := context.Background()
+	redisC, addr := setupRedis(t, ctx)
+
+	// Set up Trivy server
+	app, addr, cacheDir := setup(t, setupOptions{cacheBackend: addr})
+	defer os.RemoveAll(cacheDir)
+
+	// Test parameters
+	testArgs := args{
+		Version: "dev",
+		Input:   "testdata/fixtures/centos-7.tar.gz",
+	}
+	golden := "testdata/centos-7.json.golden"
+
+	t.Run("centos 7", func(t *testing.T) {
+		osArgs, outputFile, cleanup := setupClient(t, testArgs, addr, cacheDir, golden)
+		defer cleanup()
+
+		// Run Trivy client
+		err := app.Run(osArgs)
+		require.NoError(t, err)
+
+		compare(t, golden, outputFile)
+	})
+
+	// Terminate the Redis container
+	require.NoError(t, redisC.Terminate(ctx))
+
+	t.Run("sad path", func(t *testing.T) {
+		osArgs, _, cleanup := setupClient(t, testArgs, addr, cacheDir, golden)
+		defer cleanup()
+
+		// Run Trivy client
+		err := app.Run(osArgs)
+		require.NotNil(t, err)
+		assert.Contains(t, err.Error(), "connect: connection refused")
+	})
+}
+
+type setupOptions struct {
+	token        string
+	tokenHeader  string
+	cacheBackend string
+}
+
+func setup(t *testing.T, options setupOptions) (*cli.App, string, string) {
 	t.Helper()
 	version := "dev"
 
@@ -434,7 +486,7 @@ func setup(t *testing.T, token, tokenHeader string) (*cli.App, string, string) {
 		// Setup CLI App
 		app := internal.NewApp(version)
 		app.Writer = ioutil.Discard
-		osArgs := setupServer(addr, token, tokenHeader, cacheDir)
+		osArgs := setupServer(addr, options.token, options.tokenHeader, cacheDir, options.cacheBackend)
 
 		// Run Trivy server
 		app.Run(osArgs)
@@ -451,11 +503,14 @@ func setup(t *testing.T, token, tokenHeader string) (*cli.App, string, string) {
 	return app, addr, cacheDir
 }
 
-func setupServer(addr, token, tokenHeader, cacheDir string) []string {
+func setupServer(addr, token, tokenHeader, cacheDir, cacheBackend string) []string {
 	osArgs := []string{"trivy", "server", "--skip-update", "--cache-dir", cacheDir, "--listen", addr}
 	if token != "" {
 		osArgs = append(osArgs, []string{"--token", token, "--token-header", tokenHeader}...)
 	}
+	if cacheBackend != "" {
+		osArgs = append(osArgs, "--cache-backend", cacheBackend)
+	}
 	return osArgs
 }
 
@@ -519,6 +574,32 @@ func setupClient(t *testing.T, c args, addr string, cacheDir string, golden stri
 	return osArgs, outputFile, cleanup
 }
 
+func setupRedis(t *testing.T, ctx context.Context) (testcontainers.Container, string) {
+	t.Helper()
+	imageName := "redis:5.0"
+	port := "6379/tcp"
+	req := testcontainers.ContainerRequest{
+		Name:         "redis",
+		Image:        imageName,
+		ExposedPorts: []string{port},
+	}
+
+	redis, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+		ContainerRequest: req,
+		Started:          true,
+	})
+	require.NoError(t, err)
+
+	ip, err := redis.Host(ctx)
+	require.NoError(t, err)
+
+	p, err := redis.MappedPort(ctx, nat.Port(port))
+	require.NoError(t, err)
+
+	addr := fmt.Sprintf("redis://%s:%s", ip, p.Port())
+	return redis, addr
+}
+
 func compare(t *testing.T, wantFile, gotFile string) {
 	t.Helper()
 	// Compare want and got

internal/app.go

@@ -144,6 +144,13 @@ var (
 		EnvVars: []string{"TRIVY_CACHE_DIR"},
 	}
 
+	cacheBackendFlag = cli.StringFlag{
+		Name:    "cache-backend",
+		Value:   "fs",
+		Usage:   "cache backend (e.g. redis://localhost:6379)",
+		EnvVars: []string{"TRIVY_CACHE_BACKEND"},
+	}
+
 	ignoreFileFlag = cli.StringFlag{
 		Name:    "ignorefile",
 		Value:   vulnerability.DefaultIgnoreFile,
@@ -229,6 +236,7 @@ var (
 		&listAllPackages,
 		&skipFiles,
 		&skipDirectories,
+		&cacheBackendFlag,
 	}
 
 	// deprecated options
@@ -385,6 +393,7 @@ func NewFilesystemCommand() *cli.Command {
 			&vulnTypeFlag,
 			&ignoreFileFlag,
 			&cacheDirFlag,
+			&cacheBackendFlag,
 			&timeoutFlag,
 			&noProgressFlag,
 			&ignorePolicy,
@@ -419,6 +428,7 @@ func NewRepositoryCommand() *cli.Command {
 			&vulnTypeFlag,
 			&ignoreFileFlag,
 			&cacheDirFlag,
+			&cacheBackendFlag,
 			&timeoutFlag,
 			&noProgressFlag,
 			&ignorePolicy,
@@ -487,6 +497,7 @@ func NewServerCommand() *cli.Command {
 			&quietFlag,
 			&debugFlag,
 			&cacheDirFlag,
+			&cacheBackendFlag,
 
 			// original flags
 			&token,

internal/artifact/config/config.go

@@ -14,6 +14,7 @@ type Config struct {
 	config.DBConfig
 	config.ImageConfig
 	config.ReportConfig
+	config.CacheConfig
 
 	// deprecated
 	onlyUpdate string
@@ -36,6 +37,7 @@ func New(c *cli.Context) (Config, error) {
 		DBConfig:       config.NewDBConfig(c),
 		ImageConfig:    config.NewImageConfig(c),
 		ReportConfig:   config.NewReportConfig(c),
+		CacheConfig:    config.NewCacheConfig(c),
 
 		onlyUpdate:  c.String("only-update"),
 		refresh:     c.Bool("refresh"),
@@ -45,13 +47,11 @@ func New(c *cli.Context) (Config, error) {
 
 // Init initializes the artifact config
 func (c *Config) Init(image bool) error {
-	if err := c.ReportConfig.Init(c.Logger); err != nil {
-		return err
-	}
 	if c.onlyUpdate != "" || c.refresh || c.autoRefresh {
 		c.Logger.Warn("--only-update, --refresh and --auto-refresh are unnecessary and ignored now. These commands will be removed in the next version.")
 	}
-	if err := c.DBConfig.Init(); err != nil {
+
+	if err := c.initPreScanConfigs(); err != nil {
 		return err
 	}
 
@@ -73,6 +73,19 @@ func (c *Config) Init(image bool) error {
 	return nil
 }
 
+func (c *Config) initPreScanConfigs() error {
+	if err := c.ReportConfig.Init(c.Logger); err != nil {
+		return err
+	}
+	if err := c.DBConfig.Init(); err != nil {
+		return err
+	}
+	if err := c.CacheConfig.Init(); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (c *Config) skipScan() bool {
 	if c.ClearCache || c.DownloadDBOnly || c.Reset {
 		return true

internal/artifact/run.go

@@ -32,20 +32,18 @@ func run(c config.Config, initializeScanner InitializeScanner) error {
 
 	// configure cache dir
 	utils.SetCacheDir(c.CacheDir)
-	cacheClient, err := cache.NewFSCache(c.CacheDir)
+	cache, err := operation.NewCache(c.CacheBackend)
 	if err != nil {
 		return xerrors.Errorf("unable to initialize the cache: %w", err)
 	}
-	defer cacheClient.Close()
+	defer cache.Close()
-
-	cacheOperation := operation.NewCache(cacheClient)
 	log.Logger.Debugf("cache dir:  %s", utils.CacheDir())
 
 	if c.Reset {
-		return cacheOperation.Reset()
+		return cache.Reset()
 	}
 	if c.ClearCache {
-		return cacheOperation.ClearImages()
+		return cache.ClearImages()
 	}
 
 	// download the database file
@@ -70,7 +68,7 @@ func run(c config.Config, initializeScanner InitializeScanner) error {
 
 	ctx, cancel := context.WithTimeout(context.Background(), c.Timeout)
 	defer cancel()
-	scanner, cleanup, err := initializeScanner(ctx, target, cacheClient, cacheClient, c.Timeout)
+	scanner, cleanup, err := initializeScanner(ctx, target, cache, cache, c.Timeout)
 	if err != nil {
 		return xerrors.Errorf("unable to initialize a scanner: %w", err)
 	}

internal/config/cache.go

@@ -0,0 +1,31 @@
+package config
+
+import (
+	"strings"
+
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+)
+
+// CacheConfig holds the config for cache
+type CacheConfig struct {
+	CacheBackend string
+}
+
+// NewCacheConfig returns an instance of CacheConfig
+func NewCacheConfig(c *cli.Context) CacheConfig {
+	return CacheConfig{
+		CacheBackend: c.String("cache-backend"),
+	}
+}
+
+// Init initialize the CacheConfig
+func (c *CacheConfig) Init() error {
+	// "redis://" or "fs" are allowed for now
+	// An empty value is also allowed for testability
+	if !strings.HasPrefix(c.CacheBackend, "redis://") &&
+		c.CacheBackend != "fs" && c.CacheBackend != "" {
+		return xerrors.Errorf("unsupported cache backend: %s", c.CacheBackend)
+	}
+	return nil
+}

internal/config/cache_test.go

@@ -0,0 +1,92 @@
+package config_test
+
+import (
+	"flag"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli/v2"
+
+	"github.com/aquasecurity/trivy/internal/config"
+)
+
+func TestNewCacheConfig(t *testing.T) {
+	tests := []struct {
+		name string
+		args []string
+		want config.CacheConfig
+	}{
+		{
+			name: "happy path",
+			args: []string{"--cache-backend", "redis://localhost:6379"},
+			want: config.CacheConfig{
+				CacheBackend: "redis://localhost:6379",
+			},
+		},
+		{
+			name: "default",
+			args: []string{},
+			want: config.CacheConfig{
+				CacheBackend: "fs",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			app := &cli.App{}
+			set := flag.NewFlagSet("test", 0)
+			set.String("cache-backend", "fs", "")
+
+			c := cli.NewContext(app, set, nil)
+			_ = set.Parse(tt.args)
+
+			got := config.NewCacheConfig(c)
+			assert.Equal(t, tt.want, got, tt.name)
+		})
+	}
+}
+
+func TestCacheConfig_Init(t *testing.T) {
+	type fields struct {
+		backend string
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		wantErr string
+	}{
+		{
+			name: "fs",
+			fields: fields{
+				backend: "fs",
+			},
+		},
+		{
+			name: "redis",
+			fields: fields{
+				backend: "redis://localhost:6379",
+			},
+		},
+		{
+			name: "sad path",
+			fields: fields{
+				backend: "unknown://",
+			},
+			wantErr: "unsupported cache backend: unknown://",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := &config.CacheConfig{
+				CacheBackend: tt.fields.backend,
+			}
+
+			err := c.Init()
+			if tt.wantErr != "" {
+				assert.EqualError(t, err, tt.wantErr, err)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}

internal/operation/operation.go

@@ -3,10 +3,11 @@ package operation
 import (
 	"context"
 	"os"
+	"strings"
 
-	"github.com/spf13/afero"
+	"github.com/go-redis/redis/v8"
-
 	"github.com/google/wire"
+	"github.com/spf13/afero"
 	"golang.org/x/xerrors"
 
 	"github.com/aquasecurity/fanal/cache"
@@ -24,12 +25,23 @@ var SuperSet = wire.NewSet(
 
 // Cache implements the local cache
 type Cache struct {
-	client cache.LocalArtifactCache
+	cache.Cache
 }
 
 // NewCache is the factory method for Cache
-func NewCache(client cache.LocalArtifactCache) Cache {
+func NewCache(backend string) (Cache, error) {
-	return Cache{client: client}
+	if strings.HasPrefix(backend, "redis://") {
+		log.Logger.Info("Redis cache: %s", backend)
+		redisCache := cache.NewRedisCache(&redis.Options{
+			Addr: strings.TrimPrefix(backend, "redis://"),
+		})
+		return Cache{Cache: redisCache}, nil
+	}
+	fsCache, err := cache.NewFSCache(utils.CacheDir())
+	if err != nil {
+		return Cache{}, xerrors.Errorf("unable to initialize fs cache: %w", err)
+	}
+	return Cache{Cache: fsCache}, nil
 }
 
 // Reset resets the cache
@@ -55,7 +67,7 @@ func (c Cache) ClearDB() (err error) {
 // ClearImages clears the cache images
 func (c Cache) ClearImages() error {
 	log.Logger.Info("Removing image caches...")
-	if err := c.client.Clear(); err != nil {
+	if err := c.Clear(); err != nil {
 		return xerrors.Errorf("failed to remove the cache: %w", err)
 	}
 	return nil

internal/server/config/config.go

@@ -10,19 +10,21 @@ import (
 type Config struct {
 	config.GlobalConfig
 	config.DBConfig
+	config.CacheConfig
 
 	Listen      string
 	Token       string
 	TokenHeader string
 }
 
-// New is the factory method to return cofig
+// New is the factory method to return config
 func New(c *cli.Context) Config {
 	// the error is ignored because logger is unnecessary
 	gc, _ := config.NewGlobalConfig(c) // nolint: errcheck
 	return Config{
 		GlobalConfig: gc,
 		DBConfig:     config.NewDBConfig(c),
+		CacheConfig:  config.NewCacheConfig(c),
 
 		Listen:      c.String("listen"),
 		Token:       c.String("token"),
@@ -30,11 +32,14 @@ func New(c *cli.Context) Config {
 	}
 }
 
-// Init initializes the DB config
+// Init initializes the config
 func (c *Config) Init() (err error) {
 	if err := c.DBConfig.Init(); err != nil {
 		return err
 	}
+	if err := c.CacheConfig.Init(); err != nil {
+		return err
+	}
 
 	return nil
 }

internal/server/run.go

@@ -4,7 +4,6 @@ import (
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
-	"github.com/aquasecurity/fanal/cache"
 	"github.com/aquasecurity/trivy-db/pkg/db"
 	"github.com/aquasecurity/trivy/internal/operation"
 	"github.com/aquasecurity/trivy/internal/server/config"
@@ -30,17 +29,15 @@ func run(c config.Config) (err error) {
 
 	// configure cache dir
 	utils.SetCacheDir(c.CacheDir)
+	cache, err := operation.NewCache(c.CacheBackend)
+	if err != nil {
+		return xerrors.Errorf("server cache error: %w", err)
+	}
+	defer cache.Close()
 	log.Logger.Debugf("cache dir:  %s", utils.CacheDir())
 
-	fsCache, err := cache.NewFSCache(utils.CacheDir())
-	if err != nil {
-		return xerrors.Errorf("unable to initialize cache: %w", err)
-	}
-
-	// server doesn't have image cache
-	cacheOperation := operation.NewCache(fsCache)
 	if c.Reset {
-		return cacheOperation.ClearDB()
+		return cache.ClearDB()
 	}
 
 	// download the database file
@@ -56,5 +53,5 @@ func run(c config.Config) (err error) {
 		return xerrors.Errorf("error in vulnerability DB initialize: %w", err)
 	}
 
-	return server.ListenAndServe(c, fsCache)
+	return server.ListenAndServe(c, cache)
 }

pkg/rpc/server/listen.go

@@ -31,7 +31,7 @@ var DBWorkerSuperSet = wire.NewSet(
 )
 
 // ListenAndServe starts Trivy server
-func ListenAndServe(c config.Config, fsCache cache.FSCache) error {
+func ListenAndServe(c config.Config, serverCache cache.Cache) error {
 	requestWg := &sync.WaitGroup{}
 	dbUpdateWg := &sync.WaitGroup{}
 
@@ -46,13 +46,13 @@ func ListenAndServe(c config.Config, fsCache cache.FSCache) error {
 		}
 	}()
 
-	mux := newServeMux(fsCache, dbUpdateWg, requestWg, c.Token, c.TokenHeader)
+	mux := newServeMux(serverCache, dbUpdateWg, requestWg, c.Token, c.TokenHeader)
 	log.Logger.Infof("Listening %s...", c.Listen)
 
 	return http.ListenAndServe(c.Listen, mux)
 }
 
-func newServeMux(fsCache cache.FSCache, dbUpdateWg, requestWg *sync.WaitGroup, token, tokenHeader string) *http.ServeMux {
+func newServeMux(serverCache cache.Cache, dbUpdateWg, requestWg *sync.WaitGroup, token, tokenHeader string) *http.ServeMux {
 	withWaitGroup := func(base http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			// Stop processing requests during DB update
@@ -69,10 +69,10 @@ func newServeMux(fsCache cache.FSCache, dbUpdateWg, requestWg *sync.WaitGroup, t
 
 	mux := http.NewServeMux()
 
-	scanHandler := rpcScanner.NewScannerServer(initializeScanServer(fsCache), nil)
+	scanHandler := rpcScanner.NewScannerServer(initializeScanServer(serverCache), nil)
 	mux.Handle(rpcScanner.ScannerPathPrefix, withToken(withWaitGroup(scanHandler), token, tokenHeader))
 
-	layerHandler := rpcCache.NewCacheServer(NewCacheServer(fsCache), nil)
+	layerHandler := rpcCache.NewCacheServer(NewCacheServer(serverCache), nil)
 	mux.Handle(rpcCache.CachePathPrefix, withToken(withWaitGroup(layerHandler), token, tokenHeader))
 
 	// osHandler is for backward compatibility