Cache: Save only required files (fanal#51)

.gitignore

@@ -17,3 +17,4 @@ vendor
 cmd/fanal/fanal
 *.tar
 *.gz
+*.db

analyzer/analyzer.go

@@ -7,14 +7,11 @@ import (
 	"io"
 	"os"
 
-	"github.com/aquasecurity/fanal/utils"
-
-	"github.com/aquasecurity/fanal/types"
-
-	"golang.org/x/xerrors"
-
 	"github.com/aquasecurity/fanal/extractor"
+	"github.com/aquasecurity/fanal/types"
+	"github.com/aquasecurity/fanal/utils"
 	godeptypes "github.com/aquasecurity/go-dep-parser/pkg/types"
+	"golang.org/x/xerrors"
 )
 
 var (

analyzer/analyzer_test.go

@@ -8,7 +8,6 @@ import (
 	"testing"
 
 	"github.com/aquasecurity/fanal/extractor"
-
 	"github.com/stretchr/testify/assert"
 )
 
@@ -93,6 +92,12 @@ func TestConfig_Analyze(t *testing.T) {
 		},
 	}
 
+	// cleanup global state from other tests
+	commandAnalyzers = []CommandAnalyzer{}
+	pkgAnalyzers = []PkgAnalyzer{}
+	osAnalyzers = []OSAnalyzer{}
+	libAnalyzers = []LibraryAnalyzer{}
+
 	for _, tc := range testCases {
 		RegisterOSAnalyzer(mockOSAnalyzer{})
 

cache/cache.go

@@ -1,57 +1,25 @@
 package cache
 
 import (
-	"io"
 	"os"
-	"path/filepath"
-	"strings"
 
 	"golang.org/x/xerrors"
 )
 
-var (
-	replacer = strings.NewReplacer("/", "_")
-)
-
 type Cache interface {
-	Get(key string) io.Reader
-	Set(key string, file io.Reader) (io.Reader, error)
 	Clear() error
 }
 
-type FSCache struct {
+type RealCache struct {
 	Directory string
 }
 
 func Initialize(cacheDir string) Cache {
-	return &FSCache{Directory: cacheDir}
+	return &RealCache{Directory: cacheDir}
 }
 
-func (fs FSCache) Get(key string) io.Reader {
-	filePath := filepath.Join(fs.Directory, replacer.Replace(key))
-	f, err := os.Open(filePath)
-	if err != nil {
-		return nil
-	}
-	return f
-}
-
-func (fs FSCache) Set(key string, file io.Reader) (io.Reader, error) {
-	filePath := filepath.Join(fs.Directory, replacer.Replace(key))
-	if err := os.MkdirAll(fs.Directory, os.ModePerm); err != nil {
-		return nil, xerrors.Errorf("failed to mkdir all: %w", err)
-	}
-	cacheFile, err := os.Create(filePath)
-	if err != nil {
-		return file, xerrors.Errorf("failed to create cache file: %w", err)
-	}
-
-	tee := io.TeeReader(file, cacheFile)
-	return tee, nil
-}
-
-func (fs FSCache) Clear() error {
-	if err := os.RemoveAll(fs.Directory); err != nil {
+func (rc RealCache) Clear() error {
+	if err := os.RemoveAll(rc.Directory); err != nil {
 		return xerrors.New("failed to remove cache")
 	}
 	return nil

cache/cache_test.go

@@ -1,7 +1,6 @@
 package cache
 
 import (
-	"bytes"
 	"io/ioutil"
 	"os"
 	"testing"
@@ -9,32 +8,15 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestSetAndGetAndClear(t *testing.T) {
-	tempCacheDir, _ := ioutil.TempDir("", "TestCacheDir-*")
-	f, _ := ioutil.TempFile(tempCacheDir, "foo.bar.baz-*")
-
-	c := Initialize(tempCacheDir)
-
-	// set
-	expectedCacheContents := "foo bar baz"
-	var buf bytes.Buffer
-	buf.Write([]byte(expectedCacheContents))
-
-	r, err := c.Set(f.Name(), &buf)
-	assert.NoError(t, err)
-
-	b, _ := ioutil.ReadAll(r)
-	assert.Equal(t, expectedCacheContents, string(b))
-
-	// get
-	actualFile := c.Get(f.Name())
-	actualBytes, _ := ioutil.ReadAll(actualFile)
-	assert.Equal(t, expectedCacheContents, string(actualBytes))
-
-	// clear
+func TestRealCache_Clear(t *testing.T) {
+	d, _ := ioutil.TempDir("", "TestRealCache_Clear")
+	c := Initialize(d)
 	assert.NoError(t, c.Clear())
-
-	// confirm that no cachedir remains
-	_, err = os.Stat(tempCacheDir)
+	_, err := os.Stat(d)
 	assert.True(t, os.IsNotExist(err))
+
+	t.Run("sad path, cache dir doesn't exist", func(t *testing.T) {
+		c := Initialize(".")
+		assert.Equal(t, "failed to remove cache", c.Clear().Error())
+	})
 }

cmd/fanal/main.go

@@ -8,6 +8,10 @@ import (
 	"os"
 	"time"
 
+	"github.com/aquasecurity/fanal/cache"
+	"github.com/aquasecurity/fanal/utils"
+	"golang.org/x/xerrors"
+
 	"github.com/aquasecurity/fanal/analyzer"
 	_ "github.com/aquasecurity/fanal/analyzer/command/apk"
 	_ "github.com/aquasecurity/fanal/analyzer/library/bundler"
@@ -25,13 +29,10 @@ import (
 	_ "github.com/aquasecurity/fanal/analyzer/pkg/apk"
 	_ "github.com/aquasecurity/fanal/analyzer/pkg/dpkg"
 	_ "github.com/aquasecurity/fanal/analyzer/pkg/rpm"
-	"github.com/aquasecurity/fanal/cache"
 	"github.com/aquasecurity/fanal/extractor"
 	"github.com/aquasecurity/fanal/extractor/docker"
 	"github.com/aquasecurity/fanal/types"
-	"github.com/aquasecurity/fanal/utils"
 	"golang.org/x/crypto/ssh/terminal"
-	"golang.org/x/xerrors"
 )
 
 func main() {
@@ -46,11 +47,11 @@ func run() (err error) {
 	clearCache := flag.Bool("clear", false, "clear cache")
 	flag.Parse()
 
-	c := cache.Initialize(utils.CacheDir())
+	c := cache.Initialize(utils.CacheDir() + "/cache.db")
 
 	if *clearCache {
 		if err = c.Clear(); err != nil {
-			return xerrors.Errorf("error in cache clear: %w", err)
+			return xerrors.Errorf("%w", err)
 		}
 	}
 

extractor/docker/docker.go

@@ -2,9 +2,11 @@ package docker
 
 import (
 	"archive/tar"
+	"bytes"
 	"compress/gzip"
 	"context"
 	"encoding/json"
+	"errors"
 	"io"
 	"io/ioutil"
 	"log"
@@ -12,27 +14,35 @@ import (
 	"strings"
 	"time"
 
+	"github.com/simar7/gokv/encoding"
+
 	"github.com/aquasecurity/fanal/analyzer/library"
-	"github.com/aquasecurity/fanal/utils"
-
-	"github.com/opencontainers/go-digest"
-
 	"github.com/aquasecurity/fanal/extractor"
 	"github.com/aquasecurity/fanal/extractor/docker/token/ecr"
 	"github.com/aquasecurity/fanal/extractor/docker/token/gcr"
 	"github.com/aquasecurity/fanal/types"
-
-	"github.com/aquasecurity/fanal/cache"
+	"github.com/aquasecurity/fanal/utils"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/docker/docker/client"
 	"github.com/genuinetools/reg/registry"
+	"github.com/klauspost/compress/zstd"
 	"github.com/knqyf263/nested"
+	"github.com/opencontainers/go-digest"
+	bolt "github.com/simar7/gokv/bbolt"
+	kvtypes "github.com/simar7/gokv/types"
 	"golang.org/x/xerrors"
 )
 
 const (
 	opq string = ".wh..wh..opq"
 	wh  string = ".wh."
+
+	KVImageBucket   string = "imagebucket"
+	LayerTarsBucket string = "layertars"
+)
+
+var (
+	ErrFailedCacheWrite = errors.New("failed to write to cache")
 )
 
 type manifest struct {
@@ -62,11 +72,11 @@ type layer struct {
 
 type Extractor struct {
 	Client *client.Client
-	Cache  cache.Cache
+	Cache  *bolt.Store
 	Option types.DockerOption
 }
 
-func NewDockerExtractor(option types.DockerOption) (Extractor, error) {
+func NewDockerExtractorWithCache(option types.DockerOption, cacheOptions bolt.Options) (Extractor, error) {
 	RegisterRegistry(&gcr.GCR{})
 	RegisterRegistry(&ecr.ECR{})
 
@@ -75,13 +85,26 @@ func NewDockerExtractor(option types.DockerOption) (Extractor, error) {
 		return Extractor{}, xerrors.Errorf("error initializing docker extractor: %w", err)
 	}
 
+	var kv *bolt.Store
+	if kv, err = bolt.NewStore(cacheOptions); err != nil {
+		return Extractor{}, xerrors.Errorf("error initializing cache: %w", err)
+	}
+
 	return Extractor{
 		Option: option,
 		Client: cli,
-		Cache:  cache.Initialize(utils.CacheDir()),
+		Cache:  kv,
 	}, nil
 }
 
+func NewDockerExtractor(option types.DockerOption) (Extractor, error) {
+	return NewDockerExtractorWithCache(option, bolt.Options{
+		RootBucketName: "fanal",
+		Path:           utils.CacheDir() + "/cache.db", // TODO: Make this configurable via a public method
+		Codec:          encoding.Gob,
+	})
+}
+
 func applyLayers(layerPaths []string, filesInLayers map[string]extractor.FileMap, opqInLayers map[string]extractor.OPQDirs) (extractor.FileMap, error) {
 	sep := "/"
 	nestedMap := nested.Nested{}
@@ -145,21 +168,54 @@ func (d Extractor) createRegistryClient(ctx context.Context, domain string) (*re
 }
 
 func (d Extractor) SaveLocalImage(ctx context.Context, imageName string) (io.Reader, error) {
-	var err error
-	r := d.Cache.Get(imageName)
-	if r == nil {
-		// Save the image
-		r, err = d.saveLocalImage(ctx, imageName)
+	var storedReader io.Reader
+
+	var storedImageBytes []byte
+	found, err := d.Cache.Get(kvtypes.GetItemInput{
+		BucketName: KVImageBucket,
+		Key:        imageName,
+		Value:      &storedImageBytes,
+	})
+
+	if found {
+		dec, _ := zstd.NewReader(nil)
+		storedImage, err := dec.DecodeAll(storedImageBytes, nil)
+		if err == nil {
+			return bytes.NewReader(storedImage), nil
+		}
+
+		// bad cache, redownload
+		found = false
+	}
+
+	var savedImage []byte
+	if err != nil || !found {
+		storedReader, err = d.saveLocalImage(ctx, imageName)
 		if err != nil {
 			return nil, xerrors.Errorf("failed to save the image: %w", err)
 		}
-		r, err = d.Cache.Set(imageName, r)
+
+		savedImage, err = ioutil.ReadAll(storedReader)
 		if err != nil {
-			log.Print(err)
+			return nil, xerrors.Errorf("failed to read saved image: %w", err)
+		}
+
+		e, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedDefault))
+		if err != nil {
+			return nil, err
+		}
+
+		dst := e.EncodeAll(savedImage, nil)
+		if err := d.Cache.BatchSet(kvtypes.BatchSetItemInput{
+			BucketName: "imagebucket",
+			Keys:       []string{imageName},
+			Values:     dst,
+		}); err != nil {
+			log.Println(err)
 		}
 	}
 
-	return r, nil
+	return bytes.NewReader(savedImage), nil
 }
 
 func (d Extractor) saveLocalImage(ctx context.Context, imageName string) (io.ReadCloser, error) {
@@ -171,7 +227,7 @@ func (d Extractor) saveLocalImage(ctx context.Context, imageName string) (io.Rea
 }
 
 func (d Extractor) Extract(ctx context.Context, imageName string, filenames []string) (extractor.FileMap, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), d.Option.Timeout)
+	ctx, cancel := context.WithTimeout(ctx, d.Option.Timeout)
 	defer cancel()
 
 	image, err := registry.ParseImage(imageName)
@@ -196,14 +252,14 @@ func (d Extractor) Extract(ctx context.Context, imageName string, filenames []st
 	for _, ref := range m.Manifest.Layers {
 		layerIDs = append(layerIDs, string(ref.Digest))
 		go func(dig digest.Digest) {
-			d.extractLayerWorker(dig, r, ctx, image, errCh, layerCh)
+			d.extractLayerWorker(dig, r, ctx, image, errCh, layerCh, filenames)
 		}(ref.Digest)
 	}
 
-	filesInLayers := make(map[string]extractor.FileMap)
-	opqInLayers := make(map[string]extractor.OPQDirs)
+	filesInLayers := map[string]extractor.FileMap{}
+	opqInLayers := map[string]extractor.OPQDirs{}
 	for i := 0; i < len(m.Manifest.Layers); i++ {
-		if err := d.extractLayerFiles(layerCh, errCh, ctx, filenames, filesInLayers, opqInLayers); err != nil {
+		if err := d.extractLayerFiles(ctx, layerCh, errCh, filesInLayers, opqInLayers, filenames); err != nil {
 			return nil, err
 		}
 	}
@@ -237,7 +293,7 @@ func downloadConfigFile(ctx context.Context, r *registry.Registry, image registr
 	return config, nil
 }
 
-func (d Extractor) extractLayerFiles(layerCh chan layer, errCh chan error, ctx context.Context, filenames []string, filesInLayers map[string]extractor.FileMap, opqInLayers map[string]extractor.OPQDirs) error {
+func (d Extractor) extractLayerFiles(ctx context.Context, layerCh chan layer, errCh chan error, filesInLayers map[string]extractor.FileMap, opqInLayers map[string]extractor.OPQDirs, filenames []string) error {
 	var l layer
 	select {
 	case l = <-layerCh:
@@ -257,29 +313,124 @@ func (d Extractor) extractLayerFiles(layerCh chan layer, errCh chan error, ctx c
 	return nil
 }
 
-func (d Extractor) extractLayerWorker(dig digest.Digest, r *registry.Registry, ctx context.Context, image registry.Image, errCh chan error, layerCh chan layer) {
-	var rc io.Reader
-	// Use cache
-	rc = d.Cache.Get(string(dig))
-	if rc == nil {
-		// Download the layer.
-		layerRC, err := r.DownloadLayer(ctx, image.Path, dig)
+func (d Extractor) extractLayerWorker(dig digest.Digest, r *registry.Registry, ctx context.Context, image registry.Image, errCh chan error, layerCh chan layer, filenames []string) {
+	var tarContent bytes.Buffer
+	var cacheContent []byte
+	var cacheBuf bytes.Buffer
+
+	found, _ := d.Cache.Get(kvtypes.GetItemInput{
+		BucketName: LayerTarsBucket,
+		Key:        string(dig),
+		Value:      &cacheContent,
+	})
+
+	if found {
+		b, errTar := extractTarFromTarZstd(cacheContent)
+		n, errWrite := cacheBuf.Write(b)
+		if errTar != nil || len(b) <= 0 || errWrite != nil || n <= 0 {
+			found = false
+		}
+	}
+
+	if !found {
+		rc, err := r.DownloadLayer(ctx, image.Path, dig)
 		if err != nil {
 			errCh <- xerrors.Errorf("failed to download the layer(%s): %w", dig, err)
 			return
 		}
+		defer rc.Close()
 
-		rc, err = d.Cache.Set(string(dig), layerRC)
+		// read the incoming gzip from the layer
+		gzipReader, err := gzip.NewReader(rc)
 		if err != nil {
-			log.Print(err)
+			errCh <- xerrors.Errorf("could not init gzip reader: %w", err)
+			return
+		}
+		defer gzipReader.Close()
+
+		tarReader := tar.NewReader(io.TeeReader(gzipReader, &tarContent))
+
+		if len(filenames) > 0 {
+			if cacheBuf, err = getFilteredTarballBuffer(tarReader, filenames); err != nil {
+				errCh <- err
+				return
+			}
+		}
+
+		d.storeLayerInCache(cacheBuf, dig)
+	}
+
+	layerCh <- layer{ID: dig, Content: ioutil.NopCloser(&cacheBuf)}
+	return
+}
+
+func extractTarFromTarZstd(cacheContent []byte) ([]byte, error) {
+	var tarContent []byte
+
+	dec, err := zstd.NewReader(nil)
+	if err != nil {
+		return nil, err
+	}
+
+	tarContent, err = dec.DecodeAll(cacheContent, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return tarContent, nil
+}
+
+func getFilteredTarballBuffer(tr *tar.Reader, requiredFilenames []string) (bytes.Buffer, error) {
+	var cacheBuf bytes.Buffer
+	// Create a new tar to store in the cache
+	twc := tar.NewWriter(&cacheBuf)
+	defer twc.Close()
+
+	// check what files are inside the tar
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break // end of archive
+		}
+		if err != nil {
+			return cacheBuf, xerrors.Errorf("%s: invalid tar: %w", ErrFailedCacheWrite, err)
+		}
+		if !utils.StringInSlice(hdr.Name, requiredFilenames) {
+			continue
+		}
+
+		hdrtwc := &tar.Header{
+			Name: hdr.Name,
+			Mode: 0600,
+			Size: hdr.Size,
+		}
+
+		if err := twc.WriteHeader(hdrtwc); err != nil {
+			return cacheBuf, xerrors.Errorf("%s: %s", ErrFailedCacheWrite, err)
+		}
+
+		_, err = io.Copy(twc, tr)
+		if err != nil {
+			return cacheBuf, xerrors.Errorf("%s: %s", ErrFailedCacheWrite, err)
 		}
 	}
-	gzipReader, err := gzip.NewReader(rc)
-	if err != nil {
-		errCh <- xerrors.Errorf("invalid gzip: %w", err)
-		return
+	return cacheBuf, nil
+}
+
+func (d Extractor) storeLayerInCache(cacheBuf bytes.Buffer, dig digest.Digest) {
+	// compress tar to zstd before storing to cache
+	var dst bytes.Buffer
+	w, _ := zstd.NewWriter(&dst)
+	_, _ = io.Copy(w, &cacheBuf)
+	_ = w.Close()
+
+	if err := d.Cache.BatchSet(kvtypes.BatchSetItemInput{
+		BucketName: LayerTarsBucket,
+		Keys:       []string{string(dig)},
+		Values:     dst.Bytes(),
+	}); err != nil {
+		log.Printf("an error occurred while caching: %s", err)
 	}
-	layerCh <- layer{ID: dig, Content: gzipReader}
 }
 
 func getValidManifest(ctx context.Context, r *registry.Registry, image registry.Image) (*schema2.DeserializedManifest, error) {
@@ -365,11 +516,11 @@ func (d Extractor) ExtractFromFile(ctx context.Context, r io.Reader, filenames [
 	return fileMap, nil
 }
 
-func (d Extractor) ExtractFiles(layer io.Reader, filenames []string) (extractor.FileMap, extractor.OPQDirs, error) {
+func (d Extractor) ExtractFiles(layerReader io.Reader, filenames []string) (extractor.FileMap, extractor.OPQDirs, error) {
 	data := make(map[string][]byte)
 	opqDirs := extractor.OPQDirs{}
 
-	tr := tar.NewReader(layer)
+	tr := tar.NewReader(layerReader)
 	for {
 		hdr, err := tr.Next()
 		if err == io.EOF {

extractor/docker/docker_test.go

@@ -13,13 +13,32 @@ import (
 	"testing"
 	"time"
 
-	"github.com/aquasecurity/fanal/cache"
+	"github.com/klauspost/compress/zstd"
+
 	"github.com/aquasecurity/fanal/extractor"
 	"github.com/aquasecurity/fanal/types"
 	"github.com/docker/docker/client"
+	"github.com/genuinetools/reg/registry"
+	"github.com/opencontainers/go-digest"
+	bolt "github.com/simar7/gokv/bbolt"
+	kvtypes "github.com/simar7/gokv/types"
 	"github.com/stretchr/testify/assert"
 )
 
+// TODO: Use a memory based FS rather than actual fs
+// context: https://github.com/aquasecurity/fanal/pull/51#discussion_r352337762
+func setupCache() (*bolt.Store, *os.File, error) {
+	f, err := ioutil.TempFile(".", "Bolt_TestStore-*")
+	if err != nil {
+		return nil, nil, err
+	}
+
+	s, err := bolt.NewStore(bolt.Options{
+		Path: f.Name(),
+	})
+	return s, f, err
+}
+
 func TestExtractFromFile(t *testing.T) {
 	vectors := []struct {
 		file      string            // Test input file
@@ -187,35 +206,81 @@ func TestExtractFiles(t *testing.T) {
 }
 
 func TestDockerExtractor_SaveLocalImage(t *testing.T) {
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		httpPath := r.URL.String()
-		switch {
-		case strings.Contains(httpPath, "images/get?names=fooimage"):
-			_, _ = fmt.Fprint(w, "foocontent")
-		default:
-			assert.FailNow(t, "unexpected path accessed: ", r.URL.String())
-		}
-	}))
-	defer ts.Close()
-
-	c, err := client.NewClientWithOpts(client.WithHost(ts.URL))
-	assert.NoError(t, err)
-
-	// setup cache
-	tempCacheDir, _ := ioutil.TempDir("", "TestDockerExtractor_SaveLocalImage-*")
-	defer func() {
-		_ = os.RemoveAll(tempCacheDir)
-	}()
-
-	de := Extractor{
-		Option: types.DockerOption{},
-		Client: c,
-		Cache:  cache.Initialize(tempCacheDir),
+	testCases := []struct {
+		name              string
+		expectedImageData string
+		cacheHit          bool
+	}{
+		{
+			name:              "happy path with cache miss",
+			expectedImageData: "foofromdocker",
+		},
+		{
+			name:              "happy path with cache hit",
+			cacheHit:          true,
+			expectedImageData: "foofromcache",
+		},
 	}
 
-	r, err := de.SaveLocalImage(context.TODO(), "fooimage")
-	assert.NotNil(t, r)
-	assert.NoError(t, err)
+	for _, tc := range testCases {
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			httpPath := r.URL.String()
+			switch {
+			case strings.Contains(httpPath, "images/get?names=fooimage"):
+				_, _ = fmt.Fprint(w, "foofromdocker")
+			default:
+				assert.FailNow(t, "unexpected path accessed: ", r.URL.String())
+			}
+		}))
+		defer ts.Close()
+
+		c, err := client.NewClientWithOpts(client.WithHost(ts.URL))
+		assert.NoError(t, err)
+
+		// setup cache
+		s, f, err := setupCache()
+		defer func() {
+			_ = f.Close()
+			_ = os.RemoveAll(f.Name())
+		}()
+		assert.NoError(t, err)
+
+		if tc.cacheHit {
+			e, _ := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedDefault))
+			dst := e.EncodeAll([]byte("foofromcache"), nil)
+			_ = s.Set(kvtypes.SetItemInput{
+				BucketName: "imagebucket",
+				Key:        "fooimage",
+				Value:      dst,
+			})
+		}
+
+		de := Extractor{
+			Option: types.DockerOption{},
+			Client: c,
+			Cache:  s,
+		}
+
+		r, err := de.SaveLocalImage(context.TODO(), "fooimage")
+		actualSavedTarBytes, _ := ioutil.ReadAll(r)
+		assert.Equal(t, []byte(tc.expectedImageData), actualSavedTarBytes[:], tc.name)
+		assert.NoError(t, err, tc.name)
+
+		// check the cache for what was stored
+		var actualValue []byte
+		found, err := de.Cache.Get(kvtypes.GetItemInput{
+			BucketName: "imagebucket",
+			Key:        "fooimage",
+			Value:      &actualValue,
+		})
+
+		assert.NoError(t, err, tc.name)
+		assert.True(t, found, tc.name)
+
+		dec, _ := zstd.NewReader(nil)
+		actualStoredValue, _ := dec.DecodeAll(actualValue, nil)
+		assert.Equal(t, tc.expectedImageData, string(actualStoredValue), tc.name)
+	}
 }
 
 func TestDockerExtractor_Extract(t *testing.T) {
@@ -232,22 +297,28 @@ func TestDockerExtractor_Extract(t *testing.T) {
 		{
 			name: "happy path",
 			manifestResp: `{
-		  "schemaVersion": 2,
-		  "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
-		  "layers": [
-		     {
-		        "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
-		        "size": 153263,
-		        "digest": "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b"
-		     }
-		  ]
+		 "schemaVersion": 2,
+		 "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+		 "layers": [
+		    {
+		       "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+		       "size": 153263,
+		       "digest": "sha256:shafortestdirslashhelloworlddottxt"
+		    },
+		    {
+		       "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+		       "size": 153263,
+		       "digest": "sha256:shafortestdirslashbadworlddottxt"
+		    }
+		 ]
 		}`,
 			fileName:      "testdata/testdir.tar.gz", // includes helloworld.txt and badworld.txt
 			blobData:      "foo",
-			fileToExtract: []string{"testdir/helloworld.txt"},
+			fileToExtract: []string{"testdir/helloworld.txt", "testdir/badworld.txt"},
 			expectedFileMap: extractor.FileMap{
 				"/config":                []uint8{0x66, 0x6f, 0x6f},
 				"testdir/helloworld.txt": []uint8{0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0xa},
+				"testdir/badworld.txt":   []uint8{0x62, 0x61, 0x64, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0xa},
 			},
 		},
 		{
@@ -263,20 +334,20 @@ func TestDockerExtractor_Extract(t *testing.T) {
 		{
 			name: "sad path: corrupt layer data invalid gzip",
 			manifestResp: `{
-		 "schemaVersion": 2,
-		 "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
-		 "layers": [
-		    {
-		       "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
-		       "size": 153263,
-		       "digest": "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b"
-		    }
-		 ]
+		"schemaVersion": 2,
+		"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+		"layers": [
+		 {
+		    "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+		    "size": 153263,
+		    "digest": "sha256:shaforinvalidgzipfile"
+		 }
+		]
 		}`,
 			fileName:        "testdata/opq.tar",
 			blobData:        "foo",
 			expectedFileMap: extractor.FileMap(nil),
-			expectedError:   "invalid gzip: gzip: invalid header",
+			expectedError:   "could not init gzip reader: gzip: invalid header",
 		},
 	}
 
@@ -287,8 +358,14 @@ func TestDockerExtractor_Extract(t *testing.T) {
 			case strings.Contains(httpPath, "/v2/library/fooimage/manifests/latest"):
 				w.Header().Set("Content-Type", "application/vnd.docker.distribution.manifest.v2+json")
 				_, _ = fmt.Fprint(w, tc.manifestResp)
-			case strings.Contains(httpPath, "/v2/library/fooimage/blobs/sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b"):
-				b, _ := ioutil.ReadFile(tc.fileName)
+			case strings.Contains(httpPath, "/v2/library/fooimage/blobs/sha256:shafortestdirslashhelloworlddottxt"):
+				b, _ := ioutil.ReadFile("testdata/helloworld.tar.gz")
+				_, _ = w.Write(b)
+			case strings.Contains(httpPath, "/v2/library/fooimage/blobs/sha256:shafortestdirslashbadworlddottxt"):
+				b, _ := ioutil.ReadFile("testdata/badworld.tar.gz")
+				_, _ = w.Write(b)
+			case strings.Contains(httpPath, "/v2/library/fooimage/blobs/sha256:shaforinvalidgzipfile"):
+				b, _ := ioutil.ReadFile("testdata/opq.tar")
 				_, _ = w.Write(b)
 			case strings.Contains(httpPath, "/v2/library/fooimage/blobs/"):
 				_, _ = w.Write([]byte(tc.blobData))
@@ -302,10 +379,12 @@ func TestDockerExtractor_Extract(t *testing.T) {
 		assert.NoError(t, err)
 
 		// setup cache
-		tempCacheDir, _ := ioutil.TempDir("", "TestDockerExtractor_Extract-*")
+		s, f, err := setupCache()
 		defer func() {
-			_ = os.RemoveAll(tempCacheDir)
+			_ = f.Close()
+			_ = os.RemoveAll(f.Name())
 		}()
+		assert.NoError(t, err)
 
 		de := Extractor{
 			Option: types.DockerOption{
@@ -315,7 +394,7 @@ func TestDockerExtractor_Extract(t *testing.T) {
 				Timeout:  time.Second * 1000,
 			},
 			Client: c,
-			Cache:  cache.Initialize(tempCacheDir),
+			Cache:  s,
 		}
 
 		tsURL := strings.TrimPrefix(ts.URL, "http://")
@@ -338,3 +417,173 @@ func TestDockerExtractor_Extract(t *testing.T) {
 		assert.Equal(t, tc.expectedFileMap, fm, tc.name)
 	}
 }
+
+func TestDocker_ExtractLayerWorker(t *testing.T) {
+	goodtarzstdgolden, _ := ioutil.ReadFile("testdata/testdir.tar.zstd")
+	goodReturnedTarContent, _ := ioutil.ReadFile("testdata/goodTarContent.golden")
+
+	testCases := []struct {
+		name                       string
+		cacheHit                   bool
+		garbageCache               bool
+		requiredFiles              []string
+		expectedCacheContents      []byte
+		expectedReturnedTarContent []byte
+	}{
+		{
+			name:                       "happy path with cache miss and write back",
+			cacheHit:                   false,
+			requiredFiles:              []string{"testdir/helloworld.txt", "testdir/badworld.txt"},
+			expectedCacheContents:      goodtarzstdgolden,
+			expectedReturnedTarContent: goodReturnedTarContent,
+		},
+		{
+			name:                       "happy path with cache hit with garbage cache and write back",
+			cacheHit:                   true,
+			garbageCache:               true,
+			requiredFiles:              []string{"testdir/helloworld.txt", "testdir/badworld.txt"},
+			expectedCacheContents:      goodtarzstdgolden,
+			expectedReturnedTarContent: goodReturnedTarContent,
+		},
+		{
+			name:                       "happy path with cache hit",
+			cacheHit:                   true,
+			expectedCacheContents:      goodtarzstdgolden,
+			expectedReturnedTarContent: goodReturnedTarContent,
+		},
+		{
+			name:                       "happy path with cache miss but no write back",
+			cacheHit:                   false,
+			expectedCacheContents:      []byte{0x28, 0xb5, 0x2f, 0xfd, 0x4, 0x60, 0x1, 0x0, 0x0, 0x99, 0xe9, 0xd8, 0x51}, // just the empty tar header
+			expectedReturnedTarContent: []byte{},
+		},
+	}
+
+	for _, tc := range testCases {
+		inputDigest := digest.Digest("sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b")
+
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			httpPath := r.URL.String()
+			switch {
+			case strings.Contains(httpPath, "/v2/library/fooimage/blobs/sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b"):
+				layerData, _ := ioutil.ReadFile("testdata/testdir.tar.gz")
+				_, _ = w.Write(layerData)
+			default:
+				assert.FailNow(t, "unexpected path accessed: ", fmt.Sprintf("%s %s", r.URL.String(), tc.name))
+			}
+		}))
+		defer ts.Close()
+
+		c, err := client.NewClientWithOpts(client.WithHost(ts.URL))
+		assert.NoError(t, err)
+
+		// setup cache
+		s, f, err := setupCache()
+		defer func() {
+			_ = f.Close()
+			_ = os.RemoveAll(f.Name())
+		}()
+		assert.NoError(t, err, tc.name)
+
+		if tc.cacheHit {
+			switch tc.garbageCache {
+			case true:
+				garbage, _ := ioutil.ReadFile("testdata/invalidgzvalidtar.tar.gz")
+				assert.NoError(t, s.Set(kvtypes.SetItemInput{
+					BucketName: LayerTarsBucket,
+					Key:        string(inputDigest),
+					Value:      garbage,
+				}), tc.name)
+			default:
+				assert.NoError(t, s.Set(kvtypes.SetItemInput{
+					BucketName: LayerTarsBucket,
+					Key:        string(inputDigest),
+					Value:      goodtarzstdgolden,
+				}), tc.name)
+			}
+		}
+
+		de := Extractor{
+			Option: types.DockerOption{
+				AuthURL:  ts.URL,
+				NonSSL:   true,
+				SkipPing: true,
+				Timeout:  time.Second * 1000,
+			},
+			Client: c,
+			Cache:  s,
+		}
+
+		tsUrl := strings.TrimPrefix(ts.URL, "http://")
+		inputImage := registry.Image{
+			Domain: tsUrl,
+			Path:   "library/fooimage",
+			Tag:    "latest",
+		}
+
+		layerCh := make(chan layer)
+		errCh := make(chan error)
+		r, err := de.createRegistryClient(context.TODO(), inputImage.Domain)
+		go func() {
+			de.extractLayerWorker(inputDigest, r, context.TODO(), inputImage, errCh, layerCh, tc.requiredFiles)
+		}()
+
+		var errRecieved error
+		var layerReceived layer
+
+		select {
+		case errRecieved = <-errCh:
+			assert.FailNow(t, "unexpected error received, err: ", fmt.Sprintf("%s, %s", errRecieved, tc.name))
+		case layerReceived = <-layerCh:
+			assert.Equal(t, inputDigest, layerReceived.ID, tc.name)
+			got, _ := ioutil.ReadAll(layerReceived.Content)
+			assert.Equal(t, tc.expectedReturnedTarContent, got, tc.name)
+		}
+
+		// check cache contents
+		var actualCacheContents []byte
+		found, err := s.Get(kvtypes.GetItemInput{
+			BucketName: LayerTarsBucket,
+			Key:        string(inputDigest),
+			Value:      &actualCacheContents,
+		})
+
+		assert.True(t, found, tc.name)
+		assert.NoError(t, err, tc.name)
+		assert.Equal(t, tc.expectedCacheContents, actualCacheContents, tc.name)
+	}
+}
+
+func TestDocker_ExtractLayerFiles(t *testing.T) {
+	de := Extractor{}
+
+	layerCh := make(chan layer)
+	errCh := make(chan error)
+	inputFilenames := []string{"var/foo", "etc/test/bar"}
+
+	f, _ := os.Open("testdata/opq2.tar")
+	defer f.Close()
+
+	go func() {
+		layerCh <- layer{
+			ID:      "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b",
+			Content: f,
+		}
+	}()
+
+	filesInLayers := map[string]extractor.FileMap{}
+	opqInLayers := map[string]extractor.OPQDirs{}
+	err := de.extractLayerFiles(context.TODO(), layerCh, errCh, filesInLayers, opqInLayers, inputFilenames)
+	assert.NoError(t, err)
+	assert.Equal(t, map[string]extractor.FileMap{
+		"sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b": {
+			"etc/test/bar": {0x62, 0x61, 0x72, 0xa},
+			"var/.wh.foo":  {},
+		},
+	}, filesInLayers)
+	assert.Equal(t, map[string]extractor.OPQDirs{
+		"sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b": {
+			"etc/test",
+		},
+	}, opqInLayers)
+}

extractor/docker/testdata/invalidgzvalidtar.tar.gz

@@ -0,0 +1 @@
+badtardata

go.mod

@@ -6,19 +6,21 @@ require (
 	cloud.google.com/go v0.37.4 // indirect
 	github.com/GoogleCloudPlatform/docker-credential-gcr v1.5.0
 	github.com/aquasecurity/go-dep-parser v0.0.0-20190819075924-ea223f0ef24b
-	github.com/aws/aws-sdk-go v1.19.11
+	github.com/aws/aws-sdk-go v1.25.31
 	github.com/deckarep/golang-set v1.7.1
 	github.com/docker/distribution v2.7.1+incompatible
 	github.com/docker/docker v0.0.0-20180924202107-a9c061deec0f
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/genuinetools/reg v0.16.0
+	github.com/klauspost/compress v1.9.3
 	github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d
 	github.com/knqyf263/go-rpmdb v0.0.0-20190501070121-10a1c42a10dc
 	github.com/knqyf263/nested v0.0.1
 	github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348
 	github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2
 	github.com/pkg/errors v0.8.1
-	github.com/stretchr/testify v1.3.0
+	github.com/simar7/gokv v0.3.2
+	github.com/stretchr/testify v1.4.0
 	golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5
 	golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7
 )

go.sum

@@ -17,16 +17,21 @@ github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWX
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
+github.com/alicebob/miniredis/v2 v2.11.0/go.mod h1:UA48pmi7aSazcGAvcdKcBB49z521IC9VjTTRz2nIaJE=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/aquasecurity/go-dep-parser v0.0.0-20190819075924-ea223f0ef24b h1:55Ulc/gvfWm4ylhVaR7MxOwujRjA6et7KhmUbSgUFf4=
 github.com/aquasecurity/go-dep-parser v0.0.0-20190819075924-ea223f0ef24b/go.mod h1:BpNTD9vHfrejKsED9rx04ldM1WIbeyXGYxUrqTVwxVQ=
-github.com/aws/aws-sdk-go v1.19.11 h1:tqaTGER6Byw3QvsjGW0p018U2UOqaJPeJuzoaF7jjoQ=
-github.com/aws/aws-sdk-go v1.19.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go v1.25.31 h1:14mdh3HsTgRekePPkYcCbAaEXJknc3mN7f4XfsiMMDA=
+github.com/aws/aws-sdk-go v1.25.31/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc h1:TP+534wVlf61smEIq1nwLLAjQVEK2EADoW3CX9AuT+8=
 github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
@@ -55,6 +60,7 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4=
 github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
@@ -79,6 +85,8 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.7.1-0.20190322064113-39e2c31b7ca3/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
+github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -101,6 +109,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.9.3 h1:hkFELABwacUEgBfiguNeQydKv3M9pawBq8o24Ypw9+M=
+github.com/klauspost/compress v1.9.3/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/knqyf263/berkeleydb v0.0.0-20190501065933-fafe01fb9662 h1:UGS0RbPHwXJkq8tcba8OD0nvVUWLf2h7uUJznuHPPB0=
 github.com/knqyf263/berkeleydb v0.0.0-20190501065933-fafe01fb9662/go.mod h1:bu1CcN4tUtoRcI/B/RFHhxMNKFHVq/c3SV+UTyduoXg=
 github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d h1:X4cedH4Kn3JPupAwwWuo4AzYp16P0OyLO9d7OnMZc/c=
@@ -163,6 +173,12 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/shurcooL/httpfs v0.0.0-20181222201310-74dc9339e414/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
+github.com/simar7/gokv v0.1.0 h1:5DoScIR7r6w6DCJPXFlSGKU2SOmpLeSfZkINVyVaPmI=
+github.com/simar7/gokv v0.1.0/go.mod h1:vixfnOzH16g5QN5Uij1q4zSVbNp3iJu2D6xgIPM4ZSE=
+github.com/simar7/gokv v0.3.2 h1:UAcjZs5nsoAKyGxHMOJgGZs7Q9Yeub40Gb54YRW1Fo8=
+github.com/simar7/gokv v0.3.2/go.mod h1:jXjPspRkuCDCRTRBgfGsfXvW8ofOGh3Y+tjZvoFr7XU=
+github.com/simar7/gokv v0.3.3-0.20191213063627-21f89164966d h1:lsqk9OujJz7FwE7pP7ZMjH2pWaGO4QPSPEJ7JSsuOO8=
+github.com/simar7/gokv v0.3.3-0.20191213063627-21f89164966d/go.mod h1:jXjPspRkuCDCRTRBgfGsfXvW8ofOGh3Y+tjZvoFr7XU=
 github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
@@ -174,8 +190,13 @@ github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/tomoyamachi/reg v0.16.1-0.20190706172545-2a2250fd7c00 h1:0e4vRd9YqnQBIAIAE39jLKDWffRfJWxloyWwcaMAQho=
 github.com/tomoyamachi/reg v0.16.1-0.20190706172545-2a2250fd7c00/go.mod h1:RQE7h2jyIxekQZ24/wad0c9RGP+KSq4XzHh7h83ALi8=
+github.com/yuin/gopher-lua v0.0.0-20190206043414-8bfc7677f583/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
+go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -197,6 +218,8 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJV
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20191108221443-4ba9e2ef068c h1:SRpq/kuj/xNci/RdvEs+RSvpfxqvLAzTKuKGlzoGdZQ=
+golang.org/x/net v0.0.0-20191108221443-4ba9e2ef068c/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -212,11 +235,13 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e h1:nFYrTHrdrAOpShe27kaFHjsqYSEQ0KWqdWLu3xuZJts=
 golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190506115046-ca7f33d4116e h1:bq5BY1tGuaK8HxuwN6pT6kWgTVLeJ5KwuyBpsl1CZL4=
 golang.org/x/sys v0.0.0-20190506115046-ca7f33d4116e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd h1:3x5uuvBgE6oaXJjCOvpCC1IpgJogqQ+PqGGU3ZxAgII=
+golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -254,10 +279,13 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
 google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=