Use StoreMetadata from trivy-db (#509)

* db_test: Remove cruft

Signed-off-by: Simarpreet Singh <simar@linux.com>

* db: Add StoreMetadata from trivy-db.

Signed-off-by: Simarpreet Singh <simar@linux.com>

* mod: Update trivy-db dependency

Signed-off-by: Simarpreet Singh <simar@linux.com>

* mod: Bump trivy-db version

Signed-off-by: Simarpreet Singh <simar@linux.com>

* db: Eliminate metadata.Store

Signed-off-by: Simarpreet Singh <simar@linux.com>

* db: Add a TODO to move things into trivy-db repo

Signed-off-by: Simarpreet Singh <simar@linux.com>

go.mod

@@ -5,7 +5,7 @@ go 1.13
 require (
 	github.com/aquasecurity/fanal v0.0.0-20200528202907-79693bf4a058
 	github.com/aquasecurity/go-dep-parser v0.0.0-20190819075924-ea223f0ef24b
-	github.com/aquasecurity/trivy-db v0.0.0-20200514134639-7e57e3e02470
+	github.com/aquasecurity/trivy-db v0.0.0-20200616161554-cd5b3da29bc8
 	github.com/caarlos0/env/v6 v6.0.0
 	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/cheggaaa/pb/v3 v3.0.3

go.sum

@@ -52,8 +52,8 @@ github.com/aquasecurity/go-dep-parser v0.0.0-20190819075924-ea223f0ef24b h1:55Ul
 github.com/aquasecurity/go-dep-parser v0.0.0-20190819075924-ea223f0ef24b/go.mod h1:BpNTD9vHfrejKsED9rx04ldM1WIbeyXGYxUrqTVwxVQ=
 github.com/aquasecurity/testdocker v0.0.0-20200426142840-5f05bce6f12a h1:hsw7PpiymXP64evn/K7gsj3hWzMqLrdoeE6JkqDocVg=
 github.com/aquasecurity/testdocker v0.0.0-20200426142840-5f05bce6f12a/go.mod h1:psfu0MVaiTDLpNxCoNsTeILSKY2EICBwv345f3M+Ffs=
-github.com/aquasecurity/trivy-db v0.0.0-20200514134639-7e57e3e02470 h1:6VE+g4AK2uivPqZtVk/QtcCBb2rUjAvKqDNexSgqMC0=
+github.com/aquasecurity/trivy-db v0.0.0-20200616161554-cd5b3da29bc8 h1:PvRcn3v8lpccqmEEzmJmXrm47ag47OCt8ui+9APi4hA=
-github.com/aquasecurity/trivy-db v0.0.0-20200514134639-7e57e3e02470/go.mod h1:F77bF2nRbcH4EIhhcNEP585MoAKdLpEP3dihF9V1Hbw=
+github.com/aquasecurity/trivy-db v0.0.0-20200616161554-cd5b3da29bc8/go.mod h1:EiFA908RL0ACrbYo/9HfT7f9QcdC2bZoIO5XAAcvz9A=
 github.com/aquasecurity/vuln-list-update v0.0.0-20191016075347-3d158c2bf9a2 h1:xbdUfr2KE4THsFx9CFWtWpU91lF+YhgP46moV94nYTA=
 github.com/aquasecurity/vuln-list-update v0.0.0-20191016075347-3d158c2bf9a2/go.mod h1:6NhOP0CjZJL27bZZcaHECtzWdwDDm2g6yCY0QgXEGQQ=
 github.com/araddon/dateparse v0.0.0-20190426192744-0d74ffceef83/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI=
@@ -473,7 +473,6 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -513,6 +512,7 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -538,7 +538,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

integration/integration_test.go

@@ -5,6 +5,7 @@ package integration
 import (
 	"compress/gzip"
 	"context"
+	"encoding/json"
 	"flag"
 	"io"
 	"io/ioutil"
@@ -13,8 +14,6 @@ import (
 	"path/filepath"
 	"time"
 
-	dbFile "github.com/aquasecurity/trivy/pkg/db"
-
 	"github.com/aquasecurity/trivy-db/pkg/db"
 	"github.com/spf13/afero"
 )
@@ -53,7 +52,9 @@ func gunzipDB() (string, error) {
 		return "", err
 	}
 
-	err = dbFile.NewMetadata(afero.NewOsFs(), tmpDir).Store(db.Metadata{
+	fs := afero.NewOsFs()
+	metadataFile := filepath.Join(dbDir, "metadata.json")
+	b, err := json.Marshal(db.Metadata{
 		Version:    1,
 		Type:       1,
 		NextUpdate: time.Time{},
@@ -62,6 +63,10 @@ func gunzipDB() (string, error) {
 	if err != nil {
 		return "", err
 	}
+	err = afero.WriteFile(fs, metadataFile, b, 0600)
+	if err != nil {
+		return "", err
+	}
 
 	return tmpDir, nil
 }

internal/app_test.go

@@ -2,6 +2,7 @@ package internal
 
 import (
 	"bytes"
+	"encoding/json"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -10,7 +11,6 @@ import (
 
 	"github.com/stretchr/testify/require"
 
-	dbFile "github.com/aquasecurity/trivy/pkg/db"
 	"github.com/spf13/afero"
 
 	"github.com/aquasecurity/trivy-db/pkg/db"
@@ -87,17 +87,20 @@ Vulnerability DB:
 			}
 
 			if tt.createDB {
-				m := dbFile.NewMetadata(afero.NewOsFs(), cacheDir)
+				fs := afero.NewOsFs()
 				err := os.MkdirAll(filepath.Join(cacheDir, "db"), os.ModePerm)
 				require.NoError(t, err)
+				metadataFile := filepath.Join(cacheDir, "db", "metadata.json")
 
-				err = m.Store(db.Metadata{
+				b, err := json.Marshal(db.Metadata{
 					Version:    42,
 					Type:       1,
 					NextUpdate: time.Unix(1584403020, 0),
 					UpdatedAt:  time.Unix(1584402020, 0),
 				})
 				require.NoError(t, err)
+				err = afero.WriteFile(fs, metadataFile, b, 0600)
+				require.NoError(t, err)
 			}
 
 			fw := new(bytes.Buffer)

pkg/db/db.go

@@ -59,6 +59,7 @@ type Operation interface {
 
 type dbOperation interface {
 	GetMetadata() (metadata db.Metadata, err error)
+	StoreMetadata(metadata db.Metadata, dir string) (err error)
 }
 
 type Client struct {
@@ -181,17 +182,17 @@ func (c Client) UpdateMetadata(cacheDir string) error {
 
 	metadata, err := c.dbc.GetMetadata()
 	if err != nil {
-		return xerrors.Errorf("unable to get a metadata: %w", err)
+		return xerrors.Errorf("unable to get metadata: %w", err)
 	}
 
-	if err = c.metadata.Store(metadata); err != nil {
+	if err = c.dbc.StoreMetadata(metadata, filepath.Join(cacheDir, "db")); err != nil {
 		return xerrors.Errorf("failed to store metadata: %w", err)
 	}
 
 	return nil
 }
 
-type Metadata struct {
+type Metadata struct { // TODO: Move all Metadata things to trivy-db repo
 	fs       afero.Fs
 	filePath string
 }
@@ -210,20 +211,6 @@ func MetadataPath(cacheDir string) string {
 	return filepath.Join(dbDir, metadataFile)
 }
 
-// StoreMetadata stores database metadata as a file
-func (m Metadata) Store(metadata db.Metadata) error {
-	f, err := m.fs.Create(m.filePath)
-	if err != nil {
-		return xerrors.Errorf("unable to create a metadata file: %w", err)
-	}
-	defer f.Close()
-
-	if err = json.NewEncoder(f).Encode(metadata); err != nil {
-		return xerrors.Errorf("unable to encode metadata: %w", err)
-	}
-	return nil
-}
-
 // DeleteMetadata deletes the file of database metadata
 func (m Metadata) Delete() error {
 	if err := m.fs.Remove(m.filePath); err != nil {

pkg/db/db_test.go

@@ -2,6 +2,8 @@ package db
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"io/ioutil"
 	"os"
 	"testing"
@@ -142,11 +144,13 @@ func TestClient_NeedsUpdate(t *testing.T) {
 			fs := afero.NewMemMapFs()
 			metadata := NewMetadata(fs, "/cache")
 			if tc.metadata != (db.Metadata{}) {
-				metadata.Store(tc.metadata)
+				b, err := json.Marshal(tc.metadata)
+				require.NoError(t, err)
+				err = afero.WriteFile(fs, metadata.filePath, b, 0600)
+				require.NoError(t, err)
 			}
 
 			client := Client{
-				//dbc:      mockConfig,
 				clock:    tc.clock,
 				metadata: metadata,
 			}
@@ -161,22 +165,15 @@ func TestClient_NeedsUpdate(t *testing.T) {
 			}
 
 			assert.Equal(t, tc.expected, needsUpdate)
-			//mockConfig.AssertExpectations(t)
 		})
 	}
 }
 
 func TestClient_Download(t *testing.T) {
-	type getMetadataOutput struct {
-		metadata db.Metadata
-		err      error
-	}
-
 	testCases := []struct {
 		name            string
 		light           bool
 		downloadDB      []github.DownloadDBExpectation
-		getMetadata     dbOperationGetMetadataExpectation
 		expectedContent []byte
 		expectedError   error
 	}{
@@ -191,15 +188,6 @@ func TestClient_Download(t *testing.T) {
 					},
 				},
 			},
-			getMetadata: dbOperationGetMetadataExpectation{
-				Returns: dbOperationGetMetadataReturns{
-					Metadata: db.Metadata{
-						Version:    1,
-						Type:       db.TypeFull,
-						NextUpdate: time.Date(2019, 9, 1, 0, 0, 0, 0, time.UTC),
-					},
-				},
-			},
 		},
 		{
 			name:  "DownloadDB returns an error",
@@ -235,7 +223,6 @@ func TestClient_Download(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			mockConfig := new(mockDbOperation)
-			mockConfig.ApplyGetMetadataExpectation(tc.getMetadata)
 
 			mockGitHubClient, err := github.NewMockClient(tc.downloadDB)
 			require.NoError(t, err, tc.name)
@@ -263,3 +250,99 @@ func TestClient_Download(t *testing.T) {
 		})
 	}
 }
+
+func TestClient_UpdateMetadata(t *testing.T) {
+	testCases := []struct {
+		name                     string
+		getMetadataExpectation   dbOperationGetMetadataExpectation
+		storeMetadataExpectation dbOperationStoreMetadataExpectation
+		expectedError            error
+	}{
+		{
+			name: "happy path",
+			getMetadataExpectation: dbOperationGetMetadataExpectation{
+				Returns: dbOperationGetMetadataReturns{
+					Metadata: db.Metadata{
+						Version:    1,
+						Type:       1,
+						NextUpdate: time.Date(2020, 4, 30, 23, 59, 59, 0, time.UTC),
+						UpdatedAt:  time.Date(2006, 4, 30, 23, 59, 59, 0, time.UTC),
+					},
+					Err: nil,
+				},
+			},
+			storeMetadataExpectation: dbOperationStoreMetadataExpectation{
+				Metadata: db.Metadata{
+					Version:    1,
+					Type:       1,
+					NextUpdate: time.Date(2020, 4, 30, 23, 59, 59, 0, time.UTC),
+					UpdatedAt:  time.Date(2006, 4, 30, 23, 59, 59, 0, time.UTC),
+				},
+			},
+		},
+		{
+			name: "sad path, get metadata fails",
+			getMetadataExpectation: dbOperationGetMetadataExpectation{
+				Returns: dbOperationGetMetadataReturns{
+					Err: errors.New("get metadata failed"),
+				},
+			},
+			expectedError: errors.New("unable to get metadata: get metadata failed"),
+		},
+		{
+			name: "sad path, store metadata fails",
+			getMetadataExpectation: dbOperationGetMetadataExpectation{
+				Returns: dbOperationGetMetadataReturns{
+					Metadata: db.Metadata{
+						Version:    1,
+						Type:       1,
+						NextUpdate: time.Date(2020, 4, 30, 23, 59, 59, 0, time.UTC),
+						UpdatedAt:  time.Date(2006, 4, 30, 23, 59, 59, 0, time.UTC),
+					},
+					Err: nil,
+				},
+			},
+			storeMetadataExpectation: dbOperationStoreMetadataExpectation{
+				Metadata: db.Metadata{
+					Version:    1,
+					Type:       1,
+					NextUpdate: time.Date(2020, 4, 30, 23, 59, 59, 0, time.UTC),
+					UpdatedAt:  time.Date(2006, 4, 30, 23, 59, 59, 0, time.UTC),
+				},
+				Returns: dbOperationStoreMetadataReturns{
+					Err: errors.New("store metadata failed"),
+				},
+			},
+			expectedError: errors.New("failed to store metadata: store metadata failed"),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			mockConfig := new(mockDbOperation)
+			err := log.InitLogger(false, true)
+			require.NoError(t, err, "failed to init logger")
+
+			mockConfig.ApplyGetMetadataExpectation(tc.getMetadataExpectation)
+			mockConfig.ApplyStoreMetadataExpectation(tc.storeMetadataExpectation)
+
+			fs := afero.NewMemMapFs()
+			metadata := NewMetadata(fs, "/cache")
+
+			dir, err := ioutil.TempDir("", "db")
+			require.NoError(t, err, tc.name)
+			defer os.RemoveAll(dir)
+
+			pb := indicator.NewProgressBar(true)
+			client := NewClient(mockConfig, nil, pb, nil, metadata)
+
+			err = client.UpdateMetadata(dir)
+			switch {
+			case tc.expectedError != nil:
+				assert.EqualError(t, err, tc.expectedError.Error(), tc.name)
+			default:
+				assert.NoError(t, err, tc.name)
+			}
+		})
+	}
+}

pkg/db/mock_db_operation.go

@@ -3,7 +3,7 @@
 package db
 
 import (
-	pkgdb "github.com/aquasecurity/trivy-db/pkg/db"
+	"github.com/aquasecurity/trivy-db/pkg/db"
 	mock "github.com/stretchr/testify/mock"
 )
 
@@ -13,7 +13,7 @@ type mockDbOperation struct {
 }
 
 type dbOperationGetMetadataReturns struct {
-	Metadata pkgdb.Metadata
+	Metadata db.Metadata
 	Err      error
 }
 
@@ -33,14 +33,14 @@ func (_m *mockDbOperation) ApplyGetMetadataExpectations(expectations []dbOperati
 }
 
 // GetMetadata provides a mock function with given fields:
-func (_m *mockDbOperation) GetMetadata() (pkgdb.Metadata, error) {
+func (_m *mockDbOperation) GetMetadata() (db.Metadata, error) {
 	ret := _m.Called()
 
-	var r0 pkgdb.Metadata
+	var r0 db.Metadata
-	if rf, ok := ret.Get(0).(func() pkgdb.Metadata); ok {
+	if rf, ok := ret.Get(0).(func() db.Metadata); ok {
 		r0 = rf()
 	} else {
-		r0 = ret.Get(0).(pkgdb.Metadata)
+		r0 = ret.Get(0).(db.Metadata)
 	}
 
 	var r1 error
@@ -52,3 +52,37 @@ func (_m *mockDbOperation) GetMetadata() (pkgdb.Metadata, error) {
 
 	return r0, r1
 }
+
+type dbOperationStoreMetadataReturns struct {
+	Err error
+}
+
+type dbOperationStoreMetadataExpectation struct {
+	Metadata db.Metadata
+	Dir      string
+	Returns  dbOperationStoreMetadataReturns
+}
+
+func (_m *mockDbOperation) ApplyStoreMetadataExpectation(e dbOperationStoreMetadataExpectation) {
+	_m.On("StoreMetadata", e.Metadata, mock.Anything).Return(e.Returns.Err)
+}
+
+func (_m *mockDbOperation) ApplyStoreMetadataExpectations(expectations []dbOperationStoreMetadataExpectation) {
+	for _, e := range expectations {
+		_m.ApplyStoreMetadataExpectation(e)
+	}
+}
+
+// StoreMetadata provides a mock function with given fields: metadata, dir
+func (_m *mockDbOperation) StoreMetadata(metadata db.Metadata, dir string) error {
+	ret := _m.Called(metadata, dir)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(db.Metadata, string) error); ok {
+		r0 = rf(metadata, dir)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}