diff --git a/go.mod b/go.mod index a291423c56..16e18addb8 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.13 require ( github.com/aquasecurity/fanal v0.0.0-20200528202907-79693bf4a058 github.com/aquasecurity/go-dep-parser v0.0.0-20190819075924-ea223f0ef24b - github.com/aquasecurity/trivy-db v0.0.0-20200514134639-7e57e3e02470 + github.com/aquasecurity/trivy-db v0.0.0-20200616161554-cd5b3da29bc8 github.com/caarlos0/env/v6 v6.0.0 github.com/cenkalti/backoff v2.2.1+incompatible github.com/cheggaaa/pb/v3 v3.0.3 diff --git a/go.sum b/go.sum index 7ded77841f..8f72234f40 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,8 @@ github.com/aquasecurity/go-dep-parser v0.0.0-20190819075924-ea223f0ef24b h1:55Ul github.com/aquasecurity/go-dep-parser v0.0.0-20190819075924-ea223f0ef24b/go.mod h1:BpNTD9vHfrejKsED9rx04ldM1WIbeyXGYxUrqTVwxVQ= github.com/aquasecurity/testdocker v0.0.0-20200426142840-5f05bce6f12a h1:hsw7PpiymXP64evn/K7gsj3hWzMqLrdoeE6JkqDocVg= github.com/aquasecurity/testdocker v0.0.0-20200426142840-5f05bce6f12a/go.mod h1:psfu0MVaiTDLpNxCoNsTeILSKY2EICBwv345f3M+Ffs= -github.com/aquasecurity/trivy-db v0.0.0-20200514134639-7e57e3e02470 h1:6VE+g4AK2uivPqZtVk/QtcCBb2rUjAvKqDNexSgqMC0= -github.com/aquasecurity/trivy-db v0.0.0-20200514134639-7e57e3e02470/go.mod h1:F77bF2nRbcH4EIhhcNEP585MoAKdLpEP3dihF9V1Hbw= +github.com/aquasecurity/trivy-db v0.0.0-20200616161554-cd5b3da29bc8 h1:PvRcn3v8lpccqmEEzmJmXrm47ag47OCt8ui+9APi4hA= +github.com/aquasecurity/trivy-db v0.0.0-20200616161554-cd5b3da29bc8/go.mod h1:EiFA908RL0ACrbYo/9HfT7f9QcdC2bZoIO5XAAcvz9A= github.com/aquasecurity/vuln-list-update v0.0.0-20191016075347-3d158c2bf9a2 h1:xbdUfr2KE4THsFx9CFWtWpU91lF+YhgP46moV94nYTA= github.com/aquasecurity/vuln-list-update v0.0.0-20191016075347-3d158c2bf9a2/go.mod h1:6NhOP0CjZJL27bZZcaHECtzWdwDDm2g6yCY0QgXEGQQ= github.com/araddon/dateparse v0.0.0-20190426192744-0d74ffceef83/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI= @@ -473,7 +473,6 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -513,6 +512,7 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -538,7 +538,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/integration/integration_test.go b/integration/integration_test.go index 11e010f837..da14cf7fab 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -5,6 +5,7 @@ package integration import ( "compress/gzip" "context" + "encoding/json" "flag" "io" "io/ioutil" @@ -13,8 +14,6 @@ import ( "path/filepath" "time" - dbFile "github.com/aquasecurity/trivy/pkg/db" - "github.com/aquasecurity/trivy-db/pkg/db" "github.com/spf13/afero" ) @@ -53,7 +52,9 @@ func gunzipDB() (string, error) { return "", err } - err = dbFile.NewMetadata(afero.NewOsFs(), tmpDir).Store(db.Metadata{ + fs := afero.NewOsFs() + metadataFile := filepath.Join(dbDir, "metadata.json") + b, err := json.Marshal(db.Metadata{ Version: 1, Type: 1, NextUpdate: time.Time{}, @@ -62,6 +63,10 @@ func gunzipDB() (string, error) { if err != nil { return "", err } + err = afero.WriteFile(fs, metadataFile, b, 0600) + if err != nil { + return "", err + } return tmpDir, nil } diff --git a/internal/app_test.go b/internal/app_test.go index e61236eeb5..199927cf06 100644 --- a/internal/app_test.go +++ b/internal/app_test.go @@ -2,6 +2,7 @@ package internal import ( "bytes" + "encoding/json" "io/ioutil" "os" "path/filepath" @@ -10,7 +11,6 @@ import ( "github.com/stretchr/testify/require" - dbFile "github.com/aquasecurity/trivy/pkg/db" "github.com/spf13/afero" "github.com/aquasecurity/trivy-db/pkg/db" @@ -87,17 +87,20 @@ Vulnerability DB: } if tt.createDB { - m := dbFile.NewMetadata(afero.NewOsFs(), cacheDir) + fs := afero.NewOsFs() err := os.MkdirAll(filepath.Join(cacheDir, "db"), os.ModePerm) require.NoError(t, err) + metadataFile := filepath.Join(cacheDir, "db", "metadata.json") - err = m.Store(db.Metadata{ + b, err := json.Marshal(db.Metadata{ Version: 42, Type: 1, NextUpdate: time.Unix(1584403020, 0), UpdatedAt: time.Unix(1584402020, 0), }) require.NoError(t, err) + err = afero.WriteFile(fs, metadataFile, b, 0600) + require.NoError(t, err) } fw := new(bytes.Buffer) diff --git a/pkg/db/db.go b/pkg/db/db.go index 09ff8892cb..fad8c771d2 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -59,6 +59,7 @@ type Operation interface { type dbOperation interface { GetMetadata() (metadata db.Metadata, err error) + StoreMetadata(metadata db.Metadata, dir string) (err error) } type Client struct { @@ -181,17 +182,17 @@ func (c Client) UpdateMetadata(cacheDir string) error { metadata, err := c.dbc.GetMetadata() if err != nil { - return xerrors.Errorf("unable to get a metadata: %w", err) + return xerrors.Errorf("unable to get metadata: %w", err) } - if err = c.metadata.Store(metadata); err != nil { + if err = c.dbc.StoreMetadata(metadata, filepath.Join(cacheDir, "db")); err != nil { return xerrors.Errorf("failed to store metadata: %w", err) } return nil } -type Metadata struct { +type Metadata struct { // TODO: Move all Metadata things to trivy-db repo fs afero.Fs filePath string } @@ -210,20 +211,6 @@ func MetadataPath(cacheDir string) string { return filepath.Join(dbDir, metadataFile) } -// StoreMetadata stores database metadata as a file -func (m Metadata) Store(metadata db.Metadata) error { - f, err := m.fs.Create(m.filePath) - if err != nil { - return xerrors.Errorf("unable to create a metadata file: %w", err) - } - defer f.Close() - - if err = json.NewEncoder(f).Encode(metadata); err != nil { - return xerrors.Errorf("unable to encode metadata: %w", err) - } - return nil -} - // DeleteMetadata deletes the file of database metadata func (m Metadata) Delete() error { if err := m.fs.Remove(m.filePath); err != nil { diff --git a/pkg/db/db_test.go b/pkg/db/db_test.go index 29345ac552..3ab88cd157 100644 --- a/pkg/db/db_test.go +++ b/pkg/db/db_test.go @@ -2,6 +2,8 @@ package db import ( "context" + "encoding/json" + "errors" "io/ioutil" "os" "testing" @@ -142,11 +144,13 @@ func TestClient_NeedsUpdate(t *testing.T) { fs := afero.NewMemMapFs() metadata := NewMetadata(fs, "/cache") if tc.metadata != (db.Metadata{}) { - metadata.Store(tc.metadata) + b, err := json.Marshal(tc.metadata) + require.NoError(t, err) + err = afero.WriteFile(fs, metadata.filePath, b, 0600) + require.NoError(t, err) } client := Client{ - //dbc: mockConfig, clock: tc.clock, metadata: metadata, } @@ -161,22 +165,15 @@ func TestClient_NeedsUpdate(t *testing.T) { } assert.Equal(t, tc.expected, needsUpdate) - //mockConfig.AssertExpectations(t) }) } } func TestClient_Download(t *testing.T) { - type getMetadataOutput struct { - metadata db.Metadata - err error - } - testCases := []struct { name string light bool downloadDB []github.DownloadDBExpectation - getMetadata dbOperationGetMetadataExpectation expectedContent []byte expectedError error }{ @@ -191,15 +188,6 @@ func TestClient_Download(t *testing.T) { }, }, }, - getMetadata: dbOperationGetMetadataExpectation{ - Returns: dbOperationGetMetadataReturns{ - Metadata: db.Metadata{ - Version: 1, - Type: db.TypeFull, - NextUpdate: time.Date(2019, 9, 1, 0, 0, 0, 0, time.UTC), - }, - }, - }, }, { name: "DownloadDB returns an error", @@ -235,7 +223,6 @@ func TestClient_Download(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { mockConfig := new(mockDbOperation) - mockConfig.ApplyGetMetadataExpectation(tc.getMetadata) mockGitHubClient, err := github.NewMockClient(tc.downloadDB) require.NoError(t, err, tc.name) @@ -263,3 +250,99 @@ func TestClient_Download(t *testing.T) { }) } } + +func TestClient_UpdateMetadata(t *testing.T) { + testCases := []struct { + name string + getMetadataExpectation dbOperationGetMetadataExpectation + storeMetadataExpectation dbOperationStoreMetadataExpectation + expectedError error + }{ + { + name: "happy path", + getMetadataExpectation: dbOperationGetMetadataExpectation{ + Returns: dbOperationGetMetadataReturns{ + Metadata: db.Metadata{ + Version: 1, + Type: 1, + NextUpdate: time.Date(2020, 4, 30, 23, 59, 59, 0, time.UTC), + UpdatedAt: time.Date(2006, 4, 30, 23, 59, 59, 0, time.UTC), + }, + Err: nil, + }, + }, + storeMetadataExpectation: dbOperationStoreMetadataExpectation{ + Metadata: db.Metadata{ + Version: 1, + Type: 1, + NextUpdate: time.Date(2020, 4, 30, 23, 59, 59, 0, time.UTC), + UpdatedAt: time.Date(2006, 4, 30, 23, 59, 59, 0, time.UTC), + }, + }, + }, + { + name: "sad path, get metadata fails", + getMetadataExpectation: dbOperationGetMetadataExpectation{ + Returns: dbOperationGetMetadataReturns{ + Err: errors.New("get metadata failed"), + }, + }, + expectedError: errors.New("unable to get metadata: get metadata failed"), + }, + { + name: "sad path, store metadata fails", + getMetadataExpectation: dbOperationGetMetadataExpectation{ + Returns: dbOperationGetMetadataReturns{ + Metadata: db.Metadata{ + Version: 1, + Type: 1, + NextUpdate: time.Date(2020, 4, 30, 23, 59, 59, 0, time.UTC), + UpdatedAt: time.Date(2006, 4, 30, 23, 59, 59, 0, time.UTC), + }, + Err: nil, + }, + }, + storeMetadataExpectation: dbOperationStoreMetadataExpectation{ + Metadata: db.Metadata{ + Version: 1, + Type: 1, + NextUpdate: time.Date(2020, 4, 30, 23, 59, 59, 0, time.UTC), + UpdatedAt: time.Date(2006, 4, 30, 23, 59, 59, 0, time.UTC), + }, + Returns: dbOperationStoreMetadataReturns{ + Err: errors.New("store metadata failed"), + }, + }, + expectedError: errors.New("failed to store metadata: store metadata failed"), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + mockConfig := new(mockDbOperation) + err := log.InitLogger(false, true) + require.NoError(t, err, "failed to init logger") + + mockConfig.ApplyGetMetadataExpectation(tc.getMetadataExpectation) + mockConfig.ApplyStoreMetadataExpectation(tc.storeMetadataExpectation) + + fs := afero.NewMemMapFs() + metadata := NewMetadata(fs, "/cache") + + dir, err := ioutil.TempDir("", "db") + require.NoError(t, err, tc.name) + defer os.RemoveAll(dir) + + pb := indicator.NewProgressBar(true) + client := NewClient(mockConfig, nil, pb, nil, metadata) + + err = client.UpdateMetadata(dir) + switch { + case tc.expectedError != nil: + assert.EqualError(t, err, tc.expectedError.Error(), tc.name) + default: + assert.NoError(t, err, tc.name) + } + }) + } +} diff --git a/pkg/db/mock_db_operation.go b/pkg/db/mock_db_operation.go index f0f5bd8e41..f1992efcfd 100644 --- a/pkg/db/mock_db_operation.go +++ b/pkg/db/mock_db_operation.go @@ -3,7 +3,7 @@ package db import ( - pkgdb "github.com/aquasecurity/trivy-db/pkg/db" + "github.com/aquasecurity/trivy-db/pkg/db" mock "github.com/stretchr/testify/mock" ) @@ -13,7 +13,7 @@ type mockDbOperation struct { } type dbOperationGetMetadataReturns struct { - Metadata pkgdb.Metadata + Metadata db.Metadata Err error } @@ -33,14 +33,14 @@ func (_m *mockDbOperation) ApplyGetMetadataExpectations(expectations []dbOperati } // GetMetadata provides a mock function with given fields: -func (_m *mockDbOperation) GetMetadata() (pkgdb.Metadata, error) { +func (_m *mockDbOperation) GetMetadata() (db.Metadata, error) { ret := _m.Called() - var r0 pkgdb.Metadata - if rf, ok := ret.Get(0).(func() pkgdb.Metadata); ok { + var r0 db.Metadata + if rf, ok := ret.Get(0).(func() db.Metadata); ok { r0 = rf() } else { - r0 = ret.Get(0).(pkgdb.Metadata) + r0 = ret.Get(0).(db.Metadata) } var r1 error @@ -52,3 +52,37 @@ func (_m *mockDbOperation) GetMetadata() (pkgdb.Metadata, error) { return r0, r1 } + +type dbOperationStoreMetadataReturns struct { + Err error +} + +type dbOperationStoreMetadataExpectation struct { + Metadata db.Metadata + Dir string + Returns dbOperationStoreMetadataReturns +} + +func (_m *mockDbOperation) ApplyStoreMetadataExpectation(e dbOperationStoreMetadataExpectation) { + _m.On("StoreMetadata", e.Metadata, mock.Anything).Return(e.Returns.Err) +} + +func (_m *mockDbOperation) ApplyStoreMetadataExpectations(expectations []dbOperationStoreMetadataExpectation) { + for _, e := range expectations { + _m.ApplyStoreMetadataExpectation(e) + } +} + +// StoreMetadata provides a mock function with given fields: metadata, dir +func (_m *mockDbOperation) StoreMetadata(metadata db.Metadata, dir string) error { + ret := _m.Called(metadata, dir) + + var r0 error + if rf, ok := ret.Get(0).(func(db.Metadata, string) error); ok { + r0 = rf(metadata, dir) + } else { + r0 = ret.Error(0) + } + + return r0 +}