gitea-mirror/trivy
1
0
Fork 0
mirror of https://github.com/aquasecurity/trivy.git synced 2025-12-23 07:29:00 -08:00
Code Issues Packages Projects Releases Wiki Activity

feat(cache): based on JSON (#398)

Browse Source 
* refactor(docker_conf): rename and remove unnecessary options

* feat(rpc): define new API

* fix(cli): change default timeout

* fix(import): fix package names

* refactor(vulnerability): remove old mock

* refactor(utils): remove un-needed functions

* feat(cache): implement cache communicating with a server

* refactor(scan): separate scan function as local scanner

* test(scanner): add tests for ScanImage

* refactor(scan): remove unused options

* test(vulnerability): generate mock

* refactor(server): split a file

* feat(server): implement new RPC server

* feat(client): implement new RPC client

* fix(cache): use new cache interface

* fix(standalone): use new scanner

* fix(client): use new scanner

* fix(server): pass cache

* test(integration): make sure an error is not nil before calling the method

* fix(mod): update dependencies

* test(integration): ensure the image load finishes

* feat(docker): support DOCKER_HOST and DOCKER_CERT_PATH

* chore(mod): update dependencies

* refactor(rpc): remove old client

* feat(server): support old API for backward compatibility

* fix(server): check a schema version of JSON cache

* fix(rpc): add a version to packages

* feat(rpc): add PutImage

* test: rename expectations

* refactor(cache): rename LayerCache to ImageCache

* refactor: rename ImageInfo to ImageReference

* fix(applier): pass image_id to ApplyLayer

* feat(cache): handle image cache

* chore(mod): update dependencies

* refactor(server): pass only config

* feat(cli): add -removed-pkgs option

* refactor(err): wrap errors
This commit is contained in:
Teppei Fukuda
2020-02-27 23:17:55 +02:00
committed by GitHub
parent b83174f32d
commit 18b80e3781
81 changed files with 7592 additions and 2382 deletions
Download Patch File Download Diff File Expand all files Collapse all files

652
pkg/rpc/server/server_test.go
View File

@@ -2,138 +2,255 @@ package server
import (
"context"
"io/ioutil"
"os"
"path/filepath"
"sync"
"errors"
"testing"
"time"
"github.com/golang/protobuf/ptypes/timestamp"
"github.com/golang/protobuf/ptypes"
google_protobuf "github.com/golang/protobuf/ptypes/empty"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"golang.org/x/xerrors"
"github.com/aquasecurity/trivy-db/pkg/db"
dbFile "github.com/aquasecurity/trivy/pkg/db"
"github.com/aquasecurity/trivy/pkg/log"
"github.com/aquasecurity/fanal/cache"
ftypes "github.com/aquasecurity/fanal/types"
godeptypes "github.com/aquasecurity/go-dep-parser/pkg/types"
dbTypes "github.com/aquasecurity/trivy-db/pkg/types"
"github.com/aquasecurity/trivy/pkg/report"
"github.com/aquasecurity/trivy/pkg/scanner"
"github.com/aquasecurity/trivy/pkg/types"
"github.com/aquasecurity/trivy/pkg/vulnerability"
rpcCache "github.com/aquasecurity/trivy/rpc/cache"
"github.com/aquasecurity/trivy/rpc/common"
rpcScanner "github.com/aquasecurity/trivy/rpc/scanner"
)
func TestMain(m *testing.M) {
log.InitLogger(false, false)
os.Exit(m.Run())
type mockCache struct {
cache.MockImageCache
cache.MockLocalImageCache
}
func Test_dbWorker_update(t *testing.T) {
type needsUpdateInput struct {
appVersion string
skip bool
}
type needsUpdateOutput struct {
needsUpdate bool
err error
}
type needsUpdate struct {
input needsUpdateInput
output needsUpdateOutput
}
type download struct {
call bool
err error
}
func TestScanServer_Scan(t *testing.T) {
type args struct {
appVersion string
in *rpcScanner.ScanRequest
}
tests := []struct {
name string
needsUpdate needsUpdate
download download
args args
want db.Metadata
wantErr string
name string
args args
scanExpectation scanner.ScanExpectation
fillInfoExpectation vulnerability.FillInfoExpectation
want *rpcScanner.ScanResponse
wantErr string
}{
{
name: "happy path",
needsUpdate: needsUpdate{
input: needsUpdateInput{appVersion: "1", skip: false},
output: needsUpdateOutput{needsUpdate: true},
args: args{
in: &rpcScanner.ScanRequest{
Target: "alpine:3.11",
ImageId: "sha256:e7d92cdc71feacf90708cb59182d0df1b911f8ae022d29e8e95d75ca6a99776a",
LayerIds: []string{"sha256:5216338b40a7b96416b8b9858974bbe4acc3096ee60acbc4dfb1ee02aecceb10"},
Options: &rpcScanner.ScanOptions{},
},
},
download: download{
call: true,
scanExpectation: scanner.ScanExpectation{
Args: scanner.ScanArgs{
Target: "alpine:3.11",
ImageID: "sha256:e7d92cdc71feacf90708cb59182d0df1b911f8ae022d29e8e95d75ca6a99776a",
LayerIDs: []string{"sha256:5216338b40a7b96416b8b9858974bbe4acc3096ee60acbc4dfb1ee02aecceb10"},
},
Returns: scanner.ScanReturns{
Results: report.Results{
{
Target: "alpine:3.11 (alpine 3.11)",
Vulnerabilities: []types.DetectedVulnerability{
{
VulnerabilityID: "CVE-2019-0001",
PkgName: "musl",
InstalledVersion: "1.2.3",
FixedVersion: "1.2.4",
Vulnerability: dbTypes.Vulnerability{},
},
},
},
},
OsFound: &ftypes.OS{
Family: "alpine",
Name: "3.11",
},
},
},
args: args{appVersion: "1"},
want: db.Metadata{
Version: 1,
Type: db.TypeFull,
NextUpdate: time.Date(3000, 1, 1, 0, 0, 0, 0, time.UTC),
UpdatedAt: time.Date(3000, 1, 1, 0, 0, 0, 0, time.UTC),
fillInfoExpectation: vulnerability.FillInfoExpectation{
Args: vulnerability.FillInfoArgs{
Vulns: []types.DetectedVulnerability{
{
VulnerabilityID: "CVE-2019-0001",
PkgName: "musl",
InstalledVersion: "1.2.3",
FixedVersion: "1.2.4",
Vulnerability: dbTypes.Vulnerability{},
},
},
Light: false,
},
},
want: &rpcScanner.ScanResponse{
Os: &common.OS{
Family: "alpine",
Name: "3.11",
},
Eosl: false,
Results: []*rpcScanner.Result{
{
Target: "alpine:3.11 (alpine 3.11)",
Vulnerabilities: []*common.Vulnerability{
{
VulnerabilityId: "CVE-2019-0001",
PkgName: "musl",
InstalledVersion: "1.2.3",
FixedVersion: "1.2.4",
},
},
},
},
},
},
{
name: "not update",
needsUpdate: needsUpdate{
input: needsUpdateInput{appVersion: "1", skip: false},
output: needsUpdateOutput{needsUpdate: false},
name: "sad path: Scan returns an error",
args: args{
in: &rpcScanner.ScanRequest{
Target: "alpine:3.11",
ImageId: "sha256:e7d92cdc71feacf90708cb59182d0df1b911f8ae022d29e8e95d75ca6a99776a",
LayerIds: []string{"sha256:5216338b40a7b96416b8b9858974bbe4acc3096ee60acbc4dfb1ee02aecceb10"},
Options: &rpcScanner.ScanOptions{},
},
},
args: args{appVersion: "1"},
scanExpectation: scanner.ScanExpectation{
Args: scanner.ScanArgs{
Target: "alpine:3.11",
ImageID: "sha256:e7d92cdc71feacf90708cb59182d0df1b911f8ae022d29e8e95d75ca6a99776a",
LayerIDs: []string{"sha256:5216338b40a7b96416b8b9858974bbe4acc3096ee60acbc4dfb1ee02aecceb10"},
},
Returns: scanner.ScanReturns{
Err: errors.New("error"),
},
},
wantErr: "failed scan, alpine:3.11",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mockDriver := new(scanner.MockDriver)
mockDriver.ApplyScanExpectation(tt.scanExpectation)
mockVulnClient := new(vulnerability.MockOperation)
mockVulnClient.ApplyFillInfoExpectation(tt.fillInfoExpectation)
s := NewScanServer(mockDriver, mockVulnClient)
got, err := s.Scan(context.Background(), tt.args.in)
if tt.wantErr != "" {
require.NotNil(t, err, tt.name)
assert.Contains(t, err.Error(), tt.wantErr, tt.name)
return
} else {
assert.NoError(t, err, tt.name)
}
assert.Equal(t, tt.want, got)
})
}
}
func TestCacheServer_PutImage(t *testing.T) {
type args struct {
in *rpcCache.PutImageRequest
}
tests := []struct {
name string
args args
putImage cache.ImageCachePutImageExpectation
want *google_protobuf.Empty
wantErr string
}{
{
name: "happy path",
args: args{
in: &rpcCache.PutImageRequest{
ImageId: "sha256:e7d92cdc71feacf90708cb59182d0df1b911f8ae022d29e8e95d75ca6a99776a",
ImageInfo: &rpcCache.ImageInfo{
SchemaVersion: 1,
Architecture: "amd64",
Created: func() *timestamp.Timestamp {
d := time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC)
t, _ := ptypes.TimestampProto(d)
return t
}(),
DockerVersion: "18.09",
Os: "linux",
},
},
},
putImage: cache.ImageCachePutImageExpectation{
Args: cache.ImageCachePutImageArgs{
ImageID: "sha256:e7d92cdc71feacf90708cb59182d0df1b911f8ae022d29e8e95d75ca6a99776a",
ImageInfo: ftypes.ImageInfo{
SchemaVersion: 1,
Architecture: "amd64",
Created: time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC),
DockerVersion: "18.09",
OS: "linux",
},
},
},
want: &google_protobuf.Empty{},
},
{
name: "NeedsUpdate returns an error",
needsUpdate: needsUpdate{
input: needsUpdateInput{appVersion: "1", skip: false},
output: needsUpdateOutput{err: xerrors.New("fail")},
name: "sad path",
args: args{
in: &rpcCache.PutImageRequest{
ImageId: "sha256:e7d92cdc71feacf90708cb59182d0df1b911f8ae022d29e8e95d75ca6a99776a",
ImageInfo: &rpcCache.ImageInfo{
SchemaVersion: 1,
Created: func() *timestamp.Timestamp {
d := time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC)
t, _ := ptypes.TimestampProto(d)
return t
}(),
},
},
},
args: args{appVersion: "1"},
wantErr: "failed to check if db needs an update",
putImage: cache.ImageCachePutImageExpectation{
Args: cache.ImageCachePutImageArgs{
ImageID: "sha256:e7d92cdc71feacf90708cb59182d0df1b911f8ae022d29e8e95d75ca6a99776a",
ImageInfo: ftypes.ImageInfo{
SchemaVersion: 1,
Created: time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC),
},
},
Returns: cache.ImageCachePutImageReturns{
Err: xerrors.New("error"),
},
},
wantErr: "unable to store image info in cache",
},
{
name: "Download returns an error",
needsUpdate: needsUpdate{
input: needsUpdateInput{appVersion: "1", skip: false},
output: needsUpdateOutput{needsUpdate: true},
name: "sad path: empty image info",
args: args{
in: &rpcCache.PutImageRequest{},
},
download: download{
call: true,
err: xerrors.New("fail"),
},
args: args{appVersion: "1"},
wantErr: "failed DB hot update",
wantErr: "empty image info",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cacheDir, err := ioutil.TempDir("", "server-test")
require.NoError(t, err, tt.name)
mockCache := new(mockCache)
mockCache.ApplyPutImageExpectation(tt.putImage)
require.NoError(t, db.Init(cacheDir), tt.name)
s := NewCacheServer(mockCache)
got, err := s.PutImage(context.Background(), tt.args.in)
mockDBClient := new(dbFile.MockClient)
mockDBClient.On("NeedsUpdate", mock.Anything,
tt.needsUpdate.input.appVersion, false, tt.needsUpdate.input.skip).Return(
tt.needsUpdate.output.needsUpdate, tt.needsUpdate.output.err)
if tt.download.call {
mockDBClient.On("Download", mock.Anything, mock.Anything, false).Run(
func(args mock.Arguments) {
// fake download: copy testdata/new.db to tmpDir/db/trivy.db
content, err := ioutil.ReadFile("testdata/new.db")
require.NoError(t, err, tt.name)
tmpDir := args.String(1)
dbPath := db.Path(tmpDir)
require.NoError(t, os.MkdirAll(filepath.Dir(dbPath), 0777), tt.name)
err = ioutil.WriteFile(dbPath, content, 0444)
require.NoError(t, err, tt.name)
}).Return(tt.download.err)
}
w := newDBWorker(mockDBClient)
var dbUpdateWg, requestWg sync.WaitGroup
err = w.update(context.Background(), tt.args.appVersion, cacheDir,
&dbUpdateWg, &requestWg)
if tt.wantErr != "" {
require.NotNil(t, err, tt.name)
assert.Contains(t, err.Error(), tt.wantErr, tt.name)
@@ -142,16 +259,337 @@ func Test_dbWorker_update(t *testing.T) {
assert.NoError(t, err, tt.name)
}
if !tt.download.call {
return
}
dbc := db.Config{}
got, err := dbc.GetMetadata()
assert.NoError(t, err, tt.name)
assert.Equal(t, tt.want, got, tt.name)
mockDBClient.AssertExpectations(t)
assert.Equal(t, tt.want, got)
})
}
}
func TestCacheServer_PutLayer(t *testing.T) {
type args struct {
in *rpcCache.PutLayerRequest
}
tests := []struct {
name string
args args
putLayer cache.ImageCachePutLayerExpectation
want *google_protobuf.Empty
wantErr string
}{
{
name: "happy path",
args: args{
in: &rpcCache.PutLayerRequest{
LayerId: "sha256:154ad0735c360b212b167f424d33a62305770a1fcfb6363882f5c436cfbd9812",
DecompressedLayerId: "sha256:b2a1a2d80bf0c747a4f6b0ca6af5eef23f043fcdb1ed4f3a3e750aef2dc68079",
LayerInfo: &rpcCache.LayerInfo{
SchemaVersion: 1,
Os: &common.OS{
Family: "alpine",
Name: "3.11",
},
PackageInfos: []*common.PackageInfo{
{
FilePath: "lib/apk/db/installed",
Packages: []*common.Package{
{
Name: "binary",
Version: "1.2.3",
Release: "1",
Epoch: 2,
Arch: "x86_64",
SrcName: "src",
SrcVersion: "1.2.3",
SrcRelease: "1",
SrcEpoch: 2,
},
{
Name: "vim-minimal",
Version: "7.4.160",
Release: "5.el7",
Epoch: 2,
Arch: "x86_64",
SrcName: "vim",
SrcVersion: "7.4.160",
SrcRelease: "5.el7",
SrcEpoch: 2,
},
},
},
},
Applications: []*common.Application{
{
Type: "composer",
FilePath: "php-app/composer.lock",
Libraries: []*common.Library{
{
Name: "guzzlehttp/guzzle",
Version: "6.2.0",
},
{
Name: "guzzlehttp/promises",
Version: "v1.3.1",
},
},
},
},
OpaqueDirs: []string{"etc/"},
WhiteoutFiles: []string{"etc/hostname"},
},
},
},
putLayer: cache.ImageCachePutLayerExpectation{
Args: cache.ImageCachePutLayerArgs{
LayerID: "sha256:154ad0735c360b212b167f424d33a62305770a1fcfb6363882f5c436cfbd9812",
DecompressedLayerID: "sha256:b2a1a2d80bf0c747a4f6b0ca6af5eef23f043fcdb1ed4f3a3e750aef2dc68079",
LayerInfo: ftypes.LayerInfo{
SchemaVersion: 1,
OS: &ftypes.OS{
Family: "alpine",
Name: "3.11",
},
PackageInfos: []ftypes.PackageInfo{
{
FilePath: "lib/apk/db/installed",
Packages: []ftypes.Package{
{
Name: "binary",
Version: "1.2.3",
Release: "1",
Epoch: 2,
Arch: "x86_64",
SrcName: "src",
SrcVersion: "1.2.3",
SrcRelease: "1",
SrcEpoch: 2,
},
{
Name: "vim-minimal",
Version: "7.4.160",
Release: "5.el7",
Epoch: 2,
Arch: "x86_64",
SrcName: "vim",
SrcVersion: "7.4.160",
SrcRelease: "5.el7",
SrcEpoch: 2,
},
},
},
},
Applications: []ftypes.Application{
{
Type: "composer",
FilePath: "php-app/composer.lock",
Libraries: []godeptypes.Library{
{
Name: "guzzlehttp/guzzle",
Version: "6.2.0",
},
{
Name: "guzzlehttp/promises",
Version: "v1.3.1",
},
},
},
},
OpaqueDirs: []string{"etc/"},
WhiteoutFiles: []string{"etc/hostname"},
},
},
},
want: &google_protobuf.Empty{},
},
{
name: "sad path",
args: args{
in: &rpcCache.PutLayerRequest{
LayerInfo: &rpcCache.LayerInfo{
SchemaVersion: 1,
},
},
},
putLayer: cache.ImageCachePutLayerExpectation{
Args: cache.ImageCachePutLayerArgs{
LayerIDAnything: true,
DecompressedLayerIDAnything: true,
LayerInfoAnything: true,
},
Returns: cache.ImageCachePutLayerReturns{
Err: xerrors.New("error"),
},
},
wantErr: "unable to store layer info in cache",
},
{
name: "sad path: empty layer info",
args: args{
in: &rpcCache.PutLayerRequest{},
},
putLayer: cache.ImageCachePutLayerExpectation{
Args: cache.ImageCachePutLayerArgs{
LayerIDAnything: true,
DecompressedLayerIDAnything: true,
LayerInfoAnything: true,
},
Returns: cache.ImageCachePutLayerReturns{
Err: xerrors.New("error"),
},
},
wantErr: "empty layer info",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mockCache := new(mockCache)
mockCache.ApplyPutLayerExpectation(tt.putLayer)
s := NewCacheServer(mockCache)
got, err := s.PutLayer(context.Background(), tt.args.in)
if tt.wantErr != "" {
require.NotNil(t, err, tt.name)
assert.Contains(t, err.Error(), tt.wantErr, tt.name)
return
} else {
assert.NoError(t, err, tt.name)
}
assert.Equal(t, tt.want, got)
})
}
}
func TestCacheServer_MissingLayers(t *testing.T) {
type args struct {
ctx context.Context
in *rpcCache.MissingLayersRequest
}
tests := []struct {
name string
args args
getLayerExpectations []cache.LocalImageCacheGetLayerExpectation
getImageExpectations []cache.LocalImageCacheGetImageExpectation
want *rpcCache.MissingLayersResponse
wantErr string
}{
{
name: "happy path",
args: args{
in: &rpcCache.MissingLayersRequest{
ImageId: "sha256:e7d92cdc71feacf90708cb59182d0df1b911f8ae022d29e8e95d75ca6a99776a",
LayerIds: []string{
"sha256:932da51564135c98a49a34a193d6cd363d8fa4184d957fde16c9d8527b3f3b02",
"sha256:dffd9992ca398466a663c87c92cfea2a2db0ae0cf33fcb99da60eec52addbfc5",
},
},
},
getLayerExpectations: []cache.LocalImageCacheGetLayerExpectation{
{
Args: cache.LocalImageCacheGetLayerArgs{
LayerID: "sha256:932da51564135c98a49a34a193d6cd363d8fa4184d957fde16c9d8527b3f3b02",
},
Returns: cache.LocalImageCacheGetLayerReturns{
LayerInfo: ftypes.LayerInfo{},
},
},
{
Args: cache.LocalImageCacheGetLayerArgs{
LayerID: "sha256:dffd9992ca398466a663c87c92cfea2a2db0ae0cf33fcb99da60eec52addbfc5",
},
Returns: cache.LocalImageCacheGetLayerReturns{
LayerInfo: ftypes.LayerInfo{
SchemaVersion: 1,
},
},
},
},
getImageExpectations: []cache.LocalImageCacheGetImageExpectation{
{
Args: cache.LocalImageCacheGetImageArgs{
ImageID: "sha256:e7d92cdc71feacf90708cb59182d0df1b911f8ae022d29e8e95d75ca6a99776a",
},
Returns: cache.LocalImageCacheGetImageReturns{
ImageInfo: ftypes.ImageInfo{
SchemaVersion: 1,
},
},
},
},
want: &rpcCache.MissingLayersResponse{
MissingImage: false,
MissingLayerIds: []string{"sha256:932da51564135c98a49a34a193d6cd363d8fa4184d957fde16c9d8527b3f3b02"},
},
},
{
name: "schema version doesn't match",
args: args{
in: &rpcCache.MissingLayersRequest{
ImageId: "sha256:e7d92cdc71feacf90708cb59182d0df1b911f8ae022d29e8e95d75ca6a99776a",
LayerIds: []string{
"sha256:932da51564135c98a49a34a193d6cd363d8fa4184d957fde16c9d8527b3f3b02",
"sha256:dffd9992ca398466a663c87c92cfea2a2db0ae0cf33fcb99da60eec52addbfc5",
},
},
},
getLayerExpectations: []cache.LocalImageCacheGetLayerExpectation{
{
Args: cache.LocalImageCacheGetLayerArgs{
LayerID: "sha256:932da51564135c98a49a34a193d6cd363d8fa4184d957fde16c9d8527b3f3b02",
},
Returns: cache.LocalImageCacheGetLayerReturns{
LayerInfo: ftypes.LayerInfo{
SchemaVersion: 0,
},
},
},
{
Args: cache.LocalImageCacheGetLayerArgs{
LayerID: "sha256:dffd9992ca398466a663c87c92cfea2a2db0ae0cf33fcb99da60eec52addbfc5",
},
Returns: cache.LocalImageCacheGetLayerReturns{
LayerInfo: ftypes.LayerInfo{
SchemaVersion: -1,
},
},
},
},
getImageExpectations: []cache.LocalImageCacheGetImageExpectation{
{
Args: cache.LocalImageCacheGetImageArgs{
ImageID: "sha256:e7d92cdc71feacf90708cb59182d0df1b911f8ae022d29e8e95d75ca6a99776a",
},
Returns: cache.LocalImageCacheGetImageReturns{
ImageInfo: ftypes.ImageInfo{},
},
},
},
want: &rpcCache.MissingLayersResponse{
MissingImage: true,
MissingLayerIds: []string{
"sha256:932da51564135c98a49a34a193d6cd363d8fa4184d957fde16c9d8527b3f3b02",
"sha256:dffd9992ca398466a663c87c92cfea2a2db0ae0cf33fcb99da60eec52addbfc5",
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mockCache := new(mockCache)
mockCache.ApplyGetLayerExpectations(tt.getLayerExpectations)
mockCache.ApplyGetImageExpectations(tt.getImageExpectations)
s := NewCacheServer(mockCache)
got, err := s.MissingLayers(tt.args.ctx, tt.args.in)
if tt.wantErr != "" {
require.NotNil(t, err, tt.name)
assert.Contains(t, err.Error(), tt.wantErr, tt.name)
return
} else {
assert.NoError(t, err, tt.name)
}
assert.Equal(t, tt.want, got)
mockCache.MockLocalImageCache.AssertExpectations(t)
})
}
}