gitea-mirror/trivy
1
0
Fork 0
mirror of https://github.com/aquasecurity/trivy.git synced 2025-12-21 23:00:42 -08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add auth support for downloading OCI artifacts (#3915)

Browse Source
This commit is contained in:
Teppei Fukuda
2023-03-30 05:53:24 +03:00
committed by GitHub
parent 1ee05189f0
commit f14bed4532
22 changed files with 286 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
docs/docs/vulnerability/db.md Normal file
View File

@@ -0,0 +1,38 @@
# Database
Trivy uses two types of databases for vulnerability detection:
- Vulnerability Database
- Java Index Database
This page provides detailed information about these databases.
## Vulnerability Database
Trivy utilizes a database containing vulnerability information.
This database is built every six hours on [GitHub](https://github.com/aquasecurity/trivy-db) and is distributed via [GitHub Container registry (GHCR)](https://ghcr.io/aquasecurity/trivy-db).
The database is cached and updated as needed.
As Trivy updates the database automatically during execution, users don't need to be concerned about it.
For CLI flags related to the database, please refer to [this page](./examples/db.md).
### Private Hosting
If you host the database on your own OCI registry, you can specify a different repository with the `--db-repository` flag.
The default is `ghcr.io/aquasecurity/trivy-db`.
```shell
$ trivy image --db-repository YOUR_REPO YOUR_IMAGE
```
If authentication is required, it can be configured in the same way as for private images.
Please refer to [the documentation](../advanced/private-registries/index.md) for more details.
## Java Index Database
This database is only downloaded when scanning JAR files so that Trivy can identify the groupId, artifactId, and version of JAR files.
It is built once a day on [GitHub](https://github.com/aquasecurity/trivy-java-db) and distributed via [GitHub Container registry (GHCR)](https://ghcr.io/aquasecurity/trivy-java-db).
Like the vulnerability database, it is automatically downloaded and updated when needed, so users don't need to worry about it.
### Private Hosting
If you host the database on your own OCI registry, you can specify a different repository with the `--java-db-repository` flag.
The default is `ghcr.io/aquasecurity/trivy-java-db`.
If authentication is required, you need to run `docker login YOUR_REGISTRY`.
Currently, specifying a username and password is not supported.

4
docs/docs/vulnerability/examples/db.md
View File

@@ -1,9 +1,7 @@
# Vulnerability DB
## Skip update of vulnerability DB
`Trivy` downloads its vulnerability database every 12 hours when it starts operating.
This is usually fast, as the size of the DB is only 10~30MB.
But if you want to skip even that, use the `--skip-db-update` option.
If you want to skip downloading the vulnerability database, use the `--skip-db-update` option.
```
$ trivy image --skip-db-update python:3.4-alpine3.9

1
mkdocs.yml
View File

@@ -52,6 +52,7 @@ nav:
- OS Packages: docs/vulnerability/detection/os.md
- Language-specific Packages: docs/vulnerability/detection/language.md
- Data Sources: docs/vulnerability/detection/data-source.md
- Database: docs/vulnerability/db.md
- Examples:
- Vulnerability Filtering: docs/vulnerability/examples/filter.md
- Report Formats: docs/vulnerability/examples/report.md

2
pkg/commands/app.go
View File

@@ -759,7 +759,7 @@ func NewModuleCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command {
if err != nil {
return xerrors.Errorf("flag error: %w", err)
}
return module.Install(cmd.Context(), opts.ModuleDir, repo, opts.Quiet, opts.Insecure)
return module.Install(cmd.Context(), opts.ModuleDir, repo, opts.Quiet, opts.Remote())
},
},
&cobra.Command{

15
pkg/commands/artifact/run.go
View File

@@ -19,7 +19,6 @@ import (
"github.com/aquasecurity/trivy/pkg/fanal/analyzer/config"
"github.com/aquasecurity/trivy/pkg/fanal/artifact"
"github.com/aquasecurity/trivy/pkg/fanal/cache"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/flag"
"github.com/aquasecurity/trivy/pkg/javadb"
"github.com/aquasecurity/trivy/pkg/log"
@@ -315,7 +314,7 @@ func (r *runner) initDB(opts flag.Options) error {
// download the database file
noProgress := opts.Quiet || opts.NoProgress
if err := operation.DownloadDB(opts.AppVersion, opts.CacheDir, opts.DBRepository, noProgress, opts.Insecure, opts.SkipDBUpdate); err != nil {
if err := operation.DownloadDB(opts.AppVersion, opts.CacheDir, opts.DBRepository, noProgress, opts.SkipDBUpdate, opts.Remote()); err != nil {
return err
}
@@ -610,13 +609,7 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
}
}
remoteOption := ftypes.RemoteOptions{
Credentials: opts.Credentials,
RegistryToken: opts.RegistryToken,
Insecure: opts.Insecure,
Platform: opts.Platform,
AWSRegion: opts.AWSOptions.Region,
}
remoteOpts := opts.Remote()
return ScannerConfig{
Target: target,
@@ -643,8 +636,8 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
Slow: opts.Slow,
AWSRegion: opts.Region,
// For container registries
RemoteOptions: remoteOption,
// For OCI registries
RemoteOptions: remoteOpts,
// For misconfiguration scanning
MisconfScannerOption: configScannerOptions,

9
pkg/commands/operation/operation.go
View File

@@ -15,6 +15,7 @@ import (
"github.com/aquasecurity/trivy-db/pkg/metadata"
"github.com/aquasecurity/trivy/pkg/db"
"github.com/aquasecurity/trivy/pkg/fanal/cache"
"github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/flag"
"github.com/aquasecurity/trivy/pkg/log"
"github.com/aquasecurity/trivy/pkg/policy"
@@ -101,8 +102,8 @@ func (c Cache) ClearArtifacts() error {
}
// DownloadDB downloads the DB
func DownloadDB(appVersion, cacheDir, dbRepository string, quiet, insecure, skipUpdate bool) error {
client := db.NewClient(cacheDir, quiet, insecure, db.WithDBRepository(dbRepository))
func DownloadDB(appVersion, cacheDir, dbRepository string, quiet, skipUpdate bool, opt types.RemoteOptions) error {
client := db.NewClient(cacheDir, quiet, db.WithDBRepository(dbRepository))
ctx := context.Background()
needsUpdate, err := client.NeedsUpdate(appVersion, skipUpdate)
if err != nil {
@@ -113,7 +114,7 @@ func DownloadDB(appVersion, cacheDir, dbRepository string, quiet, insecure, skip
log.Logger.Info("Need to update DB")
log.Logger.Infof("DB Repository: %s", dbRepository)
log.Logger.Info("Downloading DB...")
if err = client.Download(ctx, cacheDir); err != nil {
if err = client.Download(ctx, cacheDir, opt); err != nil {
return xerrors.Errorf("failed to download vulnerability DB: %w", err)
}
}
@@ -145,7 +146,7 @@ func InitBuiltinPolicies(ctx context.Context, cacheDir string, quiet, skipUpdate
needsUpdate := false
if !skipUpdate {
needsUpdate, err = client.NeedsUpdate()
needsUpdate, err = client.NeedsUpdate(ctx)
if err != nil {
return nil, xerrors.Errorf("unable to check if built-in policies need to be updated: %w", err)
}

7
pkg/commands/server/run.go
View File

@@ -35,7 +35,7 @@ func Run(ctx context.Context, opts flag.Options) (err error) {
// download the database file
if err = operation.DownloadDB(opts.AppVersion, opts.CacheDir, opts.DBRepository,
true, opts.Insecure, opts.SkipDBUpdate); err != nil {
true, opts.SkipDBUpdate, opts.Remote()); err != nil {
return err
}
@@ -57,6 +57,7 @@ func Run(ctx context.Context, opts flag.Options) (err error) {
}
m.Register()
server := rpcServer.NewServer(opts.AppVersion, opts.Listen, opts.CacheDir, opts.Token, opts.TokenHeader, opts.DBRepository)
return server.ListenAndServe(cache, opts.Insecure, opts.SkipDBUpdate)
server := rpcServer.NewServer(opts.AppVersion, opts.Listen, opts.CacheDir, opts.Token, opts.TokenHeader,
opts.DBRepository, opts.Remote())
return server.ListenAndServe(cache, opts.SkipDBUpdate)
}

31
pkg/db/db.go
View File

@@ -12,6 +12,7 @@ import (
"github.com/aquasecurity/trivy-db/pkg/db"
"github.com/aquasecurity/trivy-db/pkg/metadata"
"github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/log"
"github.com/aquasecurity/trivy/pkg/oci"
)
@@ -24,7 +25,7 @@ const (
// Operation defines the DB operations
type Operation interface {
NeedsUpdate(cliVersion string, skip bool) (need bool, err error)
Download(ctx context.Context, dst string) (err error)
Download(ctx context.Context, dst string, opt types.RemoteOptions) (err error)
}
type options struct {
@@ -61,14 +62,13 @@ func WithClock(clock clock.Clock) Option {
type Client struct {
*options
cacheDir string
metadata metadata.Client
quiet bool
insecureSkipTLSVerify bool
cacheDir string
metadata metadata.Client
quiet bool
}
// NewClient is the factory method for DB client
func NewClient(cacheDir string, quiet, insecure bool, opts ...Option) *Client {
func NewClient(cacheDir string, quiet bool, opts ...Option) *Client {
o := &options{
clock: clock.RealClock{},
dbRepository: defaultDBRepository,
@@ -79,11 +79,10 @@ func NewClient(cacheDir string, quiet, insecure bool, opts ...Option) *Client {
}
return &Client{
options: o,
cacheDir: cacheDir,
metadata: metadata.NewClient(cacheDir),
quiet: quiet,
insecureSkipTLSVerify: insecure, // insecure skip for download DB
options: o,
cacheDir: cacheDir,
metadata: metadata.NewClient(cacheDir),
quiet: quiet,
}
}
@@ -144,18 +143,18 @@ func (c *Client) isNewDB(meta metadata.Metadata) bool {
}
// Download downloads the DB file
func (c *Client) Download(ctx context.Context, dst string) error {
func (c *Client) Download(ctx context.Context, dst string, opt types.RemoteOptions) error {
// Remove the metadata file under the cache directory before downloading DB
if err := c.metadata.Delete(); err != nil {
log.Logger.Debug("no metadata file")
}
art, err := c.initOCIArtifact()
art, err := c.initOCIArtifact(opt)
if err != nil {
return xerrors.Errorf("OCI artifact error: %w", err)
}
if err = art.Download(ctx, db.Dir(dst)); err != nil {
if err = art.Download(ctx, db.Dir(dst), oci.DownloadOption{MediaType: dbMediaType}); err != nil {
return xerrors.Errorf("database download error: %w", err)
}
@@ -184,13 +183,13 @@ func (c *Client) updateDownloadedAt(dst string) error {
return nil
}
func (c *Client) initOCIArtifact() (*oci.Artifact, error) {
func (c *Client) initOCIArtifact(opt types.RemoteOptions) (*oci.Artifact, error) {
if c.artifact != nil {
return c.artifact, nil
}
repo := fmt.Sprintf("%s:%d", c.dbRepository, db.SchemaVersion)
art, err := oci.NewArtifact(repo, dbMediaType, "", c.quiet, c.insecureSkipTLSVerify)
art, err := oci.NewArtifact(repo, c.quiet, opt)
if err != nil {
var terr *transport.Error
if errors.As(err, &terr) {

12
pkg/db/db_test.go
View File

@@ -18,6 +18,7 @@ import (
tdb "github.com/aquasecurity/trivy-db/pkg/db"
"github.com/aquasecurity/trivy-db/pkg/metadata"
"github.com/aquasecurity/trivy/pkg/db"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/oci"
)
@@ -152,7 +153,7 @@ func TestClient_NeedsUpdate(t *testing.T) {
require.NoError(t, err)
}
client := db.NewClient(cacheDir, true, false, db.WithClock(tt.clock))
client := db.NewClient(cacheDir, true, db.WithClock(tt.clock))
needsUpdate, err := client.NeedsUpdate("test", tt.skip)
switch {
@@ -218,11 +219,14 @@ func TestClient_Download(t *testing.T) {
}, nil)
// Mock OCI artifact
art, err := oci.NewArtifact("db", mediaType, "", true, false, oci.WithImage(img))
opt := ftypes.RemoteOptions{
Insecure: false,
}
art, err := oci.NewArtifact("db", true, opt, oci.WithImage(img))
require.NoError(t, err)
client := db.NewClient(cacheDir, true, false, db.WithOCIArtifact(art), db.WithClock(timeDownloadedAt))
err = client.Download(context.Background(), cacheDir)
client := db.NewClient(cacheDir, true, db.WithOCIArtifact(art), db.WithClock(timeDownloadedAt))
err = client.Download(context.Background(), cacheDir, opt)
if tt.wantErr != "" {
require.Error(t, err)
assert.Contains(t, err.Error(), tt.wantErr)

6
pkg/db/mock_operation.go
View File

@@ -6,6 +6,8 @@ import (
context "context"
mock "github.com/stretchr/testify/mock"
"github.com/aquasecurity/trivy/pkg/fanal/types"
)
// MockOperation is an autogenerated mock type for the Operation type
@@ -51,8 +53,8 @@ func (_m *MockOperation) ApplyDownloadExpectations(expectations []OperationDownl
}
// Download provides a mock function with given fields: ctx, dst
func (_m *MockOperation) Download(ctx context.Context, dst string) error {
ret := _m.Called(ctx, dst)
func (_m *MockOperation) Download(ctx context.Context, dst string, opt types.RemoteOptions) error {
ret := _m.Called(ctx, dst, opt)
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, string) error); ok {

2
pkg/fanal/artifact/artifact.go
View File

@@ -28,7 +28,7 @@ type Option struct {
Slow bool // Lower CPU and memory
AWSRegion string
// For container registries
// For OCI registries
types.RemoteOptions
MisconfScannerOption misconf.ScannerOption

7
pkg/fanal/artifact/image/remote_sbom.go
View File

@@ -81,7 +81,7 @@ func (a Artifact) inspectOCIReferrerSBOM(ctx context.Context) (ftypes.ArtifactRe
func (a Artifact) parseReferrer(ctx context.Context, repo string, desc v1.Descriptor) (ftypes.ArtifactReference, error) {
const fileName string = "referrer.sbom"
repoName := fmt.Sprintf("%s@%s", repo, desc.Digest)
referrer, err := oci.NewArtifact(repoName, desc.ArtifactType, fileName, true, a.artifactOption.Insecure)
referrer, err := oci.NewArtifact(repoName, true, a.artifactOption.RemoteOptions)
if err != nil {
return ftypes.ArtifactReference{}, xerrors.Errorf("OCI error: %w", err)
}
@@ -93,7 +93,10 @@ func (a Artifact) parseReferrer(ctx context.Context, repo string, desc v1.Descri
defer os.RemoveAll(tmpDir)
// Download SBOM to local filesystem
if err = referrer.Download(ctx, tmpDir); err != nil {
if err = referrer.Download(ctx, tmpDir, oci.DownloadOption{
MediaType: desc.ArtifactType,
Filename: fileName,
}); err != nil {
return ftypes.ArtifactReference{}, xerrors.Errorf("SBOM download error: %w", err)
}

12
pkg/flag/options.go
View File

@@ -15,6 +15,7 @@ import (
"golang.org/x/xerrors"
"github.com/aquasecurity/trivy/pkg/fanal/analyzer"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/log"
"github.com/aquasecurity/trivy/pkg/report"
)
@@ -121,6 +122,17 @@ func (o *Options) Align() {
}
}
// Remote returns options for OCI registries
func (o *Options) Remote() ftypes.RemoteOptions {
return ftypes.RemoteOptions{
Credentials: o.Credentials,
RegistryToken: o.RegistryToken,
Insecure: o.Insecure,
Platform: o.Platform,
AWSRegion: o.AWSOptions.Region,
}
}
func addFlag(cmd *cobra.Command, flag *Flag) {
if flag == nil || flag.Name == "" {
return

6
pkg/javadb/client.go
View File

@@ -14,6 +14,7 @@ import (
"github.com/aquasecurity/go-dep-parser/pkg/java/jar"
"github.com/aquasecurity/trivy-java-db/pkg/db"
"github.com/aquasecurity/trivy-java-db/pkg/types"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/log"
"github.com/aquasecurity/trivy/pkg/oci"
)
@@ -51,11 +52,12 @@ func (u *Updater) Update() error {
log.Logger.Infof("Java DB Repository: %s", u.repo)
log.Logger.Info("Downloading the Java DB...")
// TODO: support remote options
var a *oci.Artifact
if a, err = oci.NewArtifact(u.repo, mediaType, "", u.quiet, u.insecure); err != nil {
if a, err = oci.NewArtifact(u.repo, u.quiet, ftypes.RemoteOptions{}); err != nil {
return xerrors.Errorf("oci error: %w", err)
}
if err = a.Download(context.Background(), dbDir); err != nil {
if err = a.Download(context.Background(), dbDir, oci.DownloadOption{MediaType: mediaType}); err != nil {
return xerrors.Errorf("DB download error: %w", err)
}

7
pkg/module/command.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/google/go-containerregistry/pkg/name"
"golang.org/x/xerrors"
"github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/log"
"github.com/aquasecurity/trivy/pkg/oci"
)
@@ -15,14 +16,14 @@ import (
const mediaType = "application/vnd.module.wasm.content.layer.v1+wasm"
// Install installs a module
func Install(ctx context.Context, dir, repo string, quiet, insecure bool) error {
func Install(ctx context.Context, dir, repo string, quiet bool, opt types.RemoteOptions) error {
ref, err := name.ParseReference(repo)
if err != nil {
return xerrors.Errorf("repository parse error: %w", err)
}
log.Logger.Infof("Installing the module from %s...", repo)
artifact, err := oci.NewArtifact(repo, mediaType, "", quiet, insecure)
artifact, err := oci.NewArtifact(repo, quiet, opt)
if err != nil {
return xerrors.Errorf("module initialize error: %w", err)
}
@@ -30,7 +31,7 @@ func Install(ctx context.Context, dir, repo string, quiet, insecure bool) error
dst := filepath.Join(dir, ref.Context().Name())
log.Logger.Debugf("Installing the module to %s...", dst)
if err = artifact.Download(ctx, dst); err != nil {
if err = artifact.Download(ctx, dst, oci.DownloadOption{MediaType: mediaType}); err != nil {
return xerrors.Errorf("module download error: %w", err)
}

134
pkg/oci/artifact.go
View File

@@ -2,20 +2,19 @@ package oci
import (
"context"
"crypto/tls"
"io"
"net/http"
"os"
"path/filepath"
"sync"
"github.com/cheggaaa/pb/v3"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/remote"
"golang.org/x/xerrors"
"github.com/aquasecurity/trivy/pkg/downloader"
"github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/remote"
)
const (
@@ -35,79 +34,95 @@ var SupportedSBOMArtifactTypes = []string{
SPDXArtifactType,
}
type options struct {
img v1.Image
}
// Option is a functional option
type Option func(*options)
type Option func(*Artifact)
// WithImage takes an OCI v1 Image
func WithImage(img v1.Image) Option {
return func(opts *options) {
opts.img = img
return func(a *Artifact) {
a.image = img
}
}
// Artifact is used to download artifacts such as vulnerability database and policies from OCI registries.
type Artifact struct {
fileName string
image v1.Image
layer v1.Layer // Take the first layer as OCI artifact
quiet bool
m sync.Mutex
repository string
quiet bool
// For OCI registries
types.RemoteOptions
image v1.Image // For testing
}
// NewArtifact returns a new artifact
func NewArtifact(repo, mediaType, fileName string, quiet, insecure bool, opts ...Option) (*Artifact, error) {
o := &options{}
for _, opt := range opts {
opt(o)
func NewArtifact(repo string, quiet bool, remoteOpt types.RemoteOptions, opts ...Option) (*Artifact, error) {
art := &Artifact{
repository: repo,
quiet: quiet,
RemoteOptions: remoteOpt,
}
if o.img == nil {
ref, err := name.ParseReference(repo)
if err != nil {
return nil, xerrors.Errorf("repository name error (%s): %w", repo, err)
}
for _, o := range opts {
o(art)
}
return art, nil
}
remoteOpts := []remote.Option{remote.WithAuthFromKeychain(authn.DefaultKeychain)}
if insecure {
t := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
Proxy: http.ProxyFromEnvironment,
}
remoteOpts = append(remoteOpts, remote.WithTransport(t))
}
o.img, err = remote.Image(ref, remoteOpts...)
if err != nil {
return nil, xerrors.Errorf("OCI repository error: %w", err)
}
func (a *Artifact) populate(ctx context.Context, opt types.RemoteOptions) error {
if a.image != nil {
return nil
}
layers, err := o.img.Layers()
a.m.Lock()
defer a.m.Unlock()
ref, err := name.ParseReference(a.repository)
if err != nil {
return nil, xerrors.Errorf("OCI layer error: %w", err)
return xerrors.Errorf("repository name error (%s): %w", a.repository, err)
}
manifest, err := o.img.Manifest()
a.image, err = remote.Image(ctx, ref, opt)
if err != nil {
return nil, xerrors.Errorf("OCI manifest error: %w", err)
return xerrors.Errorf("OCI repository error: %w", err)
}
return nil
}
type DownloadOption struct {
MediaType string // Accept any media type if not specified
Filename string // Use the annotation if not specified
}
func (a *Artifact) Download(ctx context.Context, dir string, opt DownloadOption) error {
if err := a.populate(ctx, a.RemoteOptions); err != nil {
return err
}
layers, err := a.image.Layers()
if err != nil {
return xerrors.Errorf("OCI layer error: %w", err)
}
manifest, err := a.image.Manifest()
if err != nil {
return xerrors.Errorf("OCI manifest error: %w", err)
}
// A single layer is only supported now.
if len(layers) != 1 || len(manifest.Layers) != 1 {
return nil, xerrors.Errorf("OCI artifact must be a single layer")
return xerrors.Errorf("OCI artifact must be a single layer")
}
// Take the first layer
layer := layers[0]
// Take the file name of the first layer if not specified
fileName := opt.Filename
if fileName == "" {
if v, ok := manifest.Layers[0].Annotations[titleAnnotation]; !ok {
return nil, xerrors.Errorf("annotation %s is missing", titleAnnotation)
return xerrors.Errorf("annotation %s is missing", titleAnnotation)
} else {
fileName = v
}
@@ -115,26 +130,25 @@ func NewArtifact(repo, mediaType, fileName string, quiet, insecure bool, opts ..
layerMediaType, err := layer.MediaType()
if err != nil {
return nil, xerrors.Errorf("media type error: %w", err)
} else if mediaType != string(layerMediaType) {
return nil, xerrors.Errorf("unacceptable media type: %s", string(layerMediaType))
return xerrors.Errorf("media type error: %w", err)
} else if opt.MediaType != "" && opt.MediaType != string(layerMediaType) {
return xerrors.Errorf("unacceptable media type: %s", string(layerMediaType))
}
return &Artifact{
fileName: fileName,
image: o.img,
layer: layer,
quiet: quiet,
}, nil
if err = a.download(ctx, layer, fileName, dir); err != nil {
return xerrors.Errorf("oci download error: %w", err)
}
return nil
}
func (a Artifact) Download(ctx context.Context, dir string) error {
size, err := a.layer.Size()
func (a *Artifact) download(ctx context.Context, layer v1.Layer, fileName, dir string) error {
size, err := layer.Size()
if err != nil {
return xerrors.Errorf("size error: %w", err)
}
rc, err := a.layer.Compressed()
rc, err := layer.Compressed()
if err != nil {
return xerrors.Errorf("failed to fetch the layer: %w", err)
}
@@ -154,7 +168,7 @@ func (a Artifact) Download(ctx context.Context, dir string) error {
return xerrors.Errorf("failed to create a temp dir: %w", err)
}
f, err := os.Create(filepath.Join(tempDir, a.fileName))
f, err := os.Create(filepath.Join(tempDir, fileName))
if err != nil {
return xerrors.Errorf("failed to create a temp file: %w", err)
}
@@ -176,7 +190,11 @@ func (a Artifact) Download(ctx context.Context, dir string) error {
return nil
}
func (a Artifact) Digest() (string, error) {
func (a *Artifact) Digest(ctx context.Context) (string, error) {
if err := a.populate(ctx, a.RemoteOptions); err != nil {
return "", err
}
digest, err := a.image.Digest()
if err != nil {
return "", xerrors.Errorf("digest error: %w", err)

91
pkg/oci/artifact_test.go
View File

@@ -14,6 +14,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/oci"
"github.com/aquasecurity/trivy/pkg/utils/fsutils"
)
@@ -26,10 +27,13 @@ func (f fakeLayer) MediaType() (types.MediaType, error) {
return "application/vnd.cncf.openpolicyagent.layer.v1.tar+gzip", nil
}
func TestNewArtifact(t *testing.T) {
func TestArtifact_Download(t *testing.T) {
layer, err := tarball.LayerFromFile("testdata/test.tar.gz")
require.NoError(t, err)
txtLayer, err := tarball.LayerFromFile("testdata/test.txt")
require.NoError(t, err)
flayer := fakeLayer{layer}
type layersReturns struct {
@@ -38,16 +42,20 @@ func TestNewArtifact(t *testing.T) {
}
tests := []struct {
name string
input string
mediaType string
layersReturns layersReturns
want string
wantErr string
}{
{
name: "happy path",
input: "testdata/test.tar.gz",
mediaType: "application/vnd.cncf.openpolicyagent.layer.v1.tar+gzip",
layersReturns: layersReturns{
layers: []v1.Layer{flayer},
},
want: "Hello, world",
},
{
name: "sad: two layers",
@@ -68,6 +76,14 @@ func TestNewArtifact(t *testing.T) {
},
wantErr: "OCI layer error",
},
{
name: "invalid gzip",
input: "testdata/test.txt",
layersReturns: layersReturns{
layers: []v1.Layer{txtLayer},
},
wantErr: "unexpected EOF",
},
{
name: "sad: media type doesn't match",
mediaType: "unknown",
@@ -102,73 +118,14 @@ func TestNewArtifact(t *testing.T) {
},
}, nil)
_, err = oci.NewArtifact("repo", tt.mediaType, "", true, false, oci.WithImage(img))
artifact, err := oci.NewArtifact("repo", true, ftypes.RemoteOptions{}, oci.WithImage(img))
require.NoError(t, err)
err = artifact.Download(context.Background(), tempDir, oci.DownloadOption{
MediaType: tt.mediaType,
})
if tt.wantErr != "" {
require.Error(t, err)
assert.Contains(t, err.Error(), tt.wantErr)
return
}
require.NoError(t, err)
})
}
}
func TestArtifact_Download(t *testing.T) {
tests := []struct {
name string
input string
want string
wantErr string
}{
{
name: "happy path",
input: "testdata/test.tar.gz",
want: "Hello, world",
},
{
name: "invalid gzip",
input: "testdata/test.txt",
wantErr: "unexpected EOF",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tempDir := t.TempDir()
fsutils.SetCacheDir(tempDir)
// Mock layer
layer, err := tarball.LayerFromFile(tt.input)
require.NoError(t, err)
flayer := fakeLayer{layer}
// Mock image
img := new(fakei.FakeImage)
img.LayersReturns([]v1.Layer{flayer}, nil)
img.ManifestReturns(&v1.Manifest{
Layers: []v1.Descriptor{
{
MediaType: "application/vnd.cncf.openpolicyagent.layer.v1.tar+gzip",
Size: 100,
Digest: v1.Hash{
Algorithm: "sha256",
Hex: "cba33656188782852f58993f45b68bfb8577f64cdcf02a604e3fc2afbeb5f2d8",
},
Annotations: map[string]string{
"org.opencontainers.image.title": "bundle.tar.gz",
},
},
},
}, nil)
mediaType := "application/vnd.cncf.openpolicyagent.layer.v1.tar+gzip"
artifact, err := oci.NewArtifact("repo", mediaType, "", true, false, oci.WithImage(img))
require.NoError(t, err)
err = artifact.Download(context.Background(), tempDir)
if tt.wantErr != "" {
require.Error(t, err)
assert.Contains(t, err.Error(), tt.wantErr)
assert.ErrorContains(t, err, tt.wantErr)
return
}
require.NoError(t, err)

19
pkg/policy/policy.go
View File

@@ -9,12 +9,12 @@ import (
"time"
"github.com/open-policy-agent/opa/bundle"
"github.com/aquasecurity/trivy/pkg/log"
"github.com/aquasecurity/trivy/pkg/oci"
"golang.org/x/xerrors"
"k8s.io/utils/clock"
"github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/log"
"github.com/aquasecurity/trivy/pkg/oci"
)
const (
@@ -51,7 +51,6 @@ type Client struct {
*options
policyDir string
quiet bool
insecure bool
}
// Metadata holds default policy metadata
@@ -80,7 +79,7 @@ func NewClient(cacheDir string, quiet bool, opts ...Option) (*Client, error) {
func (c *Client) populateOCIArtifact() error {
if c.artifact == nil {
repo := fmt.Sprintf("%s:%d", bundleRepository, bundleVersion)
art, err := oci.NewArtifact(repo, policyMediaType, "", c.quiet, c.insecure)
art, err := oci.NewArtifact(repo, c.quiet, types.RemoteOptions{})
if err != nil {
return xerrors.Errorf("OCI artifact error: %w", err)
}
@@ -96,11 +95,11 @@ func (c *Client) DownloadBuiltinPolicies(ctx context.Context) error {
}
dst := c.contentDir()
if err := c.artifact.Download(ctx, dst); err != nil {
if err := c.artifact.Download(ctx, dst, oci.DownloadOption{MediaType: policyMediaType}); err != nil {
return xerrors.Errorf("download error: %w", err)
}
digest, err := c.artifact.Digest()
digest, err := c.artifact.Digest(ctx)
if err != nil {
return xerrors.Errorf("digest error: %w", err)
}
@@ -142,7 +141,7 @@ func (c *Client) LoadBuiltinPolicies() ([]string, error) {
}
// NeedsUpdate returns if the default policy should be updated
func (c *Client) NeedsUpdate() (bool, error) {
func (c *Client) NeedsUpdate(ctx context.Context) (bool, error) {
meta, err := c.GetMetadata()
if err != nil {
return true, nil
@@ -157,7 +156,7 @@ func (c *Client) NeedsUpdate() (bool, error) {
return false, xerrors.Errorf("OPA bundle error: %w", err)
}
digest, err := c.artifact.Digest()
digest, err := c.artifact.Digest(ctx)
if err != nil {
return false, xerrors.Errorf("digest error: %w", err)
}

12
pkg/policy/policy_test.go
View File

@@ -19,6 +19,7 @@ import (
"k8s.io/utils/clock"
fake "k8s.io/utils/clock/testing"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/oci"
"github.com/aquasecurity/trivy/pkg/policy"
)
@@ -115,8 +116,7 @@ func TestClient_LoadBuiltinPolicies(t *testing.T) {
}, nil)
// Mock OCI artifact
mediaType := "application/vnd.cncf.openpolicyagent.layer.v1.tar+gzip"
art, err := oci.NewArtifact("repo", mediaType, "", true, true, oci.WithImage(img))
art, err := oci.NewArtifact("repo", true, ftypes.RemoteOptions{}, oci.WithImage(img))
require.NoError(t, err)
c, err := policy.NewClient(tt.cacheDir, true, policy.WithOCIArtifact(art))
@@ -257,15 +257,14 @@ func TestClient_NeedsUpdate(t *testing.T) {
require.NoError(t, err)
}
mediaType := "application/vnd.cncf.openpolicyagent.layer.v1.tar+gzip"
art, err := oci.NewArtifact("repo", mediaType, "", true, true, oci.WithImage(img))
art, err := oci.NewArtifact("repo", true, ftypes.RemoteOptions{}, oci.WithImage(img))
require.NoError(t, err)
c, err := policy.NewClient(tmpDir, true, policy.WithOCIArtifact(art), policy.WithClock(tt.clock))
require.NoError(t, err)
// Assert results
got, err := c.NeedsUpdate()
got, err := c.NeedsUpdate(context.Background())
assert.Equal(t, tt.wantErr, err != nil)
assert.Equal(t, tt.want, got)
})
@@ -362,8 +361,7 @@ func TestClient_DownloadBuiltinPolicies(t *testing.T) {
}, nil)
// Mock OCI artifact
mediaType := "application/vnd.cncf.openpolicyagent.layer.v1.tar+gzip"
art, err := oci.NewArtifact("repo", mediaType, "", true, true, oci.WithImage(img))
art, err := oci.NewArtifact("repo", true, ftypes.RemoteOptions{}, oci.WithImage(img))
require.NoError(t, err)
c, err := policy.NewClient(tempDir, true, policy.WithClock(tt.clock), policy.WithOCIArtifact(art))

54
pkg/remote/remote.go
View File

@@ -27,12 +27,15 @@ type Descriptor = remote.Descriptor
// Get is a wrapper of google/go-containerregistry/pkg/v1/remote.Get
// so that it can try multiple authentication methods.
func Get(ctx context.Context, ref name.Reference, option types.RemoteOptions) (*Descriptor, error) {
remoteOpts := []remote.Option{remote.WithTransport(transport(option.Insecure))}
transport := httpTransport(option.Insecure)
var errs error
// Try each authentication method until it succeeds
for _, authOpt := range authOptions(ctx, ref, option) {
// Try each authentication method until it succeeds
remoteOpts = append(remoteOpts, authOpt)
remoteOpts := []remote.Option{
remote.WithTransport(transport),
authOpt,
}
if option.Platform != "" {
s, err := parsePlatform(ref, option.Platform, remoteOpts)
@@ -58,13 +61,42 @@ func Get(ctx context.Context, ref name.Reference, option types.RemoteOptions) (*
return nil, errs
}
func Referrers(ctx context.Context, d name.Digest, option types.RemoteOptions) (*v1.IndexManifest, error) {
remoteOpts := []remote.Option{remote.WithTransport(transport(option.Insecure))}
// Image is a wrapper of google/go-containerregistry/pkg/v1/remote.Image
// so that it can try multiple authentication methods.
func Image(ctx context.Context, ref name.Reference, option types.RemoteOptions) (v1.Image, error) {
transport := httpTransport(option.Insecure)
var errs error
// Try each authentication method until it succeeds
for _, authOpt := range authOptions(ctx, ref, option) {
remoteOpts := []remote.Option{
remote.WithTransport(transport),
authOpt,
}
index, err := remote.Image(ref, remoteOpts...)
if err != nil {
errs = multierror.Append(errs, err)
continue
}
return index, nil
}
// No authentication succeeded
return nil, errs
}
// Referrers is a wrapper of google/go-containerregistry/pkg/v1/remote.Referrers
// so that it can try multiple authentication methods.
func Referrers(ctx context.Context, d name.Digest, option types.RemoteOptions) (*v1.IndexManifest, error) {
transport := httpTransport(option.Insecure)
var errs error
// Try each authentication method until it succeeds
for _, authOpt := range authOptions(ctx, d, option) {
// Try each authentication method until it succeeds
remoteOpts = append(remoteOpts, authOpt)
remoteOpts := []remote.Option{
remote.WithTransport(transport),
authOpt,
}
index, err := remote.Referrers(d, remoteOpts...)
if err != nil {
errs = multierror.Append(errs, err)
@@ -77,7 +109,7 @@ func Referrers(ctx context.Context, d name.Digest, option types.RemoteOptions) (
return nil, errs
}
func transport(insecure bool) *http.Transport {
func httpTransport(insecure bool) *http.Transport {
d := &net.Dialer{
Timeout: 10 * time.Minute,
}
@@ -105,13 +137,13 @@ func authOptions(ctx context.Context, ref name.Reference, option types.RemoteOpt
}
switch {
case len(opts) > 0:
return opts
case option.RegistryToken != "":
bearer := authn.Bearer{Token: option.RegistryToken}
return []remote.Option{remote.WithAuth(&bearer)}
default:
return []remote.Option{remote.WithAuthFromKeychain(authn.DefaultKeychain)}
// Use the keychain anyway at the end
opts = append(opts, remote.WithAuthFromKeychain(authn.DefaultKeychain))
return opts
}
}

33
pkg/rpc/server/listen.go
View File

@@ -16,6 +16,7 @@ import (
dbFile "github.com/aquasecurity/trivy/pkg/db"
dbc "github.com/aquasecurity/trivy/pkg/db"
"github.com/aquasecurity/trivy/pkg/fanal/cache"
"github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/log"
"github.com/aquasecurity/trivy/pkg/utils/fsutils"
rpcCache "github.com/aquasecurity/trivy/rpc/cache"
@@ -32,31 +33,35 @@ type Server struct {
token string
tokenHeader string
dbRepository string
// For OCI registries
types.RemoteOptions
}
// NewServer returns an instance of Server
func NewServer(appVersion, addr, cacheDir, token, tokenHeader, dbRepository string) Server {
func NewServer(appVersion, addr, cacheDir, token, tokenHeader, dbRepository string, opt types.RemoteOptions) Server {
return Server{
appVersion: appVersion,
addr: addr,
cacheDir: cacheDir,
token: token,
tokenHeader: tokenHeader,
dbRepository: dbRepository,
appVersion: appVersion,
addr: addr,
cacheDir: cacheDir,
token: token,
tokenHeader: tokenHeader,
dbRepository: dbRepository,
RemoteOptions: opt,
}
}
// ListenAndServe starts Trivy server
func (s Server) ListenAndServe(serverCache cache.Cache, insecure, skipDBUpdate bool) error {
func (s Server) ListenAndServe(serverCache cache.Cache, skipDBUpdate bool) error {
requestWg := &sync.WaitGroup{}
dbUpdateWg := &sync.WaitGroup{}
go func() {
worker := newDBWorker(dbc.NewClient(s.cacheDir, true, insecure, dbc.WithDBRepository(s.dbRepository)))
worker := newDBWorker(dbc.NewClient(s.cacheDir, true, dbc.WithDBRepository(s.dbRepository)))
ctx := context.Background()
for {
time.Sleep(updateInterval)
if err := worker.update(ctx, s.appVersion, s.cacheDir, skipDBUpdate, dbUpdateWg, requestWg); err != nil {
if err := worker.update(ctx, s.appVersion, s.cacheDir, skipDBUpdate, dbUpdateWg, requestWg, s.RemoteOptions); err != nil {
log.Logger.Errorf("%+v\n", err)
}
}
@@ -121,7 +126,7 @@ func newDBWorker(dbClient dbFile.Operation) dbWorker {
}
func (w dbWorker) update(ctx context.Context, appVersion, cacheDir string,
skipDBUpdate bool, dbUpdateWg, requestWg *sync.WaitGroup) error {
skipDBUpdate bool, dbUpdateWg, requestWg *sync.WaitGroup, opt types.RemoteOptions) error {
log.Logger.Debug("Check for DB update...")
needsUpdate, err := w.dbClient.NeedsUpdate(appVersion, skipDBUpdate)
if err != nil {
@@ -131,20 +136,20 @@ func (w dbWorker) update(ctx context.Context, appVersion, cacheDir string,
}
log.Logger.Info("Updating DB...")
if err = w.hotUpdate(ctx, cacheDir, dbUpdateWg, requestWg); err != nil {
if err = w.hotUpdate(ctx, cacheDir, dbUpdateWg, requestWg, opt); err != nil {
return xerrors.Errorf("failed DB hot update: %w", err)
}
return nil
}
func (w dbWorker) hotUpdate(ctx context.Context, cacheDir string, dbUpdateWg, requestWg *sync.WaitGroup) error {
func (w dbWorker) hotUpdate(ctx context.Context, cacheDir string, dbUpdateWg, requestWg *sync.WaitGroup, opt types.RemoteOptions) error {
tmpDir, err := os.MkdirTemp("", "db")
if err != nil {
return xerrors.Errorf("failed to create a temp dir: %w", err)
}
defer os.RemoveAll(tmpDir)
if err = w.dbClient.Download(ctx, tmpDir); err != nil {
if err = w.dbClient.Download(ctx, tmpDir, opt); err != nil {
return xerrors.Errorf("failed to download vulnerability DB: %w", err)
}

5
pkg/rpc/server/listen_test.go
View File

@@ -19,6 +19,7 @@ import (
"github.com/aquasecurity/trivy-db/pkg/metadata"
dbFile "github.com/aquasecurity/trivy/pkg/db"
"github.com/aquasecurity/trivy/pkg/fanal/cache"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/utils/fsutils"
rpcCache "github.com/aquasecurity/trivy/rpc/cache"
)
@@ -140,7 +141,7 @@ func Test_dbWorker_update(t *testing.T) {
defer func() { _ = db.Close() }()
if tt.download.call {
mockDBClient.On("Download", mock.Anything, mock.Anything).Run(
mockDBClient.On("Download", mock.Anything, mock.Anything, mock.Anything).Run(
func(args mock.Arguments) {
// fake download: copy testdata/new.db to tmpDir/db/trivy.db
tmpDir := args.String(1)
@@ -160,7 +161,7 @@ func Test_dbWorker_update(t *testing.T) {
var dbUpdateWg, requestWg sync.WaitGroup
err := w.update(context.Background(), tt.args.appVersion, cacheDir,
tt.needsUpdate.input.skip, &dbUpdateWg, &requestWg)
tt.needsUpdate.input.skip, &dbUpdateWg, &requestWg, ftypes.RemoteOptions{})
if tt.wantErr != "" {
require.NotNil(t, err, tt.name)
assert.Contains(t, err.Error(), tt.wantErr, tt.name)