mirror of
https://github.com/aquasecurity/trivy.git
synced 2025-12-06 04:41:18 -08:00
Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
15e15fe399 | ||
|
|
904ada351d | ||
|
|
d24caa409d | ||
|
|
bfa830061e | ||
|
|
50503b00ef | ||
|
|
1da7b3a09a |
11
go.mod
11
go.mod
@@ -76,8 +76,7 @@ require (
|
||||
github.com/owenrumney/go-sarif/v2 v2.2.0
|
||||
github.com/package-url/packageurl-go v0.1.1
|
||||
github.com/samber/lo v1.38.1
|
||||
github.com/saracen/walker v0.1.3
|
||||
github.com/secure-systems-lab/go-securesystemslib v0.6.0
|
||||
github.com/secure-systems-lab/go-securesystemslib v0.7.0
|
||||
github.com/sigstore/rekor v1.2.1
|
||||
github.com/sirupsen/logrus v1.9.0
|
||||
github.com/sosedoff/gitkit v0.3.0
|
||||
@@ -96,8 +95,8 @@ require (
|
||||
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
|
||||
golang.org/x/mod v0.11.0
|
||||
golang.org/x/sync v0.3.0
|
||||
golang.org/x/term v0.9.0
|
||||
golang.org/x/text v0.10.0
|
||||
golang.org/x/term v0.10.0
|
||||
golang.org/x/text v0.11.0
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
|
||||
google.golang.org/protobuf v1.31.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
@@ -356,10 +355,10 @@ require (
|
||||
go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
|
||||
go.uber.org/atomic v1.10.0 // indirect
|
||||
go.uber.org/multierr v1.9.0 // indirect
|
||||
golang.org/x/crypto v0.10.0 // indirect
|
||||
golang.org/x/crypto v0.11.0 // indirect
|
||||
golang.org/x/net v0.11.0 // indirect
|
||||
golang.org/x/oauth2 v0.7.0 // indirect
|
||||
golang.org/x/sys v0.9.0 // indirect
|
||||
golang.org/x/sys v0.10.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
golang.org/x/tools v0.8.0 // indirect
|
||||
google.golang.org/api v0.121.0 // indirect
|
||||
|
||||
22
go.sum
22
go.sum
@@ -1544,14 +1544,12 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
|
||||
github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
|
||||
github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM=
|
||||
github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
|
||||
github.com/saracen/walker v0.1.3 h1:YtcKKmpRPy6XJTHJ75J2QYXXZYWnZNQxPCVqZSHVV/g=
|
||||
github.com/saracen/walker v0.1.3/go.mod h1:FU+7qU8DeQQgSZDmmThMJi93kPkLFgy0oVAcLxurjIk=
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
|
||||
github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
|
||||
github.com/secure-systems-lab/go-securesystemslib v0.6.0 h1:T65atpAVCJQK14UA57LMdZGpHi4QYSH/9FZyNGqMYIA=
|
||||
github.com/secure-systems-lab/go-securesystemslib v0.6.0/go.mod h1:8Mtpo9JKks/qhPG4HGZ2LGMvrPbzuxwfz/f/zLfEWkk=
|
||||
github.com/secure-systems-lab/go-securesystemslib v0.7.0 h1:OwvJ5jQf9LnIAS83waAjPbcMsODrTQUpJ02eNLUoxBg=
|
||||
github.com/secure-systems-lab/go-securesystemslib v0.7.0/go.mod h1:/2gYnlnHVQ6xeGtfIqFy7Do03K4cdCY0A/GlJLDKLHI=
|
||||
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
|
||||
github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
@@ -1799,8 +1797,8 @@ golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4
|
||||
golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
|
||||
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
|
||||
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
|
||||
golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM=
|
||||
golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
|
||||
golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
|
||||
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
@@ -2093,8 +2091,8 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
|
||||
golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
|
||||
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -2102,8 +2100,8 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
|
||||
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
|
||||
golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28=
|
||||
golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo=
|
||||
golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
|
||||
golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -2118,8 +2116,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
|
||||
golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
|
||||
golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
|
||||
@@ -636,10 +636,9 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
|
||||
RepoTag: opts.RepoTag,
|
||||
SBOMSources: opts.SBOMSources,
|
||||
RekorURL: opts.RekorURL,
|
||||
//Platform: opts.Platform,
|
||||
Slow: opts.Slow,
|
||||
AWSRegion: opts.Region,
|
||||
FileChecksum: fileChecksum,
|
||||
Parallel: 5, // Set the default value
|
||||
AWSRegion: opts.Region,
|
||||
FileChecksum: fileChecksum,
|
||||
|
||||
// For image scanning
|
||||
ImageOption: ftypes.ImageOptions{
|
||||
|
||||
@@ -41,7 +41,7 @@ var (
|
||||
// AnalyzerOptions is used to initialize analyzers
|
||||
type AnalyzerOptions struct {
|
||||
Group Group
|
||||
Slow bool
|
||||
Parallel int
|
||||
FilePatterns []string
|
||||
DisabledAnalyzers []Type
|
||||
MisconfScannerOption misconf.ScannerOption
|
||||
|
||||
@@ -2,6 +2,7 @@ package jar
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/samber/lo"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -35,14 +36,14 @@ var requiredExtensions = []string{
|
||||
|
||||
// javaLibraryAnalyzer analyzes jar/war/ear/par files
|
||||
type javaLibraryAnalyzer struct {
|
||||
once sync.Once
|
||||
client *javadb.DB
|
||||
slow bool
|
||||
once sync.Once
|
||||
client *javadb.DB
|
||||
parallel int
|
||||
}
|
||||
|
||||
func newJavaLibraryAnalyzer(options analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) {
|
||||
return &javaLibraryAnalyzer{
|
||||
slow: options.Slow,
|
||||
parallel: lo.Ternary(options.Parallel > 0, options.Parallel, 5),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -82,7 +83,7 @@ func (a *javaLibraryAnalyzer) PostAnalyze(ctx context.Context, input analyzer.Po
|
||||
return nil
|
||||
}
|
||||
|
||||
if err = parallel.WalkDir(ctx, input.FS, ".", a.slow, onFile, onResult); err != nil {
|
||||
if err = parallel.WalkDir(ctx, input.FS, ".", a.parallel, onFile, onResult); err != nil {
|
||||
return nil, xerrors.Errorf("walk dir error: %w", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -131,7 +131,7 @@ func Test_javaLibraryAnalyzer_Analyze(t *testing.T) {
|
||||
// init java-trivy-db with skip update
|
||||
javadb.Init("testdata", defaultJavaDBRepository, true, false, false)
|
||||
|
||||
a := javaLibraryAnalyzer{slow: true}
|
||||
a := javaLibraryAnalyzer{parallel: 1}
|
||||
ctx := context.Background()
|
||||
|
||||
mfs := mapfs.New()
|
||||
|
||||
@@ -23,7 +23,7 @@ type Option struct {
|
||||
AppDirs []string
|
||||
SBOMSources []string
|
||||
RekorURL string
|
||||
Slow bool // Lower CPU and memory
|
||||
Parallel int
|
||||
AWSRegion string
|
||||
FileChecksum bool // For SPDX
|
||||
|
||||
@@ -40,13 +40,7 @@ type Option struct {
|
||||
LicenseScannerOption analyzer.LicenseScannerOption
|
||||
|
||||
// File walk
|
||||
WalkOption WalkOption
|
||||
}
|
||||
|
||||
// WalkOption is a struct that allows users to define a custom walking behavior.
|
||||
// This option is only available when using Trivy as an imported library and not through CLI flags.
|
||||
type WalkOption struct {
|
||||
ErrorCallback walker.ErrorCallback
|
||||
WalkOption walker.Option
|
||||
}
|
||||
|
||||
func (o *Option) Sort() {
|
||||
|
||||
@@ -3,6 +3,7 @@ package image
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"golang.org/x/sync/semaphore"
|
||||
"io"
|
||||
"os"
|
||||
"reflect"
|
||||
@@ -23,7 +24,6 @@ import (
|
||||
"github.com/aquasecurity/trivy/pkg/fanal/types"
|
||||
"github.com/aquasecurity/trivy/pkg/fanal/walker"
|
||||
"github.com/aquasecurity/trivy/pkg/parallel"
|
||||
"github.com/aquasecurity/trivy/pkg/semaphore"
|
||||
)
|
||||
|
||||
type Artifact struct {
|
||||
@@ -51,7 +51,7 @@ func NewArtifact(img types.Image, c cache.ArtifactCache, opt artifact.Option) (a
|
||||
|
||||
a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{
|
||||
Group: opt.AnalyzerGroup,
|
||||
Slow: opt.Slow,
|
||||
Parallel: opt.Parallel,
|
||||
FilePatterns: opt.FilePatterns,
|
||||
DisabledAnalyzers: opt.DisabledAnalyzers,
|
||||
MisconfScannerOption: opt.MisconfScannerOption,
|
||||
@@ -75,7 +75,7 @@ func NewArtifact(img types.Image, c cache.ArtifactCache, opt artifact.Option) (a
|
||||
return Artifact{
|
||||
image: img,
|
||||
cache: c,
|
||||
walker: walker.NewLayerTar(opt.SkipFiles, opt.SkipDirs, opt.Slow),
|
||||
walker: walker.NewLayerTar(opt.SkipFiles, opt.SkipDirs),
|
||||
analyzer: a,
|
||||
configAnalyzer: ca,
|
||||
handlerManager: handlerManager,
|
||||
@@ -215,7 +215,7 @@ func (a Artifact) inspect(ctx context.Context, missingImage string, layerKeys, b
|
||||
layerKeyMap map[string]LayerInfo, configFile *v1.ConfigFile) error {
|
||||
|
||||
var osFound types.OS
|
||||
workers := lo.Ternary(a.artifactOption.Slow, 1, 5)
|
||||
workers := lo.Ternary(a.artifactOption.Parallel > 0, a.artifactOption.Parallel, 5)
|
||||
p := parallel.NewPipeline(workers, false, layerKeys, func(ctx context.Context, layerKey string) (any, error) {
|
||||
layer := layerKeyMap[layerKey]
|
||||
|
||||
@@ -268,7 +268,8 @@ func (a Artifact) inspectLayer(ctx context.Context, layerInfo LayerInfo, disable
|
||||
FileChecksum: a.artifactOption.FileChecksum,
|
||||
}
|
||||
result := analyzer.NewAnalysisResult()
|
||||
limit := semaphore.New(a.artifactOption.Slow)
|
||||
parallel := lo.Ternary(a.artifactOption.Parallel > 0, a.artifactOption.Parallel, 5)
|
||||
limit := semaphore.NewWeighted(int64(parallel))
|
||||
|
||||
// Prepare filesystem for post analysis
|
||||
composite, err := a.analyzer.PostAnalyzerFS()
|
||||
|
||||
@@ -1847,7 +1847,7 @@ func TestArtifact_Inspect(t *testing.T) {
|
||||
name: "sad path, PutBlob returns an error with multiple layers and Slow enabled",
|
||||
imagePath: "../../test/testdata/vuln-image.tar.gz",
|
||||
artifactOpt: artifact.Option{
|
||||
Slow: true,
|
||||
Parallel: 1,
|
||||
},
|
||||
missingBlobsExpectation: cache.ArtifactCacheMissingBlobsExpectation{
|
||||
Args: cache.ArtifactCacheMissingBlobsArgs{
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"github.com/samber/lo"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
@@ -11,6 +12,7 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/opencontainers/go-digest"
|
||||
"golang.org/x/sync/semaphore"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/aquasecurity/trivy/pkg/fanal/analyzer"
|
||||
@@ -20,7 +22,6 @@ import (
|
||||
"github.com/aquasecurity/trivy/pkg/fanal/types"
|
||||
"github.com/aquasecurity/trivy/pkg/fanal/walker"
|
||||
"github.com/aquasecurity/trivy/pkg/log"
|
||||
"github.com/aquasecurity/trivy/pkg/semaphore"
|
||||
)
|
||||
|
||||
type Artifact struct {
|
||||
@@ -41,7 +42,6 @@ func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (a
|
||||
|
||||
a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{
|
||||
Group: opt.AnalyzerGroup,
|
||||
Slow: opt.Slow,
|
||||
FilePatterns: opt.FilePatterns,
|
||||
DisabledAnalyzers: opt.DisabledAnalyzers,
|
||||
MisconfScannerOption: opt.MisconfScannerOption,
|
||||
@@ -56,7 +56,7 @@ func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (a
|
||||
rootPath: filepath.ToSlash(filepath.Clean(rootPath)),
|
||||
cache: c,
|
||||
walker: walker.NewFS(buildPathsToSkip(rootPath, opt.SkipFiles), buildPathsToSkip(rootPath, opt.SkipDirs),
|
||||
opt.Slow, opt.WalkOption.ErrorCallback),
|
||||
opt.WalkOption),
|
||||
analyzer: a,
|
||||
handlerManager: handlerManager,
|
||||
|
||||
@@ -122,7 +122,8 @@ func buildPathsToSkip(base string, paths []string) []string {
|
||||
func (a Artifact) Inspect(ctx context.Context) (types.ArtifactReference, error) {
|
||||
var wg sync.WaitGroup
|
||||
result := analyzer.NewAnalysisResult()
|
||||
limit := semaphore.New(a.artifactOption.Slow)
|
||||
parallel := lo.Ternary(a.artifactOption.Parallel > 0, a.artifactOption.Parallel, 5)
|
||||
limit := semaphore.NewWeighted(int64(parallel))
|
||||
opts := analyzer.AnalysisOptions{
|
||||
Offline: a.artifactOption.Offline,
|
||||
FileChecksum: a.artifactOption.FileChecksum,
|
||||
|
||||
@@ -158,7 +158,7 @@ func TestArtifact_Inspect(t *testing.T) {
|
||||
fields: fields{
|
||||
dir: "./testdata/unknown",
|
||||
},
|
||||
wantErr: "walk error",
|
||||
wantErr: "walk dir error",
|
||||
},
|
||||
{
|
||||
name: "happy path with single file",
|
||||
|
||||
@@ -2,6 +2,8 @@ package vm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/samber/lo"
|
||||
"golang.org/x/sync/semaphore"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
@@ -15,7 +17,6 @@ import (
|
||||
"github.com/aquasecurity/trivy/pkg/fanal/handler"
|
||||
"github.com/aquasecurity/trivy/pkg/fanal/types"
|
||||
"github.com/aquasecurity/trivy/pkg/fanal/walker"
|
||||
"github.com/aquasecurity/trivy/pkg/semaphore"
|
||||
)
|
||||
|
||||
type Type string
|
||||
@@ -41,7 +42,8 @@ type Storage struct {
|
||||
|
||||
func (a *Storage) Analyze(ctx context.Context, r *io.SectionReader) (types.BlobInfo, error) {
|
||||
var wg sync.WaitGroup
|
||||
limit := semaphore.New(a.artifactOption.Slow)
|
||||
parallel := lo.Ternary(a.artifactOption.Parallel > 0, a.artifactOption.Parallel, 5)
|
||||
limit := semaphore.NewWeighted(int64(parallel))
|
||||
result := analyzer.NewAnalysisResult()
|
||||
|
||||
opts := analyzer.AnalysisOptions{
|
||||
@@ -135,7 +137,7 @@ func NewArtifact(target string, c cache.ArtifactCache, opt artifact.Option) (art
|
||||
cache: c,
|
||||
analyzer: a,
|
||||
handlerManager: handlerManager,
|
||||
walker: walker.NewVM(opt.SkipFiles, opt.SkipDirs, opt.Slow),
|
||||
walker: walker.NewVM(opt.SkipFiles, opt.SkipDirs),
|
||||
artifactOption: opt,
|
||||
}
|
||||
|
||||
|
||||
@@ -1,29 +1,32 @@
|
||||
package walker
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"golang.org/x/xerrors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
swalker "github.com/saracen/walker"
|
||||
"golang.org/x/xerrors"
|
||||
"time"
|
||||
|
||||
dio "github.com/aquasecurity/go-dep-parser/pkg/io"
|
||||
"github.com/aquasecurity/trivy/pkg/log"
|
||||
)
|
||||
|
||||
type ErrorCallback func(pathname string, err error) error
|
||||
|
||||
type FS struct {
|
||||
walker
|
||||
errCallback ErrorCallback
|
||||
opt Option
|
||||
}
|
||||
|
||||
func NewFS(skipFiles, skipDirs []string, slow bool, errCallback ErrorCallback) FS {
|
||||
if errCallback == nil {
|
||||
errCallback = func(pathname string, err error) error {
|
||||
func NewFS(skipFiles, skipDirs []string, opt Option) FS {
|
||||
if opt.ErrCallback == nil {
|
||||
opt.ErrCallback = func(pathname string, err error) error {
|
||||
switch {
|
||||
// Unwrap fs.SkipDir error
|
||||
case errors.Is(err, fs.SkipDir):
|
||||
return fs.SkipDir
|
||||
// ignore permission errors
|
||||
if os.IsPermission(err) {
|
||||
case os.IsPermission(err):
|
||||
return nil
|
||||
}
|
||||
// halt traversal on any other error
|
||||
@@ -32,77 +35,20 @@ func NewFS(skipFiles, skipDirs []string, slow bool, errCallback ErrorCallback) F
|
||||
}
|
||||
|
||||
return FS{
|
||||
walker: newWalker(skipFiles, skipDirs, slow),
|
||||
errCallback: errCallback,
|
||||
walker: newWalker(skipFiles, skipDirs),
|
||||
opt: opt,
|
||||
}
|
||||
}
|
||||
|
||||
// Walk walks the file tree rooted at root, calling WalkFunc for each file or
|
||||
// Walk walks the file tree rooted at root, calling WalkDirFunc for each file or
|
||||
// directory in the tree, including root, but a directory to be ignored will be skipped.
|
||||
func (w FS) Walk(root string, fn WalkFunc) error {
|
||||
// walk function called for every path found
|
||||
walkFn := func(pathname string, fi os.FileInfo) error {
|
||||
pathname = filepath.Clean(pathname)
|
||||
|
||||
// For exported rootfs (e.g. images/alpine/etc/alpine-release)
|
||||
relPath, err := filepath.Rel(root, pathname)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("filepath rel (%s): %w", relPath, err)
|
||||
}
|
||||
relPath = filepath.ToSlash(relPath)
|
||||
|
||||
if fi.IsDir() {
|
||||
if w.shouldSkipDir(relPath) {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
} else if !fi.Mode().IsRegular() {
|
||||
return nil
|
||||
} else if w.shouldSkipFile(relPath) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err = fn(relPath, fi, w.fileOpener(pathname)); err != nil {
|
||||
return xerrors.Errorf("failed to analyze file: %w", err)
|
||||
walkDir := w.walkDirFunc(root, fn)
|
||||
err := filepath.WalkDir(root, func(filePath string, d fs.DirEntry, err error) error {
|
||||
if walkErr := walkDir(filePath, d, err); walkErr != nil {
|
||||
return w.opt.ErrCallback(filePath, walkErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if w.slow {
|
||||
// In series: fast, with higher CPU/memory
|
||||
return w.walkSlow(root, walkFn)
|
||||
}
|
||||
|
||||
// In parallel: slow, with lower CPU/memory
|
||||
return w.walkFast(root, walkFn)
|
||||
}
|
||||
|
||||
type fastWalkFunc func(pathname string, fi os.FileInfo) error
|
||||
|
||||
func (w FS) walkFast(root string, walkFn fastWalkFunc) error {
|
||||
// error function called for every error encountered
|
||||
errorCallbackOption := swalker.WithErrorCallback(w.errCallback)
|
||||
|
||||
// Multiple goroutines stat the filesystem concurrently. The provided
|
||||
// walkFn must be safe for concurrent use.
|
||||
log.Logger.Debugf("Walk the file tree rooted at '%s' in parallel", root)
|
||||
if err := swalker.Walk(root, walkFn, errorCallbackOption); err != nil {
|
||||
return xerrors.Errorf("walk error: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w FS) walkSlow(root string, walkFn fastWalkFunc) error {
|
||||
log.Logger.Debugf("Walk the file tree rooted at '%s' in series", root)
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return w.errCallback(path, err)
|
||||
}
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return xerrors.Errorf("file info error: %w", err)
|
||||
}
|
||||
return walkFn(path, info)
|
||||
})
|
||||
if err != nil {
|
||||
return xerrors.Errorf("walk dir error: %w", err)
|
||||
@@ -110,6 +56,46 @@ func (w FS) walkSlow(root string, walkFn fastWalkFunc) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w FS) walkDirFunc(root string, fn WalkFunc) fs.WalkDirFunc {
|
||||
return func(filePath string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return w.opt.ErrCallback(filePath, err)
|
||||
}
|
||||
time.Sleep(w.opt.Delay)
|
||||
|
||||
filePath = filepath.Clean(filePath)
|
||||
|
||||
// For exported rootfs (e.g. images/alpine/etc/alpine-release)
|
||||
relPath, err := filepath.Rel(root, filePath)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("filepath rel (%s): %w", relPath, err)
|
||||
}
|
||||
relPath = filepath.ToSlash(relPath)
|
||||
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return xerrors.Errorf("file info error: %w", err)
|
||||
}
|
||||
|
||||
switch {
|
||||
case info.IsDir():
|
||||
if w.shouldSkipDir(relPath) {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
case !info.Mode().IsRegular():
|
||||
return nil
|
||||
case w.shouldSkipFile(relPath):
|
||||
return nil
|
||||
}
|
||||
|
||||
if err = fn(relPath, info, w.fileOpener(filePath)); err != nil {
|
||||
return xerrors.Errorf("failed to analyze file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// fileOpener returns a function opening a file.
|
||||
func (w *walker) fileOpener(pathname string) func() (dio.ReadSeekCloserAt, error) {
|
||||
return func() (dio.ReadSeekCloserAt, error) {
|
||||
|
||||
@@ -3,6 +3,7 @@ package walker_test
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -77,15 +78,29 @@ func TestDir_Walk(t *testing.T) {
|
||||
return nil
|
||||
},
|
||||
},
|
||||
analyzeFn: func(filePath string, info os.FileInfo, opener analyzer.Opener) error {
|
||||
// Ignore errors
|
||||
analyzeFn: func(string, os.FileInfo, analyzer.Opener) error {
|
||||
return nil
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ignore analysis errors",
|
||||
rootDir: "testdata/fs",
|
||||
fields: fields{
|
||||
errCallback: func(pathname string, err error) error {
|
||||
if errors.Is(err, fs.ErrClosed) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
},
|
||||
},
|
||||
analyzeFn: func(string, os.FileInfo, analyzer.Opener) error {
|
||||
return fs.ErrClosed
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "sad path",
|
||||
rootDir: "testdata/fs",
|
||||
analyzeFn: func(filePath string, info os.FileInfo, opener analyzer.Opener) error {
|
||||
analyzeFn: func(string, os.FileInfo, analyzer.Opener) error {
|
||||
return errors.New("error")
|
||||
},
|
||||
wantErr: "failed to analyze file",
|
||||
@@ -93,8 +108,9 @@ func TestDir_Walk(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, true, tt.fields.errCallback)
|
||||
|
||||
w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, walker.Option{
|
||||
ErrCallback: tt.fields.errCallback,
|
||||
})
|
||||
err := w.Walk(tt.rootDir, tt.analyzeFn)
|
||||
if tt.wantErr != "" {
|
||||
require.Error(t, err)
|
||||
|
||||
@@ -25,14 +25,10 @@ type LayerTar struct {
|
||||
threshold int64
|
||||
}
|
||||
|
||||
func NewLayerTar(skipFiles, skipDirs []string, slow bool) LayerTar {
|
||||
func NewLayerTar(skipFiles, skipDirs []string) LayerTar {
|
||||
threshold := defaultSizeThreshold
|
||||
if slow {
|
||||
threshold = slowSizeThreshold
|
||||
}
|
||||
|
||||
return LayerTar{
|
||||
walker: newWalker(skipFiles, skipDirs, slow),
|
||||
walker: newWalker(skipFiles, skipDirs),
|
||||
threshold: threshold,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,7 +83,7 @@ func TestLayerTar_Walk(t *testing.T) {
|
||||
f, err := os.Open("testdata/test.tar")
|
||||
require.NoError(t, err)
|
||||
|
||||
w := walker.NewLayerTar(tt.fields.skipFiles, tt.fields.skipDirs, true)
|
||||
w := walker.NewLayerTar(tt.fields.skipFiles, tt.fields.skipDirs)
|
||||
|
||||
gotOpqDirs, gotWhFiles, err := w.Walk(f, tt.analyzeFn)
|
||||
if tt.wantErr != "" {
|
||||
|
||||
@@ -39,14 +39,10 @@ type VM struct {
|
||||
analyzeFn WalkFunc
|
||||
}
|
||||
|
||||
func NewVM(skipFiles, skipDirs []string, slow bool) VM {
|
||||
func NewVM(skipFiles, skipDirs []string) VM {
|
||||
threshold := defaultSizeThreshold
|
||||
if slow {
|
||||
threshold = slowSizeThreshold
|
||||
}
|
||||
|
||||
return VM{
|
||||
walker: newWalker(skipFiles, skipDirs, slow),
|
||||
walker: newWalker(skipFiles, skipDirs),
|
||||
threshold: threshold,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/bmatcuk/doublestar"
|
||||
|
||||
@@ -26,13 +27,19 @@ const (
|
||||
|
||||
type WalkFunc func(filePath string, info os.FileInfo, opener analyzer.Opener) error
|
||||
|
||||
// Option allows users to define a custom walking behavior.
|
||||
// This option is only available when using Trivy as an imported library and not through CLI flags.
|
||||
type Option struct {
|
||||
Delay time.Duration
|
||||
ErrCallback ErrorCallback
|
||||
}
|
||||
|
||||
type walker struct {
|
||||
skipFiles []string
|
||||
skipDirs []string
|
||||
slow bool
|
||||
}
|
||||
|
||||
func newWalker(skipFiles, skipDirs []string, slow bool) walker {
|
||||
func newWalker(skipFiles, skipDirs []string) walker {
|
||||
var cleanSkipFiles, cleanSkipDirs []string
|
||||
for _, skipFile := range skipFiles {
|
||||
skipFile = filepath.ToSlash(filepath.Clean(skipFile))
|
||||
@@ -49,7 +56,6 @@ func newWalker(skipFiles, skipDirs []string, slow bool) walker {
|
||||
return walker{
|
||||
skipFiles: cleanSkipFiles,
|
||||
skipDirs: cleanSkipDirs,
|
||||
slow: slow,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -54,7 +54,7 @@ func Test_shouldSkipFile(t *testing.T) {
|
||||
|
||||
for i, tc := range testCases {
|
||||
t.Run(fmt.Sprint(i), func(t *testing.T) {
|
||||
w := newWalker(tc.skipFiles, nil, false)
|
||||
w := newWalker(tc.skipFiles, nil)
|
||||
for file, skipResult := range tc.skipMap {
|
||||
assert.Equal(t, skipResult, w.shouldSkipFile(filepath.ToSlash(filepath.Clean(file))), fmt.Sprintf("skipFiles: %s, file: %s", tc.skipFiles, file))
|
||||
}
|
||||
@@ -108,7 +108,7 @@ func Test_shouldSkipDir(t *testing.T) {
|
||||
|
||||
for i, tc := range testCases {
|
||||
t.Run(fmt.Sprint(i), func(t *testing.T) {
|
||||
w := newWalker(nil, tc.skipDirs, false)
|
||||
w := newWalker(nil, tc.skipDirs)
|
||||
for dir, skipResult := range tc.skipMap {
|
||||
assert.Equal(t, skipResult, w.shouldSkipDir(filepath.ToSlash(filepath.Clean(dir))), fmt.Sprintf("skipDirs: %s, dir: %s", tc.skipDirs, dir))
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
type onFile[T any] func(string, fs.FileInfo, dio.ReadSeekerAt) (T, error)
|
||||
type onWalkResult[T any] func(T) error
|
||||
|
||||
func WalkDir[T any](ctx context.Context, fsys fs.FS, root string, slow bool,
|
||||
func WalkDir[T any](ctx context.Context, fsys fs.FS, root string, parallel int,
|
||||
onFile onFile[T], onResult onWalkResult[T]) error {
|
||||
|
||||
g, ctx := errgroup.WithContext(ctx)
|
||||
@@ -54,11 +54,7 @@ func WalkDir[T any](ctx context.Context, fsys fs.FS, root string, slow bool,
|
||||
|
||||
// Start a fixed number of goroutines to read and digest files.
|
||||
c := make(chan T)
|
||||
limit := 10
|
||||
if slow {
|
||||
limit = 1
|
||||
}
|
||||
for i := 0; i < limit; i++ {
|
||||
for i := 0; i < parallel; i++ {
|
||||
g.Go(func() error {
|
||||
for path := range paths {
|
||||
if err := walk(ctx, fsys, path, c, onFile); err != nil {
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
package semaphore
|
||||
|
||||
import "golang.org/x/sync/semaphore"
|
||||
|
||||
const defaultSize = 5
|
||||
|
||||
type options struct {
|
||||
size int64
|
||||
}
|
||||
|
||||
type option func(*options)
|
||||
|
||||
func WithDefault(n int64) option {
|
||||
return func(opts *options) {
|
||||
opts.size = defaultSize
|
||||
}
|
||||
}
|
||||
|
||||
func New(slow bool, opts ...option) *semaphore.Weighted {
|
||||
o := &options{size: defaultSize}
|
||||
for _, opt := range opts {
|
||||
opt(o)
|
||||
}
|
||||
if slow {
|
||||
// Process in series
|
||||
return semaphore.NewWeighted(1)
|
||||
}
|
||||
// Process in parallel
|
||||
return semaphore.NewWeighted(o.size)
|
||||
}
|
||||
Reference in New Issue
Block a user