Compare commits

...

6 Commits

Author SHA1 Message Date
knqyf263
15e15fe399 feat: re-add time.Sleep
Signed-off-by: knqyf263 <knqyf263@gmail.com>
2023-12-07 18:25:59 +04:00
knqyf263
904ada351d refactor: replace if-else with switch
Signed-off-by: knqyf263 <knqyf263@gmail.com>
2023-12-07 15:38:42 +04:00
knqyf263
d24caa409d fix(walk): call error callback on all errors
Signed-off-by: knqyf263 <knqyf263@gmail.com>
2023-12-07 15:37:06 +04:00
knqyf263
bfa830061e refactor: remove parallel walk
Signed-off-by: knqyf263 <knqyf263@gmail.com>
2023-12-07 15:34:52 +04:00
knqyf263
50503b00ef feat: configure parallelism
Signed-off-by: knqyf263 <knqyf263@gmail.com>
2023-12-05 19:38:56 +04:00
knqyf263
1da7b3a09a feat: configure delay
Signed-off-by: knqyf263 <knqyf263@gmail.com>
2023-12-05 19:16:06 +04:00
21 changed files with 144 additions and 183 deletions

11
go.mod
View File

@@ -76,8 +76,7 @@ require (
github.com/owenrumney/go-sarif/v2 v2.2.0
github.com/package-url/packageurl-go v0.1.1
github.com/samber/lo v1.38.1
github.com/saracen/walker v0.1.3
github.com/secure-systems-lab/go-securesystemslib v0.6.0
github.com/secure-systems-lab/go-securesystemslib v0.7.0
github.com/sigstore/rekor v1.2.1
github.com/sirupsen/logrus v1.9.0
github.com/sosedoff/gitkit v0.3.0
@@ -96,8 +95,8 @@ require (
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
golang.org/x/mod v0.11.0
golang.org/x/sync v0.3.0
golang.org/x/term v0.9.0
golang.org/x/text v0.10.0
golang.org/x/term v0.10.0
golang.org/x/text v0.11.0
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
google.golang.org/protobuf v1.31.0
gopkg.in/yaml.v3 v3.0.1
@@ -356,10 +355,10 @@ require (
go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
golang.org/x/crypto v0.10.0 // indirect
golang.org/x/crypto v0.11.0 // indirect
golang.org/x/net v0.11.0 // indirect
golang.org/x/oauth2 v0.7.0 // indirect
golang.org/x/sys v0.9.0 // indirect
golang.org/x/sys v0.10.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.8.0 // indirect
google.golang.org/api v0.121.0 // indirect

22
go.sum
View File

@@ -1544,14 +1544,12 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM=
github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
github.com/saracen/walker v0.1.3 h1:YtcKKmpRPy6XJTHJ75J2QYXXZYWnZNQxPCVqZSHVV/g=
github.com/saracen/walker v0.1.3/go.mod h1:FU+7qU8DeQQgSZDmmThMJi93kPkLFgy0oVAcLxurjIk=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
github.com/secure-systems-lab/go-securesystemslib v0.6.0 h1:T65atpAVCJQK14UA57LMdZGpHi4QYSH/9FZyNGqMYIA=
github.com/secure-systems-lab/go-securesystemslib v0.6.0/go.mod h1:8Mtpo9JKks/qhPG4HGZ2LGMvrPbzuxwfz/f/zLfEWkk=
github.com/secure-systems-lab/go-securesystemslib v0.7.0 h1:OwvJ5jQf9LnIAS83waAjPbcMsODrTQUpJ02eNLUoxBg=
github.com/secure-systems-lab/go-securesystemslib v0.7.0/go.mod h1:/2gYnlnHVQ6xeGtfIqFy7Do03K4cdCY0A/GlJLDKLHI=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
@@ -1799,8 +1797,8 @@ golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4
golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM=
golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -2093,8 +2091,8 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -2102,8 +2100,8 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28=
golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo=
golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -2118,8 +2116,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

View File

@@ -636,10 +636,9 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
RepoTag: opts.RepoTag,
SBOMSources: opts.SBOMSources,
RekorURL: opts.RekorURL,
//Platform: opts.Platform,
Slow: opts.Slow,
AWSRegion: opts.Region,
FileChecksum: fileChecksum,
Parallel: 5, // Set the default value
AWSRegion: opts.Region,
FileChecksum: fileChecksum,
// For image scanning
ImageOption: ftypes.ImageOptions{

View File

@@ -41,7 +41,7 @@ var (
// AnalyzerOptions is used to initialize analyzers
type AnalyzerOptions struct {
Group Group
Slow bool
Parallel int
FilePatterns []string
DisabledAnalyzers []Type
MisconfScannerOption misconf.ScannerOption

View File

@@ -2,6 +2,7 @@ package jar
import (
"context"
"github.com/samber/lo"
"io/fs"
"os"
"path/filepath"
@@ -35,14 +36,14 @@ var requiredExtensions = []string{
// javaLibraryAnalyzer analyzes jar/war/ear/par files
type javaLibraryAnalyzer struct {
once sync.Once
client *javadb.DB
slow bool
once sync.Once
client *javadb.DB
parallel int
}
func newJavaLibraryAnalyzer(options analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) {
return &javaLibraryAnalyzer{
slow: options.Slow,
parallel: lo.Ternary(options.Parallel > 0, options.Parallel, 5),
}, nil
}
@@ -82,7 +83,7 @@ func (a *javaLibraryAnalyzer) PostAnalyze(ctx context.Context, input analyzer.Po
return nil
}
if err = parallel.WalkDir(ctx, input.FS, ".", a.slow, onFile, onResult); err != nil {
if err = parallel.WalkDir(ctx, input.FS, ".", a.parallel, onFile, onResult); err != nil {
return nil, xerrors.Errorf("walk dir error: %w", err)
}

View File

@@ -131,7 +131,7 @@ func Test_javaLibraryAnalyzer_Analyze(t *testing.T) {
// init java-trivy-db with skip update
javadb.Init("testdata", defaultJavaDBRepository, true, false, false)
a := javaLibraryAnalyzer{slow: true}
a := javaLibraryAnalyzer{parallel: 1}
ctx := context.Background()
mfs := mapfs.New()

View File

@@ -23,7 +23,7 @@ type Option struct {
AppDirs []string
SBOMSources []string
RekorURL string
Slow bool // Lower CPU and memory
Parallel int
AWSRegion string
FileChecksum bool // For SPDX
@@ -40,13 +40,7 @@ type Option struct {
LicenseScannerOption analyzer.LicenseScannerOption
// File walk
WalkOption WalkOption
}
// WalkOption is a struct that allows users to define a custom walking behavior.
// This option is only available when using Trivy as an imported library and not through CLI flags.
type WalkOption struct {
ErrorCallback walker.ErrorCallback
WalkOption walker.Option
}
func (o *Option) Sort() {

View File

@@ -3,6 +3,7 @@ package image
import (
"context"
"errors"
"golang.org/x/sync/semaphore"
"io"
"os"
"reflect"
@@ -23,7 +24,6 @@ import (
"github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/fanal/walker"
"github.com/aquasecurity/trivy/pkg/parallel"
"github.com/aquasecurity/trivy/pkg/semaphore"
)
type Artifact struct {
@@ -51,7 +51,7 @@ func NewArtifact(img types.Image, c cache.ArtifactCache, opt artifact.Option) (a
a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{
Group: opt.AnalyzerGroup,
Slow: opt.Slow,
Parallel: opt.Parallel,
FilePatterns: opt.FilePatterns,
DisabledAnalyzers: opt.DisabledAnalyzers,
MisconfScannerOption: opt.MisconfScannerOption,
@@ -75,7 +75,7 @@ func NewArtifact(img types.Image, c cache.ArtifactCache, opt artifact.Option) (a
return Artifact{
image: img,
cache: c,
walker: walker.NewLayerTar(opt.SkipFiles, opt.SkipDirs, opt.Slow),
walker: walker.NewLayerTar(opt.SkipFiles, opt.SkipDirs),
analyzer: a,
configAnalyzer: ca,
handlerManager: handlerManager,
@@ -215,7 +215,7 @@ func (a Artifact) inspect(ctx context.Context, missingImage string, layerKeys, b
layerKeyMap map[string]LayerInfo, configFile *v1.ConfigFile) error {
var osFound types.OS
workers := lo.Ternary(a.artifactOption.Slow, 1, 5)
workers := lo.Ternary(a.artifactOption.Parallel > 0, a.artifactOption.Parallel, 5)
p := parallel.NewPipeline(workers, false, layerKeys, func(ctx context.Context, layerKey string) (any, error) {
layer := layerKeyMap[layerKey]
@@ -268,7 +268,8 @@ func (a Artifact) inspectLayer(ctx context.Context, layerInfo LayerInfo, disable
FileChecksum: a.artifactOption.FileChecksum,
}
result := analyzer.NewAnalysisResult()
limit := semaphore.New(a.artifactOption.Slow)
parallel := lo.Ternary(a.artifactOption.Parallel > 0, a.artifactOption.Parallel, 5)
limit := semaphore.NewWeighted(int64(parallel))
// Prepare filesystem for post analysis
composite, err := a.analyzer.PostAnalyzerFS()

View File

@@ -1847,7 +1847,7 @@ func TestArtifact_Inspect(t *testing.T) {
name: "sad path, PutBlob returns an error with multiple layers and Slow enabled",
imagePath: "../../test/testdata/vuln-image.tar.gz",
artifactOpt: artifact.Option{
Slow: true,
Parallel: 1,
},
missingBlobsExpectation: cache.ArtifactCacheMissingBlobsExpectation{
Args: cache.ArtifactCacheMissingBlobsArgs{

View File

@@ -4,6 +4,7 @@ import (
"context"
"crypto/sha256"
"encoding/json"
"github.com/samber/lo"
"os"
"path"
"path/filepath"
@@ -11,6 +12,7 @@ import (
"sync"
"github.com/opencontainers/go-digest"
"golang.org/x/sync/semaphore"
"golang.org/x/xerrors"
"github.com/aquasecurity/trivy/pkg/fanal/analyzer"
@@ -20,7 +22,6 @@ import (
"github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/fanal/walker"
"github.com/aquasecurity/trivy/pkg/log"
"github.com/aquasecurity/trivy/pkg/semaphore"
)
type Artifact struct {
@@ -41,7 +42,6 @@ func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (a
a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{
Group: opt.AnalyzerGroup,
Slow: opt.Slow,
FilePatterns: opt.FilePatterns,
DisabledAnalyzers: opt.DisabledAnalyzers,
MisconfScannerOption: opt.MisconfScannerOption,
@@ -56,7 +56,7 @@ func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (a
rootPath: filepath.ToSlash(filepath.Clean(rootPath)),
cache: c,
walker: walker.NewFS(buildPathsToSkip(rootPath, opt.SkipFiles), buildPathsToSkip(rootPath, opt.SkipDirs),
opt.Slow, opt.WalkOption.ErrorCallback),
opt.WalkOption),
analyzer: a,
handlerManager: handlerManager,
@@ -122,7 +122,8 @@ func buildPathsToSkip(base string, paths []string) []string {
func (a Artifact) Inspect(ctx context.Context) (types.ArtifactReference, error) {
var wg sync.WaitGroup
result := analyzer.NewAnalysisResult()
limit := semaphore.New(a.artifactOption.Slow)
parallel := lo.Ternary(a.artifactOption.Parallel > 0, a.artifactOption.Parallel, 5)
limit := semaphore.NewWeighted(int64(parallel))
opts := analyzer.AnalysisOptions{
Offline: a.artifactOption.Offline,
FileChecksum: a.artifactOption.FileChecksum,

View File

@@ -158,7 +158,7 @@ func TestArtifact_Inspect(t *testing.T) {
fields: fields{
dir: "./testdata/unknown",
},
wantErr: "walk error",
wantErr: "walk dir error",
},
{
name: "happy path with single file",

View File

@@ -2,6 +2,8 @@ package vm
import (
"context"
"github.com/samber/lo"
"golang.org/x/sync/semaphore"
"io"
"os"
"strings"
@@ -15,7 +17,6 @@ import (
"github.com/aquasecurity/trivy/pkg/fanal/handler"
"github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/fanal/walker"
"github.com/aquasecurity/trivy/pkg/semaphore"
)
type Type string
@@ -41,7 +42,8 @@ type Storage struct {
func (a *Storage) Analyze(ctx context.Context, r *io.SectionReader) (types.BlobInfo, error) {
var wg sync.WaitGroup
limit := semaphore.New(a.artifactOption.Slow)
parallel := lo.Ternary(a.artifactOption.Parallel > 0, a.artifactOption.Parallel, 5)
limit := semaphore.NewWeighted(int64(parallel))
result := analyzer.NewAnalysisResult()
opts := analyzer.AnalysisOptions{
@@ -135,7 +137,7 @@ func NewArtifact(target string, c cache.ArtifactCache, opt artifact.Option) (art
cache: c,
analyzer: a,
handlerManager: handlerManager,
walker: walker.NewVM(opt.SkipFiles, opt.SkipDirs, opt.Slow),
walker: walker.NewVM(opt.SkipFiles, opt.SkipDirs),
artifactOption: opt,
}

View File

@@ -1,29 +1,32 @@
package walker
import (
"errors"
"golang.org/x/xerrors"
"io/fs"
"os"
"path/filepath"
swalker "github.com/saracen/walker"
"golang.org/x/xerrors"
"time"
dio "github.com/aquasecurity/go-dep-parser/pkg/io"
"github.com/aquasecurity/trivy/pkg/log"
)
type ErrorCallback func(pathname string, err error) error
type FS struct {
walker
errCallback ErrorCallback
opt Option
}
func NewFS(skipFiles, skipDirs []string, slow bool, errCallback ErrorCallback) FS {
if errCallback == nil {
errCallback = func(pathname string, err error) error {
func NewFS(skipFiles, skipDirs []string, opt Option) FS {
if opt.ErrCallback == nil {
opt.ErrCallback = func(pathname string, err error) error {
switch {
// Unwrap fs.SkipDir error
case errors.Is(err, fs.SkipDir):
return fs.SkipDir
// ignore permission errors
if os.IsPermission(err) {
case os.IsPermission(err):
return nil
}
// halt traversal on any other error
@@ -32,77 +35,20 @@ func NewFS(skipFiles, skipDirs []string, slow bool, errCallback ErrorCallback) F
}
return FS{
walker: newWalker(skipFiles, skipDirs, slow),
errCallback: errCallback,
walker: newWalker(skipFiles, skipDirs),
opt: opt,
}
}
// Walk walks the file tree rooted at root, calling WalkFunc for each file or
// Walk walks the file tree rooted at root, calling WalkDirFunc for each file or
// directory in the tree, including root, but a directory to be ignored will be skipped.
func (w FS) Walk(root string, fn WalkFunc) error {
// walk function called for every path found
walkFn := func(pathname string, fi os.FileInfo) error {
pathname = filepath.Clean(pathname)
// For exported rootfs (e.g. images/alpine/etc/alpine-release)
relPath, err := filepath.Rel(root, pathname)
if err != nil {
return xerrors.Errorf("filepath rel (%s): %w", relPath, err)
}
relPath = filepath.ToSlash(relPath)
if fi.IsDir() {
if w.shouldSkipDir(relPath) {
return filepath.SkipDir
}
return nil
} else if !fi.Mode().IsRegular() {
return nil
} else if w.shouldSkipFile(relPath) {
return nil
}
if err = fn(relPath, fi, w.fileOpener(pathname)); err != nil {
return xerrors.Errorf("failed to analyze file: %w", err)
walkDir := w.walkDirFunc(root, fn)
err := filepath.WalkDir(root, func(filePath string, d fs.DirEntry, err error) error {
if walkErr := walkDir(filePath, d, err); walkErr != nil {
return w.opt.ErrCallback(filePath, walkErr)
}
return nil
}
if w.slow {
// In series: fast, with higher CPU/memory
return w.walkSlow(root, walkFn)
}
// In parallel: slow, with lower CPU/memory
return w.walkFast(root, walkFn)
}
type fastWalkFunc func(pathname string, fi os.FileInfo) error
func (w FS) walkFast(root string, walkFn fastWalkFunc) error {
// error function called for every error encountered
errorCallbackOption := swalker.WithErrorCallback(w.errCallback)
// Multiple goroutines stat the filesystem concurrently. The provided
// walkFn must be safe for concurrent use.
log.Logger.Debugf("Walk the file tree rooted at '%s' in parallel", root)
if err := swalker.Walk(root, walkFn, errorCallbackOption); err != nil {
return xerrors.Errorf("walk error: %w", err)
}
return nil
}
func (w FS) walkSlow(root string, walkFn fastWalkFunc) error {
log.Logger.Debugf("Walk the file tree rooted at '%s' in series", root)
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return w.errCallback(path, err)
}
info, err := d.Info()
if err != nil {
return xerrors.Errorf("file info error: %w", err)
}
return walkFn(path, info)
})
if err != nil {
return xerrors.Errorf("walk dir error: %w", err)
@@ -110,6 +56,46 @@ func (w FS) walkSlow(root string, walkFn fastWalkFunc) error {
return nil
}
func (w FS) walkDirFunc(root string, fn WalkFunc) fs.WalkDirFunc {
return func(filePath string, d fs.DirEntry, err error) error {
if err != nil {
return w.opt.ErrCallback(filePath, err)
}
time.Sleep(w.opt.Delay)
filePath = filepath.Clean(filePath)
// For exported rootfs (e.g. images/alpine/etc/alpine-release)
relPath, err := filepath.Rel(root, filePath)
if err != nil {
return xerrors.Errorf("filepath rel (%s): %w", relPath, err)
}
relPath = filepath.ToSlash(relPath)
info, err := d.Info()
if err != nil {
return xerrors.Errorf("file info error: %w", err)
}
switch {
case info.IsDir():
if w.shouldSkipDir(relPath) {
return filepath.SkipDir
}
return nil
case !info.Mode().IsRegular():
return nil
case w.shouldSkipFile(relPath):
return nil
}
if err = fn(relPath, info, w.fileOpener(filePath)); err != nil {
return xerrors.Errorf("failed to analyze file: %w", err)
}
return nil
}
}
// fileOpener returns a function opening a file.
func (w *walker) fileOpener(pathname string) func() (dio.ReadSeekCloserAt, error) {
return func() (dio.ReadSeekCloserAt, error) {

View File

@@ -3,6 +3,7 @@ package walker_test
import (
"errors"
"io"
"io/fs"
"os"
"strings"
"testing"
@@ -77,15 +78,29 @@ func TestDir_Walk(t *testing.T) {
return nil
},
},
analyzeFn: func(filePath string, info os.FileInfo, opener analyzer.Opener) error {
// Ignore errors
analyzeFn: func(string, os.FileInfo, analyzer.Opener) error {
return nil
},
},
{
name: "ignore analysis errors",
rootDir: "testdata/fs",
fields: fields{
errCallback: func(pathname string, err error) error {
if errors.Is(err, fs.ErrClosed) {
return nil
}
return err
},
},
analyzeFn: func(string, os.FileInfo, analyzer.Opener) error {
return fs.ErrClosed
},
},
{
name: "sad path",
rootDir: "testdata/fs",
analyzeFn: func(filePath string, info os.FileInfo, opener analyzer.Opener) error {
analyzeFn: func(string, os.FileInfo, analyzer.Opener) error {
return errors.New("error")
},
wantErr: "failed to analyze file",
@@ -93,8 +108,9 @@ func TestDir_Walk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, true, tt.fields.errCallback)
w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, walker.Option{
ErrCallback: tt.fields.errCallback,
})
err := w.Walk(tt.rootDir, tt.analyzeFn)
if tt.wantErr != "" {
require.Error(t, err)

View File

@@ -25,14 +25,10 @@ type LayerTar struct {
threshold int64
}
func NewLayerTar(skipFiles, skipDirs []string, slow bool) LayerTar {
func NewLayerTar(skipFiles, skipDirs []string) LayerTar {
threshold := defaultSizeThreshold
if slow {
threshold = slowSizeThreshold
}
return LayerTar{
walker: newWalker(skipFiles, skipDirs, slow),
walker: newWalker(skipFiles, skipDirs),
threshold: threshold,
}
}

View File

@@ -83,7 +83,7 @@ func TestLayerTar_Walk(t *testing.T) {
f, err := os.Open("testdata/test.tar")
require.NoError(t, err)
w := walker.NewLayerTar(tt.fields.skipFiles, tt.fields.skipDirs, true)
w := walker.NewLayerTar(tt.fields.skipFiles, tt.fields.skipDirs)
gotOpqDirs, gotWhFiles, err := w.Walk(f, tt.analyzeFn)
if tt.wantErr != "" {

View File

@@ -39,14 +39,10 @@ type VM struct {
analyzeFn WalkFunc
}
func NewVM(skipFiles, skipDirs []string, slow bool) VM {
func NewVM(skipFiles, skipDirs []string) VM {
threshold := defaultSizeThreshold
if slow {
threshold = slowSizeThreshold
}
return VM{
walker: newWalker(skipFiles, skipDirs, slow),
walker: newWalker(skipFiles, skipDirs),
threshold: threshold,
}
}

View File

@@ -5,6 +5,7 @@ import (
"path"
"path/filepath"
"strings"
"time"
"github.com/bmatcuk/doublestar"
@@ -26,13 +27,19 @@ const (
type WalkFunc func(filePath string, info os.FileInfo, opener analyzer.Opener) error
// Option allows users to define a custom walking behavior.
// This option is only available when using Trivy as an imported library and not through CLI flags.
type Option struct {
Delay time.Duration
ErrCallback ErrorCallback
}
type walker struct {
skipFiles []string
skipDirs []string
slow bool
}
func newWalker(skipFiles, skipDirs []string, slow bool) walker {
func newWalker(skipFiles, skipDirs []string) walker {
var cleanSkipFiles, cleanSkipDirs []string
for _, skipFile := range skipFiles {
skipFile = filepath.ToSlash(filepath.Clean(skipFile))
@@ -49,7 +56,6 @@ func newWalker(skipFiles, skipDirs []string, slow bool) walker {
return walker{
skipFiles: cleanSkipFiles,
skipDirs: cleanSkipDirs,
slow: slow,
}
}

View File

@@ -54,7 +54,7 @@ func Test_shouldSkipFile(t *testing.T) {
for i, tc := range testCases {
t.Run(fmt.Sprint(i), func(t *testing.T) {
w := newWalker(tc.skipFiles, nil, false)
w := newWalker(tc.skipFiles, nil)
for file, skipResult := range tc.skipMap {
assert.Equal(t, skipResult, w.shouldSkipFile(filepath.ToSlash(filepath.Clean(file))), fmt.Sprintf("skipFiles: %s, file: %s", tc.skipFiles, file))
}
@@ -108,7 +108,7 @@ func Test_shouldSkipDir(t *testing.T) {
for i, tc := range testCases {
t.Run(fmt.Sprint(i), func(t *testing.T) {
w := newWalker(nil, tc.skipDirs, false)
w := newWalker(nil, tc.skipDirs)
for dir, skipResult := range tc.skipMap {
assert.Equal(t, skipResult, w.shouldSkipDir(filepath.ToSlash(filepath.Clean(dir))), fmt.Sprintf("skipDirs: %s, dir: %s", tc.skipDirs, dir))
}

View File

@@ -15,7 +15,7 @@ import (
type onFile[T any] func(string, fs.FileInfo, dio.ReadSeekerAt) (T, error)
type onWalkResult[T any] func(T) error
func WalkDir[T any](ctx context.Context, fsys fs.FS, root string, slow bool,
func WalkDir[T any](ctx context.Context, fsys fs.FS, root string, parallel int,
onFile onFile[T], onResult onWalkResult[T]) error {
g, ctx := errgroup.WithContext(ctx)
@@ -54,11 +54,7 @@ func WalkDir[T any](ctx context.Context, fsys fs.FS, root string, slow bool,
// Start a fixed number of goroutines to read and digest files.
c := make(chan T)
limit := 10
if slow {
limit = 1
}
for i := 0; i < limit; i++ {
for i := 0; i < parallel; i++ {
g.Go(func() error {
for path := range paths {
if err := walk(ctx, fsys, path, c, onFile); err != nil {

View File

@@ -1,30 +0,0 @@
package semaphore
import "golang.org/x/sync/semaphore"
const defaultSize = 5
type options struct {
size int64
}
type option func(*options)
func WithDefault(n int64) option {
return func(opts *options) {
opts.size = defaultSize
}
}
func New(slow bool, opts ...option) *semaphore.Weighted {
o := &options{size: defaultSize}
for _, opt := range opts {
opt(o)
}
if slow {
// Process in series
return semaphore.NewWeighted(1)
}
// Process in parallel
return semaphore.NewWeighted(o.size)
}