gitea-mirror/trivy
1
0
Fork 0
mirror of https://github.com/aquasecurity/trivy.git synced 2025-12-05 20:40:16 -08:00
Code Issues Packages Projects Releases Wiki Activity

feat(flag): replace '--slow' with '--parallel' (#5572)

Browse Source 
Signed-off-by: knqyf263 <knqyf263@gmail.com>
This commit is contained in:
Teppei Fukuda
2023-11-15 15:41:13 +09:00
committed by GitHub
parent 5372067611
commit ac0e327492
31 changed files with 115 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/docs/references/configuration/cli/trivy_filesystem.md
View File

@@ -54,6 +54,7 @@ trivy filesystem [flags] PATH
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
-o, --output string output file name
--parallel int number of goroutines enabled for parallel scanning, set 0 to auto-detect parallelism (default 5)
--password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons.
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0")
--policy-namespaces strings Rego namespaces
@@ -76,7 +77,6 @@ trivy filesystem [flags] PATH
--skip-files strings specify the files or glob patterns to skip
--skip-java-db-update skip updating Java index database
--skip-policy-update skip fetching rego policy updates
--slow scan over time with lower CPU and memory utilization
-t, --template string output template
--tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules
--tf-vars strings specify paths to override the Terraform tfvars files

2
docs/docs/references/configuration/cli/trivy_image.md
View File

@@ -73,6 +73,7 @@ trivy image [flags] IMAGE_NAME
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
-o, --output string output file name
--parallel int number of goroutines enabled for parallel scanning, set 0 to auto-detect parallelism (default 5)
--password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons.
--platform string set platform in the form os/arch if image is multi-platform capable
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0")
@@ -97,7 +98,6 @@ trivy image [flags] IMAGE_NAME
--skip-files strings specify the files or glob patterns to skip
--skip-java-db-update skip updating Java index database
--skip-policy-update skip fetching rego policy updates
--slow scan over time with lower CPU and memory utilization
-t, --template string output template
--tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules
--tf-vars strings specify paths to override the Terraform tfvars files

3
docs/docs/references/configuration/cli/trivy_kubernetes.md
View File

@@ -65,7 +65,7 @@ trivy kubernetes [flags] { cluster | all | specific resources like kubectl. eg:
--node-collector-namespace string specify the namespace in which the node-collector job should be deployed (default "trivy-temp")
--offline-scan do not issue API requests to identify dependencies
-o, --output string output file name
--parallel int number (between 1-20) of goroutines enabled for parallel scanning (default 5)
--parallel int number of goroutines enabled for parallel scanning, set 0 to auto-detect parallelism (default 5)
--password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons.
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0")
--policy-namespaces strings Rego namespaces
@@ -87,7 +87,6 @@ trivy kubernetes [flags] { cluster | all | specific resources like kubectl. eg:
--skip-files strings specify the files or glob patterns to skip
--skip-java-db-update skip updating Java index database
--skip-policy-update skip fetching rego policy updates
--slow scan over time with lower CPU and memory utilization
-t, --template string output template
--tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules
--tf-vars strings specify paths to override the Terraform tfvars files

2
docs/docs/references/configuration/cli/trivy_repository.md
View File

@@ -54,6 +54,7 @@ trivy repository [flags] (REPO_PATH | REPO_URL)
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
-o, --output string output file name
--parallel int number of goroutines enabled for parallel scanning, set 0 to auto-detect parallelism (default 5)
--password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons.
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0")
--policy-namespaces strings Rego namespaces
@@ -75,7 +76,6 @@ trivy repository [flags] (REPO_PATH | REPO_URL)
--skip-files strings specify the files or glob patterns to skip
--skip-java-db-update skip updating Java index database
--skip-policy-update skip fetching rego policy updates
--slow scan over time with lower CPU and memory utilization
--tag string pass the tag name to be scanned
-t, --template string output template
--tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules

2
docs/docs/references/configuration/cli/trivy_rootfs.md
View File

@@ -56,6 +56,7 @@ trivy rootfs [flags] ROOTDIR
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
-o, --output string output file name
--parallel int number of goroutines enabled for parallel scanning, set 0 to auto-detect parallelism (default 5)
--password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons.
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0")
--policy-namespaces strings Rego namespaces
@@ -77,7 +78,6 @@ trivy rootfs [flags] ROOTDIR
--skip-files strings specify the files or glob patterns to skip
--skip-java-db-update skip updating Java index database
--skip-policy-update skip fetching rego policy updates
--slow scan over time with lower CPU and memory utilization
-t, --template string output template
--tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules
--tf-vars strings specify paths to override the Terraform tfvars files

1
docs/docs/references/configuration/cli/trivy_sbom.md
View File

@@ -55,7 +55,6 @@ trivy sbom [flags] SBOM_PATH
--skip-dirs strings specify the directories or glob patterns to skip
--skip-files strings specify the files or glob patterns to skip
--skip-java-db-update skip updating Java index database
--slow scan over time with lower CPU and memory utilization
-t, --template string output template
--token string for authentication in client/server mode
--token-header string specify a header name for token in client/server mode (default "Trivy-Token")

2
docs/docs/references/configuration/cli/trivy_vm.md
View File

@@ -51,6 +51,7 @@ trivy vm [flags] VM_IMAGE
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
-o, --output string output file name
--parallel int number of goroutines enabled for parallel scanning, set 0 to auto-detect parallelism (default 5)
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0")
--redis-ca string redis ca file location, if using redis as cache backend
--redis-cert string redis certificate file location, if using redis as cache backend
@@ -68,7 +69,6 @@ trivy vm [flags] VM_IMAGE
--skip-dirs strings specify the directories or glob patterns to skip
--skip-files strings specify the files or glob patterns to skip
--skip-java-db-update skip updating Java index database
--slow scan over time with lower CPU and memory utilization
-t, --template string output template
--tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules
--tf-vars strings specify paths to override the Terraform tfvars files

18
integration/repo_test.go
View File

@@ -3,6 +3,7 @@
package integration
import (
"fmt"
"os"
"path/filepath"
"strings"
@@ -36,6 +37,7 @@ func TestRepository(t *testing.T) {
command string
format types.Format
includeDevDeps bool
parallel int
}
tests := []struct {
name string
@@ -69,6 +71,15 @@ func TestRepository(t *testing.T) {
},
golden: "testdata/gomod-skip.json.golden",
},
{
name: "gomod in series",
args: args{
scanner: types.VulnerabilityScanner,
input: "testdata/fixtures/repo/gomod",
parallel: 1,
},
golden: "testdata/gomod.json.golden",
},
{
name: "npm",
args: args{
@@ -396,13 +407,12 @@ func TestRepository(t *testing.T) {
osArgs := []string{
"-q",
"--cache-dir",
cacheDir,
"--cache-dir", cacheDir,
command,
"--skip-db-update",
"--skip-policy-update",
"--format",
string(format),
"--format", string(format),
"--parallel", fmt.Sprint(tt.args.parallel),
"--offline-scan",
}

1
pkg/commands/app.go
View File

@@ -1098,6 +1098,7 @@ func NewSBOMCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command {
scanFlagGroup := flag.NewScanFlagGroup()
scanFlagGroup.Scanners = nil // disable '--scanners' as it always scans for vulnerabilities
scanFlagGroup.IncludeDevDeps = nil // disable '--include-dev-deps'
scanFlagGroup.Parallel = nil // disable '--parallel'
sbomFlags := &flag.Flags{
CacheFlagGroup: flag.NewCacheFlagGroup(),

2
pkg/commands/artifact/run.go
View File

@@ -637,7 +637,7 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
SBOMSources: opts.SBOMSources,
RekorURL: opts.RekorURL,
//Platform: opts.Platform,
Slow: opts.Slow,
Parallel: opts.Parallel,
AWSRegion: opts.Region,
AWSEndpoint: opts.Endpoint,
FileChecksum: fileChecksum,

2
pkg/fanal/analyzer/analyzer.go
View File

@@ -41,7 +41,7 @@ var (
// AnalyzerOptions is used to initialize analyzers
type AnalyzerOptions struct {
Group Group
Slow bool
Parallel int
FilePatterns []string
DisabledAnalyzers []Type
MisconfScannerOption misconf.ScannerOption

6
pkg/fanal/analyzer/language/java/jar/jar.go
View File

@@ -33,12 +33,12 @@ var requiredExtensions = []string{
// javaLibraryAnalyzer analyzes jar/war/ear/par files
type javaLibraryAnalyzer struct {
slow bool
parallel int
}
func newJavaLibraryAnalyzer(options analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) {
return &javaLibraryAnalyzer{
slow: options.Slow,
parallel: options.Parallel,
}, nil
}
@@ -70,7 +70,7 @@ func (a *javaLibraryAnalyzer) PostAnalyze(ctx context.Context, input analyzer.Po
return nil
}
if err = parallel.WalkDir(ctx, input.FS, ".", a.slow, onFile, onResult); err != nil {
if err = parallel.WalkDir(ctx, input.FS, ".", a.parallel, onFile, onResult); err != nil {
return nil, xerrors.Errorf("walk dir error: %w", err)
}

2
pkg/fanal/analyzer/language/java/jar/jar_test.go
View File

@@ -132,7 +132,7 @@ func Test_javaLibraryAnalyzer_Analyze(t *testing.T) {
// init java-trivy-db with skip update
javadb.Init("testdata", defaultJavaDBRepository, true, false, types.RegistryOptions{Insecure: false})
a := javaLibraryAnalyzer{slow: true}
a := javaLibraryAnalyzer{}
ctx := context.Background()
mfs := mapfs.New()

23
pkg/fanal/artifact/artifact.go
View File

@@ -23,7 +23,7 @@ type Option struct {
AppDirs []string
SBOMSources []string
RekorURL string
Slow bool // Lower CPU and memory
Parallel int
AWSRegion string
AWSEndpoint string
FileChecksum bool // For SPDX
@@ -50,6 +50,27 @@ type WalkOption struct {
ErrorCallback walker.ErrorCallback
}
func (o *Option) AnalyzerOptions() analyzer.AnalyzerOptions {
return analyzer.AnalyzerOptions{
Group: o.AnalyzerGroup,
FilePatterns: o.FilePatterns,
Parallel: o.Parallel,
DisabledAnalyzers: o.DisabledAnalyzers,
MisconfScannerOption: o.MisconfScannerOption,
SecretScannerOption: o.SecretScannerOption,
LicenseScannerOption: o.LicenseScannerOption,
}
}
func (o *Option) ConfigAnalyzerOptions() analyzer.ConfigAnalyzerOptions {
return analyzer.ConfigAnalyzerOptions{
FilePatterns: o.FilePatterns,
DisabledAnalyzers: o.DisabledAnalyzers,
MisconfScannerOption: o.MisconfScannerOption,
SecretScannerOption: o.SecretScannerOption,
}
}
func (o *Option) Sort() {
sort.Slice(o.DisabledAnalyzers, func(i, j int) bool {
return o.DisabledAnalyzers[i] < o.DisabledAnalyzers[j]

24
pkg/fanal/artifact/image/image.go
View File

@@ -49,25 +49,12 @@ func NewArtifact(img types.Image, c cache.ArtifactCache, opt artifact.Option) (a
return nil, xerrors.Errorf("handler init error: %w", err)
}
a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{
Group: opt.AnalyzerGroup,
Slow: opt.Slow,
FilePatterns: opt.FilePatterns,
DisabledAnalyzers: opt.DisabledAnalyzers,
MisconfScannerOption: opt.MisconfScannerOption,
SecretScannerOption: opt.SecretScannerOption,
LicenseScannerOption: opt.LicenseScannerOption,
})
a, err := analyzer.NewAnalyzerGroup(opt.AnalyzerOptions())
if err != nil {
return nil, xerrors.Errorf("analyzer group error: %w", err)
}
ca, err := analyzer.NewConfigAnalyzerGroup(analyzer.ConfigAnalyzerOptions{
FilePatterns: opt.FilePatterns,
DisabledAnalyzers: opt.DisabledAnalyzers,
MisconfScannerOption: opt.MisconfScannerOption,
SecretScannerOption: opt.SecretScannerOption,
})
ca, err := analyzer.NewConfigAnalyzerGroup(opt.ConfigAnalyzerOptions())
if err != nil {
return nil, xerrors.Errorf("config analyzer group error: %w", err)
}
@@ -75,7 +62,7 @@ func NewArtifact(img types.Image, c cache.ArtifactCache, opt artifact.Option) (a
return Artifact{
image: img,
cache: c,
walker: walker.NewLayerTar(opt.SkipFiles, opt.SkipDirs, opt.Slow),
walker: walker.NewLayerTar(opt.SkipFiles, opt.SkipDirs),
analyzer: a,
configAnalyzer: ca,
handlerManager: handlerManager,
@@ -215,8 +202,7 @@ func (a Artifact) inspect(ctx context.Context, missingImage string, layerKeys, b
layerKeyMap map[string]LayerInfo, configFile *v1.ConfigFile) error {
var osFound types.OS
workers := lo.Ternary(a.artifactOption.Slow, 1, 5)
p := parallel.NewPipeline(workers, false, layerKeys, func(ctx context.Context, layerKey string) (any, error) {
p := parallel.NewPipeline(a.artifactOption.Parallel, false, layerKeys, func(ctx context.Context, layerKey string) (any, error) {
layer := layerKeyMap[layerKey]
// If it is a base layer, secret scanning should not be performed.
@@ -268,7 +254,7 @@ func (a Artifact) inspectLayer(ctx context.Context, layerInfo LayerInfo, disable
FileChecksum: a.artifactOption.FileChecksum,
}
result := analyzer.NewAnalysisResult()
limit := semaphore.New(a.artifactOption.Slow)
limit := semaphore.New(a.artifactOption.Parallel)
// Prepare filesystem for post analysis
composite, err := a.analyzer.PostAnalyzerFS()

5
pkg/fanal/artifact/image/image_test.go
View File

@@ -2035,11 +2035,8 @@ func TestArtifact_Inspect(t *testing.T) {
wantErr: "put layer failed",
},
{
name: "sad path, PutBlob returns an error with multiple layers and Slow enabled",
name: "sad path, PutBlob returns an error with multiple layers",
imagePath: "../../test/testdata/vuln-image.tar.gz",
artifactOpt: artifact.Option{
Slow: true,
},
missingBlobsExpectation: cache.ArtifactCacheMissingBlobsExpectation{
Args: cache.ArtifactCacheMissingBlobsArgs{
ArtifactID: "sha256:33f9415ed2cd5a9cef5d5144333619745b9ec0f851f0684dd45fa79c6b26a650",

14
pkg/fanal/artifact/local/fs.go
View File

@@ -39,15 +39,7 @@ func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (a
return nil, xerrors.Errorf("handler initialize error: %w", err)
}
a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{
Group: opt.AnalyzerGroup,
Slow: opt.Slow,
FilePatterns: opt.FilePatterns,
DisabledAnalyzers: opt.DisabledAnalyzers,
MisconfScannerOption: opt.MisconfScannerOption,
SecretScannerOption: opt.SecretScannerOption,
LicenseScannerOption: opt.LicenseScannerOption,
})
a, err := analyzer.NewAnalyzerGroup(opt.AnalyzerOptions())
if err != nil {
return nil, xerrors.Errorf("analyzer group error: %w", err)
}
@@ -56,7 +48,7 @@ func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (a
rootPath: filepath.ToSlash(filepath.Clean(rootPath)),
cache: c,
walker: walker.NewFS(buildPathsToSkip(rootPath, opt.SkipFiles), buildPathsToSkip(rootPath, opt.SkipDirs),
opt.Slow, opt.WalkOption.ErrorCallback),
opt.Parallel, opt.WalkOption.ErrorCallback),
analyzer: a,
handlerManager: handlerManager,
@@ -122,7 +114,7 @@ func buildPathsToSkip(base string, paths []string) []string {
func (a Artifact) Inspect(ctx context.Context) (types.ArtifactReference, error) {
var wg sync.WaitGroup
result := analyzer.NewAnalysisResult()
limit := semaphore.New(a.artifactOption.Slow)
limit := semaphore.New(a.artifactOption.Parallel)
opts := analyzer.AnalysisOptions{
Offline: a.artifactOption.Offline,
FileChecksum: a.artifactOption.FileChecksum,

2
pkg/fanal/artifact/local/fs_test.go
View File

@@ -166,7 +166,7 @@ func TestArtifact_Inspect(t *testing.T) {
fields: fields{
dir: "./testdata/unknown",
},
wantErr: "walk error",
wantErr: "walk dir error",
},
{
name: "happy path with single file",

13
pkg/fanal/artifact/vm/vm.go
View File

@@ -41,7 +41,7 @@ type Storage struct {
func (a *Storage) Analyze(ctx context.Context, r *io.SectionReader) (types.BlobInfo, error) {
var wg sync.WaitGroup
limit := semaphore.New(a.artifactOption.Slow)
limit := semaphore.New(a.artifactOption.Parallel)
result := analyzer.NewAnalysisResult()
opts := analyzer.AnalysisOptions{
@@ -119,14 +119,7 @@ func NewArtifact(target string, c cache.ArtifactCache, opt artifact.Option) (art
if err != nil {
return nil, xerrors.Errorf("handler init error: %w", err)
}
a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{
Group: opt.AnalyzerGroup,
FilePatterns: opt.FilePatterns,
DisabledAnalyzers: opt.DisabledAnalyzers,
MisconfScannerOption: opt.MisconfScannerOption,
SecretScannerOption: opt.SecretScannerOption,
LicenseScannerOption: opt.LicenseScannerOption,
})
a, err := analyzer.NewAnalyzerGroup(opt.AnalyzerOptions())
if err != nil {
return nil, xerrors.Errorf("analyzer group error: %w", err)
}
@@ -135,7 +128,7 @@ func NewArtifact(target string, c cache.ArtifactCache, opt artifact.Option) (art
cache: c,
analyzer: a,
handlerManager: handlerManager,
walker: walker.NewVM(opt.SkipFiles, opt.SkipDirs, opt.Slow),
walker: walker.NewVM(opt.SkipFiles, opt.SkipDirs),
artifactOption: opt,
}

8
pkg/fanal/walker/fs.go
View File

@@ -16,10 +16,11 @@ type ErrorCallback func(pathname string, err error) error
type FS struct {
walker
parallel int
errCallback ErrorCallback
}
func NewFS(skipFiles, skipDirs []string, slow bool, errCallback ErrorCallback) FS {
func NewFS(skipFiles, skipDirs []string, parallel int, errCallback ErrorCallback) FS {
if errCallback == nil {
errCallback = func(pathname string, err error) error {
// ignore permission errors
@@ -32,7 +33,8 @@ func NewFS(skipFiles, skipDirs []string, slow bool, errCallback ErrorCallback) F
}
return FS{
walker: newWalker(skipFiles, skipDirs, slow),
walker: newWalker(skipFiles, skipDirs),
parallel: parallel,
errCallback: errCallback,
}
}
@@ -69,7 +71,7 @@ func (w FS) Walk(root string, fn WalkFunc) error {
return nil
}
if w.slow {
if w.parallel <= 1 {
// In series: fast, with higher CPU/memory
return w.walkSlow(root, walkFn)
}

2
pkg/fanal/walker/fs_test.go
View File

@@ -93,7 +93,7 @@ func TestDir_Walk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, true, tt.fields.errCallback)
w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, 1, tt.fields.errCallback)
err := w.Walk(tt.rootDir, tt.analyzeFn)
if tt.wantErr != "" {

8
pkg/fanal/walker/tar.go
View File

@@ -25,14 +25,10 @@ type LayerTar struct {
threshold int64
}
func NewLayerTar(skipFiles, skipDirs []string, slow bool) LayerTar {
func NewLayerTar(skipFiles, skipDirs []string) LayerTar {
threshold := defaultSizeThreshold
if slow {
threshold = slowSizeThreshold
}
return LayerTar{
walker: newWalker(skipFiles, skipDirs, slow),
walker: newWalker(skipFiles, skipDirs),
threshold: threshold,
}
}

2
pkg/fanal/walker/tar_test.go
View File

@@ -81,7 +81,7 @@ func TestLayerTar_Walk(t *testing.T) {
f, err := os.Open("testdata/test.tar")
require.NoError(t, err)
w := walker.NewLayerTar(tt.fields.skipFiles, tt.fields.skipDirs, true)
w := walker.NewLayerTar(tt.fields.skipFiles, tt.fields.skipDirs)
gotOpqDirs, gotWhFiles, err := w.Walk(f, tt.analyzeFn)
if tt.wantErr != "" {

8
pkg/fanal/walker/vm.go
View File

@@ -39,14 +39,10 @@ type VM struct {
analyzeFn WalkFunc
}
func NewVM(skipFiles, skipDirs []string, slow bool) VM {
func NewVM(skipFiles, skipDirs []string) VM {
threshold := defaultSizeThreshold
if slow {
threshold = slowSizeThreshold
}
return VM{
walker: newWalker(skipFiles, skipDirs, slow),
walker: newWalker(skipFiles, skipDirs),
threshold: threshold,
}
}

9
pkg/fanal/walker/walk.go
View File

@@ -22,20 +22,16 @@ var (
}
)
const (
defaultSizeThreshold = int64(200) << 20 // 200MB
slowSizeThreshold = int64(100) << 10 // 10KB
)
const defaultSizeThreshold = int64(200) << 20 // 200MB
type WalkFunc func(filePath string, info os.FileInfo, opener analyzer.Opener) error
type walker struct {
skipFiles []string
skipDirs []string
slow bool
}
func newWalker(skipFiles, skipDirs []string, slow bool) walker {
func newWalker(skipFiles, skipDirs []string) walker {
var cleanSkipFiles, cleanSkipDirs []string
for _, skipFile := range skipFiles {
skipFile = filepath.ToSlash(filepath.Clean(skipFile))
@@ -52,7 +48,6 @@ func newWalker(skipFiles, skipDirs []string, slow bool) walker {
return walker{
skipFiles: cleanSkipFiles,
skipDirs: cleanSkipDirs,
slow: slow,
}
}

4
pkg/fanal/walker/walk_test.go
View File

@@ -54,7 +54,7 @@ func Test_shouldSkipFile(t *testing.T) {
for i, tc := range testCases {
t.Run(fmt.Sprint(i), func(t *testing.T) {
w := newWalker(tc.skipFiles, nil, false)
w := newWalker(tc.skipFiles, nil)
for file, skipResult := range tc.skipMap {
assert.Equal(t, skipResult, w.shouldSkipFile(filepath.ToSlash(filepath.Clean(file))), fmt.Sprintf("skipFiles: %s, file: %s", tc.skipFiles, file))
}
@@ -115,7 +115,7 @@ func Test_shouldSkipDir(t *testing.T) {
for i, tc := range testCases {
t.Run(fmt.Sprint(i), func(t *testing.T) {
w := newWalker(nil, tc.skipDirs, false)
w := newWalker(nil, tc.skipDirs)
for dir, skipResult := range tc.skipMap {
assert.Equal(t, skipResult, w.shouldSkipDir(filepath.ToSlash(filepath.Clean(dir))), fmt.Sprintf("skipDirs: %s, dir: %s", tc.skipDirs, dir))
}

21
pkg/flag/kubernetes_flags.go
View File

@@ -6,7 +6,6 @@ import (
"strings"
"github.com/samber/lo"
"golang.org/x/xerrors"
corev1 "k8s.io/api/core/v1"
)
@@ -52,12 +51,6 @@ var (
Default: "",
Usage: "specify k8s version to validate outdated api by it (example: 1.21.0)",
}
ParallelFlag = Flag{
Name: "parallel",
ConfigName: "kubernetes.parallel",
Default: 5,
Usage: "number (between 1-20) of goroutines enabled for parallel scanning",
}
TolerationsFlag = Flag{
Name: "tolerations",
ConfigName: "kubernetes.tolerations",
@@ -97,7 +90,6 @@ type K8sFlagGroup struct {
KubeConfig *Flag
Components *Flag
K8sVersion *Flag
Parallel *Flag
Tolerations *Flag
AllNamespaces *Flag
NodeCollectorNamespace *Flag
@@ -111,7 +103,6 @@ type K8sOptions struct {
KubeConfig string
Components []string
K8sVersion string
Parallel int
Tolerations []corev1.Toleration
AllNamespaces bool
NodeCollectorNamespace string
@@ -126,7 +117,6 @@ func NewK8sFlagGroup() *K8sFlagGroup {
KubeConfig: &KubeConfigFlag,
Components: &ComponentsFlag,
K8sVersion: &K8sVersionFlag,
Parallel: &ParallelFlag,
Tolerations: &TolerationsFlag,
AllNamespaces: &AllNamespaces,
NodeCollectorNamespace: &NodeCollectorNamespace,
@@ -146,7 +136,6 @@ func (f *K8sFlagGroup) Flags() []*Flag {
f.KubeConfig,
f.Components,
f.K8sVersion,
f.Parallel,
f.Tolerations,
f.AllNamespaces,
f.NodeCollectorNamespace,
@@ -160,14 +149,7 @@ func (f *K8sFlagGroup) ToOptions() (K8sOptions, error) {
if err != nil {
return K8sOptions{}, err
}
var parallel int
if f.Parallel != nil {
parallel = getInt(f.Parallel)
// check parallel flag is a valid number between 1-20
if parallel < 1 || parallel > 20 {
return K8sOptions{}, xerrors.Errorf("unable to parse parallel value, please ensure that the value entered is a valid number between 1-20.")
}
}
exludeNodeLabels := make(map[string]string)
exludeNodes := getStringSlice(f.ExcludeNodes)
for _, exludeNodeValue := range exludeNodes {
@@ -184,7 +166,6 @@ func (f *K8sFlagGroup) ToOptions() (K8sOptions, error) {
KubeConfig: getString(f.KubeConfig),
Components: getStringSlice(f.Components),
K8sVersion: getString(f.K8sVersion),
Parallel: parallel,
Tolerations: tolerations,
AllNamespaces: getBool(f.AllNamespaces),
NodeCollectorNamespace: getString(f.NodeCollectorNamespace),

26
pkg/flag/scan_flags.go
View File

@@ -1,6 +1,8 @@
package flag
import (
"runtime"
"github.com/aquasecurity/trivy/pkg/log"
"github.com/aquasecurity/trivy/pkg/types"
xstrings "github.com/aquasecurity/trivy/pkg/x/strings"
@@ -70,6 +72,13 @@ var (
ConfigName: "scan.slow",
Default: false,
Usage: "scan over time with lower CPU and memory utilization",
Deprecated: true,
}
ParallelFlag = Flag{
Name: "parallel",
ConfigName: "scan.parallel",
Default: 5,
Usage: "number of goroutines enabled for parallel scanning, set 0 to auto-detect parallelism",
}
SBOMSourcesFlag = Flag{
Name: "sbom-sources",
@@ -98,7 +107,8 @@ type ScanFlagGroup struct {
OfflineScan *Flag
Scanners *Flag
FilePatterns *Flag
Slow *Flag
Slow *Flag // deprecated
Parallel *Flag
SBOMSources *Flag
RekorURL *Flag
IncludeDevDeps *Flag
@@ -111,7 +121,7 @@ type ScanOptions struct {
OfflineScan bool
Scanners types.Scanners
FilePatterns []string
Slow bool
Parallel int
SBOMSources []string
RekorURL string
IncludeDevDeps bool
@@ -124,10 +134,11 @@ func NewScanFlagGroup() *ScanFlagGroup {
OfflineScan: &OfflineScanFlag,
Scanners: &ScannersFlag,
FilePatterns: &FilePatternsFlag,
Slow: &SlowFlag,
Parallel: &ParallelFlag,
SBOMSources: &SBOMSourcesFlag,
RekorURL: &RekorURLFlag,
IncludeDevDeps: &IncludeDevDepsFlag,
Slow: &SlowFlag,
}
}
@@ -143,6 +154,7 @@ func (f *ScanFlagGroup) Flags() []*Flag {
f.Scanners,
f.FilePatterns,
f.Slow,
f.Parallel,
f.SBOMSources,
f.RekorURL,
f.IncludeDevDeps,
@@ -155,6 +167,12 @@ func (f *ScanFlagGroup) ToOptions(args []string) (ScanOptions, error) {
target = args[0]
}
parallel := getInt(f.Parallel)
if f.Parallel != nil && parallel == 0 {
log.Logger.Infof("Set '--parallel' to the number of CPUs (%d)", runtime.NumCPU())
parallel = runtime.NumCPU()
}
return ScanOptions{
Target: target,
SkipDirs: getStringSlice(f.SkipDirs),
@@ -162,7 +180,7 @@ func (f *ScanFlagGroup) ToOptions(args []string) (ScanOptions, error) {
OfflineScan: getBool(f.OfflineScan),
Scanners: getUnderlyingStringSlice[types.Scanner](f.Scanners),
FilePatterns: getStringSlice(f.FilePatterns),
Slow: getBool(f.Slow),
Parallel: parallel,
SBOMSources: getStringSlice(f.SBOMSources),
RekorURL: getString(f.RekorURL),
IncludeDevDeps: getBool(f.IncludeDevDeps),

5
pkg/parallel/pipeline.go
View File

@@ -7,6 +7,8 @@ import (
"golang.org/x/sync/errgroup"
)
const defaultWorkers = 5
// Pipeline represents a structure for performing parallel processing.
// T represents the input element type and U represents the output element type.
type Pipeline[T, U any] struct {
@@ -29,6 +31,9 @@ func NewPipeline[T, U any](numWorkers int, progress bool, items []T,
// In case where there is no need to process the return values
fn2 = func(_ U) error { return nil }
}
if numWorkers == 0 {
numWorkers = defaultWorkers
}
return Pipeline[T, U]{
numWorkers: numWorkers,
progress: progress,

11
pkg/parallel/walk.go
View File

@@ -12,10 +12,12 @@ import (
"github.com/aquasecurity/trivy/pkg/log"
)
const defaultParallel = 5
type onFile[T any] func(string, fs.FileInfo, dio.ReadSeekerAt) (T, error)
type onWalkResult[T any] func(T) error
func WalkDir[T any](ctx context.Context, fsys fs.FS, root string, slow bool,
func WalkDir[T any](ctx context.Context, fsys fs.FS, root string, parallel int,
onFile onFile[T], onResult onWalkResult[T]) error {
g, ctx := errgroup.WithContext(ctx)
@@ -54,11 +56,10 @@ func WalkDir[T any](ctx context.Context, fsys fs.FS, root string, slow bool,
// Start a fixed number of goroutines to read and digest files.
c := make(chan T)
limit := 10
if slow {
limit = 1
if parallel == 0 {
parallel = defaultParallel
}
for i := 0; i < limit; i++ {
for i := 0; i < parallel; i++ {
g.Go(func() error {
for path := range paths {
if err := walk(ctx, fsys, path, c, onFile); err != nil {

26
pkg/semaphore/semaphore.go
View File

@@ -4,27 +4,9 @@ import "golang.org/x/sync/semaphore"
const defaultSize = 5
type options struct {
size int64
}
type option func(*options)
func WithDefault(n int64) option {
return func(opts *options) {
opts.size = defaultSize
func New(parallel int) *semaphore.Weighted {
if parallel == 0 {
parallel = defaultSize
}
}
func New(slow bool, opts ...option) *semaphore.Weighted {
o := &options{size: defaultSize}
for _, opt := range opts {
opt(o)
}
if slow {
// Process in series
return semaphore.NewWeighted(1)
}
// Process in parallel
return semaphore.NewWeighted(o.size)
return semaphore.NewWeighted(int64(parallel))
}