feat: add slow mode (#3084)

Co-authored-by: AMF <work@afdesk.com>

pkg/commands/artifact/run.go

@@ -519,6 +519,7 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
 			SBOMSources:       opts.SBOMSources,
 			RekorURL:          opts.RekorURL,
 			Platform:          opts.Platform,
+			Slow:              opts.Slow,
 
 			// For misconfiguration scanning
 			MisconfScannerOption: configScannerOptions,

pkg/fanal/artifact/artifact.go

@@ -26,6 +26,7 @@ type Option struct {
 	SBOMSources       []string
 	RekorURL          string
 	Platform          string
+	Slow              bool // Lower CPU and memory
 
 	MisconfScannerOption misconf.ScannerOption
 	SecretScannerOption  analyzer.SecretScannerOption

pkg/fanal/artifact/image/image.go

@@ -64,7 +64,7 @@ func NewArtifact(img types.Image, c cache.ArtifactCache, opt artifact.Option) (a
 	return Artifact{
 		image:          img,
 		cache:          c,
-		walker:         walker.NewLayerTar(opt.SkipFiles, opt.SkipDirs),
+		walker:         walker.NewLayerTar(opt.SkipFiles, opt.SkipDirs, opt.Slow),
 		analyzer:       a,
 		handlerManager: handlerManager,
 
@@ -206,9 +206,24 @@ func (a Artifact) inspect(ctx context.Context, missingImage string, layerKeys, b
 	done := make(chan struct{})
 	errCh := make(chan error)
 
+	limit := semaphore.NewWeighted(parallel)
+	if a.artifactOption.Slow {
+		// Inspect layers in series
+		limit = semaphore.NewWeighted(1)
+	}
+
 	var osFound types.OS
 	for _, k := range layerKeys {
+		if err := limit.Acquire(ctx, 1); err != nil {
+			return xerrors.Errorf("semaphore acquire: %w", err)
+		}
+
 		go func(ctx context.Context, layerKey string) {
+			defer func() {
+				limit.Release(1)
+				done <- struct{}{}
+			}()
+
 			layer := layerKeyMap[layerKey]
 
 			// If it is a base layer, secret scanning should not be performed.
@@ -229,7 +244,6 @@ func (a Artifact) inspect(ctx context.Context, missingImage string, layerKeys, b
 			if layerInfo.OS != nil {
 				osFound = *layerInfo.OS
 			}
-			done <- struct{}{}
 		}(ctx, k)
 	}
 
@@ -266,6 +280,10 @@ func (a Artifact) inspectLayer(ctx context.Context, layerInfo LayerInfo, disable
 	opts := analyzer.AnalysisOptions{Offline: a.artifactOption.Offline}
 	result := analyzer.NewAnalysisResult()
 	limit := semaphore.NewWeighted(parallel)
+	if a.artifactOption.Slow {
+		// Analyze files in series
+		limit = semaphore.NewWeighted(1)
+	}
 
 	// Walk a tar layer
 	opqDirs, whFiles, err := a.walker.Walk(r, func(filePath string, info os.FileInfo, opener analyzer.Opener) error {

pkg/fanal/artifact/local/fs.go

@@ -55,7 +55,7 @@ func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (a
 	return Artifact{
 		rootPath:       filepath.Clean(rootPath),
 		cache:          c,
-		walker:         walker.NewFS(buildAbsPaths(rootPath, opt.SkipFiles), buildAbsPaths(rootPath, opt.SkipDirs)),
+		walker:         walker.NewFS(buildAbsPaths(rootPath, opt.SkipFiles), buildAbsPaths(rootPath, opt.SkipDirs), opt.Slow),
 		analyzer:       a,
 		handlerManager: handlerManager,
 
@@ -79,6 +79,10 @@ func (a Artifact) Inspect(ctx context.Context) (types.ArtifactReference, error)
 	var wg sync.WaitGroup
 	result := analyzer.NewAnalysisResult()
 	limit := semaphore.NewWeighted(parallel)
+	if a.artifactOption.Slow {
+		// Analyze files in series
+		limit = semaphore.NewWeighted(1)
+	}
 
 	err := a.walker.Walk(a.rootPath, func(filePath string, info os.FileInfo, opener analyzer.Opener) error {
 		directory := a.rootPath

pkg/fanal/walker/fs.go

@@ -1,6 +1,7 @@
 package walker
 
 import (
+	"io/fs"
 	"os"
 	"path/filepath"
 
@@ -8,15 +9,16 @@ import (
 	"golang.org/x/xerrors"
 
 	dio "github.com/aquasecurity/go-dep-parser/pkg/io"
+	"github.com/aquasecurity/trivy/pkg/log"
 )
 
 type FS struct {
 	walker
 }
 
-func NewFS(skipFiles, skipDirs []string) FS {
+func NewFS(skipFiles, skipDirs []string, slow bool) FS {
 	return FS{
-		walker: newWalker(skipFiles, skipDirs),
+		walker: newWalker(skipFiles, skipDirs, slow),
 	}
 }
 
@@ -44,6 +46,18 @@ func (w FS) Walk(root string, fn WalkFunc) error {
 		return nil
 	}
 
+	if w.slow {
+		// In series: fast, with higher CPU/memory
+		return walkSlow(root, walkFn)
+	}
+
+	// In parallel: slow, with lower CPU/memory
+	return walkFast(root, walkFn)
+}
+
+type fastWalkFunc func(pathname string, fi os.FileInfo) error
+
+func walkFast(root string, walkFn fastWalkFunc) error {
 	// error function called for every error encountered
 	errorCallbackOption := swalker.WithErrorCallback(func(pathname string, err error) error {
 		// ignore permission errors
@@ -56,12 +70,28 @@ func (w FS) Walk(root string, fn WalkFunc) error {
 
 	// Multiple goroutines stat the filesystem concurrently. The provided
 	// walkFn must be safe for concurrent use.
+	log.Logger.Debugf("Walk the file tree rooted at '%s' in parallel", root)
 	if err := swalker.Walk(root, walkFn, errorCallbackOption); err != nil {
 		return xerrors.Errorf("walk error: %w", err)
 	}
 	return nil
 }
 
+func walkSlow(root string, walkFn fastWalkFunc) error {
+	log.Logger.Debugf("Walk the file tree rooted at '%s' in series", root)
+	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		info, err := d.Info()
+		if err != nil {
+			return xerrors.Errorf("file info error: %w", err)
+		}
+		return walkFn(path, info)
+	})
+	if err != nil {
+		return xerrors.Errorf("walk dir error: %w", err)
+	}
+	return nil
+}
+
 // fileOpener returns a function opening a file.
 func (w *walker) fileOpener(pathname string) func() (dio.ReadSeekCloserAt, error) {
 	return func() (dio.ReadSeekCloserAt, error) {

pkg/fanal/walker/fs_test.go

@@ -79,7 +79,7 @@ func TestDir_Walk(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs)
+			w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, true)
 
 			err := w.Walk(tt.rootDir, tt.analyzeFn)
 			if tt.wantErr != "" {

pkg/fanal/walker/tar.go

@@ -22,15 +22,23 @@ const (
 
 type LayerTar struct {
 	walker
+	threshold int64
 }
 
-func NewLayerTar(skipFiles, skipDirs []string) LayerTar {
+func NewLayerTar(skipFiles, skipDirs []string, slow bool) LayerTar {
+	threshold := defaultSizeThreshold
+	if slow {
+		threshold = slowSizeThreshold
+	}
+
 	return LayerTar{
-		walker: newWalker(skipFiles, skipDirs),
+		walker:    newWalker(skipFiles, skipDirs, slow),
+		threshold: threshold,
 	}
 }
 
 func (w LayerTar) Walk(layer io.Reader, analyzeFn WalkFunc) ([]string, []string, error) {
+
 	var opqDirs, whFiles, skipDirs []string
 	tr := tar.NewReader(layer)
 	for {
@@ -86,7 +94,7 @@ func (w LayerTar) Walk(layer io.Reader, analyzeFn WalkFunc) ([]string, []string,
 }
 
 func (w LayerTar) processFile(filePath string, tr *tar.Reader, fi fs.FileInfo, analyzeFn WalkFunc) error {
-	tf := newTarFile(fi.Size(), tr)
+	tf := newTarFile(fi.Size(), tr, w.threshold)
 	defer func() {
 		// nolint
 		_ = tf.Clean()
@@ -120,14 +128,17 @@ type tarFile struct {
 	size   int64
 	reader io.Reader
 
+	threshold int64 //　Files larger than this threshold are written to file without being read into memory.
+
 	content  []byte // It will be populated if this file is small
 	filePath string // It will be populated if this file is large
 }
 
-func newTarFile(size int64, r io.Reader) tarFile {
+func newTarFile(size int64, r io.Reader, threshold int64) tarFile {
 	return tarFile{
-		size:   size,
+		size:      size,
-		reader: r,
+		reader:    r,
+		threshold: threshold,
 	}
 }
 
@@ -137,7 +148,7 @@ func newTarFile(size int64, r io.Reader) tarFile {
 func (o *tarFile) Open() (dio.ReadSeekCloserAt, error) {
 	o.once.Do(func() {
 		// When the file is large, it will be written down to a temp file.
-		if o.size >= ThresholdSize {
+		if o.size >= o.threshold {
 			f, err := os.CreateTemp("", "fanal-*")
 			if err != nil {
 				o.err = xerrors.Errorf("failed to create the temp file: %w", err)

pkg/fanal/walker/tar_test.go

@@ -82,7 +82,7 @@ func TestLayerTar_Walk(t *testing.T) {
 			f, err := os.Open("testdata/test.tar")
 			require.NoError(t, err)
 
-			w := walker.NewLayerTar(tt.fields.skipFiles, tt.fields.skipDirs)
+			w := walker.NewLayerTar(tt.fields.skipFiles, tt.fields.skipDirs, true)
 
 			gotOpqDirs, gotWhFiles, err := w.Walk(f, tt.analyzeFn)
 			if tt.wantErr != "" {

pkg/fanal/walker/walk.go

@@ -15,16 +15,20 @@ var (
 	SystemDirs = []string{"proc", "sys", "dev"}
 )
 
-const ThresholdSize = int64(200) << 20
+const (
+	defaultSizeThreshold = int64(200) << 20 // 200MB
+	slowSizeThreshold    = int64(200) << 20 // 10KB
+)
 
 type WalkFunc func(filePath string, info os.FileInfo, opener analyzer.Opener) error
 
 type walker struct {
 	skipFiles []string
 	skipDirs  []string
+	slow      bool
 }
 
-func newWalker(skipFiles, skipDirs []string) walker {
+func newWalker(skipFiles, skipDirs []string, slow bool) walker {
 	var cleanSkipFiles, cleanSkipDirs []string
 	for _, skipFile := range skipFiles {
 		skipFile = filepath.ToSlash(filepath.Clean(skipFile))
@@ -41,6 +45,7 @@ func newWalker(skipFiles, skipDirs []string) walker {
 	return walker{
 		skipFiles: cleanSkipFiles,
 		skipDirs:  cleanSkipDirs,
+		slow:      slow,
 	}
 }
 

pkg/flag/scan_flags.go

@@ -38,6 +38,12 @@ var (
 		Value:      []string{},
 		Usage:      "specify config file patterns",
 	}
+	SlowFlag = Flag{
+		Name:       "slow",
+		ConfigName: "scan.slow",
+		Value:      false,
+		Usage:      "scan over time with lower CPU and memory utilization",
+	}
 	SBOMSourcesFlag = Flag{
 		Name:       "sbom-sources",
 		ConfigName: "scan.sbom-sources",
@@ -58,6 +64,7 @@ type ScanFlagGroup struct {
 	OfflineScan    *Flag
 	SecurityChecks *Flag
 	FilePatterns   *Flag
+	Slow           *Flag
 	SBOMSources    *Flag
 	RekorURL       *Flag
 }
@@ -69,6 +76,7 @@ type ScanOptions struct {
 	OfflineScan    bool
 	SecurityChecks []string
 	FilePatterns   []string
+	Slow           bool
 	SBOMSources    []string
 	RekorURL       string
 }
@@ -80,6 +88,7 @@ func NewScanFlagGroup() *ScanFlagGroup {
 		OfflineScan:    &OfflineScanFlag,
 		SecurityChecks: &SecurityChecksFlag,
 		FilePatterns:   &FilePatternsFlag,
+		Slow:           &SlowFlag,
 		SBOMSources:    &SBOMSourcesFlag,
 		RekorURL:       &RekorURLFlag,
 	}
@@ -90,7 +99,8 @@ func (f *ScanFlagGroup) Name() string {
 }
 
 func (f *ScanFlagGroup) Flags() []*Flag {
-	return []*Flag{f.SkipDirs, f.SkipFiles, f.OfflineScan, f.SecurityChecks, f.FilePatterns, f.SBOMSources, f.RekorURL}
+	return []*Flag{f.SkipDirs, f.SkipFiles, f.OfflineScan, f.SecurityChecks, f.FilePatterns,
+		f.Slow, f.SBOMSources, f.RekorURL}
 }
 
 func (f *ScanFlagGroup) ToOptions(args []string) (ScanOptions, error) {
@@ -115,6 +125,7 @@ func (f *ScanFlagGroup) ToOptions(args []string) (ScanOptions, error) {
 		OfflineScan:    getBool(f.OfflineScan),
 		SecurityChecks: securityChecks,
 		FilePatterns:   getStringSlice(f.FilePatterns),
+		Slow:           getBool(f.Slow),
 		SBOMSources:    sbomSources,
 		RekorURL:       getString(f.RekorURL),
 	}, nil