feat(config): support YAML files (fanal#155)

* feat: add config

* feat(analyzer/config): add yaml analyzer

* chore(mod): update

* chore(ci): bump up Go to 1.15

* test(analyzer/config): add anchors yaml test

* test(analyzer/config): add circular referneces yaml test

* refactor(analyzer/config) change yaml interface

* test(analyzer/config) add multiple yaml test

* chore(analyzer) change comment

Co-authored-by: masahiro331 <mur4m4s4.331@gmail.com>

.github/workflows/bench.yml

@@ -6,10 +6,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.13
+    - name: Set up Go 1.15
       uses: actions/setup-go@v1
       with:
-        go-version: 1.13
+        go-version: 1.15
       id: go
 
     - name: Check out code into the Go module directory

.github/workflows/test.yml

@@ -9,10 +9,10 @@ jobs:
     name: Unit Test
     runs-on: ubuntu-latest
     steps:
-    - name: Set up Go 1.13
+    - name: Set up Go 1.15
       uses: actions/setup-go@v1
       with:
-        go-version: 1.13
+        go-version: 1.15
       id: go
 
     - name: Check out code into the Go module directory
@@ -28,10 +28,10 @@ jobs:
     name: Integration Test
     runs-on: ubuntu-latest
     steps:
-    - name: Set up Go 1.13
+    - name: Set up Go 1.15
       uses: actions/setup-go@v1
       with:
-        go-version: 1.13
+        go-version: 1.15
       id: go
 
     - name: Check out code into the Go module directory

analyzer/analyzer.go

@@ -56,10 +56,11 @@ type AnalysisResult struct {
 	OS           *types.OS
 	PackageInfos []types.PackageInfo
 	Applications []types.Application
+	Configs      []types.Config
 }
 
 func (r *AnalysisResult) isEmpty() bool {
-	return r.OS == nil && len(r.PackageInfos) == 0 && len(r.Applications) == 0
+	return r.OS == nil && len(r.PackageInfos) == 0 && len(r.Applications) == 0 && len(r.Configs) == 0
 }
 
 func (r *AnalysisResult) Sort() {
@@ -70,6 +71,10 @@ func (r *AnalysisResult) Sort() {
 	sort.Slice(r.Applications, func(i, j int) bool {
 		return r.Applications[i].FilePath < r.Applications[j].FilePath
 	})
+
+	sort.Slice(r.Configs, func(i, j int) bool {
+		return r.Configs[i].FilePath < r.Configs[j].FilePath
+	})
 }
 
 func (r *AnalysisResult) Merge(new *AnalysisResult) {
@@ -97,6 +102,10 @@ func (r *AnalysisResult) Merge(new *AnalysisResult) {
 	if len(new.Applications) > 0 {
 		r.Applications = append(r.Applications, new.Applications...)
 	}
+
+	if len(new.Configs) > 0 {
+		r.Configs = append(r.Configs, new.Configs...)
+	}
 }
 
 func AnalyzeFile(wg *sync.WaitGroup, result *AnalysisResult, filePath string, info os.FileInfo, opener Opener,

analyzer/config/const.go

@@ -0,0 +1,5 @@
+package config
+
+const (
+	YAML = "yaml"
+)

analyzer/config/yaml/testdata/anchor.yaml

@@ -0,0 +1,15 @@
+default: &default
+  line: single line
+
+john: &J
+  john_name: john
+fred: &F
+  fred_name: fred
+
+main:
+  <<: *default
+  name:
+    <<: [*J, *F]
+  comment: |
+    multi
+    line

analyzer/config/yaml/testdata/broken.yaml

@@ -0,0 +1 @@
+apiVersion": foo: bar

analyzer/config/yaml/testdata/circular_references.yaml

@@ -0,0 +1,3 @@
+circular: &circular
+  name:
+    <<: *circular

analyzer/config/yaml/testdata/deployment.yaml

@@ -0,0 +1,6 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hello-kubernetes
+spec:
+  replicas: 3

analyzer/config/yaml/testdata/multiple.yaml

@@ -0,0 +1,18 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hello-kubernetes
+spec:
+  replicas: 3
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: hello-kubernetes
+spec:
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 8080

analyzer/config/yaml/yaml.go

@@ -0,0 +1,55 @@
+package yaml
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/open-policy-agent/conftest/parser/yaml"
+	"golang.org/x/xerrors"
+
+	"github.com/aquasecurity/fanal/analyzer"
+	"github.com/aquasecurity/fanal/analyzer/config"
+	"github.com/aquasecurity/fanal/types"
+)
+
+func init() {
+	analyzer.RegisterAnalyzer(&yamlConfigAnalyzer{
+		parser: &yaml.Parser{},
+	})
+}
+
+var (
+	requiredExts = []string{".yaml", ".yml"}
+)
+
+type yamlConfigAnalyzer struct {
+	parser *yaml.Parser
+}
+
+func (a yamlConfigAnalyzer) Analyze(target analyzer.AnalysisTarget) (*analyzer.AnalysisResult, error) {
+	var parsed interface{}
+	if err := a.parser.Unmarshal(target.Content, &parsed); err != nil {
+		return nil, xerrors.Errorf("unable to parse YAML (%s): %w", target.FilePath, err)
+	}
+	return &analyzer.AnalysisResult{
+		Configs: []types.Config{{
+			Type:     config.YAML,
+			FilePath: target.FilePath,
+			Content:  parsed,
+		}},
+	}, nil
+}
+
+func (a yamlConfigAnalyzer) Required(filePath string, _ os.FileInfo) bool {
+	ext := filepath.Ext(filePath)
+	for _, required := range requiredExts {
+		if ext == required {
+			return true
+		}
+	}
+	return false
+}
+
+func (a yamlConfigAnalyzer) Type() analyzer.Type {
+	return analyzer.TypeYaml
+}

analyzer/config/yaml/yaml_test.go

@@ -0,0 +1,184 @@
+package yaml
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/open-policy-agent/conftest/parser/yaml"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/aquasecurity/fanal/analyzer"
+	"github.com/aquasecurity/fanal/analyzer/config"
+	"github.com/aquasecurity/fanal/types"
+)
+
+func Test_yamlConfigAnalyzer_Analyze(t *testing.T) {
+	tests := []struct {
+		name      string
+		inputFile string
+		want      *analyzer.AnalysisResult
+		wantErr   string
+	}{
+		{
+			name:      "happy path",
+			inputFile: "testdata/deployment.yaml",
+			want: &analyzer.AnalysisResult{
+				Configs: []types.Config{
+					{
+						Type:     config.YAML,
+						FilePath: "testdata/deployment.yaml",
+						Content: map[string]interface{}{
+							"apiVersion": "apps/v1",
+							"kind":       "Deployment",
+							"metadata": map[string]interface{}{
+								"name": "hello-kubernetes",
+							},
+							"spec": map[string]interface{}{
+								"replicas": float64(3),
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name:      "happy path using anchors",
+			inputFile: "testdata/anchor.yaml",
+			want: &analyzer.AnalysisResult{
+				Configs: []types.Config{
+					{
+						Type:     config.YAML,
+						FilePath: "testdata/anchor.yaml",
+						Content: map[string]interface{}{
+							"default": map[string]interface{}{
+								"line": "single line",
+							},
+							"john": map[string]interface{}{
+								"john_name": "john",
+							},
+							"fred": map[string]interface{}{
+								"fred_name": "fred",
+							},
+							"main": map[string]interface{}{
+								"line": "single line",
+								"name": map[string]interface{}{
+									"john_name": "john",
+									"fred_name": "fred",
+								},
+								"comment": "multi\nline\n",
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name:      "happy path using multiple yaml",
+			inputFile: "testdata/multiple.yaml",
+			want: &analyzer.AnalysisResult{
+				Configs: []types.Config{
+					{
+						Type:     config.YAML,
+						FilePath: "testdata/multiple.yaml",
+						Content: []interface{}{
+							map[string]interface{}{
+								"apiVersion": "apps/v1",
+								"kind":       "Deployment",
+								"metadata": map[string]interface{}{
+									"name": "hello-kubernetes",
+								},
+								"spec": map[string]interface{}{
+									"replicas": float64(3),
+								},
+							},
+							map[string]interface{}{
+								"apiVersion": "v1",
+								"kind":       "Service",
+								"metadata": map[string]interface{}{
+									"name": "hello-kubernetes",
+								},
+								"spec": map[string]interface{}{
+									"ports": []interface{}{
+										map[string]interface{}{
+											"protocol":   "TCP",
+											"port":       float64(80),
+											"targetPort": float64(8080),
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name:      "broken YAML",
+			inputFile: "testdata/broken.yaml",
+			wantErr:   "unmarshal yaml",
+		},
+		{
+			name:      "invalid circular references yaml",
+			inputFile: "testdata/circular_references.yaml",
+			wantErr:   "yaml: anchor 'circular' value contains itself",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			b, err := ioutil.ReadFile(tt.inputFile)
+			require.NoError(t, err)
+
+			a := yamlConfigAnalyzer{
+				parser: &yaml.Parser{},
+			}
+
+			got, err := a.Analyze(analyzer.AnalysisTarget{
+				FilePath: tt.inputFile,
+				Content:  b,
+			})
+
+			if tt.wantErr != "" {
+				require.NotNil(t, err)
+				assert.Contains(t, err.Error(), tt.wantErr)
+				return
+			}
+			assert.NoError(t, err)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func Test_yamlConfigAnalyzer_Required(t *testing.T) {
+	tests := []struct {
+		name     string
+		filePath string
+		want     bool
+	}{
+		{
+			name:     "yaml",
+			filePath: "deployment.yaml",
+			want:     true,
+		},
+		{
+			name:     "yml",
+			filePath: "deployment.yml",
+			want:     true,
+		},
+		{
+			name:     "json",
+			filePath: "deployment.json",
+			want:     false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			a := yamlConfigAnalyzer{
+				parser: &yaml.Parser{},
+			}
+
+			got := a.Required(tt.filePath, nil)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}

analyzer/const.go

@@ -31,6 +31,9 @@ const (
 	TypePoetry   = Type("poetry")
 	TypeYarn     = Type("yarn")
 
-	// Config
+	// Image Config
 	TypeApkCommand = Type("apk-command")
+
+	// Structured Config
+	TypeYaml = Type("yaml")
 )

applier/docker.go

@@ -88,6 +88,13 @@ func ApplyLayers(layers []types.BlobInfo) types.ArtifactDetail {
 		for _, app := range layer.Applications {
 			nestedMap.SetByString(app.FilePath, sep, app)
 		}
+		for _, config := range layer.Configs {
+			config.Layer = types.Layer{
+				Digest: layer.Digest,
+				DiffID: layer.DiffID,
+			}
+			nestedMap.SetByString(config.FilePath, sep, config)
+		}
 	}
 
 	_ = nestedMap.Walk(func(keys []string, value interface{}) error {
@@ -96,6 +103,8 @@ func ApplyLayers(layers []types.BlobInfo) types.ArtifactDetail {
 			mergedLayer.Packages = append(mergedLayer.Packages, v.Packages...)
 		case types.Application:
 			mergedLayer.Applications = append(mergedLayer.Applications, v)
+		case types.Config:
+			mergedLayer.Configs = append(mergedLayer.Configs, v)
 		}
 		return nil
 	})

artifact/image/image.go

@@ -137,6 +137,7 @@ func (a Artifact) inspectLayer(diffID string) (types.BlobInfo, error) {
 		OS:            result.OS,
 		PackageInfos:  result.PackageInfos,
 		Applications:  result.Applications,
+		Configs:       result.Configs,
 		OpaqueDirs:    opqDirs,
 		WhiteoutFiles: whFiles,
 	}

artifact/local/fs.go

@@ -63,6 +63,7 @@ func (a Artifact) Inspect(_ context.Context) (types.ArtifactReference, error) {
 		OS:            result.OS,
 		PackageInfos:  result.PackageInfos,
 		Applications:  result.Applications,
+		Configs:       result.Configs,
 	}
 
 	// calculate hash of JSON and use it as pseudo artifactID and blobID

cmd/fanal/main.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/aquasecurity/fanal/analyzer"
 	_ "github.com/aquasecurity/fanal/analyzer/command/apk"
+	_ "github.com/aquasecurity/fanal/analyzer/config/yaml"
 	_ "github.com/aquasecurity/fanal/analyzer/library/bundler"
 	_ "github.com/aquasecurity/fanal/analyzer/library/cargo"
 	_ "github.com/aquasecurity/fanal/analyzer/library/composer"

go.mod

@@ -4,34 +4,45 @@ go 1.13
 
 require (
 	github.com/GoogleCloudPlatform/docker-credential-gcr v1.5.0
-	github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 // indirect
+	github.com/Microsoft/go-winio v0.4.16 // indirect
+	github.com/Microsoft/hcsshim v0.8.14 // indirect
 	github.com/alicebob/miniredis/v2 v2.14.1
 	github.com/aquasecurity/go-dep-parser v0.0.0-20210214113128-b97635cfd627
 	github.com/aquasecurity/testdocker v0.0.0-20210106133225-0b17fe083674
-	github.com/aws/aws-sdk-go v1.27.1
+	github.com/aws/aws-sdk-go v1.31.6
+	github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68 // indirect
+	github.com/containerd/containerd v1.4.3 // indirect
+	github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e // indirect
 	github.com/deckarep/golang-set v1.7.1
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
-	github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7
+	github.com/docker/docker v20.10.3+incompatible
 	github.com/docker/go-connections v0.4.0
 	github.com/go-git/go-git/v5 v5.0.0
 	github.com/go-redis/redis/v8 v8.4.0
-	github.com/google/go-containerregistry v0.0.0-20200331213917-3d03ed9b1ca2
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/google/go-containerregistry v0.1.2
 	github.com/hashicorp/go-multierror v1.1.0
 	github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f
 	github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d
 	github.com/knqyf263/go-rpmdb v0.0.0-20201215100354-a9e3110d8ee1
 	github.com/knqyf263/nested v0.0.1
-	github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348
-	github.com/opencontainers/go-digest v1.0.0-rc1
+	github.com/kylelemons/godebug v1.1.0
+	github.com/magefile/mage v1.11.0 // indirect
+	github.com/open-policy-agent/conftest v0.23.0
+	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6
 	github.com/saracen/walker v0.0.0-20191201085201-324a081bae7e
+	github.com/sirupsen/logrus v1.8.0 // indirect
 	github.com/sosedoff/gitkit v0.2.0
-	github.com/stretchr/testify v1.6.1
-	github.com/testcontainers/testcontainers-go v0.3.1
+	github.com/stretchr/testify v1.7.0
+	github.com/testcontainers/testcontainers-go v0.9.1-0.20210218153226-c8e070a2f18d
 	github.com/urfave/cli/v2 v2.2.0
-	go.etcd.io/bbolt v1.3.3
-	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
+	go.etcd.io/bbolt v1.3.5
+	go.opencensus.io v0.22.6 // indirect
+	golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
+	google.golang.org/genproto v0.0.0-20210219173056-d891e3cb3b5b // indirect
+	google.golang.org/grpc v1.35.0 // indirect
 )
-
-// https://github.com/moby/term/issues/15
-replace golang.org/x/sys => golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6

types/image.go

@@ -41,6 +41,13 @@ type PackageInfo struct {
 	Packages []Package
 }
 
+type Config struct {
+	Type     string
+	FilePath string
+	Content  interface{} `json:",omitempty"`
+	Layer    Layer       `json:",omitempty"`
+}
+
 type LibraryInfo struct {
 	Library godeptypes.Library `json:",omitempty"`
 	Layer   Layer              `json:",omitempty"`
@@ -81,6 +88,7 @@ type BlobInfo struct {
 	OS            *OS           `json:",omitempty"`
 	PackageInfos  []PackageInfo `json:",omitempty"`
 	Applications  []Application `json:",omitempty"`
+	Configs       []Config      `json:",omitempty"`
 	OpaqueDirs    []string      `json:",omitempty"`
 	WhiteoutFiles []string      `json:",omitempty"`
 }
@@ -90,6 +98,7 @@ type ArtifactDetail struct {
 	OS           *OS           `json:",omitempty"`
 	Packages     []Package     `json:",omitempty"`
 	Applications []Application `json:",omitempty"`
+	Configs      []Config      `json:",omitempty"`
 
 	// HistoryPackages are packages extracted from RUN instructions
 	HistoryPackages []Package `json:",omitempty"`