From e55ec73aba72842c6e723be306eac86b18f2bc27 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Wed, 1 May 2019 15:24:08 +0900 Subject: [PATCH] Support library --- analyzer/analyzer.go | 33 +++++++++++++++ analyzer/library/bundler/bundler.go | 43 +++++++++++++++++++ analyzer/library/composer/composer.go | 43 +++++++++++++++++++ analyzer/library/npm/npm.go | 43 +++++++++++++++++++ analyzer/library/pipenv/pipenv.go | 43 +++++++++++++++++++ cmd/fanal/main.go | 60 ++++++++++++++++++++------- extractor/docker.go | 2 +- utils/utils.go | 10 +++++ 8 files changed, 261 insertions(+), 16 deletions(-) create mode 100644 analyzer/library/bundler/bundler.go create mode 100644 analyzer/library/composer/composer.go create mode 100644 analyzer/library/npm/npm.go create mode 100644 analyzer/library/pipenv/pipenv.go create mode 100644 utils/utils.go diff --git a/analyzer/analyzer.go b/analyzer/analyzer.go index ca65029c5a..00ce71ca9c 100644 --- a/analyzer/analyzer.go +++ b/analyzer/analyzer.go @@ -5,13 +5,17 @@ import ( "io" "time" + "golang.org/x/xerrors" + "github.com/knqyf263/fanal/extractor" + "github.com/knqyf263/go-dep-parser/pkg/types" "github.com/pkg/errors" ) var ( osAnalyzers []OSAnalyzer pkgAnalyzers []PkgAnalyzer + libAnalyzers []LibraryAnalyzer // ErrUnknownOS occurs when unknown OS is analyzed. ErrUnknownOS = errors.New("Unknown OS") @@ -29,6 +33,13 @@ type PkgAnalyzer interface { RequiredFiles() []string } +type FilePath string + +type LibraryAnalyzer interface { + Analyze(extractor.FileMap) (map[FilePath][]types.Library, error) + RequiredFiles() []string +} + type OS struct { Name string Family string @@ -61,6 +72,10 @@ func RegisterPkgAnalyzer(analyzer PkgAnalyzer) { pkgAnalyzers = append(pkgAnalyzers, analyzer) } +func RegisterLibraryAnalyzer(analyzer LibraryAnalyzer) { + libAnalyzers = append(libAnalyzers, analyzer) +} + func RequiredFilenames() []string { filenames := []string{} for _, analyzer := range osAnalyzers { @@ -69,6 +84,9 @@ func RequiredFilenames() []string { for _, analyzer := range pkgAnalyzers { filenames = append(filenames, analyzer.RequiredFiles()...) } + for _, analyzer := range libAnalyzers { + filenames = append(filenames, analyzer.RequiredFiles()...) + } return filenames } @@ -116,3 +134,18 @@ func GetPackages(filesMap extractor.FileMap) ([]Package, error) { func CheckPackage(pkg *Package) bool { return pkg.Name != "" && pkg.Version != "" } + +func GetLibraries(filesMap extractor.FileMap) (map[FilePath][]types.Library, error) { + results := map[FilePath][]types.Library{} + for _, analyzer := range libAnalyzers { + libMap, err := analyzer.Analyze(filesMap) + if err != nil { + return nil, xerrors.Errorf("failed to analyze libraries: %w", err) + } + + for filePath, libs := range libMap { + results[filePath] = libs + } + } + return results, nil +} diff --git a/analyzer/library/bundler/bundler.go b/analyzer/library/bundler/bundler.go new file mode 100644 index 0000000000..969596de57 --- /dev/null +++ b/analyzer/library/bundler/bundler.go @@ -0,0 +1,43 @@ +package bundler + +import ( + "bytes" + "path/filepath" + + "github.com/knqyf263/fanal/analyzer" + "github.com/knqyf263/fanal/extractor" + "github.com/knqyf263/fanal/utils" + "github.com/knqyf263/go-dep-parser/pkg/bundler" + "github.com/knqyf263/go-dep-parser/pkg/types" + "golang.org/x/xerrors" +) + +func init() { + analyzer.RegisterLibraryAnalyzer(&bundlerLibraryAnalyzer{}) +} + +type bundlerLibraryAnalyzer struct{} + +func (a bundlerLibraryAnalyzer) Analyze(fileMap extractor.FileMap) (map[analyzer.FilePath][]types.Library, error) { + libMap := map[analyzer.FilePath][]types.Library{} + requiredFiles := a.RequiredFiles() + + for filename, content := range fileMap { + basename := filepath.Base(filename) + if !utils.StringInSlice(basename, requiredFiles) { + continue + } + + r := bytes.NewBuffer(content) + libs, err := bundler.Parse(r) + if err != nil { + return nil, xerrors.Errorf("invalid Gemfile.lock format: %w", err) + } + libMap[analyzer.FilePath(filename)] = libs + } + return libMap, nil +} + +func (a bundlerLibraryAnalyzer) RequiredFiles() []string { + return []string{"Gemfile.lock"} +} diff --git a/analyzer/library/composer/composer.go b/analyzer/library/composer/composer.go new file mode 100644 index 0000000000..25d3a50547 --- /dev/null +++ b/analyzer/library/composer/composer.go @@ -0,0 +1,43 @@ +package composer + +import ( + "bytes" + "path/filepath" + + "github.com/knqyf263/fanal/analyzer" + "github.com/knqyf263/fanal/extractor" + "github.com/knqyf263/fanal/utils" + "github.com/knqyf263/go-dep-parser/pkg/composer" + "github.com/knqyf263/go-dep-parser/pkg/types" + "golang.org/x/xerrors" +) + +func init() { + analyzer.RegisterLibraryAnalyzer(&composerLibraryAnalyzer{}) +} + +type composerLibraryAnalyzer struct{} + +func (a composerLibraryAnalyzer) Analyze(fileMap extractor.FileMap) (map[analyzer.FilePath][]types.Library, error) { + libMap := map[analyzer.FilePath][]types.Library{} + requiredFiles := a.RequiredFiles() + + for filename, content := range fileMap { + basename := filepath.Base(filename) + if !utils.StringInSlice(basename, requiredFiles) { + continue + } + + r := bytes.NewBuffer(content) + libs, err := composer.Parse(r) + if err != nil { + return nil, xerrors.Errorf("invalid composer.lock format: %w", err) + } + libMap[analyzer.FilePath(filename)] = libs + } + return libMap, nil +} + +func (a composerLibraryAnalyzer) RequiredFiles() []string { + return []string{"composer.lock"} +} diff --git a/analyzer/library/npm/npm.go b/analyzer/library/npm/npm.go new file mode 100644 index 0000000000..3f119b7750 --- /dev/null +++ b/analyzer/library/npm/npm.go @@ -0,0 +1,43 @@ +package npm + +import ( + "bytes" + "path/filepath" + + "github.com/knqyf263/fanal/analyzer" + "github.com/knqyf263/fanal/extractor" + "github.com/knqyf263/fanal/utils" + "github.com/knqyf263/go-dep-parser/pkg/npm" + "github.com/knqyf263/go-dep-parser/pkg/types" + "golang.org/x/xerrors" +) + +func init() { + analyzer.RegisterLibraryAnalyzer(&npmLibraryAnalyzer{}) +} + +type npmLibraryAnalyzer struct{} + +func (a npmLibraryAnalyzer) Analyze(fileMap extractor.FileMap) (map[analyzer.FilePath][]types.Library, error) { + libMap := map[analyzer.FilePath][]types.Library{} + requiredFiles := a.RequiredFiles() + + for filename, content := range fileMap { + basename := filepath.Base(filename) + if !utils.StringInSlice(basename, requiredFiles) { + continue + } + + r := bytes.NewBuffer(content) + libs, err := npm.Parse(r) + if err != nil { + return nil, xerrors.Errorf("invalid package-lock.json format: %w", err) + } + libMap[analyzer.FilePath(filename)] = libs + } + return libMap, nil +} + +func (a npmLibraryAnalyzer) RequiredFiles() []string { + return []string{"package-lock.json"} +} diff --git a/analyzer/library/pipenv/pipenv.go b/analyzer/library/pipenv/pipenv.go new file mode 100644 index 0000000000..97f377d612 --- /dev/null +++ b/analyzer/library/pipenv/pipenv.go @@ -0,0 +1,43 @@ +package pipenv + +import ( + "bytes" + "path/filepath" + + "github.com/knqyf263/fanal/analyzer" + "github.com/knqyf263/fanal/extractor" + "github.com/knqyf263/fanal/utils" + "github.com/knqyf263/go-dep-parser/pkg/pipenv" + "github.com/knqyf263/go-dep-parser/pkg/types" + "golang.org/x/xerrors" +) + +func init() { + analyzer.RegisterLibraryAnalyzer(&pipenvLibraryAnalyzer{}) +} + +type pipenvLibraryAnalyzer struct{} + +func (a pipenvLibraryAnalyzer) Analyze(fileMap extractor.FileMap) (map[analyzer.FilePath][]types.Library, error) { + libMap := map[analyzer.FilePath][]types.Library{} + requiredFiles := a.RequiredFiles() + + for filename, content := range fileMap { + basename := filepath.Base(filename) + if !utils.StringInSlice(basename, requiredFiles) { + continue + } + + r := bytes.NewBuffer(content) + libs, err := pipenv.Parse(r) + if err != nil { + return nil, xerrors.Errorf("invalid Pipfile.lock format: %w", err) + } + libMap[analyzer.FilePath(filename)] = libs + } + return libMap, nil +} + +func (a pipenvLibraryAnalyzer) RequiredFiles() []string { + return []string{"Pipfile.lock"} +} diff --git a/cmd/fanal/main.go b/cmd/fanal/main.go index f5a2f4146d..942acc54ba 100644 --- a/cmd/fanal/main.go +++ b/cmd/fanal/main.go @@ -3,10 +3,15 @@ package main import ( "context" "flag" + "fmt" "log" "os" "github.com/knqyf263/fanal/analyzer" + _ "github.com/knqyf263/fanal/analyzer/library/bundler" + _ "github.com/knqyf263/fanal/analyzer/library/composer" + _ "github.com/knqyf263/fanal/analyzer/library/npm" + _ "github.com/knqyf263/fanal/analyzer/library/pipenv" _ "github.com/knqyf263/fanal/analyzer/os/alpine" _ "github.com/knqyf263/fanal/analyzer/os/amazonlinux" _ "github.com/knqyf263/fanal/analyzer/os/debian" @@ -16,36 +21,61 @@ import ( _ "github.com/knqyf263/fanal/analyzer/pkg/apk" _ "github.com/knqyf263/fanal/analyzer/pkg/dpkg" _ "github.com/knqyf263/fanal/analyzer/pkg/rpm" + "github.com/knqyf263/fanal/extractor" "golang.org/x/crypto/ssh/terminal" ) func main() { - ctx := context.Background() - imageName := os.Args[1] - - files, err := analyzer.Analyze(ctx, imageName) - if err != nil { + if err := run(); err != nil { log.Fatal(err) } - analyzer.GetOS(files) - analyzer.GetPackages(files) } -func main2() { +func run() (err error) { ctx := context.Background() tarPath := flag.String("f", "-", "layer.tar path") flag.Parse() - rc, err := openStream(*tarPath) - if err != nil { - log.Fatal(err) + + args := flag.Args() + + var files extractor.FileMap + if len(args) > 0 { + files, err = analyzer.Analyze(ctx, args[1]) + if err != nil { + return err + } + } else { + rc, err := openStream(*tarPath) + if err != nil { + return err + } + + files, err = analyzer.AnalyzeFromFile(ctx, rc) + if err != nil { + return err + } } - files, err := analyzer.AnalyzeFromFile(ctx, rc) + os, err := analyzer.GetOS(files) if err != nil { - log.Fatal(err) + return err } - analyzer.GetOS(files) - analyzer.GetPackages(files) + fmt.Printf("%+v\n", os) + + pkgs, err := analyzer.GetPackages(files) + if err != nil { + return err + } + fmt.Printf("Packages: %d\n", len(pkgs)) + + libs, err := analyzer.GetLibraries(files) + if err != nil { + return err + } + for filepath, libList := range libs { + fmt.Printf("%s: %d\n", filepath, len(libList)) + } + return nil } func openStream(path string) (*os.File, error) { diff --git a/extractor/docker.go b/extractor/docker.go index 304329cd54..d61a6d0d4b 100644 --- a/extractor/docker.go +++ b/extractor/docker.go @@ -266,7 +266,7 @@ func (d DockerExtractor) ExtractFiles(layer io.Reader, filenames []string) (File // Determine if we should extract the element extract := false for _, s := range filenames { - if s == filePath || strings.HasPrefix(fileName, wh) { + if s == filePath || s == fileName || strings.HasPrefix(fileName, wh) { extract = true break } diff --git a/utils/utils.go b/utils/utils.go new file mode 100644 index 0000000000..bcde9f53af --- /dev/null +++ b/utils/utils.go @@ -0,0 +1,10 @@ +package utils + +func StringInSlice(a string, list []string) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +}