From c8127c458fa9f1eedd095afe0ddb430cbc6cb9f4 Mon Sep 17 00:00:00 2001 From: Tomoya AMACHI Date: Thu, 9 May 2019 11:06:20 +0900 Subject: [PATCH] GetToken only run extractor/docker --- analyzer/analyzer.go | 19 ++++- extractor/{ => docker}/docker.go | 58 +++++++------ extractor/{ => docker}/docker_test.go | 44 +++++----- extractor/{ => docker}/testdata/image1.tar | Bin extractor/{ => docker}/testdata/image2.tar | Bin extractor/{ => docker}/testdata/image3.tar | Bin extractor/{ => docker}/testdata/image4.tar | Bin extractor/{ => docker}/testdata/normal.tar | Bin extractor/{ => docker}/testdata/opq.tar | Bin extractor/{ => docker}/testdata/opq2.tar | Bin extractor/docker/token.go | 44 ++++++++++ .../docker/token/dockerhub}/docker.go | 2 +- extractor/docker/token/ecr/ecr.go | 79 ++++++++++++++++++ .../docker/token/ecr}/ecr_test.go | 2 +- {token => extractor/docker/token/gcr}/gcr.go | 31 ++++--- extractor/extractor.go | 2 +- token/ecr.go | 48 ----------- token/token.go | 39 --------- 18 files changed, 214 insertions(+), 154 deletions(-) rename extractor/{ => docker}/docker.go (85%) rename extractor/{ => docker}/docker_test.go (66%) rename extractor/{ => docker}/testdata/image1.tar (100%) rename extractor/{ => docker}/testdata/image2.tar (100%) rename extractor/{ => docker}/testdata/image3.tar (100%) rename extractor/{ => docker}/testdata/image4.tar (100%) rename extractor/{ => docker}/testdata/normal.tar (100%) rename extractor/{ => docker}/testdata/opq.tar (100%) rename extractor/{ => docker}/testdata/opq2.tar (100%) create mode 100644 extractor/docker/token.go rename {token => extractor/docker/token/dockerhub}/docker.go (91%) create mode 100644 extractor/docker/token/ecr/ecr.go rename {token => extractor/docker/token/ecr}/ecr_test.go (99%) rename {token => extractor/docker/token/gcr}/gcr.go (63%) delete mode 100644 token/ecr.go delete mode 100644 token/token.go diff --git a/analyzer/analyzer.go b/analyzer/analyzer.go index 330e56ccdd..8982a07f08 100644 --- a/analyzer/analyzer.go +++ b/analyzer/analyzer.go @@ -8,6 +8,9 @@ import ( "golang.org/x/xerrors" "github.com/knqyf263/fanal/extractor" + "github.com/knqyf263/fanal/extractor/docker" + _ "github.com/knqyf263/fanal/extractor/docker/token/ecr" + _ "github.com/knqyf263/fanal/extractor/docker/token/gcr" "github.com/knqyf263/go-dep-parser/pkg/types" ) @@ -89,8 +92,18 @@ func RequiredFilenames() []string { return filenames } -func Analyze(ctx context.Context, imageName string) (filesMap extractor.FileMap, err error) { - e := extractor.NewDockerExtractor(extractor.DockerOption{Timeout: 600 * time.Second}) +func Analyze(ctx context.Context, imageName string, opts ...docker.DockerOption) (filesMap extractor.FileMap, err error) { + var opt docker.DockerOption + if len(opts) > 0 { + opt = opts[0] + } else { + // default docker option + opt = docker.DockerOption{ + Timeout: 600 * time.Second, + } + } + + e := docker.NewDockerExtractor(opt) r, err := e.SaveLocalImage(ctx, imageName) if err != nil { // when no docker daemon is installed or no image exists in the local machine @@ -109,7 +122,7 @@ func Analyze(ctx context.Context, imageName string) (filesMap extractor.FileMap, } func AnalyzeFromFile(ctx context.Context, r io.ReadCloser) (filesMap extractor.FileMap, err error) { - e := extractor.NewDockerExtractor(extractor.DockerOption{}) + e := docker.NewDockerExtractor(docker.DockerOption{}) filesMap, err = e.ExtractFromFile(ctx, r, RequiredFilenames()) if err != nil { return nil, xerrors.Errorf("failed to extract files from tar: %w", err) diff --git a/extractor/docker.go b/extractor/docker/docker.go similarity index 85% rename from extractor/docker.go rename to extractor/docker/docker.go index b1c92bd426..8e641d4cf5 100644 --- a/extractor/docker.go +++ b/extractor/docker/docker.go @@ -1,4 +1,4 @@ -package extractor +package docker import ( "archive/tar" @@ -12,14 +12,14 @@ import ( "strings" "time" + "github.com/knqyf263/fanal/extractor" + "github.com/docker/distribution/manifest/schema2" "github.com/docker/docker/client" "github.com/genuinetools/reg/registry" - "github.com/genuinetools/reg/repoutils" "github.com/knqyf263/fanal/cache" - "github.com/knqyf263/fanal/token" "github.com/knqyf263/nested" - digest "github.com/opencontainers/go-digest" + "github.com/opencontainers/go-digest" "golang.org/x/xerrors" ) @@ -45,22 +45,25 @@ type DockerExtractor struct { } type DockerOption struct { - AuthURL string - UserName string - Password string - Credential string - Insecure bool - Debug bool - SkipPing bool - NonSSL bool - Timeout time.Duration + AuthURL string + UserName string + Password string + GCRCredPath string + AwsAccessKey string + AwsSecretKey string + AwsRegion string + Insecure bool + Debug bool + SkipPing bool + NonSSL bool + Timeout time.Duration } func NewDockerExtractor(option DockerOption) DockerExtractor { return DockerExtractor{Option: option} } -func applyLayers(layerIDs []string, filesInLayers map[string]FileMap, opqInLayers map[string]opqDirs) (FileMap, error) { +func applyLayers(layerIDs []string, filesInLayers map[string]extractor.FileMap, opqInLayers map[string]opqDirs) (extractor.FileMap, error) { sep := "/" nestedMap := nested.Nested{} for _, layerID := range layerIDs { @@ -83,7 +86,7 @@ func applyLayers(layerIDs []string, filesInLayers map[string]FileMap, opqInLayer } } - fileMap := FileMap{} + fileMap := extractor.FileMap{} walkFn := func(keys []string, value interface{}) error { content, ok := value.([]byte) if !ok { @@ -102,19 +105,14 @@ func applyLayers(layerIDs []string, filesInLayers map[string]FileMap, opqInLayer } func (d DockerExtractor) createRegistryClient(ctx context.Context, domain string) (*registry.Registry, error) { - // Use the auth-url domain if provided. - authDomain := d.Option.AuthURL - if authDomain == "" { - authDomain = domain - } - auth, err := repoutils.GetAuthConfig(d.Option.UserName, d.Option.Password, authDomain) + auth, err := GetToken(ctx, domain, d.Option) if err != nil { return nil, xerrors.Errorf("failed to get auth config: %w", err) } - auth = token.GetToken(ctx, auth, d.Option.Credential) + // Prevent non-ssl unless explicitly forced if !d.Option.NonSSL && strings.HasPrefix(auth.ServerAddress, "http:") { - return nil, xerrors.New("attempted to use insecure protocol! Use force-non-ssl option to force") + return nil, xerrors.New("attempted to us1e insecure protocol! Use force-non-ssl option to force") } // Create the registry client. @@ -159,7 +157,7 @@ func (d DockerExtractor) saveLocalImage(ctx context.Context, imageName string) ( return r, nil } -func (d DockerExtractor) Extract(ctx context.Context, imageName string, filenames []string) (FileMap, error) { +func (d DockerExtractor) Extract(ctx context.Context, imageName string, filenames []string) (extractor.FileMap, error) { ctx, cancel := context.WithTimeout(context.Background(), d.Option.Timeout) defer cancel() @@ -211,7 +209,7 @@ func (d DockerExtractor) Extract(ctx context.Context, imageName string, filename }(ref.Digest) } - filesInLayers := make(map[string]FileMap) + filesInLayers := make(map[string]extractor.FileMap) opqInLayers := make(map[string]opqDirs) for i := 0; i < len(m.Manifest.Layers); i++ { var l layer @@ -234,9 +232,9 @@ func (d DockerExtractor) Extract(ctx context.Context, imageName string, filename return applyLayers(layerIDs, filesInLayers, opqInLayers) } -func (d DockerExtractor) ExtractFromFile(ctx context.Context, r io.Reader, filenames []string) (FileMap, error) { +func (d DockerExtractor) ExtractFromFile(ctx context.Context, r io.Reader, filenames []string) (extractor.FileMap, error) { manifests := make([]manifest, 0) - filesInLayers := make(map[string]FileMap) + filesInLayers := make(map[string]extractor.FileMap) opqInLayers := make(map[string]opqDirs) tr := tar.NewReader(r) @@ -246,7 +244,7 @@ func (d DockerExtractor) ExtractFromFile(ctx context.Context, r io.Reader, filen break } if err != nil { - return nil, ErrCouldNotExtract + return nil, extractor.ErrCouldNotExtract } switch { case header.Name == "manifest.json": @@ -271,7 +269,7 @@ func (d DockerExtractor) ExtractFromFile(ctx context.Context, r io.Reader, filen return applyLayers(manifests[0].Layers, filesInLayers, opqInLayers) } -func (d DockerExtractor) ExtractFiles(layer io.Reader, filenames []string) (FileMap, opqDirs, error) { +func (d DockerExtractor) ExtractFiles(layer io.Reader, filenames []string) (extractor.FileMap, opqDirs, error) { data := make(map[string][]byte) opqDirs := opqDirs{} @@ -282,7 +280,7 @@ func (d DockerExtractor) ExtractFiles(layer io.Reader, filenames []string) (File break } if err != nil { - return data, nil, ErrCouldNotExtract + return data, nil, extractor.ErrCouldNotExtract } filePath := hdr.Name diff --git a/extractor/docker_test.go b/extractor/docker/docker_test.go similarity index 66% rename from extractor/docker_test.go rename to extractor/docker/docker_test.go index 6179a95ed4..bb600ffdf5 100644 --- a/extractor/docker_test.go +++ b/extractor/docker/docker_test.go @@ -1,41 +1,43 @@ -package extractor +package docker import ( "os" "path" "reflect" "testing" + + "github.com/knqyf263/fanal/extractor" ) func TestExtractFromFile(t *testing.T) { vectors := []struct { - file string // Test input file - filenames []string // Target files - fileMap FileMap // Expected output - err error // Expected error to occur + file string // Test input file + filenames []string // Target files + FileMap extractor.FileMap // Expected output + err error // Expected error to occur }{ { file: "testdata/image1.tar", filenames: []string{"var/foo", "etc/test/bar"}, - fileMap: FileMap{"etc/test/bar": []byte("bar\n")}, + FileMap: extractor.FileMap{"etc/test/bar": []byte("bar\n")}, err: nil, }, { file: "testdata/image2.tar", filenames: []string{"home/app/Gemfile", "home/app2/Gemfile"}, - fileMap: FileMap{"home/app2/Gemfile": []byte("gem")}, + FileMap: extractor.FileMap{"home/app2/Gemfile": []byte("gem")}, err: nil, }, { file: "testdata/image3.tar", filenames: []string{"home/app/Gemfile", "home/app2/Pipfile", "home/app/Pipfile"}, - fileMap: FileMap{"home/app/Pipfile": []byte("pip")}, + FileMap: extractor.FileMap{"home/app/Pipfile": []byte("pip")}, err: nil, }, { file: "testdata/image4.tar", filenames: []string{".abc", ".def", "foo/.abc", "foo/.def", ".foo/.abc"}, - fileMap: FileMap{ + FileMap: extractor.FileMap{ ".def": []byte("def"), "foo/.abc": []byte("abc"), }, @@ -56,8 +58,8 @@ func TestExtractFromFile(t *testing.T) { if v.err != err { t.Errorf("err: got %v, want %v", v.err, err) } - if !reflect.DeepEqual(fm, v.fileMap) { - t.Errorf("FilesMap: got %v, want %v", fm, v.fileMap) + if !reflect.DeepEqual(fm, v.FileMap) { + t.Errorf("FilesMap: got %v, want %v", fm, v.FileMap) } }) } @@ -65,23 +67,23 @@ func TestExtractFromFile(t *testing.T) { func TestExtractFiles(t *testing.T) { vectors := []struct { - file string // Test input file - filenames []string // Target files - fileMap FileMap // Expected output - opqDirs opqDirs // Expected output - err error // Expected error to occur + file string // Test input file + filenames []string // Target files + FileMap extractor.FileMap // Expected output + opqDirs opqDirs // Expected output + err error // Expected error to occur }{ { file: "testdata/normal.tar", filenames: []string{"var/foo"}, - fileMap: FileMap{"var/foo": []byte{}}, + FileMap: extractor.FileMap{"var/foo": []byte{}}, opqDirs: []string{}, err: nil, }, { file: "testdata/opq.tar", filenames: []string{"var/foo"}, - fileMap: FileMap{ + FileMap: extractor.FileMap{ "var/.wh.foo": []byte{}, }, opqDirs: []string{"etc/test"}, @@ -90,7 +92,7 @@ func TestExtractFiles(t *testing.T) { { file: "testdata/opq2.tar", filenames: []string{"var/foo", "etc/test/bar"}, - fileMap: FileMap{ + FileMap: extractor.FileMap{ "etc/test/bar": []byte("bar\n"), "var/.wh.foo": []byte{}, }, @@ -115,8 +117,8 @@ func TestExtractFiles(t *testing.T) { if !reflect.DeepEqual(opqDirs, v.opqDirs) { t.Errorf("opqDirs: got %v, want %v", opqDirs, v.opqDirs) } - if !reflect.DeepEqual(fm, v.fileMap) { - t.Errorf("FilesMap: got %v, want %v", fm, v.fileMap) + if !reflect.DeepEqual(fm, v.FileMap) { + t.Errorf("FilesMap: got %v, want %v", fm, v.FileMap) } }) } diff --git a/extractor/testdata/image1.tar b/extractor/docker/testdata/image1.tar similarity index 100% rename from extractor/testdata/image1.tar rename to extractor/docker/testdata/image1.tar diff --git a/extractor/testdata/image2.tar b/extractor/docker/testdata/image2.tar similarity index 100% rename from extractor/testdata/image2.tar rename to extractor/docker/testdata/image2.tar diff --git a/extractor/testdata/image3.tar b/extractor/docker/testdata/image3.tar similarity index 100% rename from extractor/testdata/image3.tar rename to extractor/docker/testdata/image3.tar diff --git a/extractor/testdata/image4.tar b/extractor/docker/testdata/image4.tar similarity index 100% rename from extractor/testdata/image4.tar rename to extractor/docker/testdata/image4.tar diff --git a/extractor/testdata/normal.tar b/extractor/docker/testdata/normal.tar similarity index 100% rename from extractor/testdata/normal.tar rename to extractor/docker/testdata/normal.tar diff --git a/extractor/testdata/opq.tar b/extractor/docker/testdata/opq.tar similarity index 100% rename from extractor/testdata/opq.tar rename to extractor/docker/testdata/opq.tar diff --git a/extractor/testdata/opq2.tar b/extractor/docker/testdata/opq2.tar similarity index 100% rename from extractor/testdata/opq2.tar rename to extractor/docker/testdata/opq2.tar diff --git a/extractor/docker/token.go b/extractor/docker/token.go new file mode 100644 index 0000000000..bb94868e13 --- /dev/null +++ b/extractor/docker/token.go @@ -0,0 +1,44 @@ +package docker + +import ( + "context" + + "github.com/docker/docker/api/types" + "github.com/genuinetools/reg/repoutils" +) + +var ( + registries []Registry +) + +type Registry interface { + CheckOptions(domain string, option DockerOption) error + GetCredential(ctx context.Context) (string, string, error) +} + +func RegisterRegistry(registry Registry) { + registries = append(registries, registry) +} + +func GetToken(ctx context.Context, domain string, opt DockerOption) (auth types.AuthConfig, err error) { + authDomain := opt.AuthURL + if authDomain == "" { + authDomain = domain + } + auth.ServerAddress = authDomain + // check registry which particular to get credential + for _, registry := range registries { + err := registry.CheckOptions(authDomain, opt) + if err != nil { + continue + } + auth.Username, auth.Password, err = registry.GetCredential(ctx) + if err != nil { + // only skip check registry if error occured + break + } else { + return auth, nil + } + } + return repoutils.GetAuthConfig(opt.UserName, opt.Password, authDomain) +} diff --git a/token/docker.go b/extractor/docker/token/dockerhub/docker.go similarity index 91% rename from token/docker.go rename to extractor/docker/token/dockerhub/docker.go index 46d403cd68..a84e64bb75 100644 --- a/token/docker.go +++ b/extractor/docker/token/dockerhub/docker.go @@ -1,4 +1,4 @@ -package token +package dockerhub import "context" diff --git a/extractor/docker/token/ecr/ecr.go b/extractor/docker/token/ecr/ecr.go new file mode 100644 index 0000000000..7567fb29aa --- /dev/null +++ b/extractor/docker/token/ecr/ecr.go @@ -0,0 +1,79 @@ +package ecr + +import ( + "context" + "encoding/base64" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + + "github.com/knqyf263/fanal/extractor/docker" + + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ecr" + "github.com/aws/aws-sdk-go/service/ecr/ecriface" + "golang.org/x/xerrors" +) + +const ecrURL = "amazonaws.com" + +func init() { + docker.RegisterRegistry(&ECR{}) +} + +type ECR struct { + Client ecriface.ECRAPI +} + +func getSession(option docker.DockerOption) (*session.Session, error) { + // create custom credential information if option is valid + if option.AwsSecretKey != "" && option.AwsAccessKey != "" && option.AwsRegion != "" { + return session.NewSessionWithOptions( + session.Options{ + Config: aws.Config{ + Region: aws.String(option.AwsRegion), + Credentials: credentials.NewStaticCredentialsFromCreds( + credentials.Value{ + AccessKeyID: option.AwsAccessKey, + SecretAccessKey: option.AwsSecretKey, + }, + ), + }, + }) + } + // use shared configuration normally + return session.NewSessionWithOptions(session.Options{ + SharedConfigState: session.SharedConfigEnable, + }) +} + +func (e *ECR) CheckOptions(domain string, option docker.DockerOption) error { + if !strings.HasSuffix(domain, ecrURL) { + return xerrors.New("invalid ECR url pattern") + } + sess := session.Must(getSession(option)) + svc := ecr.New(sess) + e.Client = svc + return nil +} + +func (e *ECR) GetCredential(ctx context.Context) (username, password string, err error) { + input := &ecr.GetAuthorizationTokenInput{} + result, err := e.Client.GetAuthorizationTokenWithContext(ctx, input) + if err != nil { + return "", "", xerrors.Errorf("failed to get authorization token: %w", err) + } + for _, data := range result.AuthorizationData { + b, err := base64.StdEncoding.DecodeString(*data.AuthorizationToken) + if err != nil { + return "", "", xerrors.Errorf("base64 decode failed: %w", err) + } + // e.g. AWS:eyJwYXlsb2... + split := strings.SplitN(string(b), ":", 2) + if len(split) == 2 { + return split[0], split[1], nil + } + } + return "", "", nil +} diff --git a/token/ecr_test.go b/extractor/docker/token/ecr/ecr_test.go similarity index 99% rename from token/ecr_test.go rename to extractor/docker/token/ecr/ecr_test.go index bb57245aea..31320f4c8f 100644 --- a/token/ecr_test.go +++ b/extractor/docker/token/ecr/ecr_test.go @@ -1,4 +1,4 @@ -package token +package ecr import ( "context" diff --git a/token/gcr.go b/extractor/docker/token/gcr/gcr.go similarity index 63% rename from token/gcr.go rename to extractor/docker/token/gcr/gcr.go index d96799ba79..59170b6e0e 100644 --- a/token/gcr.go +++ b/extractor/docker/token/gcr/gcr.go @@ -1,8 +1,12 @@ -package token +package gcr import ( "context" - "fmt" + "strings" + + "golang.org/x/xerrors" + + "github.com/knqyf263/fanal/extractor/docker" "github.com/docker/docker/api/types" @@ -16,14 +20,22 @@ type GCR struct { Auth types.AuthConfig } -func NewGCR(auth types.AuthConfig, credPath string) *GCR { - if credPath != "" { - return &GCR{ - Store: store.NewGCRCredStore(credPath), - Auth: auth, - } +const gcrURL = "gcr.io" + +func init() { + docker.RegisterRegistry(&GCR{}) +} + +func (g *GCR) CheckOptions(domain string, d docker.DockerOption) error { + if !strings.HasSuffix(domain, gcrURL) { + return xerrors.New("invalid GCR url pattern") } - return &GCR{Auth: auth} + + g.Auth = types.AuthConfig{} + if d.GCRCredPath != "" { + g.Store = store.NewGCRCredStore(d.GCRCredPath) + } + return nil } func (g *GCR) GetCredential(ctx context.Context) (username, password string, err error) { @@ -40,7 +52,6 @@ func (g *GCR) GetCredential(ctx context.Context) (username, password string, err if err != nil { return "", "", err } - fmt.Printf("%v, %v \n", credStore, userCfg) helper := credhelper.NewGCRCredentialHelper(credStore, userCfg) return helper.Get(g.Auth.ServerAddress) } diff --git a/extractor/extractor.go b/extractor/extractor.go index acfdabb527..2bfa3f7e35 100644 --- a/extractor/extractor.go +++ b/extractor/extractor.go @@ -16,5 +16,5 @@ type FileMap map[string][]byte type Extractor interface { Extract(ctx context.Context, imageName string, filenames []string) (FileMap, error) - ExtractFromFile(ctx context.Context, r io.ReadCloser, filenames []string) (FileMap, error) + ExtractFromFile(ctx context.Context, r io.Reader, filenames []string) (FileMap, error) } diff --git a/token/ecr.go b/token/ecr.go deleted file mode 100644 index 7cbd845473..0000000000 --- a/token/ecr.go +++ /dev/null @@ -1,48 +0,0 @@ -package token - -import ( - "context" - "encoding/base64" - "strings" - - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ecr" - "github.com/aws/aws-sdk-go/service/ecr/ecriface" - "golang.org/x/xerrors" -) - -type ECR struct { - Client ecriface.ECRAPI -} - -func NewECR() *ECR { - sess := session.Must(session.NewSessionWithOptions(session.Options{ - SharedConfigState: session.SharedConfigEnable, - })) - svc := ecr.New(sess) - return &ECR{ - Client: svc, - } -} - -func (e *ECR) GetCredential(ctx context.Context) (username, password string, err error) { - input := &ecr.GetAuthorizationTokenInput{} - - result, err := e.Client.GetAuthorizationTokenWithContext(ctx, input) - if err != nil { - return "", "", xerrors.Errorf("failed to get authorization token: %w", err) - } - - for _, data := range result.AuthorizationData { - b, err := base64.StdEncoding.DecodeString(*data.AuthorizationToken) - if err != nil { - return "", "", xerrors.Errorf("base64 decode failed: %w", err) - } - // e.g. AWS:eyJwYXlsb2... - split := strings.SplitN(string(b), ":", 2) - if len(split) == 2 { - return split[0], split[1], nil - } - } - return "", "", nil -} diff --git a/token/token.go b/token/token.go deleted file mode 100644 index b97dd860d4..0000000000 --- a/token/token.go +++ /dev/null @@ -1,39 +0,0 @@ -package token - -import ( - "context" - "log" - "strings" - - "github.com/docker/docker/api/types" -) - -const ( - ecrURL = "amazonaws.com" - gcrURL = "gcr.io" -) - -type Registry interface { - GetCredential(ctx context.Context) (string, string, error) -} - -func GetToken(ctx context.Context, auth types.AuthConfig, credPath string) types.AuthConfig { - if auth.Username != "" || auth.Password != "" { - return auth - } - var registry Registry - switch { - case strings.HasSuffix(auth.ServerAddress, ecrURL): - registry = NewECR() - case strings.HasSuffix(auth.ServerAddress, gcrURL): - registry = NewGCR(auth, credPath) - default: - registry = NewDocker() - } - var err error - auth.Username, auth.Password, err = registry.GetCredential(ctx) - if err != nil { - log.Printf("failed to get token: %s", err) - } - return auth -}