GetToken only run extractor/docker

analyzer/analyzer.go

@@ -8,6 +8,9 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/knqyf263/fanal/extractor"
+	"github.com/knqyf263/fanal/extractor/docker"
+	_ "github.com/knqyf263/fanal/extractor/docker/token/ecr"
+	_ "github.com/knqyf263/fanal/extractor/docker/token/gcr"
 	"github.com/knqyf263/go-dep-parser/pkg/types"
 )
 
@@ -89,8 +92,18 @@ func RequiredFilenames() []string {
 	return filenames
 }
 
-func Analyze(ctx context.Context, imageName string) (filesMap extractor.FileMap, err error) {
+func Analyze(ctx context.Context, imageName string, opts ...docker.DockerOption) (filesMap extractor.FileMap, err error) {
-	e := extractor.NewDockerExtractor(extractor.DockerOption{Timeout: 600 * time.Second})
+	var opt docker.DockerOption
+	if len(opts) > 0 {
+		opt = opts[0]
+	} else {
+		// default docker option
+		opt = docker.DockerOption{
+			Timeout: 600 * time.Second,
+		}
+	}
+
+	e := docker.NewDockerExtractor(opt)
 	r, err := e.SaveLocalImage(ctx, imageName)
 	if err != nil {
 		// when no docker daemon is installed or no image exists in the local machine
@@ -109,7 +122,7 @@ func Analyze(ctx context.Context, imageName string) (filesMap extractor.FileMap,
 }
 
 func AnalyzeFromFile(ctx context.Context, r io.ReadCloser) (filesMap extractor.FileMap, err error) {
-	e := extractor.NewDockerExtractor(extractor.DockerOption{})
+	e := docker.NewDockerExtractor(docker.DockerOption{})
 	filesMap, err = e.ExtractFromFile(ctx, r, RequiredFilenames())
 	if err != nil {
 		return nil, xerrors.Errorf("failed to extract files from tar: %w", err)

extractor/docker/docker.go

@@ -1,4 +1,4 @@
-package extractor
+package docker
 
 import (
 	"archive/tar"
@@ -12,14 +12,14 @@ import (
 	"strings"
 	"time"
 
+	"github.com/knqyf263/fanal/extractor"
+
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/docker/docker/client"
 	"github.com/genuinetools/reg/registry"
-	"github.com/genuinetools/reg/repoutils"
 	"github.com/knqyf263/fanal/cache"
-	"github.com/knqyf263/fanal/token"
 	"github.com/knqyf263/nested"
-	digest "github.com/opencontainers/go-digest"
+	"github.com/opencontainers/go-digest"
 	"golang.org/x/xerrors"
 )
 
@@ -45,22 +45,25 @@ type DockerExtractor struct {
 }
 
 type DockerOption struct {
-	AuthURL    string
+	AuthURL      string
-	UserName   string
+	UserName     string
-	Password   string
+	Password     string
-	Credential string
+	GCRCredPath  string
-	Insecure   bool
+	AwsAccessKey string
-	Debug      bool
+	AwsSecretKey string
-	SkipPing   bool
+	AwsRegion    string
-	NonSSL     bool
+	Insecure     bool
-	Timeout    time.Duration
+	Debug        bool
+	SkipPing     bool
+	NonSSL       bool
+	Timeout      time.Duration
 }
 
 func NewDockerExtractor(option DockerOption) DockerExtractor {
 	return DockerExtractor{Option: option}
 }
 
-func applyLayers(layerIDs []string, filesInLayers map[string]FileMap, opqInLayers map[string]opqDirs) (FileMap, error) {
+func applyLayers(layerIDs []string, filesInLayers map[string]extractor.FileMap, opqInLayers map[string]opqDirs) (extractor.FileMap, error) {
 	sep := "/"
 	nestedMap := nested.Nested{}
 	for _, layerID := range layerIDs {
@@ -83,7 +86,7 @@ func applyLayers(layerIDs []string, filesInLayers map[string]FileMap, opqInLayer
 		}
 	}
 
-	fileMap := FileMap{}
+	fileMap := extractor.FileMap{}
 	walkFn := func(keys []string, value interface{}) error {
 		content, ok := value.([]byte)
 		if !ok {
@@ -102,19 +105,14 @@ func applyLayers(layerIDs []string, filesInLayers map[string]FileMap, opqInLayer
 }
 
 func (d DockerExtractor) createRegistryClient(ctx context.Context, domain string) (*registry.Registry, error) {
-	// Use the auth-url domain if provided.
+	auth, err := GetToken(ctx, domain, d.Option)
-	authDomain := d.Option.AuthURL
-	if authDomain == "" {
-		authDomain = domain
-	}
-	auth, err := repoutils.GetAuthConfig(d.Option.UserName, d.Option.Password, authDomain)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to get auth config: %w", err)
 	}
-	auth = token.GetToken(ctx, auth, d.Option.Credential)
+
 	// Prevent non-ssl unless explicitly forced
 	if !d.Option.NonSSL && strings.HasPrefix(auth.ServerAddress, "http:") {
-		return nil, xerrors.New("attempted to use insecure protocol! Use force-non-ssl option to force")
+		return nil, xerrors.New("attempted to us1e insecure protocol! Use force-non-ssl option to force")
 	}
 
 	// Create the registry client.
@@ -159,7 +157,7 @@ func (d DockerExtractor) saveLocalImage(ctx context.Context, imageName string) (
 	return r, nil
 }
 
-func (d DockerExtractor) Extract(ctx context.Context, imageName string, filenames []string) (FileMap, error) {
+func (d DockerExtractor) Extract(ctx context.Context, imageName string, filenames []string) (extractor.FileMap, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), d.Option.Timeout)
 	defer cancel()
 
@@ -211,7 +209,7 @@ func (d DockerExtractor) Extract(ctx context.Context, imageName string, filename
 		}(ref.Digest)
 	}
 
-	filesInLayers := make(map[string]FileMap)
+	filesInLayers := make(map[string]extractor.FileMap)
 	opqInLayers := make(map[string]opqDirs)
 	for i := 0; i < len(m.Manifest.Layers); i++ {
 		var l layer
@@ -234,9 +232,9 @@ func (d DockerExtractor) Extract(ctx context.Context, imageName string, filename
 	return applyLayers(layerIDs, filesInLayers, opqInLayers)
 }
 
-func (d DockerExtractor) ExtractFromFile(ctx context.Context, r io.Reader, filenames []string) (FileMap, error) {
+func (d DockerExtractor) ExtractFromFile(ctx context.Context, r io.Reader, filenames []string) (extractor.FileMap, error) {
 	manifests := make([]manifest, 0)
-	filesInLayers := make(map[string]FileMap)
+	filesInLayers := make(map[string]extractor.FileMap)
 	opqInLayers := make(map[string]opqDirs)
 
 	tr := tar.NewReader(r)
@@ -246,7 +244,7 @@ func (d DockerExtractor) ExtractFromFile(ctx context.Context, r io.Reader, filen
 			break
 		}
 		if err != nil {
-			return nil, ErrCouldNotExtract
+			return nil, extractor.ErrCouldNotExtract
 		}
 		switch {
 		case header.Name == "manifest.json":
@@ -271,7 +269,7 @@ func (d DockerExtractor) ExtractFromFile(ctx context.Context, r io.Reader, filen
 
 	return applyLayers(manifests[0].Layers, filesInLayers, opqInLayers)
 }
-func (d DockerExtractor) ExtractFiles(layer io.Reader, filenames []string) (FileMap, opqDirs, error) {
+func (d DockerExtractor) ExtractFiles(layer io.Reader, filenames []string) (extractor.FileMap, opqDirs, error) {
 	data := make(map[string][]byte)
 	opqDirs := opqDirs{}
 
@@ -282,7 +280,7 @@ func (d DockerExtractor) ExtractFiles(layer io.Reader, filenames []string) (File
 			break
 		}
 		if err != nil {
-			return data, nil, ErrCouldNotExtract
+			return data, nil, extractor.ErrCouldNotExtract
 		}
 
 		filePath := hdr.Name

extractor/docker/docker_test.go

@@ -1,41 +1,43 @@
-package extractor
+package docker
 
 import (
 	"os"
 	"path"
 	"reflect"
 	"testing"
+
+	"github.com/knqyf263/fanal/extractor"
 )
 
 func TestExtractFromFile(t *testing.T) {
 	vectors := []struct {
-		file      string   // Test input file
+		file      string            // Test input file
-		filenames []string // Target files
+		filenames []string          // Target files
-		fileMap   FileMap  // Expected output
+		FileMap   extractor.FileMap // Expected output
-		err       error    // Expected error to occur
+		err       error             // Expected error to occur
 	}{
 		{
 			file:      "testdata/image1.tar",
 			filenames: []string{"var/foo", "etc/test/bar"},
-			fileMap:   FileMap{"etc/test/bar": []byte("bar\n")},
+			FileMap:   extractor.FileMap{"etc/test/bar": []byte("bar\n")},
 			err:       nil,
 		},
 		{
 			file:      "testdata/image2.tar",
 			filenames: []string{"home/app/Gemfile", "home/app2/Gemfile"},
-			fileMap:   FileMap{"home/app2/Gemfile": []byte("gem")},
+			FileMap:   extractor.FileMap{"home/app2/Gemfile": []byte("gem")},
 			err:       nil,
 		},
 		{
 			file:      "testdata/image3.tar",
 			filenames: []string{"home/app/Gemfile", "home/app2/Pipfile", "home/app/Pipfile"},
-			fileMap:   FileMap{"home/app/Pipfile": []byte("pip")},
+			FileMap:   extractor.FileMap{"home/app/Pipfile": []byte("pip")},
 			err:       nil,
 		},
 		{
 			file:      "testdata/image4.tar",
 			filenames: []string{".abc", ".def", "foo/.abc", "foo/.def", ".foo/.abc"},
-			fileMap: FileMap{
+			FileMap: extractor.FileMap{
 				".def":     []byte("def"),
 				"foo/.abc": []byte("abc"),
 			},
@@ -56,8 +58,8 @@ func TestExtractFromFile(t *testing.T) {
 			if v.err != err {
 				t.Errorf("err: got %v, want %v", v.err, err)
 			}
-			if !reflect.DeepEqual(fm, v.fileMap) {
+			if !reflect.DeepEqual(fm, v.FileMap) {
-				t.Errorf("FilesMap: got %v, want %v", fm, v.fileMap)
+				t.Errorf("FilesMap: got %v, want %v", fm, v.FileMap)
 			}
 		})
 	}
@@ -65,23 +67,23 @@ func TestExtractFromFile(t *testing.T) {
 
 func TestExtractFiles(t *testing.T) {
 	vectors := []struct {
-		file      string   // Test input file
+		file      string            // Test input file
-		filenames []string // Target files
+		filenames []string          // Target files
-		fileMap   FileMap  // Expected output
+		FileMap   extractor.FileMap // Expected output
-		opqDirs   opqDirs  // Expected output
+		opqDirs   opqDirs           // Expected output
-		err       error    // Expected error to occur
+		err       error             // Expected error to occur
 	}{
 		{
 			file:      "testdata/normal.tar",
 			filenames: []string{"var/foo"},
-			fileMap:   FileMap{"var/foo": []byte{}},
+			FileMap:   extractor.FileMap{"var/foo": []byte{}},
 			opqDirs:   []string{},
 			err:       nil,
 		},
 		{
 			file:      "testdata/opq.tar",
 			filenames: []string{"var/foo"},
-			fileMap: FileMap{
+			FileMap: extractor.FileMap{
 				"var/.wh.foo": []byte{},
 			},
 			opqDirs: []string{"etc/test"},
@@ -90,7 +92,7 @@ func TestExtractFiles(t *testing.T) {
 		{
 			file:      "testdata/opq2.tar",
 			filenames: []string{"var/foo", "etc/test/bar"},
-			fileMap: FileMap{
+			FileMap: extractor.FileMap{
 				"etc/test/bar": []byte("bar\n"),
 				"var/.wh.foo":  []byte{},
 			},
@@ -115,8 +117,8 @@ func TestExtractFiles(t *testing.T) {
 			if !reflect.DeepEqual(opqDirs, v.opqDirs) {
 				t.Errorf("opqDirs: got %v, want %v", opqDirs, v.opqDirs)
 			}
-			if !reflect.DeepEqual(fm, v.fileMap) {
+			if !reflect.DeepEqual(fm, v.FileMap) {
-				t.Errorf("FilesMap: got %v, want %v", fm, v.fileMap)
+				t.Errorf("FilesMap: got %v, want %v", fm, v.FileMap)
 			}
 		})
 	}

extractor/docker/token.go

@@ -0,0 +1,44 @@
+package docker
+
+import (
+	"context"
+
+	"github.com/docker/docker/api/types"
+	"github.com/genuinetools/reg/repoutils"
+)
+
+var (
+	registries []Registry
+)
+
+type Registry interface {
+	CheckOptions(domain string, option DockerOption) error
+	GetCredential(ctx context.Context) (string, string, error)
+}
+
+func RegisterRegistry(registry Registry) {
+	registries = append(registries, registry)
+}
+
+func GetToken(ctx context.Context, domain string, opt DockerOption) (auth types.AuthConfig, err error) {
+	authDomain := opt.AuthURL
+	if authDomain == "" {
+		authDomain = domain
+	}
+	auth.ServerAddress = authDomain
+	// check registry which particular to get credential
+	for _, registry := range registries {
+		err := registry.CheckOptions(authDomain, opt)
+		if err != nil {
+			continue
+		}
+		auth.Username, auth.Password, err = registry.GetCredential(ctx)
+		if err != nil {
+			// only skip check registry if error occured
+			break
+		} else {
+			return auth, nil
+		}
+	}
+	return repoutils.GetAuthConfig(opt.UserName, opt.Password, authDomain)
+}

extractor/docker/token/dockerhub/docker.go

@@ -1,4 +1,4 @@
-package token
+package dockerhub
 
 import "context"
 

extractor/docker/token/ecr/ecr.go

@@ -0,0 +1,79 @@
+package ecr
+
+import (
+	"context"
+	"encoding/base64"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+
+	"github.com/knqyf263/fanal/extractor/docker"
+
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ecr"
+	"github.com/aws/aws-sdk-go/service/ecr/ecriface"
+	"golang.org/x/xerrors"
+)
+
+const ecrURL = "amazonaws.com"
+
+func init() {
+	docker.RegisterRegistry(&ECR{})
+}
+
+type ECR struct {
+	Client ecriface.ECRAPI
+}
+
+func getSession(option docker.DockerOption) (*session.Session, error) {
+	// create custom credential information if option is valid
+	if option.AwsSecretKey != "" && option.AwsAccessKey != "" && option.AwsRegion != "" {
+		return session.NewSessionWithOptions(
+			session.Options{
+				Config: aws.Config{
+					Region: aws.String(option.AwsRegion),
+					Credentials: credentials.NewStaticCredentialsFromCreds(
+						credentials.Value{
+							AccessKeyID:     option.AwsAccessKey,
+							SecretAccessKey: option.AwsSecretKey,
+						},
+					),
+				},
+			})
+	}
+	// use shared configuration normally
+	return session.NewSessionWithOptions(session.Options{
+		SharedConfigState: session.SharedConfigEnable,
+	})
+}
+
+func (e *ECR) CheckOptions(domain string, option docker.DockerOption) error {
+	if !strings.HasSuffix(domain, ecrURL) {
+		return xerrors.New("invalid ECR url pattern")
+	}
+	sess := session.Must(getSession(option))
+	svc := ecr.New(sess)
+	e.Client = svc
+	return nil
+}
+
+func (e *ECR) GetCredential(ctx context.Context) (username, password string, err error) {
+	input := &ecr.GetAuthorizationTokenInput{}
+	result, err := e.Client.GetAuthorizationTokenWithContext(ctx, input)
+	if err != nil {
+		return "", "", xerrors.Errorf("failed to get authorization token: %w", err)
+	}
+	for _, data := range result.AuthorizationData {
+		b, err := base64.StdEncoding.DecodeString(*data.AuthorizationToken)
+		if err != nil {
+			return "", "", xerrors.Errorf("base64 decode failed: %w", err)
+		}
+		// e.g. AWS:eyJwYXlsb2...
+		split := strings.SplitN(string(b), ":", 2)
+		if len(split) == 2 {
+			return split[0], split[1], nil
+		}
+	}
+	return "", "", nil
+}

extractor/docker/token/ecr/ecr_test.go

@@ -1,4 +1,4 @@
-package token
+package ecr
 
 import (
 	"context"

extractor/docker/token/gcr/gcr.go

@@ -1,8 +1,12 @@
-package token
+package gcr
 
 import (
 	"context"
-	"fmt"
+	"strings"
+
+	"golang.org/x/xerrors"
+
+	"github.com/knqyf263/fanal/extractor/docker"
 
 	"github.com/docker/docker/api/types"
 
@@ -16,14 +20,22 @@ type GCR struct {
 	Auth  types.AuthConfig
 }
 
-func NewGCR(auth types.AuthConfig, credPath string) *GCR {
+const gcrURL = "gcr.io"
-	if credPath != "" {
+
-		return &GCR{
+func init() {
-			Store: store.NewGCRCredStore(credPath),
+	docker.RegisterRegistry(&GCR{})
-			Auth:  auth,
+}
-		}
+
+func (g *GCR) CheckOptions(domain string, d docker.DockerOption) error {
+	if !strings.HasSuffix(domain, gcrURL) {
+		return xerrors.New("invalid GCR url pattern")
 	}
-	return &GCR{Auth: auth}
+
+	g.Auth = types.AuthConfig{}
+	if d.GCRCredPath != "" {
+		g.Store = store.NewGCRCredStore(d.GCRCredPath)
+	}
+	return nil
 }
 
 func (g *GCR) GetCredential(ctx context.Context) (username, password string, err error) {
@@ -40,7 +52,6 @@ func (g *GCR) GetCredential(ctx context.Context) (username, password string, err
 	if err != nil {
 		return "", "", err
 	}
-	fmt.Printf("%v, %v \n", credStore, userCfg)
 	helper := credhelper.NewGCRCredentialHelper(credStore, userCfg)
 	return helper.Get(g.Auth.ServerAddress)
 }

extractor/extractor.go

@@ -16,5 +16,5 @@ type FileMap map[string][]byte
 
 type Extractor interface {
 	Extract(ctx context.Context, imageName string, filenames []string) (FileMap, error)
-	ExtractFromFile(ctx context.Context, r io.ReadCloser, filenames []string) (FileMap, error)
+	ExtractFromFile(ctx context.Context, r io.Reader, filenames []string) (FileMap, error)
 }

token/ecr.go

@@ -1,48 +0,0 @@
-package token
-
-import (
-	"context"
-	"encoding/base64"
-	"strings"
-
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/ecr"
-	"github.com/aws/aws-sdk-go/service/ecr/ecriface"
-	"golang.org/x/xerrors"
-)
-
-type ECR struct {
-	Client ecriface.ECRAPI
-}
-
-func NewECR() *ECR {
-	sess := session.Must(session.NewSessionWithOptions(session.Options{
-		SharedConfigState: session.SharedConfigEnable,
-	}))
-	svc := ecr.New(sess)
-	return &ECR{
-		Client: svc,
-	}
-}
-
-func (e *ECR) GetCredential(ctx context.Context) (username, password string, err error) {
-	input := &ecr.GetAuthorizationTokenInput{}
-
-	result, err := e.Client.GetAuthorizationTokenWithContext(ctx, input)
-	if err != nil {
-		return "", "", xerrors.Errorf("failed to get authorization token: %w", err)
-	}
-
-	for _, data := range result.AuthorizationData {
-		b, err := base64.StdEncoding.DecodeString(*data.AuthorizationToken)
-		if err != nil {
-			return "", "", xerrors.Errorf("base64 decode failed: %w", err)
-		}
-		// e.g. AWS:eyJwYXlsb2...
-		split := strings.SplitN(string(b), ":", 2)
-		if len(split) == 2 {
-			return split[0], split[1], nil
-		}
-	}
-	return "", "", nil
-}

token/token.go

@@ -1,39 +0,0 @@
-package token
-
-import (
-	"context"
-	"log"
-	"strings"
-
-	"github.com/docker/docker/api/types"
-)
-
-const (
-	ecrURL = "amazonaws.com"
-	gcrURL = "gcr.io"
-)
-
-type Registry interface {
-	GetCredential(ctx context.Context) (string, string, error)
-}
-
-func GetToken(ctx context.Context, auth types.AuthConfig, credPath string) types.AuthConfig {
-	if auth.Username != "" || auth.Password != "" {
-		return auth
-	}
-	var registry Registry
-	switch {
-	case strings.HasSuffix(auth.ServerAddress, ecrURL):
-		registry = NewECR()
-	case strings.HasSuffix(auth.ServerAddress, gcrURL):
-		registry = NewGCR(auth, credPath)
-	default:
-		registry = NewDocker()
-	}
-	var err error
-	auth.Username, auth.Password, err = registry.GetCredential(ctx)
-	if err != nil {
-		log.Printf("failed to get token: %s", err)
-	}
-	return auth
-}