Support private registry and use cache

cache/cache.go

@@ -0,0 +1,42 @@
+package cache
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+
+	"golang.org/x/xerrors"
+)
+
+func init() {
+	d, err := os.UserCacheDir()
+	if err != nil {
+		d = os.TempDir()
+	}
+	cacheDir = filepath.Join(d, "fanal")
+	os.MkdirAll(cacheDir, os.ModePerm)
+}
+
+var (
+	cacheDir string
+)
+
+func Get(key string) io.Reader {
+	filePath := filepath.Join(cacheDir, key)
+	f, err := os.Open(filePath)
+	if err != nil {
+		return nil
+	}
+	return f
+}
+
+func Set(key string, file io.Reader) (io.Reader, error) {
+	filePath := filepath.Join(cacheDir, key)
+	cacheFile, err := os.Create(filePath)
+	if err != nil {
+		return file, xerrors.Errorf("failed to create cache file: %w", err)
+	}
+
+	tee := io.TeeReader(file, cacheFile)
+	return tee, nil
+}

cmd/fanal/main.go

@@ -7,11 +7,9 @@ import (
 	"os"
 
 	"github.com/knqyf263/fanal/analyzer"
-
-	"golang.org/x/crypto/ssh/terminal"
-
 	_ "github.com/knqyf263/fanal/analyzer/os/alpine"
 	_ "github.com/knqyf263/fanal/analyzer/pkg/apk"
+	"golang.org/x/crypto/ssh/terminal"
 )
 
 func main() {

extractor/docker.go

@@ -7,17 +7,18 @@ import (
 	"encoding/json"
 	"io"
 	"io/ioutil"
+	"log"
 	"path/filepath"
 	"strings"
 	"time"
 
-	digest "github.com/opencontainers/go-digest"
-
 	"github.com/docker/distribution/manifest/schema2"
-
 	"github.com/genuinetools/reg/registry"
 	"github.com/genuinetools/reg/repoutils"
+	"github.com/knqyf263/fanal/cache"
+	"github.com/knqyf263/fanal/token"
 	"github.com/knqyf263/nested"
+	digest "github.com/opencontainers/go-digest"
 	"golang.org/x/xerrors"
 )
 
@@ -108,6 +109,7 @@ func (d DockerExtractor) createRegistryClient(ctx context.Context, domain string
 	if err != nil {
 		return nil, err
 	}
+	auth = token.GetToken(ctx, auth)
 
 	// Prevent non-ssl unless explicitly forced
 	if !d.Option.NonSSL && strings.HasPrefix(auth.ServerAddress, "http:") {
@@ -154,13 +156,21 @@ func (d DockerExtractor) Extract(ctx context.Context, imageName string, filename
 	for _, ref := range m.Manifest.Layers {
 		layerIDs = append(layerIDs, string(ref.Digest))
 		go func(d digest.Digest) {
+			// Use cache
+			rc := cache.Get(string(d))
+			if rc == nil {
 				// Download the layer.
-			content, err := r.DownloadLayer(ctx, image.Path, d)
+				rc, err = r.DownloadLayer(ctx, image.Path, d)
 				if err != nil {
 					errCh <- xerrors.Errorf("failed to download the layer(%s): %w", d, err)
 					return
 				}
-			gzipReader, err := gzip.NewReader(content)
+				rc, err = cache.Set(string(d), rc)
+				if err != nil {
+					log.Print(err)
+				}
+			}
+			gzipReader, err := gzip.NewReader(rc)
 			if err != nil {
 				errCh <- xerrors.Errorf("invalid gzip: %w", err)
 				return

go.mod

@@ -3,11 +3,17 @@ module github.com/knqyf263/fanal
 go 1.12
 
 require (
+	github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect
+	github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect
+	github.com/aws/aws-sdk-go v1.19.11
 	github.com/docker/distribution v0.0.0-20180920194744-16128bbac47f
+	github.com/docker/docker v0.0.0-20180924202107-a9c061deec0f
 	github.com/genuinetools/reg v0.16.1-0.20190102165523-d959057b30da
 	github.com/knqyf263/nested v0.0.1
 	github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2
 	github.com/pkg/errors v0.8.1
+	github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e
 	golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5
 	golang.org/x/xerrors v0.0.0-20190315151331-d61658bd2e18
+	gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
 )

go.sum

@@ -5,6 +5,12 @@ github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
 github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/aws/aws-sdk-go v1.19.11 h1:tqaTGER6Byw3QvsjGW0p018U2UOqaJPeJuzoaF7jjoQ=
+github.com/aws/aws-sdk-go v1.19.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -57,6 +63,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.5.0 h1:WcmKMm43DR7RdtlkEXQJyo5ws8iTp98
 github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/knqyf263/nested v0.0.1 h1:Sv26CegUMhjt19zqbBKntjwESdxe5hxVPSk0+AKjdUc=
 github.com/knqyf263/nested v0.0.1/go.mod h1:zwhsIhMkBg90DTOJQvxPkKIypEHPYkgWHs4gybdlUmk=
@@ -125,6 +133,8 @@ google.golang.org/grpc v1.15.0 h1:Az/KuahOM4NAidTEuJCv/RonAA7rYsTPkqXVjr+8OOw=
 google.golang.org/grpc v1.15.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=

token/token.go

@@ -0,0 +1,72 @@
+package token
+
+import (
+	"context"
+	"encoding/base64"
+	"strings"
+
+	"github.com/prometheus/common/log"
+
+	"golang.org/x/xerrors"
+
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ecr"
+	"github.com/docker/docker/api/types"
+)
+
+const (
+	ecrURL = "amazonaws.com"
+	gcrURL = "grc.io"
+)
+
+func GetToken(ctx context.Context, auth types.AuthConfig) types.AuthConfig {
+	if auth.Username != "" || auth.Password != "" {
+		return auth
+	}
+
+	var username, password string
+	var err error
+
+	switch {
+	case strings.HasSuffix(auth.ServerAddress, ecrURL):
+		username, password, err = GetECRAuthorizationToken(ctx)
+	case strings.HasSuffix(auth.ServerAddress, gcrURL):
+		username, password, err = GetGCRAuthorizationToken(ctx)
+	}
+	if err != nil {
+		log.Debugf("failed to get token: %w", err)
+	}
+	auth.Username = username
+	auth.Password = password
+	return auth
+}
+
+func GetECRAuthorizationToken(ctx context.Context) (username, password string, err error) {
+	sess := session.Must(session.NewSessionWithOptions(session.Options{
+		SharedConfigState: session.SharedConfigEnable,
+	}))
+	svc := ecr.New(sess)
+	input := &ecr.GetAuthorizationTokenInput{}
+
+	result, err := svc.GetAuthorizationTokenWithContext(ctx, input)
+	if err != nil {
+		return "", "", xerrors.Errorf("failed to get authorization token: %w", err)
+	}
+
+	for _, data := range result.AuthorizationData {
+		b, err := base64.StdEncoding.DecodeString(*data.AuthorizationToken)
+		if err != nil {
+			return "", "", xerrors.Errorf("base64 decode failed: %w", err)
+		}
+		// e.g. AWS:eyJwYXlsb2...
+		split := strings.SplitN(string(b), ":", 2)
+		if len(split) == 2 {
+			return split[0], split[1], nil
+		}
+	}
+	return "", "", nil
+}
+
+func GetGCRAuthorizationToken(ctx context.Context) (username, password string, err error) {
+	return "", "", nil
+}