From c7208b3efae0b4cc67943dff51ac1cb08ece70c9 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Mon, 8 Apr 2019 00:09:18 +0900 Subject: [PATCH] Support private registry and use cache --- cache/cache.go | 42 ++++++++++++++++++++++++++ cmd/fanal/main.go | 4 +-- extractor/docker.go | 28 ++++++++++++------ go.mod | 6 ++++ go.sum | 10 +++++++ token/token.go | 72 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 150 insertions(+), 12 deletions(-) create mode 100644 cache/cache.go create mode 100644 token/token.go diff --git a/cache/cache.go b/cache/cache.go new file mode 100644 index 0000000000..97d3ea5d95 --- /dev/null +++ b/cache/cache.go @@ -0,0 +1,42 @@ +package cache + +import ( + "io" + "os" + "path/filepath" + + "golang.org/x/xerrors" +) + +func init() { + d, err := os.UserCacheDir() + if err != nil { + d = os.TempDir() + } + cacheDir = filepath.Join(d, "fanal") + os.MkdirAll(cacheDir, os.ModePerm) +} + +var ( + cacheDir string +) + +func Get(key string) io.Reader { + filePath := filepath.Join(cacheDir, key) + f, err := os.Open(filePath) + if err != nil { + return nil + } + return f +} + +func Set(key string, file io.Reader) (io.Reader, error) { + filePath := filepath.Join(cacheDir, key) + cacheFile, err := os.Create(filePath) + if err != nil { + return file, xerrors.Errorf("failed to create cache file: %w", err) + } + + tee := io.TeeReader(file, cacheFile) + return tee, nil +} diff --git a/cmd/fanal/main.go b/cmd/fanal/main.go index 6cfad38cfa..9d832331a2 100644 --- a/cmd/fanal/main.go +++ b/cmd/fanal/main.go @@ -7,11 +7,9 @@ import ( "os" "github.com/knqyf263/fanal/analyzer" - - "golang.org/x/crypto/ssh/terminal" - _ "github.com/knqyf263/fanal/analyzer/os/alpine" _ "github.com/knqyf263/fanal/analyzer/pkg/apk" + "golang.org/x/crypto/ssh/terminal" ) func main() { diff --git a/extractor/docker.go b/extractor/docker.go index 78739da9c7..08831cd2a7 100644 --- a/extractor/docker.go +++ b/extractor/docker.go @@ -7,17 +7,18 @@ import ( "encoding/json" "io" "io/ioutil" + "log" "path/filepath" "strings" "time" - digest "github.com/opencontainers/go-digest" - "github.com/docker/distribution/manifest/schema2" - "github.com/genuinetools/reg/registry" "github.com/genuinetools/reg/repoutils" + "github.com/knqyf263/fanal/cache" + "github.com/knqyf263/fanal/token" "github.com/knqyf263/nested" + digest "github.com/opencontainers/go-digest" "golang.org/x/xerrors" ) @@ -108,6 +109,7 @@ func (d DockerExtractor) createRegistryClient(ctx context.Context, domain string if err != nil { return nil, err } + auth = token.GetToken(ctx, auth) // Prevent non-ssl unless explicitly forced if !d.Option.NonSSL && strings.HasPrefix(auth.ServerAddress, "http:") { @@ -154,13 +156,21 @@ func (d DockerExtractor) Extract(ctx context.Context, imageName string, filename for _, ref := range m.Manifest.Layers { layerIDs = append(layerIDs, string(ref.Digest)) go func(d digest.Digest) { - // Download the layer. - content, err := r.DownloadLayer(ctx, image.Path, d) - if err != nil { - errCh <- xerrors.Errorf("failed to download the layer(%s): %w", d, err) - return + // Use cache + rc := cache.Get(string(d)) + if rc == nil { + // Download the layer. + rc, err = r.DownloadLayer(ctx, image.Path, d) + if err != nil { + errCh <- xerrors.Errorf("failed to download the layer(%s): %w", d, err) + return + } + rc, err = cache.Set(string(d), rc) + if err != nil { + log.Print(err) + } } - gzipReader, err := gzip.NewReader(content) + gzipReader, err := gzip.NewReader(rc) if err != nil { errCh <- xerrors.Errorf("invalid gzip: %w", err) return diff --git a/go.mod b/go.mod index 13412e7a9f..ab41d15599 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,17 @@ module github.com/knqyf263/fanal go 1.12 require ( + github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect + github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect + github.com/aws/aws-sdk-go v1.19.11 github.com/docker/distribution v0.0.0-20180920194744-16128bbac47f + github.com/docker/docker v0.0.0-20180924202107-a9c061deec0f github.com/genuinetools/reg v0.16.1-0.20190102165523-d959057b30da github.com/knqyf263/nested v0.0.1 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2 github.com/pkg/errors v0.8.1 + github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5 golang.org/x/xerrors v0.0.0-20190315151331-d61658bd2e18 + gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect ) diff --git a/go.sum b/go.sum index 6d02aa7b99..89a15c59a5 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,12 @@ github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/aws/aws-sdk-go v1.19.11 h1:tqaTGER6Byw3QvsjGW0p018U2UOqaJPeJuzoaF7jjoQ= +github.com/aws/aws-sdk-go v1.19.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -57,6 +63,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.5.0 h1:WcmKMm43DR7RdtlkEXQJyo5ws8iTp98 github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/knqyf263/nested v0.0.1 h1:Sv26CegUMhjt19zqbBKntjwESdxe5hxVPSk0+AKjdUc= github.com/knqyf263/nested v0.0.1/go.mod h1:zwhsIhMkBg90DTOJQvxPkKIypEHPYkgWHs4gybdlUmk= @@ -125,6 +133,8 @@ google.golang.org/grpc v1.15.0 h1:Az/KuahOM4NAidTEuJCv/RonAA7rYsTPkqXVjr+8OOw= google.golang.org/grpc v1.15.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= diff --git a/token/token.go b/token/token.go new file mode 100644 index 0000000000..14bd1534c9 --- /dev/null +++ b/token/token.go @@ -0,0 +1,72 @@ +package token + +import ( + "context" + "encoding/base64" + "strings" + + "github.com/prometheus/common/log" + + "golang.org/x/xerrors" + + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ecr" + "github.com/docker/docker/api/types" +) + +const ( + ecrURL = "amazonaws.com" + gcrURL = "grc.io" +) + +func GetToken(ctx context.Context, auth types.AuthConfig) types.AuthConfig { + if auth.Username != "" || auth.Password != "" { + return auth + } + + var username, password string + var err error + + switch { + case strings.HasSuffix(auth.ServerAddress, ecrURL): + username, password, err = GetECRAuthorizationToken(ctx) + case strings.HasSuffix(auth.ServerAddress, gcrURL): + username, password, err = GetGCRAuthorizationToken(ctx) + } + if err != nil { + log.Debugf("failed to get token: %w", err) + } + auth.Username = username + auth.Password = password + return auth +} + +func GetECRAuthorizationToken(ctx context.Context) (username, password string, err error) { + sess := session.Must(session.NewSessionWithOptions(session.Options{ + SharedConfigState: session.SharedConfigEnable, + })) + svc := ecr.New(sess) + input := &ecr.GetAuthorizationTokenInput{} + + result, err := svc.GetAuthorizationTokenWithContext(ctx, input) + if err != nil { + return "", "", xerrors.Errorf("failed to get authorization token: %w", err) + } + + for _, data := range result.AuthorizationData { + b, err := base64.StdEncoding.DecodeString(*data.AuthorizationToken) + if err != nil { + return "", "", xerrors.Errorf("base64 decode failed: %w", err) + } + // e.g. AWS:eyJwYXlsb2... + split := strings.SplitN(string(b), ":", 2) + if len(split) == 2 { + return split[0], split[1], nil + } + } + return "", "", nil +} + +func GetGCRAuthorizationToken(ctx context.Context) (username, password string, err error) { + return "", "", nil +}