gitea-mirror/trivy
1
0
Fork 0
mirror of https://github.com/aquasecurity/trivy.git synced 2025-12-22 15:16:33 -08:00
Code Issues Packages Projects Releases Wiki Activity
Files
7ef1e5f970221c8648c467171c66f98787a0302c
trivy/extractor/docker/docker_test.go
Teppei Fukuda 7ef1e5f970 Cache: Save only required files (fanal#51)
2019-12-16 08:44:43 +02:00

590 lines
33 KiB
Go
Raw Blame History

package docker
import (
"context"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"path"
"reflect"
"strings"
"testing"
"time"
"github.com/klauspost/compress/zstd"
"github.com/aquasecurity/fanal/extractor"
"github.com/aquasecurity/fanal/types"
"github.com/docker/docker/client"
"github.com/genuinetools/reg/registry"
"github.com/opencontainers/go-digest"
bolt "github.com/simar7/gokv/bbolt"
kvtypes "github.com/simar7/gokv/types"
"github.com/stretchr/testify/assert"
)
// TODO: Use a memory based FS rather than actual fs
// context: https://github.com/aquasecurity/fanal/pull/51#discussion_r352337762
func setupCache() (*bolt.Store, *os.File, error) {
f, err := ioutil.TempFile(".", "Bolt_TestStore-*")
if err != nil {
return nil, nil, err
}
s, err := bolt.NewStore(bolt.Options{
Path: f.Name(),
})
return s, f, err
}
func TestExtractFromFile(t *testing.T) {
vectors := []struct {
file string // Test input file
filenames []string // Target files
FileMap extractor.FileMap // Expected output
err error // Expected error to occur
}{
{
file: "testdata/image1.tar",
filenames: []string{"var/foo", "etc/test/bar"},
FileMap: extractor.FileMap{
"etc/test/bar": []byte("bar\n"),
"/config": []byte(`{"architecture":"amd64","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh"],"ArgsEscaped":true,"Image":"sha256:e641703a6c77abde58a2e2d5e506da5ac61a648bdb17fba7c3325db9d2ba4ded","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"7dfcd2c8327651024825c14e0d8752544f59c03efeca291a71e532b7e0ca66bf","container_config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","rm /var/foo \u0026\u0026 rm -rf /etc/test \u0026\u0026 mkdir /etc/test \u0026\u0026 echo bar \u003e /etc/test/bar"],"ArgsEscaped":true,"Image":"sha256:e641703a6c77abde58a2e2d5e506da5ac61a648bdb17fba7c3325db9d2ba4ded","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2019-04-07T04:27:16.291049098Z","docker_version":"18.09.2","history":[{"created":"2019-03-07T22:19:46.661698137Z","created_by":"/bin/sh -c #(nop) ADD file:38bc6b51693b13d84a63e281403e2f6d0218c44b1d7ff12157c4523f9f0ebb1e in / "},{"created":"2019-03-07T22:19:46.815331171Z","created_by":"/bin/sh -c #(nop) CMD [\"/bin/sh\"]","empty_layer":true},{"created":"2019-04-07T04:08:02.548475493Z","created_by":"/bin/sh -c mkdir /etc/test \u0026\u0026 touch /var/foo \u0026\u0026 touch /etc/test/test"},{"created":"2019-04-07T04:27:16.291049098Z","created_by":"/bin/sh -c rm /var/foo \u0026\u0026 rm -rf /etc/test \u0026\u0026 mkdir /etc/test \u0026\u0026 echo bar \u003e /etc/test/bar"}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:d9ff549177a94a413c425ffe14ae1cc0aa254bc9c7df781add08e7d2fba25d27","sha256:f75441026d68038ca80e92f342fb8f3c0f1faeec67b5a80c98f033a65beaef5a","sha256:a8b87ccf2f2f94b9e23308560800afa3f272aa6db5cc7d9b0119b6843889cff2"]}}`),
},
err: nil,
},
{
file: "testdata/image2.tar",
filenames: []string{"home/app/Gemfile", "home/app2/Gemfile"},
FileMap: extractor.FileMap{
"home/app2/Gemfile": []byte("gem"),
"/config": []byte(`{"architecture":"amd64","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh"],"ArgsEscaped":true,"Image":"sha256:4fe3bbb628df60571f88cb053db9e2c9ec2f1c1e8373db9b026d0e582ef01d6d","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"7b1b7a0cfacbce82b51230bf0c6354e64cd0068e4e51180ab717890fc805bdf5","container_config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","mv /home/app /home/app2"],"ArgsEscaped":true,"Image":"sha256:4fe3bbb628df60571f88cb053db9e2c9ec2f1c1e8373db9b026d0e582ef01d6d","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2019-04-07T05:32:59.607884934Z","docker_version":"18.09.2","history":[{"created":"2019-03-07T22:19:46.661698137Z","created_by":"/bin/sh -c #(nop) ADD file:38bc6b51693b13d84a63e281403e2f6d0218c44b1d7ff12157c4523f9f0ebb1e in / "},{"created":"2019-03-07T22:19:46.815331171Z","created_by":"/bin/sh -c #(nop) CMD [\"/bin/sh\"]","empty_layer":true},{"created":"2019-04-07T05:32:58.27180871Z","created_by":"/bin/sh -c mkdir /home/app \u0026\u0026 echo -n gem \u003e /home/app/Gemfile"},{"created":"2019-04-07T05:32:59.607884934Z","created_by":"/bin/sh -c mv /home/app /home/app2"}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:d9ff549177a94a413c425ffe14ae1cc0aa254bc9c7df781add08e7d2fba25d27","sha256:f9e7e541d5be4537a826c4c6cb68b603a8e552c22e28ac726e9be6b22f51af44","sha256:718fb3edf377530e3713cd074d141827d05f654f6389e827c344b7fcff153025"]}}`),
},
err: nil,
},
{
file: "testdata/image3.tar",
filenames: []string{"home/app/Gemfile", "home/app2/Pipfile", "home/app/Pipfile"},
FileMap: extractor.FileMap{
"home/app/Pipfile": []byte("pip"),
"/config": []byte(`{"architecture":"amd64","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh"],"ArgsEscaped":true,"Image":"sha256:53dca1cadfa555151d28ac616df868eed4fc935f21af393118f4fbc36d9fb24a","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"42b6c68c1704e06fbffecfee6ae5400978cf508790d563e2bda4d1b20ce93c6d","container_config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","rm -rf /home/app \u0026\u0026 mv /home/app2 /home/app"],"ArgsEscaped":true,"Image":"sha256:53dca1cadfa555151d28ac616df868eed4fc935f21af393118f4fbc36d9fb24a","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2019-04-07T05:36:08.899764053Z","docker_version":"18.09.2","history":[{"created":"2019-03-07T22:19:46.661698137Z","created_by":"/bin/sh -c #(nop) ADD file:38bc6b51693b13d84a63e281403e2f6d0218c44b1d7ff12157c4523f9f0ebb1e in / "},{"created":"2019-03-07T22:19:46.815331171Z","created_by":"/bin/sh -c #(nop) CMD [\"/bin/sh\"]","empty_layer":true},{"created":"2019-04-07T05:32:58.27180871Z","created_by":"/bin/sh -c mkdir /home/app \u0026\u0026 echo -n gem \u003e /home/app/Gemfile"},{"created":"2019-04-07T05:36:07.629894435Z","created_by":"/bin/sh -c mkdir /home/app2 \u0026\u0026 echo -n pip \u003e /home/app2/Pipfile"},{"created":"2019-04-07T05:36:08.899764053Z","created_by":"/bin/sh -c rm -rf /home/app \u0026\u0026 mv /home/app2 /home/app"}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:d9ff549177a94a413c425ffe14ae1cc0aa254bc9c7df781add08e7d2fba25d27","sha256:f9e7e541d5be4537a826c4c6cb68b603a8e552c22e28ac726e9be6b22f51af44","sha256:5a917ce45575a009bb5b4f462ed84522c7f642647b62a9f2b2bdfc2275f85104","sha256:50022087bbe2b08d1ce033122a56c7cf74cc1d1d6dae97a397226dd49a309c3b"]}}`),
},
err: nil,
},
{
file: "testdata/image3_gz_layers.tar",
filenames: []string{"home/app/Gemfile", "home/app2/Pipfile", "home/app/Pipfile"},
FileMap: extractor.FileMap{
"home/app/Pipfile": []byte("pip"),
"/config": []byte(`{"architecture":"amd64","container":"a0625bb53d38b712d9fe7e307c53a5b1f2528189d694a29ba37b7a27bee20029","created":"2019-07-23T08:53:13.7506797Z","docker_version":"18.06.1-ce","history":[{"author":"","created":"2019-07-11T22:20:52.139709355Z","created_by":"/bin/sh -c #(nop) ADD file:0eb5ea35741d23fe39cbac245b3a5d84856ed6384f4ff07d496369ee6d960bad in / ","comment":""},{"author":"","created":"2019-07-11T22:20:52.375286404Z","created_by":"/bin/sh -c #(nop) CMD [\"/bin/sh\"]","comment":"","empty_layer":true},{"author":"kaniko","created":"0001-01-01T00:00:00Z","created_by":"RUN mkdir /home/app \u0026\u0026 echo -n gem \u003e /home/app/Gemfile","comment":""},{"author":"kaniko","created":"0001-01-01T00:00:00Z","created_by":"RUN mkdir /home/app2 \u0026\u0026 echo -n pip \u003e /home/app2/Pipfile","comment":""},{"author":"kaniko","created":"0001-01-01T00:00:00Z","created_by":"RUN rm -rf /home/app \u0026\u0026 mv /home/app2 /home/app","comment":""}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:1bfeebd65323b8ddf5bd6a51cc7097b72788bc982e9ab3280d53d3c613adffa7","sha256:927e7087ecbc8de977df69cb60c19f1759b374bd36b2fa9c7c30cebe0fcf8156","sha256:25dbee606f1ef778e242145a8da895e416661dbd8b4243523aaa5227919458cc","sha256:ccbf319ba1f9c5f823254a36c91187db9ec2ff4cd1ec27dfc6b10bcf6996e334"]},"config":{"AttachStderr":false,"AttachStdin":false,"AttachStdout":false,"Cmd":["/bin/sh"],"Healthcheck":null,"Domainname":"","Entrypoint":null,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Hostname":"","Image":"sha256:f248cae16d3e1b44bf474ad89815438f10c395f8e532153e4fcd32cbbb150fb3","Labels":null,"OnBuild":null,"OpenStdin":false,"StdinOnce":false,"Tty":false,"User":"","Volumes":null,"WorkingDir":"","ExposedPorts":null,"ArgsEscaped":true,"NetworkDisabled":false,"MacAddress":"","StopSignal":"","Shell":null},"container_config":{"AttachStderr":false,"AttachStdin":false,"AttachStdout":false,"Cmd":["/bin/sh","-c","#(nop) ","CMD [\"/bin/sh\"]"],"Healthcheck":null,"Domainname":"","Entrypoint":null,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Hostname":"a0625bb53d38","Image":"sha256:f248cae16d3e1b44bf474ad89815438f10c395f8e532153e4fcd32cbbb150fb3","Labels":{},"OnBuild":null,"OpenStdin":false,"StdinOnce":false,"Tty":false,"User":"","Volumes":null,"WorkingDir":"","ExposedPorts":null,"ArgsEscaped":true,"NetworkDisabled":false,"MacAddress":"","StopSignal":"","Shell":null},"osversion":""}`),
},
err: nil,
},
{file: "testdata/image4.tar",
filenames: []string{".abc", ".def", "foo/.abc", "foo/.def", ".foo/.abc"},
FileMap: extractor.FileMap{
".def": []byte("def"),
"foo/.abc": []byte("abc"),
"/config": []byte(`{"architecture":"amd64","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh"],"ArgsEscaped":true,"Image":"sha256:cabfb6dd9c622b8cd0efdc7bb38ed9a9d2001a32c2b5d5c174e284784df712e8","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"8290b131834ed7ef8c388a290594afeaa5daea024031a2551c8dedfc845fd09e","container_config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","rm -rf /.foo"],"ArgsEscaped":true,"Image":"sha256:cabfb6dd9c622b8cd0efdc7bb38ed9a9d2001a32c2b5d5c174e284784df712e8","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2019-04-07T05:48:16.088980845Z","docker_version":"18.09.2","history":[{"created":"2019-03-07T22:19:46.661698137Z","created_by":"/bin/sh -c #(nop) ADD file:38bc6b51693b13d84a63e281403e2f6d0218c44b1d7ff12157c4523f9f0ebb1e in / "},{"created":"2019-03-07T22:19:46.815331171Z","created_by":"/bin/sh -c #(nop) CMD [\"/bin/sh\"]","empty_layer":true},{"created":"2019-04-07T05:48:10.560447082Z","created_by":"/bin/sh -c echo -n abc \u003e .abc \u0026\u0026 echo -n def \u003e .def"},{"created":"2019-04-07T05:48:11.938256528Z","created_by":"/bin/sh -c mkdir foo \u0026\u0026 echo -n abc \u003e /foo/.abc \u0026\u0026 echo -n def \u003e /foo/.def"},{"created":"2019-04-07T05:48:13.188275588Z","created_by":"/bin/sh -c rm .abc /foo/.def"},{"created":"2019-04-07T05:48:14.569944213Z","created_by":"/bin/sh -c mkdir .foo \u0026\u0026 echo -n abc /.foo/.abc"},{"created":"2019-04-07T05:48:16.088980845Z","created_by":"/bin/sh -c rm -rf /.foo"}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:d9ff549177a94a413c425ffe14ae1cc0aa254bc9c7df781add08e7d2fba25d27","sha256:c42355fdc6d1a90c39b26ae5ac44c85c079f6da260def6bcb781ffcfe45ce6c9","sha256:b16629f22093ce5dfec353149661886cc1ca0c62ff30c450a82eba693eaedbd2","sha256:9717a79724f7114e32f004067a9cf96493812b2772f8a88096d1c43f7898d4f9","sha256:87c73b7beca2340705c988bb35235c66ae16b2ed2a6ce5b37b215f9bb08e7dc9","sha256:99cc8353ab2a712793601465751b9f518a35763db138e8b92b54f13e0c82d8b6"]}}`),
},
err: nil,
},
{
file: "testdata/image5.tar",
// Not detect foo/baz cause set "foo"
filenames: []string{"bar", "foo/bar/", "foo"},
FileMap: extractor.FileMap{
"bar": []byte("bar"),
"foo/bar/abc": []byte("abc"),
"foo/bar/def": []byte("def"),
"/config": []byte(`{"architecture":"amd64","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh"],"ArgsEscaped":true,"Image":"sha256:961769676411f082461f9ef46626dd7a2d1e2b2a38e6a44364bcbecf51e66dd4","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"65b0a9f4cc5ba8eaad4edf5e8edd9166aa8dc31b2d6e21d84951b9737c250078","container_config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","echo -n bar \u003e bar \u0026\u0026 mkdir -p foo/bar \u0026\u0026 echo -n abc \u003e foo/bar/abc \u0026\u0026 echo -n def \u003e foo/bar/def \u0026\u0026 echo -n baz \u003e foo/baz"],"Image":"sha256:961769676411f082461f9ef46626dd7a2d1e2b2a38e6a44364bcbecf51e66dd4","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2019-09-05T17:19:53.569209Z","docker_version":"19.03.1","history":[{"created":"2019-08-20T20:19:55.062606894Z","created_by":"/bin/sh -c #(nop) ADD file:fe64057fbb83dccb960efabbf1cd8777920ef279a7fa8dbca0a8801c651bdf7c in / "},{"created":"2019-08-20T20:19:55.211423266Z","created_by":"/bin/sh -c #(nop) CMD [\"/bin/sh\"]","empty_layer":true},{"created":"2019-09-05T17:19:53.569209Z","created_by":"/bin/sh -c echo -n bar \u003e bar \u0026\u0026 mkdir -p foo/bar \u0026\u0026 echo -n abc \u003e foo/bar/abc \u0026\u0026 echo -n def \u003e foo/bar/def \u0026\u0026 echo -n baz \u003e foo/baz"}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:03901b4a2ea88eeaad62dbe59b072b28b6efa00491962b8741081c5df50c65e0","sha256:69c7f4ae201dc4669b58cbac8f1cc0593e28ddb0f5d35a21541217ab17f550fa"]}}`),
},
err: nil,
},
{
file: "testdata/image5.tar",
// Detect foo/baz cause set "foo/"
filenames: []string{"bar", "foo/"},
FileMap: extractor.FileMap{
"bar": []byte("bar"),
"foo/baz": []byte("baz"),
"/config": []byte(`{"architecture":"amd64","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh"],"ArgsEscaped":true,"Image":"sha256:961769676411f082461f9ef46626dd7a2d1e2b2a38e6a44364bcbecf51e66dd4","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"65b0a9f4cc5ba8eaad4edf5e8edd9166aa8dc31b2d6e21d84951b9737c250078","container_config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","echo -n bar \u003e bar \u0026\u0026 mkdir -p foo/bar \u0026\u0026 echo -n abc \u003e foo/bar/abc \u0026\u0026 echo -n def \u003e foo/bar/def \u0026\u0026 echo -n baz \u003e foo/baz"],"Image":"sha256:961769676411f082461f9ef46626dd7a2d1e2b2a38e6a44364bcbecf51e66dd4","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2019-09-05T17:19:53.569209Z","docker_version":"19.03.1","history":[{"created":"2019-08-20T20:19:55.062606894Z","created_by":"/bin/sh -c #(nop) ADD file:fe64057fbb83dccb960efabbf1cd8777920ef279a7fa8dbca0a8801c651bdf7c in / "},{"created":"2019-08-20T20:19:55.211423266Z","created_by":"/bin/sh -c #(nop) CMD [\"/bin/sh\"]","empty_layer":true},{"created":"2019-09-05T17:19:53.569209Z","created_by":"/bin/sh -c echo -n bar \u003e bar \u0026\u0026 mkdir -p foo/bar \u0026\u0026 echo -n abc \u003e foo/bar/abc \u0026\u0026 echo -n def \u003e foo/bar/def \u0026\u0026 echo -n baz \u003e foo/baz"}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:03901b4a2ea88eeaad62dbe59b072b28b6efa00491962b8741081c5df50c65e0","sha256:69c7f4ae201dc4669b58cbac8f1cc0593e28ddb0f5d35a21541217ab17f550fa"]}}`),
},
err: nil,
},
{
file: "testdata/image6.tar",
// Not detect package-lock.json and composer.lock under vendor/ or node_modules/"
filenames: []string{"foo", "package-lock.json", "composer.lock"},
FileMap: extractor.FileMap{
"foo": []byte("foo\n"),
"/config": []byte(`{"architecture":"amd64","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt"],"Cmd":null,"Image":"sha256:7b421e99fb75da8466c90ec04a1c43a3f8b9fd9be0bb71a341d30ca11c75e9e0","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container_config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt"],"Cmd":["/bin/sh","-c","#(nop) COPY file:745080737b5cb8cd47dbca9a8819cc8e9034b7748ccb0ed1c60bdb0b5fc6c2e8 in /app/vendor/ "],"Image":"sha256:7b421e99fb75da8466c90ec04a1c43a3f8b9fd9be0bb71a341d30ca11c75e9e0","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2019-10-30T09:59:16.9071124Z","docker_version":"19.03.4","history":[{"created":"1970-01-01T00:00:00Z","author":"Bazel","created_by":"bazel build ..."},{"created":"1970-01-01T00:00:00Z","author":"Bazel","created_by":"bazel build ..."},{"created":"2019-10-30T09:59:16.1671833Z","created_by":"/bin/sh -c #(nop) COPY file:8d7ea209a266ec183c53e0de5dad09aa6ccd217961306f174884d94eb92369ab in /foo "},{"created":"2019-10-30T09:59:16.5096275Z","created_by":"/bin/sh -c #(nop) COPY file:4772d4fa23206c27b7ddbac967e3d2e3f6b08f6dde8ba8170975efc4b6041255 in /app/node_modules/ "},{"created":"2019-10-30T09:59:16.9071124Z","created_by":"/bin/sh -c #(nop) COPY file:745080737b5cb8cd47dbca9a8819cc8e9034b7748ccb0ed1c60bdb0b5fc6c2e8 in /app/vendor/ "}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:932da51564135c98a49a34a193d6cd363d8fa4184d957fde16c9d8527b3f3b02","sha256:dffd9992ca398466a663c87c92cfea2a2db0ae0cf33fcb99da60eec52addbfc5","sha256:ca9852efab2ff2b5031a02f69021c6d4c49d351480f9cd7b08beec00d3527d1a","sha256:88d8eebab45ec2cb73ad2859a158617324eadfb904a8fe24b936b07ccd15096d","sha256:ef55f7ceb56d4ccbfb9228ab865f3e411c9a9781814fe4558003a2b2427f457d"]}}`),
},
err: nil,
},
}
for _, v := range vectors {
t.Run(path.Base(v.file), func(t *testing.T) {
f, err := os.Open(v.file)
if err != nil {
t.Fatalf("Open() error: %v", err)
}
defer f.Close()
d := Extractor{}
fm, err := d.ExtractFromFile(context.TODO(), f, v.filenames)
if v.err != err {
t.Errorf("err: got %v, want %v", v.err, err)
}
if !reflect.DeepEqual(fm, v.FileMap) {
t.Errorf("FilesMap: got %v, want %v", fm, v.FileMap)
}
})
}
}
func TestExtractFiles(t *testing.T) {
vectors := []struct {
file string // Test input file
filenames []string // Target files
FileMap extractor.FileMap // Expected output
opqDirs extractor.OPQDirs // Expected output
err error // Expected error to occur
}{
{
file: "testdata/normal.tar",
filenames: []string{"var/foo"},
FileMap: extractor.FileMap{"var/foo": []byte{}},
opqDirs: []string{},
err: nil,
},
{
file: "testdata/opq.tar",
filenames: []string{"var/foo"},
FileMap: extractor.FileMap{
"var/.wh.foo": []byte{},
},
opqDirs: []string{"etc/test"},
err: nil,
},
{
file: "testdata/opq2.tar",
filenames: []string{"var/foo", "etc/test/bar"},
FileMap: extractor.FileMap{
"etc/test/bar": []byte("bar\n"),
"var/.wh.foo": []byte{},
},
opqDirs: []string{"etc/test"},
err: nil,
},
}
for _, v := range vectors {
t.Run(path.Base(v.file), func(t *testing.T) {
f, err := os.Open(v.file)
if err != nil {
t.Fatalf("Open() error: %v", err)
}
defer f.Close()
d := Extractor{}
fm, opqDirs, err := d.ExtractFiles(f, v.filenames)
if v.err != err {
t.Errorf("err: got %v, want %v", v.err, err)
}
if !reflect.DeepEqual(opqDirs, v.opqDirs) {
t.Errorf("OPQDirs: got %v, want %v", opqDirs, v.opqDirs)
}
if !reflect.DeepEqual(fm, v.FileMap) {
t.Errorf("FilesMap: got %v, want %v", fm, v.FileMap)
}
})
}
}
func TestDockerExtractor_SaveLocalImage(t *testing.T) {
testCases := []struct {
name string
expectedImageData string
cacheHit bool
}{
{
name: "happy path with cache miss",
expectedImageData: "foofromdocker",
},
{
name: "happy path with cache hit",
cacheHit: true,
expectedImageData: "foofromcache",
},
}
for _, tc := range testCases {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
httpPath := r.URL.String()
switch {
case strings.Contains(httpPath, "images/get?names=fooimage"):
_, _ = fmt.Fprint(w, "foofromdocker")
default:
assert.FailNow(t, "unexpected path accessed: ", r.URL.String())
}
}))
defer ts.Close()
c, err := client.NewClientWithOpts(client.WithHost(ts.URL))
assert.NoError(t, err)
// setup cache
s, f, err := setupCache()
defer func() {
_ = f.Close()
_ = os.RemoveAll(f.Name())
}()
assert.NoError(t, err)
if tc.cacheHit {
e, _ := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedDefault))
dst := e.EncodeAll([]byte("foofromcache"), nil)
_ = s.Set(kvtypes.SetItemInput{
BucketName: "imagebucket",
Key: "fooimage",
Value: dst,
})
}
de := Extractor{
Option: types.DockerOption{},
Client: c,
Cache: s,
}
r, err := de.SaveLocalImage(context.TODO(), "fooimage")
actualSavedTarBytes, _ := ioutil.ReadAll(r)
assert.Equal(t, []byte(tc.expectedImageData), actualSavedTarBytes[:], tc.name)
assert.NoError(t, err, tc.name)
// check the cache for what was stored
var actualValue []byte
found, err := de.Cache.Get(kvtypes.GetItemInput{
BucketName: "imagebucket",
Key: "fooimage",
Value: &actualValue,
})
assert.NoError(t, err, tc.name)
assert.True(t, found, tc.name)
dec, _ := zstd.NewReader(nil)
actualStoredValue, _ := dec.DecodeAll(actualValue, nil)
assert.Equal(t, tc.expectedImageData, string(actualStoredValue), tc.name)
}
}
func TestDockerExtractor_Extract(t *testing.T) {
testCases := []struct {
name string
imageName string
manifestResp string
fileName string
blobData string
fileToExtract []string
expectedFileMap extractor.FileMap
expectedError string
}{
{
name: "happy path",
manifestResp: `{
"schemaVersion": 2,
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"layers": [
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"size": 153263,
"digest": "sha256:shafortestdirslashhelloworlddottxt"
},
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"size": 153263,
"digest": "sha256:shafortestdirslashbadworlddottxt"
}
]
}`,
fileName: "testdata/testdir.tar.gz", // includes helloworld.txt and badworld.txt
blobData: "foo",
fileToExtract: []string{"testdir/helloworld.txt", "testdir/badworld.txt"},
expectedFileMap: extractor.FileMap{
"/config": []uint8{0x66, 0x6f, 0x6f},
"testdir/helloworld.txt": []uint8{0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0xa},
"testdir/badworld.txt": []uint8{0x62, 0x61, 0x64, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0xa},
},
},
{
name: "sad path: invalid manifest response",
manifestResp: "badManifestResponse",
expectedError: "failed to get the v2 manifest: invalid character 'b' looking for beginning of value",
},
{
name: "sad path: bad image name",
imageName: "https://docker/very/bad/imagename",
expectedError: `failed to parse the image: parsing image "https://docker/very/bad/imagename" failed: invalid reference format`,
},
{
name: "sad path: corrupt layer data invalid gzip",
manifestResp: `{
"schemaVersion": 2,
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"layers": [
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"size": 153263,
"digest": "sha256:shaforinvalidgzipfile"
}
]
}`,
fileName: "testdata/opq.tar",
blobData: "foo",
expectedFileMap: extractor.FileMap(nil),
expectedError: "could not init gzip reader: gzip: invalid header",
},
}
for _, tc := range testCases {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
httpPath := r.URL.String()
switch {
case strings.Contains(httpPath, "/v2/library/fooimage/manifests/latest"):
w.Header().Set("Content-Type", "application/vnd.docker.distribution.manifest.v2+json")
_, _ = fmt.Fprint(w, tc.manifestResp)
case strings.Contains(httpPath, "/v2/library/fooimage/blobs/sha256:shafortestdirslashhelloworlddottxt"):
b, _ := ioutil.ReadFile("testdata/helloworld.tar.gz")
_, _ = w.Write(b)
case strings.Contains(httpPath, "/v2/library/fooimage/blobs/sha256:shafortestdirslashbadworlddottxt"):
b, _ := ioutil.ReadFile("testdata/badworld.tar.gz")
_, _ = w.Write(b)
case strings.Contains(httpPath, "/v2/library/fooimage/blobs/sha256:shaforinvalidgzipfile"):
b, _ := ioutil.ReadFile("testdata/opq.tar")
_, _ = w.Write(b)
case strings.Contains(httpPath, "/v2/library/fooimage/blobs/"):
_, _ = w.Write([]byte(tc.blobData))
default:
assert.FailNow(t, "unexpected path accessed: ", r.URL.String())
}
}))
defer ts.Close()
c, err := client.NewClientWithOpts(client.WithHost(ts.URL))
assert.NoError(t, err)
// setup cache
s, f, err := setupCache()
defer func() {
_ = f.Close()
_ = os.RemoveAll(f.Name())
}()
assert.NoError(t, err)
de := Extractor{
Option: types.DockerOption{
AuthURL: ts.URL,
NonSSL: true,
SkipPing: true,
Timeout: time.Second * 1000,
},
Client: c,
Cache: s,
}
tsURL := strings.TrimPrefix(ts.URL, "http://")
var imageName string
switch {
case tc.imageName != "":
imageName = tc.imageName
default:
imageName = tsURL + "/library/fooimage"
}
fm, err := de.Extract(context.TODO(), imageName, tc.fileToExtract)
switch {
case tc.expectedError != "":
assert.Equal(t, tc.expectedError, err.Error(), tc.name)
default:
assert.NoError(t, err, tc.name)
}
assert.Equal(t, tc.expectedFileMap, fm, tc.name)
}
}
func TestDocker_ExtractLayerWorker(t *testing.T) {
goodtarzstdgolden, _ := ioutil.ReadFile("testdata/testdir.tar.zstd")
goodReturnedTarContent, _ := ioutil.ReadFile("testdata/goodTarContent.golden")
testCases := []struct {
name string
cacheHit bool
garbageCache bool
requiredFiles []string
expectedCacheContents []byte
expectedReturnedTarContent []byte
}{
{
name: "happy path with cache miss and write back",
cacheHit: false,
requiredFiles: []string{"testdir/helloworld.txt", "testdir/badworld.txt"},
expectedCacheContents: goodtarzstdgolden,
expectedReturnedTarContent: goodReturnedTarContent,
},
{
name: "happy path with cache hit with garbage cache and write back",
cacheHit: true,
garbageCache: true,
requiredFiles: []string{"testdir/helloworld.txt", "testdir/badworld.txt"},
expectedCacheContents: goodtarzstdgolden,
expectedReturnedTarContent: goodReturnedTarContent,
},
{
name: "happy path with cache hit",
cacheHit: true,
expectedCacheContents: goodtarzstdgolden,
expectedReturnedTarContent: goodReturnedTarContent,
},
{
name: "happy path with cache miss but no write back",
cacheHit: false,
expectedCacheContents: []byte{0x28, 0xb5, 0x2f, 0xfd, 0x4, 0x60, 0x1, 0x0, 0x0, 0x99, 0xe9, 0xd8, 0x51}, // just the empty tar header
expectedReturnedTarContent: []byte{},
},
}
for _, tc := range testCases {
inputDigest := digest.Digest("sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b")
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
httpPath := r.URL.String()
switch {
case strings.Contains(httpPath, "/v2/library/fooimage/blobs/sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b"):
layerData, _ := ioutil.ReadFile("testdata/testdir.tar.gz")
_, _ = w.Write(layerData)
default:
assert.FailNow(t, "unexpected path accessed: ", fmt.Sprintf("%s %s", r.URL.String(), tc.name))
}
}))
defer ts.Close()
c, err := client.NewClientWithOpts(client.WithHost(ts.URL))
assert.NoError(t, err)
// setup cache
s, f, err := setupCache()
defer func() {
_ = f.Close()
_ = os.RemoveAll(f.Name())
}()
assert.NoError(t, err, tc.name)
if tc.cacheHit {
switch tc.garbageCache {
case true:
garbage, _ := ioutil.ReadFile("testdata/invalidgzvalidtar.tar.gz")
assert.NoError(t, s.Set(kvtypes.SetItemInput{
BucketName: LayerTarsBucket,
Key: string(inputDigest),
Value: garbage,
}), tc.name)
default:
assert.NoError(t, s.Set(kvtypes.SetItemInput{
BucketName: LayerTarsBucket,
Key: string(inputDigest),
Value: goodtarzstdgolden,
}), tc.name)
}
}
de := Extractor{
Option: types.DockerOption{
AuthURL: ts.URL,
NonSSL: true,
SkipPing: true,
Timeout: time.Second * 1000,
},
Client: c,
Cache: s,
}
tsUrl := strings.TrimPrefix(ts.URL, "http://")
inputImage := registry.Image{
Domain: tsUrl,
Path: "library/fooimage",
Tag: "latest",
}
layerCh := make(chan layer)
errCh := make(chan error)
r, err := de.createRegistryClient(context.TODO(), inputImage.Domain)
go func() {
de.extractLayerWorker(inputDigest, r, context.TODO(), inputImage, errCh, layerCh, tc.requiredFiles)
}()
var errRecieved error
var layerReceived layer
select {
case errRecieved = <-errCh:
assert.FailNow(t, "unexpected error received, err: ", fmt.Sprintf("%s, %s", errRecieved, tc.name))
case layerReceived = <-layerCh:
assert.Equal(t, inputDigest, layerReceived.ID, tc.name)
got, _ := ioutil.ReadAll(layerReceived.Content)
assert.Equal(t, tc.expectedReturnedTarContent, got, tc.name)
}
// check cache contents
var actualCacheContents []byte
found, err := s.Get(kvtypes.GetItemInput{
BucketName: LayerTarsBucket,
Key: string(inputDigest),
Value: &actualCacheContents,
})
assert.True(t, found, tc.name)
assert.NoError(t, err, tc.name)
assert.Equal(t, tc.expectedCacheContents, actualCacheContents, tc.name)
}
}
func TestDocker_ExtractLayerFiles(t *testing.T) {
de := Extractor{}
layerCh := make(chan layer)
errCh := make(chan error)
inputFilenames := []string{"var/foo", "etc/test/bar"}
f, _ := os.Open("testdata/opq2.tar")
defer f.Close()
go func() {
layerCh <- layer{
ID: "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b",
Content: f,
}
}()
filesInLayers := map[string]extractor.FileMap{}
opqInLayers := map[string]extractor.OPQDirs{}
err := de.extractLayerFiles(context.TODO(), layerCh, errCh, filesInLayers, opqInLayers, inputFilenames)
assert.NoError(t, err)
assert.Equal(t, map[string]extractor.FileMap{
"sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b": {
"etc/test/bar": {0x62, 0x61, 0x72, 0xa},
"var/.wh.foo": {},
},
}, filesInLayers)
assert.Equal(t, map[string]extractor.OPQDirs{
"sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b": {
"etc/test",
},
}, opqInLayers)
}
Reference in New Issue View Git Blame Copy Permalink