package docker

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"path"
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/klauspost/compress/zstd"

	"github.com/aquasecurity/fanal/extractor"
	"github.com/aquasecurity/fanal/types"
	"github.com/docker/docker/client"
	"github.com/genuinetools/reg/registry"
	"github.com/opencontainers/go-digest"
	bolt "github.com/simar7/gokv/bbolt"
	kvtypes "github.com/simar7/gokv/types"
	"github.com/stretchr/testify/assert"
)

// TODO: Use a memory based FS rather than actual fs
// context: https://github.com/aquasecurity/fanal/pull/51#discussion_r352337762
func setupCache() (*bolt.Store, *os.File, error) {
	f, err := ioutil.TempFile(".", "Bolt_TestStore-*")
	if err != nil {
		return nil, nil, err
	}

	s, err := bolt.NewStore(bolt.Options{
		Path: f.Name(),
	})
	return s, f, err
}

func TestExtractFromFile(t *testing.T) {
	vectors := []struct {
		file      string            // Test input file
		filenames []string          // Target files
		FileMap   extractor.FileMap // Expected output
		err       error             // Expected error to occur
	}{
		{
			file:      "testdata/image1.tar",
			filenames: []string{"var/foo", "etc/test/bar"},
			FileMap: extractor.FileMap{
				"etc/test/bar": []byte("bar\n"),
				"/config":      []byte(`{"architecture":"amd64","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh"],"ArgsEscaped":true,"Image":"sha256:e641703a6c77abde58a2e2d5e506da5ac61a648bdb17fba7c3325db9d2ba4ded","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"7dfcd2c8327651024825c14e0d8752544f59c03efeca291a71e532b7e0ca66bf","container_config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","rm /var/foo \u0026\u0026 rm -rf /etc/test \u0026\u0026 mkdir /etc/test \u0026\u0026 echo bar \u003e /etc/test/bar"],"ArgsEscaped":true,"Image":"sha256:e641703a6c77abde58a2e2d5e506da5ac61a648bdb17fba7c3325db9d2ba4ded","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2019-04-07T04:27:16.291049098Z","docker_version":"18.09.2","history":[{"created":"2019-03-07T22:19:46.661698137Z","created_by":"/bin/sh -c #(nop) ADD file:38bc6b51693b13d84a63e281403e2f6d0218c44b1d7ff12157c4523f9f0ebb1e in / "},{"created":"2019-03-07T22:19:46.815331171Z","created_by":"/bin/sh -c #(nop)  CMD [\"/bin/sh\"]","empty_layer":true},{"created":"2019-04-07T04:08:02.548475493Z","created_by":"/bin/sh -c mkdir /etc/test \u0026\u0026 touch /var/foo \u0026\u0026 touch /etc/test/test"},{"created":"2019-04-07T04:27:16.291049098Z","created_by":"/bin/sh -c rm /var/foo \u0026\u0026 rm -rf /etc/test \u0026\u0026 mkdir /etc/test \u0026\u0026 echo bar \u003e /etc/test/bar"}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:d9ff549177a94a413c425ffe14ae1cc0aa254bc9c7df781add08e7d2fba25d27","sha256:f75441026d68038ca80e92f342fb8f3c0f1faeec67b5a80c98f033a65beaef5a","sha256:a8b87ccf2f2f94b9e23308560800afa3f272aa6db5cc7d9b0119b6843889cff2"]}}`),
			},
			err: nil,
		},
		{
			file:      "testdata/image2.tar",
			filenames: []string{"home/app/Gemfile", "home/app2/Gemfile"},
			FileMap: extractor.FileMap{
				"home/app2/Gemfile": []byte("gem"),
				"/config":           []byte(`{"architecture":"amd64","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh"],"ArgsEscaped":true,"Image":"sha256:4fe3bbb628df60571f88cb053db9e2c9ec2f1c1e8373db9b026d0e582ef01d6d","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"7b1b7a0cfacbce82b51230bf0c6354e64cd0068e4e51180ab717890fc805bdf5","container_config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","mv /home/app /home/app2"],"ArgsEscaped":true,"Image":"sha256:4fe3bbb628df60571f88cb053db9e2c9ec2f1c1e8373db9b026d0e582ef01d6d","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2019-04-07T05:32:59.607884934Z","docker_version":"18.09.2","history":[{"created":"2019-03-07T22:19:46.661698137Z","created_by":"/bin/sh -c #(nop) ADD file:38bc6b51693b13d84a63e281403e2f6d0218c44b1d7ff12157c4523f9f0ebb1e in / "},{"created":"2019-03-07T22:19:46.815331171Z","created_by":"/bin/sh -c #(nop)  CMD [\"/bin/sh\"]","empty_layer":true},{"created":"2019-04-07T05:32:58.27180871Z","created_by":"/bin/sh -c mkdir /home/app \u0026\u0026 echo -n gem \u003e /home/app/Gemfile"},{"created":"2019-04-07T05:32:59.607884934Z","created_by":"/bin/sh -c mv /home/app /home/app2"}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:d9ff549177a94a413c425ffe14ae1cc0aa254bc9c7df781add08e7d2fba25d27","sha256:f9e7e541d5be4537a826c4c6cb68b603a8e552c22e28ac726e9be6b22f51af44","sha256:718fb3edf377530e3713cd074d141827d05f654f6389e827c344b7fcff153025"]}}`),
			},
			err: nil,
		},
		{
			file:      "testdata/image3.tar",
			filenames: []string{"home/app/Gemfile", "home/app2/Pipfile", "home/app/Pipfile"},
			FileMap: extractor.FileMap{
				"home/app/Pipfile": []byte("pip"),
				"/config":          []byte(`{"architecture":"amd64","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh"],"ArgsEscaped":true,"Image":"sha256:53dca1cadfa555151d28ac616df868eed4fc935f21af393118f4fbc36d9fb24a","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"42b6c68c1704e06fbffecfee6ae5400978cf508790d563e2bda4d1b20ce93c6d","container_config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","rm -rf /home/app \u0026\u0026 mv /home/app2 /home/app"],"ArgsEscaped":true,"Image":"sha256:53dca1cadfa555151d28ac616df868eed4fc935f21af393118f4fbc36d9fb24a","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2019-04-07T05:36:08.899764053Z","docker_version":"18.09.2","history":[{"created":"2019-03-07T22:19:46.661698137Z","created_by":"/bin/sh -c #(nop) ADD file:38bc6b51693b13d84a63e281403e2f6d0218c44b1d7ff12157c4523f9f0ebb1e in / "},{"created":"2019-03-07T22:19:46.815331171Z","created_by":"/bin/sh -c #(nop)  CMD [\"/bin/sh\"]","empty_layer":true},{"created":"2019-04-07T05:32:58.27180871Z","created_by":"/bin/sh -c mkdir /home/app \u0026\u0026 echo -n gem \u003e /home/app/Gemfile"},{"created":"2019-04-07T05:36:07.629894435Z","created_by":"/bin/sh -c mkdir /home/app2 \u0026\u0026 echo -n pip \u003e /home/app2/Pipfile"},{"created":"2019-04-07T05:36:08.899764053Z","created_by":"/bin/sh -c rm -rf /home/app \u0026\u0026 mv /home/app2 /home/app"}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:d9ff549177a94a413c425ffe14ae1cc0aa254bc9c7df781add08e7d2fba25d27","sha256:f9e7e541d5be4537a826c4c6cb68b603a8e552c22e28ac726e9be6b22f51af44","sha256:5a917ce45575a009bb5b4f462ed84522c7f642647b62a9f2b2bdfc2275f85104","sha256:50022087bbe2b08d1ce033122a56c7cf74cc1d1d6dae97a397226dd49a309c3b"]}}`),
			},
			err: nil,
		},
		{
			file:      "testdata/image3_gz_layers.tar",
			filenames: []string{"home/app/Gemfile", "home/app2/Pipfile", "home/app/Pipfile"},
			FileMap: extractor.FileMap{
				"home/app/Pipfile": []byte("pip"),
				"/config":          []byte(`{"architecture":"amd64","container":"a0625bb53d38b712d9fe7e307c53a5b1f2528189d694a29ba37b7a27bee20029","created":"2019-07-23T08:53:13.7506797Z","docker_version":"18.06.1-ce","history":[{"author":"","created":"2019-07-11T22:20:52.139709355Z","created_by":"/bin/sh -c #(nop) ADD file:0eb5ea35741d23fe39cbac245b3a5d84856ed6384f4ff07d496369ee6d960bad in / ","comment":""},{"author":"","created":"2019-07-11T22:20:52.375286404Z","created_by":"/bin/sh -c #(nop)  CMD [\"/bin/sh\"]","comment":"","empty_layer":true},{"author":"kaniko","created":"0001-01-01T00:00:00Z","created_by":"RUN mkdir /home/app \u0026\u0026 echo -n gem \u003e /home/app/Gemfile","comment":""},{"author":"kaniko","created":"0001-01-01T00:00:00Z","created_by":"RUN mkdir /home/app2 \u0026\u0026 echo -n pip \u003e /home/app2/Pipfile","comment":""},{"author":"kaniko","created":"0001-01-01T00:00:00Z","created_by":"RUN rm -rf /home/app \u0026\u0026 mv /home/app2 /home/app","comment":""}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:1bfeebd65323b8ddf5bd6a51cc7097b72788bc982e9ab3280d53d3c613adffa7","sha256:927e7087ecbc8de977df69cb60c19f1759b374bd36b2fa9c7c30cebe0fcf8156","sha256:25dbee606f1ef778e242145a8da895e416661dbd8b4243523aaa5227919458cc","sha256:ccbf319ba1f9c5f823254a36c91187db9ec2ff4cd1ec27dfc6b10bcf6996e334"]},"config":{"AttachStderr":false,"AttachStdin":false,"AttachStdout":false,"Cmd":["/bin/sh"],"Healthcheck":null,"Domainname":"","Entrypoint":null,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Hostname":"","Image":"sha256:f248cae16d3e1b44bf474ad89815438f10c395f8e532153e4fcd32cbbb150fb3","Labels":null,"OnBuild":null,"OpenStdin":false,"StdinOnce":false,"Tty":false,"User":"","Volumes":null,"WorkingDir":"","ExposedPorts":null,"ArgsEscaped":true,"NetworkDisabled":false,"MacAddress":"","StopSignal":"","Shell":null},"container_config":{"AttachStderr":false,"AttachStdin":false,"AttachStdout":false,"Cmd":["/bin/sh","-c","#(nop) ","CMD [\"/bin/sh\"]"],"Healthcheck":null,"Domainname":"","Entrypoint":null,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Hostname":"a0625bb53d38","Image":"sha256:f248cae16d3e1b44bf474ad89815438f10c395f8e532153e4fcd32cbbb150fb3","Labels":{},"OnBuild":null,"OpenStdin":false,"StdinOnce":false,"Tty":false,"User":"","Volumes":null,"WorkingDir":"","ExposedPorts":null,"ArgsEscaped":true,"NetworkDisabled":false,"MacAddress":"","StopSignal":"","Shell":null},"osversion":""}`),
			},
			err: nil,
		},
		{file: "testdata/image4.tar",
			filenames: []string{".abc", ".def", "foo/.abc", "foo/.def", ".foo/.abc"},
			FileMap: extractor.FileMap{
				".def":     []byte("def"),
				"foo/.abc": []byte("abc"),
				"/config":  []byte(`{"architecture":"amd64","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh"],"ArgsEscaped":true,"Image":"sha256:cabfb6dd9c622b8cd0efdc7bb38ed9a9d2001a32c2b5d5c174e284784df712e8","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"8290b131834ed7ef8c388a290594afeaa5daea024031a2551c8dedfc845fd09e","container_config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","rm -rf /.foo"],"ArgsEscaped":true,"Image":"sha256:cabfb6dd9c622b8cd0efdc7bb38ed9a9d2001a32c2b5d5c174e284784df712e8","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2019-04-07T05:48:16.088980845Z","docker_version":"18.09.2","history":[{"created":"2019-03-07T22:19:46.661698137Z","created_by":"/bin/sh -c #(nop) ADD file:38bc6b51693b13d84a63e281403e2f6d0218c44b1d7ff12157c4523f9f0ebb1e in / "},{"created":"2019-03-07T22:19:46.815331171Z","created_by":"/bin/sh -c #(nop)  CMD [\"/bin/sh\"]","empty_layer":true},{"created":"2019-04-07T05:48:10.560447082Z","created_by":"/bin/sh -c echo -n abc \u003e .abc \u0026\u0026 echo -n def \u003e .def"},{"created":"2019-04-07T05:48:11.938256528Z","created_by":"/bin/sh -c mkdir foo \u0026\u0026 echo -n abc \u003e /foo/.abc \u0026\u0026 echo -n def \u003e /foo/.def"},{"created":"2019-04-07T05:48:13.188275588Z","created_by":"/bin/sh -c rm .abc /foo/.def"},{"created":"2019-04-07T05:48:14.569944213Z","created_by":"/bin/sh -c mkdir .foo \u0026\u0026 echo -n abc /.foo/.abc"},{"created":"2019-04-07T05:48:16.088980845Z","created_by":"/bin/sh -c rm -rf /.foo"}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:d9ff549177a94a413c425ffe14ae1cc0aa254bc9c7df781add08e7d2fba25d27","sha256:c42355fdc6d1a90c39b26ae5ac44c85c079f6da260def6bcb781ffcfe45ce6c9","sha256:b16629f22093ce5dfec353149661886cc1ca0c62ff30c450a82eba693eaedbd2","sha256:9717a79724f7114e32f004067a9cf96493812b2772f8a88096d1c43f7898d4f9","sha256:87c73b7beca2340705c988bb35235c66ae16b2ed2a6ce5b37b215f9bb08e7dc9","sha256:99cc8353ab2a712793601465751b9f518a35763db138e8b92b54f13e0c82d8b6"]}}`),
			},
			err: nil,
		},
		{
			file: "testdata/image5.tar",
			// Not detect foo/baz cause set "foo"
			filenames: []string{"bar", "foo/bar/", "foo"},
			FileMap: extractor.FileMap{
				"bar":         []byte("bar"),
				"foo/bar/abc": []byte("abc"),
				"foo/bar/def": []byte("def"),
				"/config":     []byte(`{"architecture":"amd64","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh"],"ArgsEscaped":true,"Image":"sha256:961769676411f082461f9ef46626dd7a2d1e2b2a38e6a44364bcbecf51e66dd4","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"65b0a9f4cc5ba8eaad4edf5e8edd9166aa8dc31b2d6e21d84951b9737c250078","container_config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","echo -n bar \u003e bar \u0026\u0026 mkdir -p foo/bar \u0026\u0026 echo -n abc \u003e foo/bar/abc \u0026\u0026 echo -n def \u003e foo/bar/def \u0026\u0026 echo -n baz \u003e foo/baz"],"Image":"sha256:961769676411f082461f9ef46626dd7a2d1e2b2a38e6a44364bcbecf51e66dd4","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2019-09-05T17:19:53.569209Z","docker_version":"19.03.1","history":[{"created":"2019-08-20T20:19:55.062606894Z","created_by":"/bin/sh -c #(nop) ADD file:fe64057fbb83dccb960efabbf1cd8777920ef279a7fa8dbca0a8801c651bdf7c in / "},{"created":"2019-08-20T20:19:55.211423266Z","created_by":"/bin/sh -c #(nop)  CMD [\"/bin/sh\"]","empty_layer":true},{"created":"2019-09-05T17:19:53.569209Z","created_by":"/bin/sh -c echo -n bar \u003e bar \u0026\u0026 mkdir -p foo/bar \u0026\u0026 echo -n abc \u003e foo/bar/abc \u0026\u0026 echo -n def \u003e foo/bar/def \u0026\u0026 echo -n baz \u003e foo/baz"}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:03901b4a2ea88eeaad62dbe59b072b28b6efa00491962b8741081c5df50c65e0","sha256:69c7f4ae201dc4669b58cbac8f1cc0593e28ddb0f5d35a21541217ab17f550fa"]}}`),
			},
			err: nil,
		},
		{
			file: "testdata/image5.tar",
			// Detect foo/baz cause set "foo/"
			filenames: []string{"bar", "foo/"},
			FileMap: extractor.FileMap{
				"bar":     []byte("bar"),
				"foo/baz": []byte("baz"),
				"/config": []byte(`{"architecture":"amd64","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh"],"ArgsEscaped":true,"Image":"sha256:961769676411f082461f9ef46626dd7a2d1e2b2a38e6a44364bcbecf51e66dd4","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"65b0a9f4cc5ba8eaad4edf5e8edd9166aa8dc31b2d6e21d84951b9737c250078","container_config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","echo -n bar \u003e bar \u0026\u0026 mkdir -p foo/bar \u0026\u0026 echo -n abc \u003e foo/bar/abc \u0026\u0026 echo -n def \u003e foo/bar/def \u0026\u0026 echo -n baz \u003e foo/baz"],"Image":"sha256:961769676411f082461f9ef46626dd7a2d1e2b2a38e6a44364bcbecf51e66dd4","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2019-09-05T17:19:53.569209Z","docker_version":"19.03.1","history":[{"created":"2019-08-20T20:19:55.062606894Z","created_by":"/bin/sh -c #(nop) ADD file:fe64057fbb83dccb960efabbf1cd8777920ef279a7fa8dbca0a8801c651bdf7c in / "},{"created":"2019-08-20T20:19:55.211423266Z","created_by":"/bin/sh -c #(nop)  CMD [\"/bin/sh\"]","empty_layer":true},{"created":"2019-09-05T17:19:53.569209Z","created_by":"/bin/sh -c echo -n bar \u003e bar \u0026\u0026 mkdir -p foo/bar \u0026\u0026 echo -n abc \u003e foo/bar/abc \u0026\u0026 echo -n def \u003e foo/bar/def \u0026\u0026 echo -n baz \u003e foo/baz"}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:03901b4a2ea88eeaad62dbe59b072b28b6efa00491962b8741081c5df50c65e0","sha256:69c7f4ae201dc4669b58cbac8f1cc0593e28ddb0f5d35a21541217ab17f550fa"]}}`),
			},
			err: nil,
		},
		{
			file: "testdata/image6.tar",
			// Not detect package-lock.json and composer.lock under vendor/ or node_modules/"
			filenames: []string{"foo", "package-lock.json", "composer.lock"},
			FileMap: extractor.FileMap{
				"foo":     []byte("foo\n"),
				"/config": []byte(`{"architecture":"amd64","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt"],"Cmd":null,"Image":"sha256:7b421e99fb75da8466c90ec04a1c43a3f8b9fd9be0bb71a341d30ca11c75e9e0","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container_config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt"],"Cmd":["/bin/sh","-c","#(nop) COPY file:745080737b5cb8cd47dbca9a8819cc8e9034b7748ccb0ed1c60bdb0b5fc6c2e8 in /app/vendor/ "],"Image":"sha256:7b421e99fb75da8466c90ec04a1c43a3f8b9fd9be0bb71a341d30ca11c75e9e0","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2019-10-30T09:59:16.9071124Z","docker_version":"19.03.4","history":[{"created":"1970-01-01T00:00:00Z","author":"Bazel","created_by":"bazel build ..."},{"created":"1970-01-01T00:00:00Z","author":"Bazel","created_by":"bazel build ..."},{"created":"2019-10-30T09:59:16.1671833Z","created_by":"/bin/sh -c #(nop) COPY file:8d7ea209a266ec183c53e0de5dad09aa6ccd217961306f174884d94eb92369ab in /foo "},{"created":"2019-10-30T09:59:16.5096275Z","created_by":"/bin/sh -c #(nop) COPY file:4772d4fa23206c27b7ddbac967e3d2e3f6b08f6dde8ba8170975efc4b6041255 in /app/node_modules/ "},{"created":"2019-10-30T09:59:16.9071124Z","created_by":"/bin/sh -c #(nop) COPY file:745080737b5cb8cd47dbca9a8819cc8e9034b7748ccb0ed1c60bdb0b5fc6c2e8 in /app/vendor/ "}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:932da51564135c98a49a34a193d6cd363d8fa4184d957fde16c9d8527b3f3b02","sha256:dffd9992ca398466a663c87c92cfea2a2db0ae0cf33fcb99da60eec52addbfc5","sha256:ca9852efab2ff2b5031a02f69021c6d4c49d351480f9cd7b08beec00d3527d1a","sha256:88d8eebab45ec2cb73ad2859a158617324eadfb904a8fe24b936b07ccd15096d","sha256:ef55f7ceb56d4ccbfb9228ab865f3e411c9a9781814fe4558003a2b2427f457d"]}}`),
			},
			err: nil,
		},
	}

	for _, v := range vectors {
		t.Run(path.Base(v.file), func(t *testing.T) {
			f, err := os.Open(v.file)
			if err != nil {
				t.Fatalf("Open() error: %v", err)
			}
			defer f.Close()

			d := Extractor{}
			fm, err := d.ExtractFromFile(context.TODO(), f, v.filenames)
			if v.err != err {
				t.Errorf("err: got %v, want %v", v.err, err)
			}
			if !reflect.DeepEqual(fm, v.FileMap) {
				t.Errorf("FilesMap: got %v, want %v", fm, v.FileMap)
			}
		})
	}
}

func TestExtractFiles(t *testing.T) {
	vectors := []struct {
		file      string            // Test input file
		filenames []string          // Target files
		FileMap   extractor.FileMap // Expected output
		opqDirs   extractor.OPQDirs // Expected output
		err       error             // Expected error to occur
	}{
		{
			file:      "testdata/normal.tar",
			filenames: []string{"var/foo"},
			FileMap:   extractor.FileMap{"var/foo": []byte{}},
			opqDirs:   []string{},
			err:       nil,
		},
		{
			file:      "testdata/opq.tar",
			filenames: []string{"var/foo"},
			FileMap: extractor.FileMap{
				"var/.wh.foo": []byte{},
			},
			opqDirs: []string{"etc/test"},
			err:     nil,
		},
		{
			file:      "testdata/opq2.tar",
			filenames: []string{"var/foo", "etc/test/bar"},
			FileMap: extractor.FileMap{
				"etc/test/bar": []byte("bar\n"),
				"var/.wh.foo":  []byte{},
			},
			opqDirs: []string{"etc/test"},
			err:     nil,
		},
	}

	for _, v := range vectors {
		t.Run(path.Base(v.file), func(t *testing.T) {
			f, err := os.Open(v.file)
			if err != nil {
				t.Fatalf("Open() error: %v", err)
			}
			defer f.Close()

			d := Extractor{}
			fm, opqDirs, err := d.ExtractFiles(f, v.filenames)
			if v.err != err {
				t.Errorf("err: got %v, want %v", v.err, err)
			}
			if !reflect.DeepEqual(opqDirs, v.opqDirs) {
				t.Errorf("OPQDirs: got %v, want %v", opqDirs, v.opqDirs)
			}
			if !reflect.DeepEqual(fm, v.FileMap) {
				t.Errorf("FilesMap: got %v, want %v", fm, v.FileMap)
			}
		})
	}
}

func TestDockerExtractor_SaveLocalImage(t *testing.T) {
	testCases := []struct {
		name              string
		expectedImageData string
		cacheHit          bool
	}{
		{
			name:              "happy path with cache miss",
			expectedImageData: "foofromdocker",
		},
		{
			name:              "happy path with cache hit",
			cacheHit:          true,
			expectedImageData: "foofromcache",
		},
	}

	for _, tc := range testCases {
		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			httpPath := r.URL.String()
			switch {
			case strings.Contains(httpPath, "images/get?names=fooimage"):
				_, _ = fmt.Fprint(w, "foofromdocker")
			default:
				assert.FailNow(t, "unexpected path accessed: ", r.URL.String())
			}
		}))
		defer ts.Close()

		c, err := client.NewClientWithOpts(client.WithHost(ts.URL))
		assert.NoError(t, err)

		// setup cache
		s, f, err := setupCache()
		defer func() {
			_ = f.Close()
			_ = os.RemoveAll(f.Name())
		}()
		assert.NoError(t, err)

		if tc.cacheHit {
			e, _ := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedDefault))
			dst := e.EncodeAll([]byte("foofromcache"), nil)
			_ = s.Set(kvtypes.SetItemInput{
				BucketName: "imagebucket",
				Key:        "fooimage",
				Value:      dst,
			})
		}

		de := Extractor{
			Option: types.DockerOption{},
			Client: c,
			Cache:  s,
		}

		r, err := de.SaveLocalImage(context.TODO(), "fooimage")
		actualSavedTarBytes, _ := ioutil.ReadAll(r)
		assert.Equal(t, []byte(tc.expectedImageData), actualSavedTarBytes[:], tc.name)
		assert.NoError(t, err, tc.name)

		// check the cache for what was stored
		var actualValue []byte
		found, err := de.Cache.Get(kvtypes.GetItemInput{
			BucketName: "imagebucket",
			Key:        "fooimage",
			Value:      &actualValue,
		})

		assert.NoError(t, err, tc.name)
		assert.True(t, found, tc.name)

		dec, _ := zstd.NewReader(nil)
		actualStoredValue, _ := dec.DecodeAll(actualValue, nil)
		assert.Equal(t, tc.expectedImageData, string(actualStoredValue), tc.name)
	}
}

func TestDockerExtractor_Extract(t *testing.T) {
	testCases := []struct {
		name            string
		imageName       string
		manifestResp    string
		fileName        string
		blobData        string
		fileToExtract   []string
		expectedFileMap extractor.FileMap
		expectedError   string
	}{
		{
			name: "happy path",
			manifestResp: `{
		 "schemaVersion": 2,
		 "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
		 "layers": [
		    {
		       "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
		       "size": 153263,
		       "digest": "sha256:shafortestdirslashhelloworlddottxt"
		    },
		    {
		       "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
		       "size": 153263,
		       "digest": "sha256:shafortestdirslashbadworlddottxt"
		    }
		 ]
		}`,
			fileName:      "testdata/testdir.tar.gz", // includes helloworld.txt and badworld.txt
			blobData:      "foo",
			fileToExtract: []string{"testdir/helloworld.txt", "testdir/badworld.txt"},
			expectedFileMap: extractor.FileMap{
				"/config":                []uint8{0x66, 0x6f, 0x6f},
				"testdir/helloworld.txt": []uint8{0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0xa},
				"testdir/badworld.txt":   []uint8{0x62, 0x61, 0x64, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0xa},
			},
		},
		{
			name:          "sad path: invalid manifest response",
			manifestResp:  "badManifestResponse",
			expectedError: "failed to get the v2 manifest: invalid character 'b' looking for beginning of value",
		},
		{
			name:          "sad path: bad image name",
			imageName:     "https://docker/very/bad/imagename",
			expectedError: `failed to parse the image: parsing image "https://docker/very/bad/imagename" failed: invalid reference format`,
		},
		{
			name: "sad path: corrupt layer data invalid gzip",
			manifestResp: `{
		"schemaVersion": 2,
		"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
		"layers": [
		 {
		    "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
		    "size": 153263,
		    "digest": "sha256:shaforinvalidgzipfile"
		 }
		]
		}`,
			fileName:        "testdata/opq.tar",
			blobData:        "foo",
			expectedFileMap: extractor.FileMap(nil),
			expectedError:   "could not init gzip reader: gzip: invalid header",
		},
	}

	for _, tc := range testCases {
		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			httpPath := r.URL.String()
			switch {
			case strings.Contains(httpPath, "/v2/library/fooimage/manifests/latest"):
				w.Header().Set("Content-Type", "application/vnd.docker.distribution.manifest.v2+json")
				_, _ = fmt.Fprint(w, tc.manifestResp)
			case strings.Contains(httpPath, "/v2/library/fooimage/blobs/sha256:shafortestdirslashhelloworlddottxt"):
				b, _ := ioutil.ReadFile("testdata/helloworld.tar.gz")
				_, _ = w.Write(b)
			case strings.Contains(httpPath, "/v2/library/fooimage/blobs/sha256:shafortestdirslashbadworlddottxt"):
				b, _ := ioutil.ReadFile("testdata/badworld.tar.gz")
				_, _ = w.Write(b)
			case strings.Contains(httpPath, "/v2/library/fooimage/blobs/sha256:shaforinvalidgzipfile"):
				b, _ := ioutil.ReadFile("testdata/opq.tar")
				_, _ = w.Write(b)
			case strings.Contains(httpPath, "/v2/library/fooimage/blobs/"):
				_, _ = w.Write([]byte(tc.blobData))
			default:
				assert.FailNow(t, "unexpected path accessed: ", r.URL.String())
			}
		}))
		defer ts.Close()

		c, err := client.NewClientWithOpts(client.WithHost(ts.URL))
		assert.NoError(t, err)

		// setup cache
		s, f, err := setupCache()
		defer func() {
			_ = f.Close()
			_ = os.RemoveAll(f.Name())
		}()
		assert.NoError(t, err)

		de := Extractor{
			Option: types.DockerOption{
				AuthURL:  ts.URL,
				NonSSL:   true,
				SkipPing: true,
				Timeout:  time.Second * 1000,
			},
			Client: c,
			Cache:  s,
		}

		tsURL := strings.TrimPrefix(ts.URL, "http://")

		var imageName string
		switch {
		case tc.imageName != "":
			imageName = tc.imageName
		default:
			imageName = tsURL + "/library/fooimage"
		}
		fm, err := de.Extract(context.TODO(), imageName, tc.fileToExtract)

		switch {
		case tc.expectedError != "":
			assert.Equal(t, tc.expectedError, err.Error(), tc.name)
		default:
			assert.NoError(t, err, tc.name)
		}
		assert.Equal(t, tc.expectedFileMap, fm, tc.name)
	}
}

func TestDocker_ExtractLayerWorker(t *testing.T) {
	goodtarzstdgolden, _ := ioutil.ReadFile("testdata/testdir.tar.zstd")
	goodReturnedTarContent, _ := ioutil.ReadFile("testdata/goodTarContent.golden")

	testCases := []struct {
		name                       string
		cacheHit                   bool
		garbageCache               bool
		requiredFiles              []string
		expectedCacheContents      []byte
		expectedReturnedTarContent []byte
	}{
		{
			name:                       "happy path with cache miss and write back",
			cacheHit:                   false,
			requiredFiles:              []string{"testdir/helloworld.txt", "testdir/badworld.txt"},
			expectedCacheContents:      goodtarzstdgolden,
			expectedReturnedTarContent: goodReturnedTarContent,
		},
		{
			name:                       "happy path with cache hit with garbage cache and write back",
			cacheHit:                   true,
			garbageCache:               true,
			requiredFiles:              []string{"testdir/helloworld.txt", "testdir/badworld.txt"},
			expectedCacheContents:      goodtarzstdgolden,
			expectedReturnedTarContent: goodReturnedTarContent,
		},
		{
			name:                       "happy path with cache hit",
			cacheHit:                   true,
			expectedCacheContents:      goodtarzstdgolden,
			expectedReturnedTarContent: goodReturnedTarContent,
		},
		{
			name:                       "happy path with cache miss but no write back",
			cacheHit:                   false,
			expectedCacheContents:      []byte{0x28, 0xb5, 0x2f, 0xfd, 0x4, 0x60, 0x1, 0x0, 0x0, 0x99, 0xe9, 0xd8, 0x51}, // just the empty tar header
			expectedReturnedTarContent: []byte{},
		},
	}

	for _, tc := range testCases {
		inputDigest := digest.Digest("sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b")

		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			httpPath := r.URL.String()
			switch {
			case strings.Contains(httpPath, "/v2/library/fooimage/blobs/sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b"):
				layerData, _ := ioutil.ReadFile("testdata/testdir.tar.gz")
				_, _ = w.Write(layerData)
			default:
				assert.FailNow(t, "unexpected path accessed: ", fmt.Sprintf("%s %s", r.URL.String(), tc.name))
			}
		}))
		defer ts.Close()

		c, err := client.NewClientWithOpts(client.WithHost(ts.URL))
		assert.NoError(t, err)

		// setup cache
		s, f, err := setupCache()
		defer func() {
			_ = f.Close()
			_ = os.RemoveAll(f.Name())
		}()
		assert.NoError(t, err, tc.name)

		if tc.cacheHit {
			switch tc.garbageCache {
			case true:
				garbage, _ := ioutil.ReadFile("testdata/invalidgzvalidtar.tar.gz")
				assert.NoError(t, s.Set(kvtypes.SetItemInput{
					BucketName: LayerTarsBucket,
					Key:        string(inputDigest),
					Value:      garbage,
				}), tc.name)
			default:
				assert.NoError(t, s.Set(kvtypes.SetItemInput{
					BucketName: LayerTarsBucket,
					Key:        string(inputDigest),
					Value:      goodtarzstdgolden,
				}), tc.name)
			}
		}

		de := Extractor{
			Option: types.DockerOption{
				AuthURL:  ts.URL,
				NonSSL:   true,
				SkipPing: true,
				Timeout:  time.Second * 1000,
			},
			Client: c,
			Cache:  s,
		}

		tsUrl := strings.TrimPrefix(ts.URL, "http://")
		inputImage := registry.Image{
			Domain: tsUrl,
			Path:   "library/fooimage",
			Tag:    "latest",
		}

		layerCh := make(chan layer)
		errCh := make(chan error)
		r, err := de.createRegistryClient(context.TODO(), inputImage.Domain)
		go func() {
			de.extractLayerWorker(inputDigest, r, context.TODO(), inputImage, errCh, layerCh, tc.requiredFiles)
		}()

		var errRecieved error
		var layerReceived layer

		select {
		case errRecieved = <-errCh:
			assert.FailNow(t, "unexpected error received, err: ", fmt.Sprintf("%s, %s", errRecieved, tc.name))
		case layerReceived = <-layerCh:
			assert.Equal(t, inputDigest, layerReceived.ID, tc.name)
			got, _ := ioutil.ReadAll(layerReceived.Content)
			assert.Equal(t, tc.expectedReturnedTarContent, got, tc.name)
		}

		// check cache contents
		var actualCacheContents []byte
		found, err := s.Get(kvtypes.GetItemInput{
			BucketName: LayerTarsBucket,
			Key:        string(inputDigest),
			Value:      &actualCacheContents,
		})

		assert.True(t, found, tc.name)
		assert.NoError(t, err, tc.name)
		assert.Equal(t, tc.expectedCacheContents, actualCacheContents, tc.name)
	}
}

func TestDocker_ExtractLayerFiles(t *testing.T) {
	de := Extractor{}

	layerCh := make(chan layer)
	errCh := make(chan error)
	inputFilenames := []string{"var/foo", "etc/test/bar"}

	f, _ := os.Open("testdata/opq2.tar")
	defer f.Close()

	go func() {
		layerCh <- layer{
			ID:      "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b",
			Content: f,
		}
	}()

	filesInLayers := map[string]extractor.FileMap{}
	opqInLayers := map[string]extractor.OPQDirs{}
	err := de.extractLayerFiles(context.TODO(), layerCh, errCh, filesInLayers, opqInLayers, inputFilenames)
	assert.NoError(t, err)
	assert.Equal(t, map[string]extractor.FileMap{
		"sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b": {
			"etc/test/bar": {0x62, 0x61, 0x72, 0xa},
			"var/.wh.foo":  {},
		},
	}, filesInLayers)
	assert.Equal(t, map[string]extractor.OPQDirs{
		"sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b": {
			"etc/test",
		},
	}, opqInLayers)
}