chore(ci): move integration tests to GitHub Actions (#485)

* fix(standalone): add defer to close databases

* test(client/server): launch a server only once

* test(docker_engine): remove the duplicated case

* test(docker_engine): copy a database only once

* test(standalone): copy a database only once

* test(server): fix tests according to updated mock

* chore(mod): update

* chore(ci): add integration tests to GitHub Actions

* chore(ci): bump up Go to 1.14

* chore(ci): remove integration tests from CircleCI

* chore(ci): add name

* chore(ci): add new lines

.circleci/config.yml

@@ -24,18 +24,6 @@ jobs:
       - run:
           name: Test
           command: make test
-  integration-test:
-    <<: *defaults
-    parameters:
-      docker_version:
-        type: string
-    steps:
-      - checkout
-      - setup_remote_docker:
-          version: << parameters.docker_version >>
-      - run:
-          name: Integration Test
-          command: make test-integration
   release:
     <<: *defaults
     steps:
@@ -71,8 +59,6 @@ workflows:
   release:
     jobs:
       - unit-test
-      - integration-test:
-            docker_version: 18.09.3
       - release:
           filters:
             branches:

.github/workflows/test.yaml

@@ -1,15 +1,34 @@
 name: Test
 on: pull_request
 jobs:
+  integration:
+    name: Integration Test
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.14.x
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - name: Run integration tests
+      run: make test-integration
+
   build-test:
+    name: Build Test
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
       uses: actions/checkout@v2
-    - name: Set up go
+
+    - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: 1.13.x
+        go-version: 1.14.x
+
     - name: Run GoReleaser
       uses: goreleaser/goreleaser-action@v1
       with:

go.mod

@@ -3,7 +3,7 @@ module github.com/aquasecurity/trivy
 go 1.13
 
 require (
-	github.com/aquasecurity/fanal v0.0.0-20200427221647-c3528846e21c
+	github.com/aquasecurity/fanal v0.0.0-20200504143803-30a561989059
 	github.com/aquasecurity/go-dep-parser v0.0.0-20190819075924-ea223f0ef24b
 	github.com/aquasecurity/trivy-db v0.0.0-20200430091154-7c0a6e1ad398
 	github.com/caarlos0/env/v6 v6.0.0

go.sum

@@ -42,8 +42,8 @@ github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdc
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/aquasecurity/fanal v0.0.0-20200427221647-c3528846e21c h1:Rg4yt5YiL2SfOx2sbJjn3Y3jgYxOSJ+XXj7ogp+FeWk=
-github.com/aquasecurity/fanal v0.0.0-20200427221647-c3528846e21c/go.mod h1:3H3F3x2XtcdFH3o1LQJEzfu2sS/rf+XufPIngMZrKO4=
+github.com/aquasecurity/fanal v0.0.0-20200504143803-30a561989059 h1:FLQkluzBXeQvyNAMNtFpvd0qMbxLeYVdP6B/Pxx/d54=
+github.com/aquasecurity/fanal v0.0.0-20200504143803-30a561989059/go.mod h1:3H3F3x2XtcdFH3o1LQJEzfu2sS/rf+XufPIngMZrKO4=
 github.com/aquasecurity/go-dep-parser v0.0.0-20190819075924-ea223f0ef24b h1:55Ulc/gvfWm4ylhVaR7MxOwujRjA6et7KhmUbSgUFf4=
 github.com/aquasecurity/go-dep-parser v0.0.0-20190819075924-ea223f0ef24b/go.mod h1:BpNTD9vHfrejKsED9rx04ldM1WIbeyXGYxUrqTVwxVQ=
 github.com/aquasecurity/testdocker v0.0.0-20200426142840-5f05bce6f12a h1:hsw7PpiymXP64evn/K7gsj3hWzMqLrdoeE6JkqDocVg=

integration/client_server_test.go

@@ -12,10 +12,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/aquasecurity/trivy/internal"
 	"github.com/stretchr/testify/assert"
-
 	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli"
+
+	"github.com/aquasecurity/trivy/internal"
 )
 
 type args struct {
@@ -28,8 +29,6 @@ type args struct {
 	Input             string
 	ClientToken       string
 	ClientTokenHeader string
-	ServerToken       string
-	ServerTokenHeader string
 }
 
 func TestClientServer(t *testing.T) {
@@ -47,18 +46,6 @@ func TestClientServer(t *testing.T) {
 			},
 			golden: "testdata/alpine-310.json.golden",
 		},
-		{
-			name: "alpine 3.10 integration with token",
-			testArgs: args{
-				Version:           "dev",
-				Input:             "testdata/fixtures/alpine-310.tar.gz",
-				ClientToken:       "token",
-				ClientTokenHeader: "Trivy-Token",
-				ServerToken:       "token",
-				ServerTokenHeader: "Trivy-Token",
-			},
-			golden: "testdata/alpine-310.json.golden",
-		},
 		{
 			name: "alpine 3.10 integration with --ignore-unfixed option",
 			testArgs: args{
@@ -312,6 +299,41 @@ func TestClientServer(t *testing.T) {
 			},
 			golden: "testdata/busybox-with-lockfile.json.golden",
 		},
+	}
+
+	app, addr, cacheDir := setup(t, "", "")
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			osArgs, outputFile, cleanup := setupClient(t, c.testArgs, addr, cacheDir, c.golden)
+			defer cleanup()
+
+			// Run Trivy client
+			err := app.Run(osArgs)
+			require.NoError(t, err)
+
+			compare(t, c.golden, outputFile)
+		})
+	}
+}
+
+func TestClientServerWithToken(t *testing.T) {
+	cases := []struct {
+		name     string
+		testArgs args
+		golden   string
+		wantErr  string
+	}{
+		{
+			name: "alpine 3.10 integration with token",
+			testArgs: args{
+				Version:           "dev",
+				Input:             "testdata/fixtures/alpine-310.tar.gz",
+				ClientToken:       "token",
+				ClientTokenHeader: "Trivy-Token",
+			},
+			golden: "testdata/alpine-310.json.golden",
+		},
 		{
 			name: "invalid token",
 			testArgs: args{
@@ -319,8 +341,6 @@ func TestClientServer(t *testing.T) {
 				Input:             "testdata/fixtures/distroless-base.tar.gz",
 				ClientToken:       "invalidtoken",
 				ClientTokenHeader: "Trivy-Token",
-				ServerToken:       "token",
-				ServerTokenHeader: "Trivy-Token",
 			},
 			wantErr: "twirp error unauthenticated: invalid token",
 		},
@@ -331,46 +351,23 @@ func TestClientServer(t *testing.T) {
 				Input:             "testdata/fixtures/distroless-base.tar.gz",
 				ClientToken:       "valid-token",
 				ClientTokenHeader: "Trivy-Token",
-				ServerToken:       "valid-token",
-				ServerTokenHeader: "Invalid",
 			},
 			wantErr: "twirp error unauthenticated: invalid token",
 		},
 	}
 
-	for _, c := range cases {
-		t.Run(c.name, func(t *testing.T) {
-			// Copy DB file
-			cacheDir, err := gunzipDB()
-			require.NoError(t, err)
+	serverToken := "token"
+	serverTokenHeader := "Trivy-Token"
+	app, addr, cacheDir := setup(t, serverToken, serverTokenHeader)
 	defer os.RemoveAll(cacheDir)
 
-			port, err := getFreePort()
-			require.NoError(t, err, c.name)
-			addr := fmt.Sprintf("localhost:%d", port)
-
-			go func() {
-				// Setup CLI App
-				app := internal.NewApp(c.testArgs.Version)
-				app.Writer = ioutil.Discard
-				osArgs := setupServer(addr, c.testArgs.ServerToken, c.testArgs.ServerTokenHeader, cacheDir)
-
-				// Run Trivy server
-				require.NoError(t, app.Run(osArgs), c.name)
-			}()
-
-			ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
-			require.NoError(t, waitPort(ctx, addr), c.name)
-
-			// Setup CLI App
-			app := internal.NewApp(c.testArgs.Version)
-			app.Writer = ioutil.Discard
-
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
 			osArgs, outputFile, cleanup := setupClient(t, c.testArgs, addr, cacheDir, c.golden)
 			defer cleanup()
 
 			// Run Trivy client
-			err = app.Run(osArgs)
+			err := app.Run(osArgs)
 
 			if c.wantErr != "" {
 				require.NotNil(t, err, c.name)
@@ -380,17 +377,44 @@ func TestClientServer(t *testing.T) {
 				assert.NoError(t, err, c.name)
 			}
 
-			// Compare want and got
-			want, err := ioutil.ReadFile(c.golden)
-			assert.NoError(t, err)
-			got, err := ioutil.ReadFile(outputFile)
-			assert.NoError(t, err)
-
-			assert.JSONEq(t, string(want), string(got))
+			compare(t, c.golden, outputFile)
 		})
 	}
 }
 
+func setup(t *testing.T, token, tokenHeader string) (*cli.App, string, string) {
+	t.Helper()
+	version := "dev"
+
+	// Copy DB file
+	cacheDir, err := gunzipDB()
+	assert.NoError(t, err)
+
+	port, err := getFreePort()
+	assert.NoError(t, err)
+	addr := fmt.Sprintf("localhost:%d", port)
+
+	go func() {
+		// Setup CLI App
+		app := internal.NewApp(version)
+		app.Writer = ioutil.Discard
+		osArgs := setupServer(addr, token, tokenHeader, cacheDir)
+
+		// Run Trivy server
+		app.Run(osArgs)
+	}()
+
+	ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
+	err = waitPort(ctx, addr)
+	assert.NoError(t, err)
+
+	// Setup CLI App
+	app := internal.NewApp(version)
+	app.Writer = ioutil.Discard
+
+	return app, addr, cacheDir
+}
+
 func setupServer(addr, token, tokenHeader, cacheDir string) []string {
 	osArgs := []string{"trivy", "server", "--skip-update", "--cache-dir", cacheDir, "--listen", addr}
 	if token != "" {
@@ -458,3 +482,14 @@ func setupClient(t *testing.T, c args, addr string, cacheDir string, golden stri
 	osArgs = append(osArgs, []string{"--output", outputFile}...)
 	return osArgs, outputFile, cleanup
 }
+
+func compare(t *testing.T, wantFile, gotFile string) {
+	t.Helper()
+	// Compare want and got
+	want, err := ioutil.ReadFile(wantFile)
+	assert.NoError(t, err)
+	got, err := ioutil.ReadFile(gotFile)
+	assert.NoError(t, err)
+
+	assert.JSONEq(t, string(want), string(got))
+}

integration/docker_engine_test.go

@@ -82,12 +82,6 @@ func TestRun_WithDockerEngine(t *testing.T) {
 			expectedOutputFile: "testdata/centos-6.json.golden",
 			testfile:           "testdata/fixtures/centos-6.tar.gz",
 		},
-		{
-			name:               "happy path, valid image path, centos:6",
-			imageTag:           "centos:6",
-			expectedOutputFile: "testdata/centos-6.json.golden",
-			testfile:           "testdata/fixtures/centos-6.tar.gz",
-		},
 		{
 			name:               "happy path, valid image path, centos:7",
 			imageTag:           "centos:7",
@@ -240,8 +234,6 @@ func TestRun_WithDockerEngine(t *testing.T) {
 		},
 	}
 
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
 	// Copy DB file
 	cacheDir, err := gunzipDB()
 	require.NoError(t, err)
@@ -251,8 +243,10 @@ func TestRun_WithDockerEngine(t *testing.T) {
 	defer ctx.Done()
 
 	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
-			require.NoError(t, err, tc.name)
+	require.NoError(t, err)
 
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
 			if !tc.invalidImage {
 				testfile, err := os.Open(tc.testfile)
 				require.NoError(t, err, tc.name)
@@ -279,7 +273,7 @@ func TestRun_WithDockerEngine(t *testing.T) {
 
 			// run trivy
 			app := internal.NewApp("dev")
-			trivyArgs := []string{"trivy", "--skip-update", "--cache-dir", cacheDir, "--format=json"}
+			trivyArgs := []string{"trivy", "--skip-update", "--cache-dir", cacheDir, "--format=json", "--output", of.Name()}
 			if tc.ignoreUnfixed {
 				trivyArgs = append(trivyArgs, "--ignore-unfixed")
 			}
@@ -294,9 +288,6 @@ func TestRun_WithDockerEngine(t *testing.T) {
 				assert.NoError(t, err, "failed to write .trivyignore")
 				defer os.Remove(trivyIgnore)
 			}
-			if !tc.invalidImage {
-				trivyArgs = append(trivyArgs, "--output", of.Name())
-			}
 			trivyArgs = append(trivyArgs, tc.testfile)
 
 			err = app.Run(trivyArgs)
@@ -304,11 +295,11 @@ func TestRun_WithDockerEngine(t *testing.T) {
 			case tc.expectedError != "":
 				require.NotNil(t, err)
 				assert.Contains(t, err.Error(), tc.expectedError, tc.name)
+				return
 			default:
 				assert.NoError(t, err, tc.name)
 			}
 
-			if !tc.invalidImage {
 			// check for vulnerability output info
 			got, err := ioutil.ReadAll(of)
 			assert.NoError(t, err, tc.name)
@@ -326,7 +317,6 @@ func TestRun_WithDockerEngine(t *testing.T) {
 				PruneChildren: true,
 			})
 			assert.NoError(t, err, tc.name)
-			}
 		})
 	}
 }

integration/standalone_test.go

@@ -8,11 +8,10 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/aquasecurity/trivy/internal"
-
-	"github.com/stretchr/testify/assert"
 )
 
 func TestRun_WithTar(t *testing.T) {
@@ -343,17 +342,18 @@ func TestRun_WithTar(t *testing.T) {
 		},
 	}
 
-	for _, c := range cases {
-		t.Run(c.name, func(t *testing.T) {
 	// Copy DB file
 	cacheDir, err := gunzipDB()
 	require.NoError(t, err)
 	defer os.RemoveAll(cacheDir)
 
 	// Setup CLI App
-			app := internal.NewApp(c.testArgs.Version)
+	app := internal.NewApp("dev")
 	app.Writer = ioutil.Discard
 
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+
 			osArgs := []string{"trivy", "--cache-dir", cacheDir, "--format", c.testArgs.Format}
 			if c.testArgs.SkipUpdate {
 				osArgs = append(osArgs, "--skip-update")

internal/standalone/run.go

@@ -43,6 +43,7 @@ func run(c config.Config) (err error) {
 	if err != nil {
 		return xerrors.Errorf("unable to initialize the cache: %w", err)
 	}
+	defer cacheClient.Close()
 
 	cacheOperation := operation.NewCache(cacheClient)
 	log.Logger.Debugf("cache dir:  %s", utils.CacheDir())
@@ -67,6 +68,7 @@ func run(c config.Config) (err error) {
 	if err = db.Init(c.CacheDir); err != nil {
 		return xerrors.Errorf("error in vulnerability DB initialize: %w", err)
 	}
+	defer db.Close()
 
 	var scanner scanner.Scanner
 	ctx := context.Background()

pkg/rpc/server/server_test.go

@@ -497,7 +497,7 @@ func TestCacheServer_MissingLayers(t *testing.T) {
 			getLayerExpectations: []cache.LocalImageCacheGetLayerExpectation{
 				{
 					Args: cache.LocalImageCacheGetLayerArgs{
-						LayerID: "sha256:932da51564135c98a49a34a193d6cd363d8fa4184d957fde16c9d8527b3f3b02",
+						DiffID: "sha256:932da51564135c98a49a34a193d6cd363d8fa4184d957fde16c9d8527b3f3b02",
 					},
 					Returns: cache.LocalImageCacheGetLayerReturns{
 						LayerInfo: ftypes.LayerInfo{},
@@ -505,7 +505,7 @@ func TestCacheServer_MissingLayers(t *testing.T) {
 				},
 				{
 					Args: cache.LocalImageCacheGetLayerArgs{
-						LayerID: "sha256:dffd9992ca398466a663c87c92cfea2a2db0ae0cf33fcb99da60eec52addbfc5",
+						DiffID: "sha256:dffd9992ca398466a663c87c92cfea2a2db0ae0cf33fcb99da60eec52addbfc5",
 					},
 					Returns: cache.LocalImageCacheGetLayerReturns{
 						LayerInfo: ftypes.LayerInfo{
@@ -545,7 +545,7 @@ func TestCacheServer_MissingLayers(t *testing.T) {
 			getLayerExpectations: []cache.LocalImageCacheGetLayerExpectation{
 				{
 					Args: cache.LocalImageCacheGetLayerArgs{
-						LayerID: "sha256:932da51564135c98a49a34a193d6cd363d8fa4184d957fde16c9d8527b3f3b02",
+						DiffID: "sha256:932da51564135c98a49a34a193d6cd363d8fa4184d957fde16c9d8527b3f3b02",
 					},
 					Returns: cache.LocalImageCacheGetLayerReturns{
 						LayerInfo: ftypes.LayerInfo{
@@ -555,7 +555,7 @@ func TestCacheServer_MissingLayers(t *testing.T) {
 				},
 				{
 					Args: cache.LocalImageCacheGetLayerArgs{
-						LayerID: "sha256:dffd9992ca398466a663c87c92cfea2a2db0ae0cf33fcb99da60eec52addbfc5",
+						DiffID: "sha256:dffd9992ca398466a663c87c92cfea2a2db0ae0cf33fcb99da60eec52addbfc5",
 					},
 					Returns: cache.LocalImageCacheGetLayerReturns{
 						LayerInfo: ftypes.LayerInfo{