Skip to content

Commit

Permalink
integration: Add dockerless mode tests (aquasecurity#81)
Browse files Browse the repository at this point in the history 
* integration: Add dockerless mode tests

Signed-off-by: Simarpreet Singh <[email protected]>

* .github: Fix typo

Signed-off-by: Simarpreet Singh <[email protected]>

* library_test: Improve cache assertions with containers/image

Signed-off-by: Simarpreet Singh <[email protected]>

* github: Update docker version before running

Signed-off-by: Simarpreet Singh <[email protected]>

* integration: Remove un-needed random num generation

Signed-off-by: Simarpreet Singh <[email protected]>

* rebase on latest master

Signed-off-by: Simarpreet Singh <[email protected]>

* integration: improve packages check

Signed-off-by: Simarpreet Singh <[email protected]>

* integration: Remove un-needed goldenfiles

Signed-off-by: Simarpreet Singh <[email protected]>

* library_test: Rename dockerlessImageName to remoteImageName

Signed-off-by: Simarpreet Singh <[email protected]>

* github: Remove un-needed bench target

Signed-off-by: Simarpreet Singh <[email protected]>

* library_test: Rename dockerlessImageName to remoteImageName

Signed-off-by: Simarpreet Singh <[email protected]>
  • Loading branch information
@simar7
simar7 authored Mar 15, 2020
1 parent 32106ed commit 4122447
Show file tree
Hide file tree
Showing 4 changed files with 99 additions and 63 deletions.
6 changes: 2 additions & 4 deletions .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ jobs:
name: Unit Test
runs-on: ubuntu-latest
steps:

- name: Set up Go 1.13
uses: actions/setup-go@v1
with:
Expand All @@ -28,7 +27,6 @@ jobs:
name: Integration Test
runs-on: ubuntu-latest
steps:

- name: Set up Go 1.13
uses: actions/setup-go@v1
with:
Expand All @@ -41,5 +39,5 @@ jobs:
- name: Install dependencies
run: sudo apt-get install libdb-dev

- name: Run instegration tests
run: make test-integration
- name: Run integration tests
run: make test-integration
156 changes: 97 additions & 59 deletions integration/library_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"io"
"io/ioutil"
"os"
"sort"
"strings"
"testing"
"time"
Expand Down Expand Up @@ -44,6 +45,7 @@ import (
type testCase struct {
name string
imageName string
remoteImageName string
imageFile string
expectedOS types.OS
expectedPkgsFromCmds string
Expand All @@ -52,83 +54,109 @@ type testCase struct {

var testCases = []testCase{
{
name: "happy path, alpine:3.10",
imageName: "alpine:3.10",
imageFile: "testdata/fixtures/alpine-310.tar.gz",
expectedOS: types.OS{Name: "3.10.2", Family: "alpine"},
name: "happy path, alpine:3.10",
imageName: "alpine:3.10",
remoteImageName: "knqyf263/alpine:3.10",
imageFile: "testdata/fixtures/alpine-310.tar.gz",
expectedOS: types.OS{Name: "3.10.2", Family: "alpine"},
},
{
name: "happy path, debian:buster",
imageName: "debian:buster",
imageFile: "testdata/fixtures/debian-buster.tar.gz",
expectedOS: types.OS{Name: "10.1", Family: "debian"},
name: "happy path, amazonlinux:2",
imageName: "amazonlinux:2",
remoteImageName: "knqyf263/amazonlinux:2",
imageFile: "testdata/fixtures/amazon-2.tar.gz",
expectedOS: types.OS{Name: "2 (Karoo)", Family: "amazon"},
},
{
name: "happy path, ubuntu:18.04",
imageName: "ubuntu:18.04",
imageFile: "testdata/fixtures/ubuntu-1804.tar.gz",
expectedOS: types.OS{Name: "18.04", Family: "ubuntu"},
name: "happy path, debian:buster",
imageName: "debian:buster",
remoteImageName: "knqyf263/debian:buster",
imageFile: "testdata/fixtures/debian-buster.tar.gz",
expectedOS: types.OS{Name: "10.1", Family: "debian"},
},
{
name: "happy path, centos6",
imageName: "centos:6",
imageFile: "testdata/fixtures/centos-6.tar.gz",
expectedOS: types.OS{Name: "6.10", Family: "centos"},
name: "happy path, photon:1.0",
imageName: "photon:1.0-20190823",
remoteImageName: "knqyf263/photon:1.0-20190823",
imageFile: "testdata/fixtures/photon-10.tar.gz",
expectedOS: types.OS{Name: "1.0", Family: "photon"},
},
{
name: "happy path, registry.redhat.io/ubi7",
imageName: "registry.redhat.io/ubi7",
imageFile: "testdata/fixtures/ubi-7.tar.gz",
expectedOS: types.OS{Name: "7.7", Family: "redhat"},
name: "happy path, registry.redhat.io/ubi7",
imageName: "registry.redhat.io/ubi7",
remoteImageName: "knqyf263/registry.redhat.io-ubi7:latest",
imageFile: "testdata/fixtures/ubi-7.tar.gz",
expectedOS: types.OS{Name: "7.7", Family: "redhat"},
},
{
name: "happy path, amazonlinux:2",
imageName: "amazonlinux:2",
imageFile: "testdata/fixtures/amazon-2.tar.gz",
expectedOS: types.OS{Name: "2 (Karoo)", Family: "amazon"},
},
{
name: "happy path, oracle:8",
imageName: "oraclelinux:8-slim",
imageFile: "testdata/fixtures/oraclelinux-8-slim.tar.gz",
expectedOS: types.OS{Name: "8.0", Family: "oracle"},
},
{
name: "happy path, distroless",
imageName: "gcr.io/distroless/base:latest",
imageFile: "testdata/fixtures/distroless-base.tar.gz",
expectedOS: types.OS{Name: "9.9", Family: "debian"},
},
{
name: "happy path, photon:1.0",
imageName: "photon:1.0-20190823",
imageFile: "testdata/fixtures/photon-10.tar.gz",
expectedOS: types.OS{Name: "1.0", Family: "photon"},
},
{
name: "happy path, opensuse leap 15.1",
imageName: "opensuse/leap:latest",
imageFile: "testdata/fixtures/opensuse-leap-151.tar.gz",
expectedOS: types.OS{Name: "15.1", Family: "opensuse.leap"},
name: "happy path, opensuse leap 15.1",
imageName: "opensuse/leap:latest",
remoteImageName: "knqyf263/opensuse-leap:latest",
imageFile: "testdata/fixtures/opensuse-leap-151.tar.gz",
expectedOS: types.OS{Name: "15.1", Family: "opensuse.leap"},
},
{
name: "happy path, vulnimage with lock files",
imageName: "knqyf263/vuln-image:1.2.3",
remoteImageName: "knqyf263/vuln-image:1.2.3",
imageFile: "testdata/fixtures/vulnimage.tar.gz",
expectedOS: types.OS{Name: "3.7.1", Family: "alpine"},
expectedLibraries: "testdata/goldens/knqyf263vuln-image1.2.3.expectedlibs.golden",
expectedPkgsFromCmds: "testdata/goldens/knqyf263vuln-image1.2.3.expectedpkgsfromcmds.golden",
expectedLibraries: "testdata/goldens/vuln-image1.2.3.expectedlibs.golden",
expectedPkgsFromCmds: "testdata/goldens/vuln-image1.2.3.expectedpkgsfromcmds.golden",
},
}

func TestFanal_Library_DockerLessMode(t *testing.T) {
for _, tc := range testCases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
ctx := context.Background()
d, _ := ioutil.TempDir("", "TestFanal_Library_DockerLessMode_*")
defer os.RemoveAll(d)

c, err := cache.NewFSCache(d)
require.NoError(t, err, tc.name)

opt := types.DockerOption{
Timeout: 600 * time.Second,
SkipPing: true,
}

cli, err := client.NewClientWithOpts(client.FromEnv)
require.NoError(t, err, tc.name)

// remove existing Image if any
_, _ = cli.ImageRemove(ctx, tc.remoteImageName, dtypes.ImageRemoveOptions{
Force: true,
PruneChildren: true,
})

ext, cleanup, err := docker.NewDockerExtractor(ctx, tc.remoteImageName, opt)
require.NoError(t, err, tc.name)
defer cleanup()

ac := analyzer.New(ext, c)
applier := analyzer.NewApplier(c)

// run tests twice, one without cache and with cache
for i := 1; i <= 2; i++ {
runChecks(t, ctx, ac, applier, tc)
}

// clear Cache
require.NoError(t, c.Clear(), tc.name)
})
}
}

func TestFanal_Library_DockerMode(t *testing.T) {
t.Parallel()
for _, tc := range testCases {
tc := tc // save a copy of tc for use in t.Run https://gist.github.com/posener/92a55c4cd441fc5e5e85f27bca008721
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
ctx := context.Background()
d, _ := ioutil.TempDir("", "TestFanal_Library_")
d, _ := ioutil.TempDir("", "TestFanal_Library_DockerMode_*")
defer os.RemoveAll(d)
c, err := cache.NewFSCache(d)
require.NoError(t, err)
Expand Down Expand Up @@ -186,13 +214,12 @@ func TestFanal_Library_DockerMode(t *testing.T) {
}

func TestFanal_Library_TarMode(t *testing.T) {
t.Parallel()
for _, tc := range testCases {
tc := tc // save a copy of tc for use in t.Run https://gist.github.com/posener/92a55c4cd441fc5e5e85f27bca008721
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
ctx := context.Background()
d, _ := ioutil.TempDir("", "TestFanal_Library_")
d, _ := ioutil.TempDir("", "TestFanal_Library_TarMode_*")
defer os.RemoveAll(d)

c, err := cache.NewFSCache(d)
Expand Down Expand Up @@ -236,18 +263,28 @@ func commonChecks(t *testing.T, detail types.ImageDetail, tc testCase) {
func checkPackages(t *testing.T, detail types.ImageDetail, tc testCase) {
r := strings.NewReplacer("/", "-", ":", "-")
goldenFile := fmt.Sprintf("testdata/goldens/packages/%s.json.golden", r.Replace(tc.imageName))
data, err := ioutil.ReadFile(goldenFile)
require.NoError(t, err, tc.name)

data, _ := ioutil.ReadFile(goldenFile)
var expectedPkgs []types.Package
err := json.Unmarshal(data, &expectedPkgs)
err = json.Unmarshal(data, &expectedPkgs)
require.NoError(t, err)
assert.ElementsMatch(t, expectedPkgs, detail.Packages, tc.name)

require.Equal(t, len(expectedPkgs), len(detail.Packages), tc.name)
sort.Slice(expectedPkgs, func(i, j int) bool { return expectedPkgs[i].Name < expectedPkgs[j].Name })
sort.Slice(detail.Packages, func(i, j int) bool { return detail.Packages[i].Name < detail.Packages[j].Name })

for i := 0; i < len(expectedPkgs); i++ {
require.Equal(t, expectedPkgs[i].Name, detail.Packages[i].Name, tc.name)
require.Equal(t, expectedPkgs[i].Version, detail.Packages[i].Version, tc.name)
}
}

func checkLibraries(detail types.ImageDetail, t *testing.T, tc testCase) {
if tc.expectedLibraries != "" {
data, _ := ioutil.ReadFile(tc.expectedLibraries)
var expectedLibraries map[types.FilePath][]godeptypes.Library

json.Unmarshal(data, &expectedLibraries)
require.Equal(t, len(expectedLibraries), len(detail.Applications), tc.name)
} else {
Expand All @@ -259,6 +296,7 @@ func checkPackageFromCommands(t *testing.T, detail types.ImageDetail, tc testCas
if tc.expectedPkgsFromCmds != "" {
data, _ := ioutil.ReadFile(tc.expectedPkgsFromCmds)
var expectedPkgsFromCmds []types.Package

json.Unmarshal(data, &expectedPkgsFromCmds)
assert.ElementsMatch(t, expectedPkgsFromCmds, detail.HistoryPackages, tc.name)
} else {
Expand Down
0 ...qyf263vuln-image1.2.3.expectedlibs.golden → ...ldens/vuln-image1.2.3.expectedlibs.golden
View file
File renamed without changes.
0 ...ln-image1.2.3.expectedpkgsfromcmds.golden → ...ln-image1.2.3.expectedpkgsfromcmds.golden
View file
File renamed without changes.

0 comments on commit 4122447

Please sign in to comment.