Add layer id info (merge to master) (aquasecurity#88)

* analyzer: Include layerID as part of LayerInfo Signed-off-by: Simarpreet Singh <[email protected]> * Add LayerID to Package struct Signed-off-by: Simarpreet Singh <[email protected]> * analyzer: Remove ID from returned layerInfo Signed-off-by: Simarpreet Singh <[email protected]> * analyzer: Handle missing layer.ID from cached layer Signed-off-by: Simarpreet Singh <[email protected]> * extractor/docker: Cleanup logic to avoid extra slice usage Signed-off-by: Simarpreet Singh <[email protected]> * integration: Fix golden files to include LayerID Signed-off-by: Simarpreet Singh <[email protected]> * analyzer: Remove condition for adding layer.ID Signed-off-by: Simarpreet Singh <[email protected]> * types: Introduce types.LibraryInfo Signed-off-by: Simarpreet Singh <[email protected]> * docker: Add LayerID to each LibraryInfo Signed-off-by: Simarpreet Singh <[email protected]> * .github/bench: Bump up docker version Signed-off-by: Simarpreet Singh <[email protected]> * intergration/perf: Remove other OSes for the timebeing. Looks like Github CI is running out of space while running other tests. Until we find a better solution we need to comment out bigger OSes. Signed-off-by: Simarpreet Singh <[email protected]> * fix(image): call Close() via cleanup funcion * refactor(type): add omitempty * analyzer: Change to types.LibraryInfo in analzyer.go Signed-off-by: Simarpreet Singh <[email protected]> * wip: add CleanupDockerExtractorFn for cleanup Signed-off-by: Simarpreet Singh <[email protected]> * refactor(analyzer): remove un-needed function * test(cache): comment in * Revert "wip: add CleanupDockerExtractorFn for cleanup" This reverts commit dabfae1. * Revert ".github/bench: Bump up docker version" This reverts commit b982c46. * refactor(analyzer): sort imports * test(cache): remove debug code * test(cache): format * chore(image): remove debug code Co-authored-by: Teppei Fukuda <[email protected]>
lbxp1997 · Mar 4, 2020 · e868f4e · e868f4e
1 parent e6c70ea
commit e868f4e
Show file tree

Hide file tree

Showing 23 changed files with 2,251 additions and 1,140 deletions.
diff --git a/analyzer/analyzer.go b/analyzer/analyzer.go
@@ -5,16 +5,14 @@ import (
 	"encoding/json"
 	"sort"
 
-	"github.com/aquasecurity/fanal/extractor/docker"
 	"github.com/containers/image/v5/manifest"
 	digest "github.com/opencontainers/go-digest"
-
-	"github.com/aquasecurity/fanal/cache"
-	"github.com/aquasecurity/fanal/types"
-
 	"golang.org/x/xerrors"
 
+	"github.com/aquasecurity/fanal/cache"
 	"github.com/aquasecurity/fanal/extractor"
+	"github.com/aquasecurity/fanal/extractor/docker"
+	"github.com/aquasecurity/fanal/types"
 	godeptypes "github.com/aquasecurity/go-dep-parser/pkg/types"
 )
 
@@ -234,6 +232,7 @@ func (a Applier) ApplyLayers(imageID digest.Digest, layerIDs []string) (types.Im
 		if layer.SchemaVersion == 0 {
 			return types.ImageDetail{}, xerrors.Errorf("layer cache missing: %s", layerID)
 		}
+		layer.ID = digest.Digest(layerID)
 		layers = append(layers, layer)
 	}
 
@@ -312,28 +311,20 @@ func GetLibraries(filesMap extractor.FileMap) ([]types.Application, error) {
 			return nil, xerrors.Errorf("failed to get libraries: %w", err)
 		}
 
+		var lis []types.LibraryInfo
 		for filePath, libs := range libMap {
+			for _, lib := range libs {
+				lis = append(lis, types.LibraryInfo{
+					Library: lib,
+				})
+			}
+
 			results = append(results, types.Application{
 				Type:      analyzer.Name(),
 				FilePath:  string(filePath),
-				Libraries: libs,
+				Libraries: lis,
 			})
 		}
 	}
 	return results, nil
 }
-
-func mergePkgs(pkgs, pkgsFromCommands []types.Package) []types.Package {
-	// pkg has priority over pkgsFromCommands
-	uniqPkgs := map[string]struct{}{}
-	for _, pkg := range pkgs {
-		uniqPkgs[pkg.Name] = struct{}{}
-	}
-	for _, pkg := range pkgsFromCommands {
-		if _, ok := uniqPkgs[pkg.Name]; ok {
-			continue
-		}
-		pkgs = append(pkgs, pkg)
-	}
-	return pkgs
-}
diff --git a/analyzer/analyzer_test.go b/analyzer/analyzer_test.go
@@ -139,8 +139,25 @@ func TestConfig_Analyze(t *testing.T) {
 						DecompressedLayerID: "sha256:24df0d4e20c0f42d3703bf1f1db2bdd77346c7956f74f423603d651e8e5ae8a7",
 						LayerInfo: types.LayerInfo{
 							SchemaVersion: 1,
-							Applications:  []types.Application{{Type: "composer", FilePath: "php-app/composer.lock", Libraries: []depTypes.Library{{Name: "guzzlehttp/guzzle", Version: "6.2.0"}, {Name: "guzzlehttp/promises", Version: "v1.3.1"}, {Name: "guzzlehttp/psr7", Version: "1.5.2"}, {Name: "laravel/installer", Version: "v2.0.1"}, {Name: "pear/log", Version: "1.13.1"}, {Name: "pear/pear_exception", Version: "v1.0.0"}, {Name: "psr/http-message", Version: "1.0.1"}, {Name: "ralouphie/getallheaders", Version: "2.0.5"}, {Name: "symfony/console", Version: "v4.2.7"}, {Name: "symfony/contracts", Version: "v1.0.2"}, {Name: "symfony/filesystem", Version: "v4.2.7"}, {Name: "symfony/polyfill-ctype", Version: "v1.11.0"}, {Name: "symfony/polyfill-mbstring", Version: "v1.11.0"}, {Name: "symfony/process", Version: "v4.2.7"}}}},
-							OpaqueDirs:    []string{"php-app/"},
+							Applications: []types.Application{{Type: "composer", FilePath: "php-app/composer.lock",
+								Libraries: []types.LibraryInfo{
+									{Library: depTypes.Library{Name: "guzzlehttp/guzzle", Version: "6.2.0"}},
+									{Library: depTypes.Library{Name: "guzzlehttp/promises", Version: "v1.3.1"}},
+									{Library: depTypes.Library{Name: "guzzlehttp/psr7", Version: "1.5.2"}},
+									{Library: depTypes.Library{Name: "laravel/installer", Version: "v2.0.1"}},
+									{Library: depTypes.Library{Name: "pear/log", Version: "1.13.1"}},
+									{Library: depTypes.Library{Name: "pear/pear_exception", Version: "v1.0.0"}},
+									{Library: depTypes.Library{Name: "psr/http-message", Version: "1.0.1"}},
+									{Library: depTypes.Library{Name: "ralouphie/getallheaders", Version: "2.0.5"}},
+									{Library: depTypes.Library{Name: "symfony/console", Version: "v4.2.7"}},
+									{Library: depTypes.Library{Name: "symfony/contracts", Version: "v1.0.2"}},
+									{Library: depTypes.Library{Name: "symfony/filesystem", Version: "v4.2.7"}},
+									{Library: depTypes.Library{Name: "symfony/polyfill-ctype", Version: "v1.11.0"}},
+									{Library: depTypes.Library{Name: "symfony/polyfill-mbstring", Version: "v1.11.0"}},
+									{Library: depTypes.Library{Name: "symfony/process", Version: "v4.2.7"}},
+								},
+							}},
+							OpaqueDirs: []string{"php-app/"},
 						},
 					},
 				},
@@ -209,8 +226,9 @@ func TestConfig_Analyze(t *testing.T) {
 			mockCache.ApplyPutLayerExpectations(tt.putLayerExpectations)
 			mockCache.ApplyPutImageExpectations(tt.putImageExpectations)
 
-			d, err := docker.NewDockerArchiveExtractor(context.Background(), tt.imagePath, types.DockerOption{})
+			d, cleanup, err := docker.NewDockerArchiveExtractor(context.Background(), tt.imagePath, types.DockerOption{})
 			require.NoError(t, err, tt.name)
+			defer cleanup()
 
 			ac := analyzer.New(d, mockCache)
 			got, err := ac.Analyze(context.Background())
@@ -265,6 +283,7 @@ func TestApplier_ApplyLayers(t *testing.T) {
 									FilePath: "var/lib/dpkg/status.d/tzdata",
 									Packages: []types.Package{
 										{
+											LayerID:    "sha256:932da51564135c98a49a34a193d6cd363d8fa4184d957fde16c9d8527b3f3b02",
 											Name:       "tzdata",
 											Version:    "2019a-0+deb9u1",
 											SrcName:    "tzdata",
@@ -313,14 +332,18 @@ func TestApplier_ApplyLayers(t *testing.T) {
 								{
 									Type:     "composer",
 									FilePath: "php-app/composer.lock",
-									Libraries: []depTypes.Library{
+									Libraries: []types.LibraryInfo{
 										{
-											Name:    "guzzlehttp/guzzle",
-											Version: "6.2.0",
+											Library: depTypes.Library{
+												Name:    "guzzlehttp/guzzle",
+												Version: "6.2.0",
+											},
 										},
 										{
-											Name:    "symfony/process",
-											Version: "v4.2.7",
+											Library: depTypes.Library{
+												Name:    "symfony/process",
+												Version: "v4.2.7",
+											},
 										},
 									},
 								},
@@ -347,14 +370,33 @@ func TestApplier_ApplyLayers(t *testing.T) {
 					Name:   "9.9",
 				},
 				Packages: []types.Package{
-					{Name: "libc6", Version: "2.24-11+deb9u4", SrcName: "glibc", SrcVersion: "2.24-11+deb9u4"},
-					{Name: "tzdata", Version: "2019a-0+deb9u1", SrcName: "tzdata", SrcVersion: "2019a-0+deb9u1"},
+					{
+						Name: "libc6", Version: "2.24-11+deb9u4", SrcName: "glibc", SrcVersion: "2.24-11+deb9u4",
+						LayerID: "sha256:dffd9992ca398466a663c87c92cfea2a2db0ae0cf33fcb99da60eec52addbfc5",
+					},
+					{
+						Name: "tzdata", Version: "2019a-0+deb9u1", SrcName: "tzdata", SrcVersion: "2019a-0+deb9u1",
+						LayerID: "sha256:932da51564135c98a49a34a193d6cd363d8fa4184d957fde16c9d8527b3f3b02",
+					},
 				},
 				Applications: []types.Application{
 					{
 						Type: "composer", FilePath: "php-app/composer.lock",
-						Libraries: []depTypes.Library{{Name: "guzzlehttp/guzzle", Version: "6.2.0"},
-							{Name: "symfony/process", Version: "v4.2.7"},
+						Libraries: []types.LibraryInfo{
+							{
+								Library: depTypes.Library{
+									Name:    "guzzlehttp/guzzle",
+									Version: "6.2.0",
+								},
+								LayerID: "sha256:24df0d4e20c0f42d3703bf1f1db2bdd77346c7956f74f423603d651e8e5ae8a7",
+							},
+							{
+								Library: depTypes.Library{
+									Name:    "symfony/process",
+									Version: "v4.2.7",
+								},
+								LayerID: "sha256:24df0d4e20c0f42d3703bf1f1db2bdd77346c7956f74f423603d651e8e5ae8a7",
+							},
 						},
 					},
 				},
@@ -384,11 +426,11 @@ func TestApplier_ApplyLayers(t *testing.T) {
 								{
 									FilePath: "lib/apk/db/installed",
 									Packages: []types.Package{
-										{Name: "musl", Version: "1.1.22-r3"},
-										{Name: "busybox", Version: "1.30.1-r3"},
-										{Name: "openssl", Version: "1.1.1d-r2"},
-										{Name: "libcrypto1.1", Version: "1.1.1d-r2"},
-										{Name: "libssl1.1", Version: "1.1.1d-r2"},
+										{Name: "musl", Version: "1.1.22-r3", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
+										{Name: "busybox", Version: "1.30.1-r3", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
+										{Name: "openssl", Version: "1.1.1d-r2", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
+										{Name: "libcrypto1.1", Version: "1.1.1d-r2", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
+										{Name: "libssl1.1", Version: "1.1.1d-r2", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
 									},
 								},
 							},
@@ -424,11 +466,11 @@ func TestApplier_ApplyLayers(t *testing.T) {
 					Name:   "3.10.4",
 				},
 				Packages: []types.Package{
-					{Name: "busybox", Version: "1.30.1-r3"},
-					{Name: "libcrypto1.1", Version: "1.1.1d-r2"},
-					{Name: "libssl1.1", Version: "1.1.1d-r2"},
-					{Name: "musl", Version: "1.1.22-r3"},
-					{Name: "openssl", Version: "1.1.1d-r2"},
+					{Name: "busybox", Version: "1.30.1-r3", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
+					{Name: "libcrypto1.1", Version: "1.1.1d-r2", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
+					{Name: "libssl1.1", Version: "1.1.1d-r2", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
+					{Name: "musl", Version: "1.1.22-r3", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
+					{Name: "openssl", Version: "1.1.1d-r2", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
 				},
 				HistoryPackages: []types.Package{
 					{Name: "musl", Version: "1.1.23"},
@@ -544,7 +586,7 @@ func TestApplier_ApplyLayers(t *testing.T) {
 			})
 			for _, app := range got.Applications {
 				sort.Slice(app.Libraries, func(i, j int) bool {
-					return app.Libraries[i].Name < app.Libraries[j].Name
+					return app.Libraries[i].Library.Name < app.Libraries[j].Library.Name
 				})
 			}
 			assert.Equal(t, tt.want, got)

diff --git a/cache/cache_test.go b/cache/cache_test.go
@@ -114,14 +114,18 @@ func TestFSCache_GetLayer(t *testing.T) {
 					{
 						Type:     "composer",
 						FilePath: "php-app/composer.lock",
-						Libraries: []depTypes.Library{
+						Libraries: []types.LibraryInfo{
 							{
-								Name:    "guzzlehttp/guzzle",
-								Version: "6.2.0",
+								Library: depTypes.Library{
+									Name:    "guzzlehttp/guzzle",
+									Version: "6.2.0",
+								},
 							},
 							{
-								Name:    "guzzlehttp/promises",
-								Version: "v1.3.1",
+								Library: depTypes.Library{
+									Name:    "guzzlehttp/promises",
+									Version: "v1.3.1",
+								},
 							},
 						},
 					},
@@ -215,9 +219,19 @@ func TestFSCache_PutLayer(t *testing.T) {
 						{
 							Type:     "composer",
 							FilePath: "php-app/composer.lock",
-							Libraries: []depTypes.Library{
-								{Name: "guzzlehttp/guzzle", Version: "6.2.0"},
-								{Name: "guzzlehttp/promises", Version: "v1.3.1"},
+							Libraries: []types.LibraryInfo{
+								{
+									Library: depTypes.Library{
+										Name:    "guzzlehttp/guzzle",
+										Version: "6.2.0",
+									},
+								},
+								{
+									Library: depTypes.Library{
+										Name:    "guzzlehttp/promises",
+										Version: "v1.3.1",
+									},
+								},
 							},
 						},
 					},
@@ -248,14 +262,18 @@ func TestFSCache_PutLayer(t *testing.T) {
 				      "Type": "composer",
 				      "FilePath": "php-app/composer.lock",
 				      "Libraries": [
-				        {
-				          "Name": "guzzlehttp/guzzle",
-				          "Version": "6.2.0"
-				        },
-				        {
-				          "Name": "guzzlehttp/promises",
-				          "Version": "v1.3.1"
-				        }
+                        {
+                           "Library":{
+                              "Name":"guzzlehttp/guzzle",
+                              "Version":"6.2.0"
+                           }
+                        },
+                        {
+                           "Library":{
+                              "Name":"guzzlehttp/promises",
+                              "Version":"v1.3.1"
+                           }
+                        }
 				      ]
 				    }
 				  ],

diff --git a/cache/testdata/decompressed-layer-id.db b/cache/testdata/decompressed-layer-id.db
diff --git a/cmd/fanal/main.go b/cmd/fanal/main.go
@@ -68,17 +68,19 @@ func run() (err error) {
 	}
 
 	var ext extractor.Extractor
+	var cleanup func()
 	if len(args) > 0 {
-		ext, err = docker.NewDockerExtractor(ctx, args[0], opt)
+		ext, cleanup, err = docker.NewDockerExtractor(ctx, args[0], opt)
 		if err != nil {
 			return err
 		}
 	} else {
-		ext, err = docker.NewDockerArchiveExtractor(ctx, *tarPath, opt)
+		ext, cleanup, err = docker.NewDockerArchiveExtractor(ctx, *tarPath, opt)
 		if err != nil {
 			return err
 		}
 	}
+	defer cleanup()
 
 	ac := analyzer.New(ext, c)
 	imageInfo, err := ac.Analyze(ctx)

diff --git a/extractor/docker/docker.go b/extractor/docker/docker.go
@@ -52,32 +52,36 @@ func init() {
 	image.RegisterRegistry(&ecr.ECR{})
 }
 
-func NewDockerExtractor(ctx context.Context, imageName string, option types.DockerOption) (Extractor, error) {
+func NewDockerExtractor(ctx context.Context, imageName string, option types.DockerOption) (Extractor, func(), error) {
 	ref := image.Reference{Name: imageName, IsFile: false}
 	transports := []string{"docker-daemon:", "docker://"}
 	return newDockerExtractor(ctx, ref, transports, option)
 }
 
-func NewDockerArchiveExtractor(ctx context.Context, fileName string, option types.DockerOption) (Extractor, error) {
+func NewDockerArchiveExtractor(ctx context.Context, fileName string, option types.DockerOption) (Extractor, func(), error) {
 	ref := image.Reference{Name: fileName, IsFile: true}
 	transports := []string{"docker-archive:"}
 	return newDockerExtractor(ctx, ref, transports, option)
 }
 
 func newDockerExtractor(ctx context.Context, imgRef image.Reference, transports []string,
-	option types.DockerOption) (Extractor, error) {
+	option types.DockerOption) (Extractor, func(), error) {
 	ctx, cancel := context.WithTimeout(ctx, option.Timeout)
 	defer cancel()
 
 	img, err := image.NewImage(ctx, imgRef, transports, option)
 	if err != nil {
-		return Extractor{}, xerrors.Errorf("unable to initialize a image struct: %w", err)
+		return Extractor{}, nil, xerrors.Errorf("unable to initialize a image struct: %w", err)
+	}
+
+	cleanup := func() {
+		_ = img.Close()
 	}
 
 	return Extractor{
 		option: option,
 		image:  img,
-	}, nil
+	}, cleanup, nil
 }
 
 func ApplyLayers(layers []types.LayerInfo) types.ImageDetail {
@@ -98,9 +102,15 @@ func ApplyLayers(layers []types.LayerInfo) types.ImageDetail {
 		}
 
 		for _, pkgInfo := range layer.PackageInfos {
+			for i := range pkgInfo.Packages {
+				pkgInfo.Packages[i].LayerID = layer.ID
+			}
 			nestedMap.SetByString(pkgInfo.FilePath, sep, pkgInfo)
 		}
 		for _, app := range layer.Applications {
+			for i := range app.Libraries {
+				app.Libraries[i].LayerID = layer.ID
+			}
 			nestedMap.SetByString(app.FilePath, sep, app)
 		}
 	}