Skip to content

Commit

Permalink
feat: split ID into Digest and DiffID (aquasecurity#97)
Browse files Browse the repository at this point in the history
  • Loading branch information
@knqyf263
knqyf263 authored Mar 24, 2020
1 parent f28b6d2 commit c536335
Show file tree
Hide file tree
Showing 6 changed files with 174 additions and 59 deletions.
3 changes: 2 additions & 1 deletion analyzer/analyzer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,8 @@ func (ac Config) analyzeLayer(ctx context.Context, dig digest.Digest) (digest.Di
}

layerInfo := types.LayerInfo{
Digest: string(dig),
DiffID: string(decompressedLayerID),
SchemaVersion: types.LayerJSONSchemaVersion,
OS: os,
PackageInfos: pkgs,
Expand Down Expand Up @@ -232,7 +234,6 @@ func (a Applier) ApplyLayers(imageID digest.Digest, layerIDs []string) (types.Im
if layer.SchemaVersion == 0 {
return types.ImageDetail{}, xerrors.Errorf("layer cache missing: %s", layerID)
}
layer.ID = digest.Digest(layerID)
layers = append(layers, layer)
}

Expand Down
94 changes: 79 additions & 15 deletions analyzer/analyzer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ func TestConfig_Analyze(t *testing.T) {
DecompressedLayerID: "sha256:77cae8ab23bf486355d1b3191259705374f4a11d483b24964d2f729dd8c076a0",
LayerInfo: types.LayerInfo{
SchemaVersion: 1,
Digest: "sha256:77cae8ab23bf486355d1b3191259705374f4a11d483b24964d2f729dd8c076a0",
DiffID: "sha256:77cae8ab23bf486355d1b3191259705374f4a11d483b24964d2f729dd8c076a0",
OS: &types.OS{
Family: "alpine",
Name: "3.10.3",
Expand Down Expand Up @@ -118,6 +120,8 @@ func TestConfig_Analyze(t *testing.T) {
DecompressedLayerID: "sha256:d9441f7754ba1423ee11793b5db6390256e44097c2fc25e75a5fab19e6dc7911",
LayerInfo: types.LayerInfo{
SchemaVersion: 1,
Digest: "sha256:932da51564135c98a49a34a193d6cd363d8fa4184d957fde16c9d8527b3f3b02",
DiffID: "sha256:d9441f7754ba1423ee11793b5db6390256e44097c2fc25e75a5fab19e6dc7911",
OS: &types.OS{Family: "debian", Name: "9.9"},
PackageInfos: []types.PackageInfo{{FilePath: "var/lib/dpkg/status.d/base", Packages: []types.Package{{Name: "base-files", Version: "9.9+deb9u9", Release: "", Epoch: 0, Arch: "", SrcName: "base-files", SrcVersion: "9.9+deb9u9", SrcRelease: "", SrcEpoch: 0}}}, {FilePath: "var/lib/dpkg/status.d/netbase", Packages: []types.Package{{Name: "netbase", Version: "5.4", Release: "", Epoch: 0, Arch: "", SrcName: "netbase", SrcVersion: "5.4", SrcRelease: "", SrcEpoch: 0}}}, {FilePath: "var/lib/dpkg/status.d/tzdata", Packages: []types.Package{{Name: "tzdata", Version: "2019a-0+deb9u1", Release: "", Epoch: 0, Arch: "", SrcName: "tzdata", SrcVersion: "2019a-0+deb9u1", SrcRelease: "", SrcEpoch: 0}}}},
},
Expand All @@ -129,6 +133,8 @@ func TestConfig_Analyze(t *testing.T) {
DecompressedLayerID: "sha256:dab15cac9ebd43beceeeda3ce95c574d6714ed3d3969071caead678c065813ec",
LayerInfo: types.LayerInfo{
SchemaVersion: 1,
Digest: "sha256:dffd9992ca398466a663c87c92cfea2a2db0ae0cf33fcb99da60eec52addbfc5",
DiffID: "sha256:dab15cac9ebd43beceeeda3ce95c574d6714ed3d3969071caead678c065813ec",
PackageInfos: []types.PackageInfo{{FilePath: "var/lib/dpkg/status.d/libc6", Packages: []types.Package{{Name: "libc6", Version: "2.24-11+deb9u4", Release: "", Epoch: 0, Arch: "", SrcName: "glibc", SrcVersion: "2.24-11+deb9u4", SrcRelease: "", SrcEpoch: 0}}}, {FilePath: "var/lib/dpkg/status.d/libssl1", Packages: []types.Package{{Name: "libssl1.1", Version: "1.1.0k-1~deb9u1", Release: "", Epoch: 0, Arch: "", SrcName: "openssl", SrcVersion: "1.1.0k-1~deb9u1", SrcRelease: "", SrcEpoch: 0}}}, {FilePath: "var/lib/dpkg/status.d/openssl", Packages: []types.Package{{Name: "openssl", Version: "1.1.0k-1~deb9u1", Release: "", Epoch: 0, Arch: "", SrcName: "openssl", SrcVersion: "1.1.0k-1~deb9u1", SrcRelease: "", SrcEpoch: 0}}}},
},
},
Expand All @@ -139,6 +145,8 @@ func TestConfig_Analyze(t *testing.T) {
DecompressedLayerID: "sha256:24df0d4e20c0f42d3703bf1f1db2bdd77346c7956f74f423603d651e8e5ae8a7",
LayerInfo: types.LayerInfo{
SchemaVersion: 1,
Digest: "sha256:24df0d4e20c0f42d3703bf1f1db2bdd77346c7956f74f423603d651e8e5ae8a7",
DiffID: "sha256:24df0d4e20c0f42d3703bf1f1db2bdd77346c7956f74f423603d651e8e5ae8a7",
Applications: []types.Application{{Type: "composer", FilePath: "php-app/composer.lock",
Libraries: []types.LibraryInfo{
{Library: depTypes.Library{Name: "guzzlehttp/guzzle", Version: "6.2.0"}},
Expand Down Expand Up @@ -201,6 +209,8 @@ func TestConfig_Analyze(t *testing.T) {
DecompressedLayerID: "sha256:77cae8ab23bf486355d1b3191259705374f4a11d483b24964d2f729dd8c076a0",
LayerInfo: types.LayerInfo{
SchemaVersion: 1,
Digest: "sha256:77cae8ab23bf486355d1b3191259705374f4a11d483b24964d2f729dd8c076a0",
DiffID: "sha256:77cae8ab23bf486355d1b3191259705374f4a11d483b24964d2f729dd8c076a0",
OS: &types.OS{
Family: "alpine",
Name: "3.10.3",
Expand Down Expand Up @@ -274,6 +284,8 @@ func TestApplier_ApplyLayers(t *testing.T) {
Returns: cache.LocalImageCacheGetLayerReturns{
LayerInfo: types.LayerInfo{
SchemaVersion: 1,
Digest: "sha256:932da51564135c98a49a34a193d6cd363d8fa4184d957fde16c9d8527b3f3b02",
DiffID: "sha256:a187dde48cd289ac374ad8539930628314bc581a481cdb41409c9289419ddb72",
OS: &types.OS{
Family: "debian",
Name: "9.9",
Expand All @@ -283,7 +295,6 @@ func TestApplier_ApplyLayers(t *testing.T) {
FilePath: "var/lib/dpkg/status.d/tzdata",
Packages: []types.Package{
{
LayerID: "sha256:932da51564135c98a49a34a193d6cd363d8fa4184d957fde16c9d8527b3f3b02",
Name: "tzdata",
Version: "2019a-0+deb9u1",
SrcName: "tzdata",
Expand All @@ -302,6 +313,8 @@ func TestApplier_ApplyLayers(t *testing.T) {
Returns: cache.LocalImageCacheGetLayerReturns{
LayerInfo: types.LayerInfo{
SchemaVersion: 1,
Digest: "sha256:dffd9992ca398466a663c87c92cfea2a2db0ae0cf33fcb99da60eec52addbfc5",
DiffID: "sha256:aad63a9339440e7c3e1fff2b988991b9bfb81280042fa7f39a5e327023056819",
PackageInfos: []types.PackageInfo{
{
FilePath: "var/lib/dpkg/status.d/libc6",
Expand All @@ -328,6 +341,8 @@ func TestApplier_ApplyLayers(t *testing.T) {
Returns: cache.LocalImageCacheGetLayerReturns{
LayerInfo: types.LayerInfo{
SchemaVersion: 1,
Digest: "sha256:beee9f30bc1f711043e78d4a2be0668955d4b761d587d6f60c2c8dc081efb203",
DiffID: "sha256:24df0d4e20c0f42d3703bf1f1db2bdd77346c7956f74f423603d651e8e5ae8a7",
Applications: []types.Application{
{
Type: "composer",
Expand Down Expand Up @@ -372,11 +387,17 @@ func TestApplier_ApplyLayers(t *testing.T) {
Packages: []types.Package{
{
Name: "libc6", Version: "2.24-11+deb9u4", SrcName: "glibc", SrcVersion: "2.24-11+deb9u4",
LayerID: "sha256:dffd9992ca398466a663c87c92cfea2a2db0ae0cf33fcb99da60eec52addbfc5",
Layer: types.Layer{
Digest: "sha256:dffd9992ca398466a663c87c92cfea2a2db0ae0cf33fcb99da60eec52addbfc5",
DiffID: "sha256:aad63a9339440e7c3e1fff2b988991b9bfb81280042fa7f39a5e327023056819",
},
},
{
Name: "tzdata", Version: "2019a-0+deb9u1", SrcName: "tzdata", SrcVersion: "2019a-0+deb9u1",
LayerID: "sha256:932da51564135c98a49a34a193d6cd363d8fa4184d957fde16c9d8527b3f3b02",
Layer: types.Layer{
Digest: "sha256:932da51564135c98a49a34a193d6cd363d8fa4184d957fde16c9d8527b3f3b02",
DiffID: "sha256:a187dde48cd289ac374ad8539930628314bc581a481cdb41409c9289419ddb72",
},
},
},
Applications: []types.Application{
Expand All @@ -388,14 +409,20 @@ func TestApplier_ApplyLayers(t *testing.T) {
Name: "guzzlehttp/guzzle",
Version: "6.2.0",
},
LayerID: "sha256:24df0d4e20c0f42d3703bf1f1db2bdd77346c7956f74f423603d651e8e5ae8a7",
Layer: types.Layer{
Digest: "sha256:beee9f30bc1f711043e78d4a2be0668955d4b761d587d6f60c2c8dc081efb203",
DiffID: "sha256:24df0d4e20c0f42d3703bf1f1db2bdd77346c7956f74f423603d651e8e5ae8a7",
},
},
{
Library: depTypes.Library{
Name: "symfony/process",
Version: "v4.2.7",
},
LayerID: "sha256:24df0d4e20c0f42d3703bf1f1db2bdd77346c7956f74f423603d651e8e5ae8a7",
Layer: types.Layer{
Digest: "sha256:beee9f30bc1f711043e78d4a2be0668955d4b761d587d6f60c2c8dc081efb203",
DiffID: "sha256:24df0d4e20c0f42d3703bf1f1db2bdd77346c7956f74f423603d651e8e5ae8a7",
},
},
},
},
Expand All @@ -418,6 +445,8 @@ func TestApplier_ApplyLayers(t *testing.T) {
Returns: cache.LocalImageCacheGetLayerReturns{
LayerInfo: types.LayerInfo{
SchemaVersion: 1,
Digest: "sha256:a187dde48cd289ac374ad8539930628314bc581a481cdb41409c9289419ddb72",
DiffID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028",
OS: &types.OS{
Family: "alpine",
Name: "3.10.4",
Expand All @@ -426,11 +455,11 @@ func TestApplier_ApplyLayers(t *testing.T) {
{
FilePath: "lib/apk/db/installed",
Packages: []types.Package{
{Name: "musl", Version: "1.1.22-r3", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
{Name: "busybox", Version: "1.30.1-r3", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
{Name: "openssl", Version: "1.1.1d-r2", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
{Name: "libcrypto1.1", Version: "1.1.1d-r2", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
{Name: "libssl1.1", Version: "1.1.1d-r2", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
{Name: "musl", Version: "1.1.22-r3"},
{Name: "busybox", Version: "1.30.1-r3"},
{Name: "openssl", Version: "1.1.1d-r2"},
{Name: "libcrypto1.1", Version: "1.1.1d-r2"},
{Name: "libssl1.1", Version: "1.1.1d-r2"},
},
},
},
Expand Down Expand Up @@ -466,11 +495,46 @@ func TestApplier_ApplyLayers(t *testing.T) {
Name: "3.10.4",
},
Packages: []types.Package{
{Name: "busybox", Version: "1.30.1-r3", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
{Name: "libcrypto1.1", Version: "1.1.1d-r2", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
{Name: "libssl1.1", Version: "1.1.1d-r2", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
{Name: "musl", Version: "1.1.22-r3", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
{Name: "openssl", Version: "1.1.1d-r2", LayerID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028"},
{
Name: "busybox",
Version: "1.30.1-r3",
Layer: types.Layer{
Digest: "sha256:a187dde48cd289ac374ad8539930628314bc581a481cdb41409c9289419ddb72",
DiffID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028",
},
},
{
Name: "libcrypto1.1",
Version: "1.1.1d-r2",
Layer: types.Layer{
Digest: "sha256:a187dde48cd289ac374ad8539930628314bc581a481cdb41409c9289419ddb72",
DiffID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028",
},
},
{
Name: "libssl1.1",
Version: "1.1.1d-r2",
Layer: types.Layer{
Digest: "sha256:a187dde48cd289ac374ad8539930628314bc581a481cdb41409c9289419ddb72",
DiffID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028",
},
},
{
Name: "musl",
Version: "1.1.22-r3",
Layer: types.Layer{
Digest: "sha256:a187dde48cd289ac374ad8539930628314bc581a481cdb41409c9289419ddb72",
DiffID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028",
},
},
{
Name: "openssl",
Version: "1.1.1d-r2",
Layer: types.Layer{
Digest: "sha256:a187dde48cd289ac374ad8539930628314bc581a481cdb41409c9289419ddb72",
DiffID: "sha256:531743b7098cb2aaf615641007a129173f63ed86ca32fe7b5a246a1c47286028",
},
},
},
HistoryPackages: []types.Package{
{Name: "musl", Version: "1.1.23"},
Expand Down
16 changes: 12 additions & 4 deletions cache/cache_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,8 @@ func TestFSCache_PutLayer(t *testing.T) {
decompressedLayerID: "sha256:dab15cac9ebd43beceeeda3ce95c574d6714ed3d3969071caead678c065813ec",
layerInfo: types.LayerInfo{
SchemaVersion: 1,
Digest: "sha256:dffd9992ca398466a663c87c92cfea2a2db0ae0cf33fcb99da60eec52addbfc5",
DiffID: "sha256:dab15cac9ebd43beceeeda3ce95c574d6714ed3d3969071caead678c065813ec",
OS: &types.OS{
Family: "alpine",
Name: "3.10",
Expand Down Expand Up @@ -242,6 +244,8 @@ func TestFSCache_PutLayer(t *testing.T) {
want: `
{
"SchemaVersion": 1,
"Digest": "sha256:dffd9992ca398466a663c87c92cfea2a2db0ae0cf33fcb99da60eec52addbfc5",
"DiffID": "sha256:dab15cac9ebd43beceeeda3ce95c574d6714ed3d3969071caead678c065813ec",
"OS": {
"Family": "alpine",
"Name": "3.10"
Expand All @@ -252,7 +256,8 @@ func TestFSCache_PutLayer(t *testing.T) {
"Packages": [
{
"Name": "musl",
"Version": "1.1.22-r3"
"Version": "1.1.22-r3",
"Layer": {}
}
]
}
Expand All @@ -266,13 +271,15 @@ func TestFSCache_PutLayer(t *testing.T) {
"Library":{
"Name":"guzzlehttp/guzzle",
"Version":"6.2.0"
}
},
"Layer": {}
},
{
"Library":{
"Name":"guzzlehttp/promises",
"Version":"v1.3.1"
}
},
"Layer": {}
}
]
}
Expand Down Expand Up @@ -375,7 +382,8 @@ func TestFSCache_PutImage(t *testing.T) {
"HistoryPackages": [
{
"Name": "musl",
"Version": "1.2.3"
"Version": "1.2.3",
"Layer": {}
}
]
}
Expand Down
26 changes: 16 additions & 10 deletions extractor/docker/docker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,29 +104,29 @@ func containsLibrary(e godeptypes.Library, s []types.LibraryInfo) bool {
return false
}

func lookupOriginLayerForPkg(pkg types.Package, layers []types.LayerInfo) digest.Digest {
func lookupOriginLayerForPkg(pkg types.Package, layers []types.LayerInfo) (string, string) {
for _, layer := range layers {
for _, info := range layer.PackageInfos {
if containsPackage(pkg, info.Packages) {
return layer.ID
return layer.Digest, layer.DiffID
}
}
}
return ""
return "", ""
}

func lookupOriginLayerForLib(filePath string, lib godeptypes.Library, layers []types.LayerInfo) digest.Digest {
func lookupOriginLayerForLib(filePath string, lib godeptypes.Library, layers []types.LayerInfo) (string, string) {
for _, layer := range layers {
for _, layerApp := range layer.Applications {
if filePath != layerApp.FilePath {
continue
}
if containsLibrary(lib, layerApp.Libraries) {
return layer.ID
return layer.Digest, layer.DiffID
}
}
}
return ""
return "", ""
}

func ApplyLayers(layers []types.LayerInfo) types.ImageDetail {
Expand Down Expand Up @@ -165,14 +165,20 @@ func ApplyLayers(layers []types.LayerInfo) types.ImageDetail {
})

for i, pkg := range mergedLayer.Packages {
originLayerID := lookupOriginLayerForPkg(pkg, layers)
mergedLayer.Packages[i].LayerID = originLayerID
originLayerDigest, originLayerDiffID := lookupOriginLayerForPkg(pkg, layers)
mergedLayer.Packages[i].Layer = types.Layer{
Digest: originLayerDigest,
DiffID: originLayerDiffID,
}
}

for _, app := range mergedLayer.Applications {
for i, libInfo := range app.Libraries {
originLayerID := lookupOriginLayerForLib(app.FilePath, libInfo.Library, layers)
app.Libraries[i].LayerID = originLayerID
originLayerDigest, originLayerDiffID := lookupOriginLayerForLib(app.FilePath, libInfo.Library, layers)
app.Libraries[i].Layer = types.Layer{
Digest: originLayerDigest,
DiffID: originLayerDiffID,
}
}
}

Expand Down
Loading

0 comments on commit c536335

Please sign in to comment.