fix(vuln) unique vulnerabilities from different data sources (aquasec…

…urity#984) * Fix duplicate Rule in sarif template * Fix integration tests * Fixed tests * Update certs validity upto 2100 * Moved deduplication logic to Filter * Fix linting issue * Fix liniting issue * fix: deduplicate vulnerabilities * refactor * fix: add installed versions to uniq keys * Fix tests * Fix Unit tests. * Revert port change Co-authored-by: knqyf263 <[email protected]>
vishnugonela · May 12, 2021 · e26e39a · e26e39a
1 parent 04e7cca
commit e26e39a
Show file tree

Hide file tree

Showing 8 changed files with 246 additions and 86 deletions.
diff --git a/contrib/sarif.tpl b/contrib/sarif.tpl
@@ -20,7 +20,7 @@
                 ,
               {{- end }}
             {
-              "id": "{{ .VulnerabilityID }}/{{ .PkgName }}",
+              "id": "{{ .VulnerabilityID }}/{{ .PkgName }}/{{ .InstalledVersion }}",
               "name": "{{ toSarifRuleName $vulnerabilityType }}",
               "shortDescription": {
                 "text": {{ printf "%v Package: %v" .VulnerabilityID .PkgName | printf "%q" }}
@@ -66,7 +66,7 @@
             ,
           {{- end }}
         {
-          "ruleId": "{{ $vulnerability.VulnerabilityID }}/{{ $vulnerability.PkgName }}",
+          "ruleId": "{{ $vulnerability.VulnerabilityID }}/{{ $vulnerability.PkgName }}/{{ $vulnerability.InstalledVersion }}",
           "ruleIndex": {{ $index }},
           "level": "{{ toSarifErrorLevel $vulnerability.Vulnerability.Severity }}",
           "message": {
@@ -92,4 +92,4 @@
       }
     }
   ]
-}
+}
diff --git a/integration/data/certs/cert.pem b/integration/data/certs/cert.pem
@@ -1,18 +1,17 @@
 -----BEGIN CERTIFICATE-----
-MIIC+jCCAeKgAwIBAgIRAJLJ5vw48YZwoHlC8i6VdHswDQYJKoZIhvcNAQELBQAw
-EjEQMA4GA1UEChMHQWNtZSBDbzAeFw0yMDA1MDMxMTU2MzhaFw0yMTA1MDMxMTU2
-MzhaMBIxEDAOBgNVBAoTB0FjbWUgQ28wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
-ggEKAoIBAQDhnepAL1Atd1xVh/TOZpTK7yHwtOrtGWNEqNkFbcyD7x9CNgUkxjO8
-nc4ynEo4ARpLj+2VDLIwi93weCFj6mcz2tdHi7n0eiPR7+PSNMNpPFwablLOEtaX
-XVqHhJNsHcJx6okX6ullksJoRnZGu+n1LvGRMMLWjS3UJZA6+1pujoifyrx9YXLU
-qSjkRRv3Ly8HmAPJq0T19uCZiJ8qbrW1Vx3hdUILL4OlJmpjZvGKMRnolinko2Vk
-0pHH5MWz0iUbqWQjHZmQWi0rDHRAFbuCqQdmFsEneXmUzExXZbyHwrTH/mrjJTCJ
-YmtR7Eq80AxsWnXNI3Z0mVQ9/nZDsT31AgMBAAGjSzBJMA4GA1UdDwEB/wQEAwIF
-oDATBgNVHSUEDDAKBggrBgEFBQcDATAMBgNVHRMBAf8EAjAAMBQGA1UdEQQNMAuC
-CWxvY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAG7spAmpZVKOV913WhKZp8o1+
-T6v/b6KUOAZ0iWXeGWO7LwnRaulkyauGav9xJUpfEw7Z/57qwTVIGZD6mZF6tt77
-FsXXiQJA88LrQnt1BRTeNK8sRp4S3N1hrtY3akTit/dyQcfh3NSDttzkYsoUu0qT
-DUkXD0b4eDmaD47+0Z6eIVp3aEcPMzpiy6qWc5fMjMeHjtYF4lBSF0JTWzmxNUGl
-fiGhMJStQK/n73t58O7h5Adva5wRV+Km6pa+6SfOxPNUjsxXjG0LzWA9dJg/q2rs
-k/ouIE05BfB3z538ncQVBTwfPMClbIiJhAs3b6ej22+j/O+vbFBmdfkpVpFRtg==
+MIICwTCCAamgAwIBAgIJAP09YW8ChPlwMA0GCSqGSIb3DQEBCwUAMBIxEDAOBgNV
+BAoMB0FjbWUgQ28wIBcNMjEwNTEyMDQ0NzA1WhgPMjEwMDA0MTQwNDQ3MDVaMBIx
+EDAOBgNVBAoMB0FjbWUgQ28wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
+AQDNmKpDOzU8GK5Xb3GfeqU1kKQ0gBejGtqK5ydH8tlRoy2NKGvjJ95nhIxUXMKe
+e345JFlzkCen5Ekvt70LT0O253z0FecfpaFilreIiu5J2YWWNtlruMhpjp4kYVMO
+piKnujiNK9eAUcz++YeAmrog7QPBJBCgdu18xTy/yOW/Y414e1efvbRJZ4TaQb0Y
+LgXRl1nlOLPPr5ew9pgnct7DxJVXpjXtgBxCsfcjH4kZGfc9zP0IKyODqaSCFRtj
+eKH8gSpJCimBp3hpWvsSTHTRraOxAGXqhIYPhqRM83eB2QbeHnyk+YOn76pdMndb
+vqAPksmTyHcgZShkhGcHKvbVAgMBAAGjGDAWMBQGA1UdEQQNMAuCCWxvY2FsaG9z
+dDANBgkqhkiG9w0BAQsFAAOCAQEAHxXOTKGP1hl3J2jQrpha5LuYdMEbK1HFbPhV
+042k0tBmfP3wRgx0o/WQhg4f5RswQRtipdUCmMZVOAoQfos8j9LFmIKwcsboEQe/
+Fvqq2+W/5TRhsKn/1OxvCZAEurazSygtm6hyiMGwKjJLfyzwjZx+Oopn3lqRUP36
+gLQQ57szoNZFKyPN2z2unXAuDG5wpG2InX8WJvlrhaiCHGUoxO8r0rVawm58bahM
+uGPlVPCNdxl1h7K8aecKpm+7Wh8n06Nl/kOWBDFAXeI8IwrnIy1rAZLngvnjqL//
+umjXKCBWya48ed9HMoOR2aruzseXc8k6cGXuBxYFtHissPvPPQ==
 -----END CERTIFICATE-----
diff --git a/integration/data/certs/key.pem b/integration/data/certs/key.pem
@@ -1,28 +1,27 @@
------BEGIN PRIVATE KEY-----
-MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDhnepAL1Atd1xV
-h/TOZpTK7yHwtOrtGWNEqNkFbcyD7x9CNgUkxjO8nc4ynEo4ARpLj+2VDLIwi93w
-eCFj6mcz2tdHi7n0eiPR7+PSNMNpPFwablLOEtaXXVqHhJNsHcJx6okX6ullksJo
-RnZGu+n1LvGRMMLWjS3UJZA6+1pujoifyrx9YXLUqSjkRRv3Ly8HmAPJq0T19uCZ
-iJ8qbrW1Vx3hdUILL4OlJmpjZvGKMRnolinko2Vk0pHH5MWz0iUbqWQjHZmQWi0r
-DHRAFbuCqQdmFsEneXmUzExXZbyHwrTH/mrjJTCJYmtR7Eq80AxsWnXNI3Z0mVQ9
-/nZDsT31AgMBAAECggEBAKwwGhSMR3O7sdNxJIvVzF8orE2JtfXoN1OyTZcQGlLi
-z4d3tOtA/UFJapJDp30gklHy8Y6clu3oASVCebFItyTjMwPehrgn82iI3eWS8URC
-lcRySG4QAIia7bmZm+2atMi+B40icqhbnlV42VHYnpDKGAEIJtsZ+kz7shzhsj3G
-yTQMFyuqk0DUmsbSVKPjryv15DXsT9Rk2pVZYFhiRw/gQpWD58GMP/HMrSz+sjuX
-ZIlhSMGVWA4Yc7le4PpWI2qAZLR+X1EgkzxcMJ0kWvnvzEXFmofaYzkbEcNOlguF
-Bv9kP5fh35AbQbTLykGO9h4VrfDajlHequzNBJs1z60CgYEA53cwBh42pg/fSmaO
-sowpFV52ZfbfUPcuXRuaidHWougByB8P8XTMeQTse4NLt+2oat/5rdP3keGr5OR5
-8q7v8/R/KY0NQOa/93BUeRDW4ntxMECWbC2p/sq2wnRKTl+yepAWrRXzk8z9vFP/
-TZM5m65aj3IsZ3Bo1WG+SSf8bvsCgYEA+YgFxmiTauKRO4IVPuOqJ88yC5SQ83mF
-T54ILYalG3yq/Jm1TTOzoZAoKvHrJeeZqQvjS4jSY5gc5TCrUVTdsw5nXtrRKZJs
-HjtVT78qfzjCSHzImvc3Rw5+SNO2+j9yxuBSAG4tEKD3KKxSodXnKtD4CwzvRdyI
-gUyjQi3Os88CgYEAgrzegkYkhe2nKKX+6bijJ+/AHl2vy1KifHKv+jJs8nzrLLbm
-0XIwYBa44BbL+Oqi2yMBKv7z8hEuf03R15KZ9Ahgnv6Nwt/TBBcNj4hEZ45j42ZH
-0HiGcWTcj78RjW0eKX4jYMZqW0xI8Uvcg1uqCVYUzrsle5ORkxzvVvDf82sCgYEA
-4hS9tsA1IJhaoaIAgdRf7GWroBZhJlep0zMJkcX2fer8OJVDUMlRLUahPhelx9gI
-vsLIkz1J8XZ2Z6kq7yuHGp4oRibXb2T8lH+JkhFP/ah9TpPQZacq7DRTcsRvelhW
-M542bbFlHzXX+X/39i0Jnx9qPQjhGVjwTMYU/Pbn2r0CgYAkwwh2oq6BP3w7/4xe
-giaW/5zzMA8R9ZXFfkE4OXk2vig8LzUn1IO0JeGLyHQbdg8exYxTU3zygIlSvCQP
-Zbl0+RB+NHwGOqlpEDdPFXsqi1GAdWOC6FzYtBFNk9WyjFICXjB42wnfIIUQPLU1
-kQKFaehfx9KR0iW1dnm3vbFlzw==
------END PRIVATE KEY-----
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAzZiqQzs1PBiuV29xn3qlNZCkNIAXoxraiucnR/LZUaMtjShr
+4yfeZ4SMVFzCnnt+OSRZc5Anp+RJL7e9C09Dtud89BXnH6WhYpa3iIruSdmFljbZ
+a7jIaY6eJGFTDqYip7o4jSvXgFHM/vmHgJq6IO0DwSQQoHbtfMU8v8jlv2ONeHtX
+n720SWeE2kG9GC4F0ZdZ5Tizz6+XsPaYJ3Lew8SVV6Y17YAcQrH3Ix+JGRn3Pcz9
+CCsjg6mkghUbY3ih/IEqSQopgad4aVr7Ekx00a2jsQBl6oSGD4akTPN3gdkG3h58
+pPmDp++qXTJ3W76gD5LJk8h3IGUoZIRnByr21QIDAQABAoIBAQCVN2ETjIxVgqA+
+K08u7Ses2b6jr/f31AybVasnx/S8EI+F7Llo003SmdvzeqNxvLVeqagWfKCbdM89
+R8B3zd6aiCYjTSZCzMZ1tGeePR83EB2paUOhsCocmnricpSChEeQrlJO+2vb4QLE
+Z7xVtXazYPIhophCri4tKUWu+BLvNPez+TndaE5Xg77HLmu24rloZh6XhYDdFWd+
+u/eF+QiWy4/EoLUv2TLym8ivUws+r2G9yK57kcQCJw+BqlaRew7Ts0RHnam53OxV
+T4dEHJxAfXO8jC1F5NCjoBO/+0HJqrMtD0NqWH9G+fEtakL7h5oeh6vYrSQfpZGC
+V7MXojqdAoGBAOhwBi0erXOn4strtkGvSjJ6HVLwWfmm+rlfm23JGigYghYTSxBM
+ESuwppt1QPXK5jfil89RqrvDqKG9BjXV4yWyaJlIRaYeJe8/TZa3e8WkLr0XaKGH
+v1LTW+/uc73ihDJ/M2axmP4vjThCfqiG9aKXLCDM+DIgfdvIbkXUfPZjAoGBAOJw
+Fc6D3z0r09F3/UgtADhQlbD2jzs6xdcqCu7af3527F6ePXXU8CTLS5jusAiDW5xH
+ukQS/0ZM92UTUJxpQxgzHSWOImhcv3o45vzQ0C0pXSSaE+Pp8QYWaE/BdE+VoVOK
+YGAfppZywPGnKYt4R5ho5XLwAL3rrH+2m7z51mdnAoGBANP5LbjCLF64Mb1f2pOm
+f1zvPoTfyr5BSI/7n+yMJL2CNEhbie4v4MzeSeKmGPrO8grvK5EXIkQgGE5/6wT3
+rTI4tOltHo9zGRdJvMGBTXAd3b32diYxfQrU1BhIducph3PhyweRWTweM4SmJ4ob
+ojGH+edj5ckZFo50CBTIxrmPAoGBAI8SpSSsfCRJiffjadzt2iK7AColT9DrzM+r
+1+adlksQ1z7dmxXVqrqE3UpPHljyrrKrO40Bt9vyi6qIrrl1ZRhoS3VMPn9UgwO1
+6nU5dx/h7+FNnV23ljvzcotaP6R9dca0OzrhJMAQ18qYhY6DPDGXrcqWzNEzlPXJ
+KtQXxBnnAoGAFQPvW/wDahrGcm1MBw83E0TgNJpoB10tz6R1dLdKVSHJUXMfxmij
+Wj4MaF0JB0GWRRjutng+i7y7Tx+mUpu80qV8E9zAH7jGFnpqjw8A9zp5ftK00e7Y
+shRlg+lhJhlvMA5QCYNzpYj+7EXJm7nzbhC6pMtBapT9a/MUPYFte38=
+-----END RSA PRIVATE KEY-----
diff --git a/integration/testdata/alpine-310.sarif.golden b/integration/testdata/alpine-310.sarif.golden
@@ -11,7 +11,7 @@
           "version": "0.15.0",
           "rules": [
             {
-              "id": "CVE-2019-1549/libcrypto1.1",
+              "id": "CVE-2019-1549/libcrypto1.1/1.1.1c-r0",
               "name": "OS Package Vulnerability (Alpine)",
               "shortDescription": {
                 "text": "CVE-2019-1549 Package: libcrypto1.1"
@@ -37,7 +37,7 @@
               }
             },
             {
-              "id": "CVE-2019-1551/libcrypto1.1",
+              "id": "CVE-2019-1551/libcrypto1.1/1.1.1c-r0",
               "name": "OS Package Vulnerability (Alpine)",
               "shortDescription": {
                 "text": "CVE-2019-1551 Package: libcrypto1.1"
@@ -63,7 +63,7 @@
               }
             },
             {
-              "id": "CVE-2019-1563/libcrypto1.1",
+              "id": "CVE-2019-1563/libcrypto1.1/1.1.1c-r0",
               "name": "OS Package Vulnerability (Alpine)",
               "shortDescription": {
                 "text": "CVE-2019-1563 Package: libcrypto1.1"
@@ -89,7 +89,7 @@
               }
             },
             {
-              "id": "CVE-2019-1547/libcrypto1.1",
+              "id": "CVE-2019-1547/libcrypto1.1/1.1.1c-r0",
               "name": "OS Package Vulnerability (Alpine)",
               "shortDescription": {
                 "text": "CVE-2019-1547 Package: libcrypto1.1"
@@ -115,7 +115,7 @@
               }
             },
             {
-              "id": "CVE-2019-1549/libssl1.1",
+              "id": "CVE-2019-1549/libssl1.1/1.1.1c-r0",
               "name": "OS Package Vulnerability (Alpine)",
               "shortDescription": {
                 "text": "CVE-2019-1549 Package: libssl1.1"
@@ -141,7 +141,7 @@
               }
             },
             {
-              "id": "CVE-2019-1551/libssl1.1",
+              "id": "CVE-2019-1551/libssl1.1/1.1.1c-r0",
               "name": "OS Package Vulnerability (Alpine)",
               "shortDescription": {
                 "text": "CVE-2019-1551 Package: libssl1.1"
@@ -167,7 +167,7 @@
               }
             },
             {
-              "id": "CVE-2019-1563/libssl1.1",
+              "id": "CVE-2019-1563/libssl1.1/1.1.1c-r0",
               "name": "OS Package Vulnerability (Alpine)",
               "shortDescription": {
                 "text": "CVE-2019-1563 Package: libssl1.1"
@@ -193,7 +193,7 @@
               }
             },
             {
-              "id": "CVE-2019-1547/libssl1.1",
+              "id": "CVE-2019-1547/libssl1.1/1.1.1c-r0",
               "name": "OS Package Vulnerability (Alpine)",
               "shortDescription": {
                 "text": "CVE-2019-1547 Package: libssl1.1"
@@ -222,7 +222,7 @@
       },
       "results": [
         {
-          "ruleId": "CVE-2019-1549/libcrypto1.1",
+          "ruleId": "CVE-2019-1549/libcrypto1.1/1.1.1c-r0",
           "ruleIndex": 0,
           "level": "warning",
           "message": {
@@ -238,7 +238,7 @@
           }]
         },
         {
-          "ruleId": "CVE-2019-1551/libcrypto1.1",
+          "ruleId": "CVE-2019-1551/libcrypto1.1/1.1.1c-r0",
           "ruleIndex": 1,
           "level": "warning",
           "message": {
@@ -254,7 +254,7 @@
           }]
         },
         {
-          "ruleId": "CVE-2019-1563/libcrypto1.1",
+          "ruleId": "CVE-2019-1563/libcrypto1.1/1.1.1c-r0",
           "ruleIndex": 2,
           "level": "warning",
           "message": {
@@ -270,7 +270,7 @@
           }]
         },
         {
-          "ruleId": "CVE-2019-1547/libcrypto1.1",
+          "ruleId": "CVE-2019-1547/libcrypto1.1/1.1.1c-r0",
           "ruleIndex": 3,
           "level": "note",
           "message": {
@@ -286,7 +286,7 @@
           }]
         },
         {
-          "ruleId": "CVE-2019-1549/libssl1.1",
+          "ruleId": "CVE-2019-1549/libssl1.1/1.1.1c-r0",
           "ruleIndex": 4,
           "level": "warning",
           "message": {
@@ -302,7 +302,7 @@
           }]
         },
         {
-          "ruleId": "CVE-2019-1551/libssl1.1",
+          "ruleId": "CVE-2019-1551/libssl1.1/1.1.1c-r0",
           "ruleIndex": 5,
           "level": "warning",
           "message": {
@@ -318,7 +318,7 @@
           }]
         },
         {
-          "ruleId": "CVE-2019-1563/libssl1.1",
+          "ruleId": "CVE-2019-1563/libssl1.1/1.1.1c-r0",
           "ruleIndex": 6,
           "level": "warning",
           "message": {
@@ -334,7 +334,7 @@
           }]
         },
         {
-          "ruleId": "CVE-2019-1547/libssl1.1",
+          "ruleId": "CVE-2019-1547/libssl1.1/1.1.1c-r0",
           "ruleIndex": 7,
           "level": "note",
           "message": {
@@ -357,4 +357,4 @@
       }
     }
   ]
-}
+}
diff --git a/pkg/report/writer_test.go b/pkg/report/writer_test.go
@@ -409,7 +409,7 @@ func TestReportWriter_Template_SARIF(t *testing.T) {
           "version": "0.15.0",
           "rules": [
             {
-              "id": "CVE-1234-5678/foopackage",
+              "id": "CVE-1234-5678/foopackage/1.2.3",
               "name": "Other Vulnerability (Footype)",
               "shortDescription": {
                 "text": "CVE-1234-5678 Package: foopackage"
@@ -437,7 +437,7 @@ func TestReportWriter_Template_SARIF(t *testing.T) {
       },
       "results": [
         {
-          "ruleId": "CVE-1234-5678/foopackage",
+          "ruleId": "CVE-1234-5678/foopackage/1.2.3",
           "ruleIndex": 0,
           "level": "error",
           "message": {
@@ -493,7 +493,7 @@ func TestReportWriter_Template_SARIF(t *testing.T) {
           "version": "0.15.0",
           "rules": [
             {
-              "id": "CVE-1234-5678/foopackage",
+              "id": "CVE-1234-5678/foopackage/1.2.3",
               "name": "Other Vulnerability (Footype)",
               "shortDescription": {
                 "text": "CVE-1234-5678 Package: foopackage"
@@ -522,7 +522,7 @@ func TestReportWriter_Template_SARIF(t *testing.T) {
       },
       "results": [
         {
-          "ruleId": "CVE-1234-5678/foopackage",
+          "ruleId": "CVE-1234-5678/foopackage/1.2.3",
           "ruleIndex": 0,
           "level": "error",
           "message": {

diff --git a/pkg/types/vulnerability.go b/pkg/types/vulnerability.go
@@ -28,6 +28,8 @@ func (v BySeverity) Len() int { return len(v) }
 func (v BySeverity) Less(i, j int) bool {
 	if v[i].PkgName != v[j].PkgName {
 		return v[i].PkgName < v[j].PkgName
+	} else if v[i].InstalledVersion != v[j].InstalledVersion {
+		return v[i].InstalledVersion < v[j].InstalledVersion
 	}
 	ret := types.CompareSeverityString(
 		v[j].Severity, v[i].Severity,