Skip to content

Commit

Permalink
Feat(E2E): Added E2E Test to validate CSV reports (Checkmarx#5065)
Browse files Browse the repository at this point in the history
* added E2E tests to validate CSV Results

* lint fixes & new error handling

* update file close - csv

* fix - Errors unhandled

* changing csvSchema name
  • Loading branch information
cxlucas authored Mar 30, 2022
1 parent 649b892 commit 428f243
Show file tree
Hide file tree
Showing 5 changed files with 293 additions and 4 deletions.
11 changes: 11 additions & 0 deletions e2e/cli_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,17 @@ func checkExpectedOutput(t *testing.T, tt *testcases.TestCase, argIndex int) {
json := utils.XMLToJSON(t, filename, "cyclonedx")
utils.JSONSchemaValidationFromData(t, json, "result-cyclonedx.json")
}
// Check result file (CSV)
if utils.Contains(resultsFormats, "csv") || utils.Contains(resultsFormats, "csv-cis") {
filename := tt.Args.ExpectedResult[argIndex].ResultsFile + ".csv"
json := utils.CSVToJSON(t, filename)

if utils.Contains(resultsFormats, "csv-cis") {
utils.JSONSchemaValidationFromData(t, json, "result-csv-cis.json")
} else {
utils.JSONSchemaValidationFromData(t, json, "result-csv.json")
}
}
}

func prepareTemplates() testcases.TestTemplates {
Expand Down
33 changes: 33 additions & 0 deletions e2e/fixtures/schemas/result-csv-cis.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
{
"allOf": [
{
"$ref": "result-csv.json"
},
{
"type": "array",
"minItems": 1,
"items": {
"type": "object",
"required": [
"CISDescriptionIDFormatted",
"CISDescriptionTitle",
"CISDescriptionTextFormatted"
],
"properties": {
"CISDescriptionIDFormatted": {
"type": "string",
"minLength": 1
},
"CISDescriptionTitle": {
"type": "string",
"minLength": 1
},
"CISDescriptionTextFormatted": {
"type": "string",
"minLength": 1
}
}
}
}
]
}
153 changes: 153 additions & 0 deletions e2e/fixtures/schemas/result-csv.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
{
"type": "array",
"minItems": 1,
"definitions": {
"query_id_pattern": {
"type": "string",
"minLength": 1,
"pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-4{1}[a-f0-9]{3}-[89ab]{1}[a-f0-9]{3}-[a-f0-9]{12}$"
},
"similarity_id_pattern": {
"type": "string",
"minLength": 1,
"pattern": "^[A-Fa-f0-9]{64}$"
},
"description_id_pattern": {
"type": "string",
"minLength": 1,
"pattern": "^[a-f0-9]{8}$"
},
"file_name_pattern": {
"type": "string",
"oneOf": [
{
"pattern": "^(.)*(:)*(http:|https:|www\\.)(.)+$"
},
{
"pattern": "^([\\w\\-. ]+(\\\\|\\/))*([\\w\\-. ]+(\\\\|\\/).(.)*)$"
}
]
}
},
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"QueryName",
"QueryID",
"QueryURI",
"Severity",
"Platform",
"CloudProvider",
"Category",
"DescriptionID",
"Description",
"CISDescriptionIDFormatted",
"CISDescriptionTitle",
"CISDescriptionTextFormatted",
"FileName",
"SimilarityID",
"Line",
"IssueType",
"SearchKey",
"SearchLine",
"SearchValue",
"ExpectedValue",
"ActualValue"
],
"properties": {
"QueryName": {
"type": "string",
"minLength": 1
},
"QueryID": {
"$ref": "#/definitions/query_id_pattern"
},
"QueryURI": {
"type": "string",
"format": "uri"
},
"Severity": {
"type": "string",
"enum": [
"HIGH",
"MEDIUM",
"LOW",
"INFO"
]
},
"Platform": {
"type": "string",
"enum": [
"Ansible",
"AzureResourceManager",
"Buildah",
"CloudFormation",
"Common",
"Dockerfile",
"DockerCompose",
"GRPC",
"GoogleDeploymentManager",
"Kubernetes",
"OpenAPI",
"Terraform"
]
},
"CloudProvider": {
"type": "string",
"minLength": 1
},
"Category": {
"type": "string",
"minLength": 1
},
"DescriptionID": {
"$ref": "#/definitions/description_id_pattern"
},
"Description": {
"type": "string",
"minLength": 1
},
"CISDescriptionIDFormatted": {
"type": "string"
},
"CISDescriptionTitle": {
"type": "string"
},
"CISDescriptionTextFormatted": {
"type": "string"
},
"FileName": {
"$ref": "#/definitions/file_name_pattern"
},
"SimilarityID": {
"$ref": "#/definitions/similarity_id_pattern"
},
"Line": {
"type": "integer",
"minimum": 0
},
"IssueType": {
"type": "string"
},
"SearchKey": {
"type": "string"
},
"SearchLine": {
"type": "integer",
"minimum": 0
},
"SearchValue": {
"type": "string"
},
"ExpectedValue": {
"type": "string",
"minLength": 1
},
"ActualValue": {
"type": "string",
"minLength": 1
}
}
}
}
8 changes: 4 additions & 4 deletions e2e/testcases/e2e-cli-031_scan_report-formats.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,21 @@ func init() { //nolint
Args: args{
Args: []cmdArgs{
[]string{"scan", "--output-path", "/path/e2e/output", "--output-name", "E2E_CLI_031_RESULT",
"--report-formats", "json,SARIF,glsast,Html,SonarQUBE,Junit,cyclonedx,asff",
"--report-formats", "json,SARIF,glsast,Html,SonarQUBE,Junit,cyclonedx,asff,csv",
"-p", "/path/e2e/fixtures/samples/positive.yaml"},

[]string{"scan", "--output-path", "/path/e2e/output", "--output-name", "E2E_CLI_031_RESULT_CIS",
"--report-formats", "json,JUnit", "--include-queries", "275a3217-ca37-40c1-a6cf-bb57d245ab32",
"--report-formats", "json,JUnit,CSV", "--include-queries", "275a3217-ca37-40c1-a6cf-bb57d245ab32",
"-p", "/path/e2e/fixtures/samples/positive.yaml"},
},
ExpectedResult: []ResultsValidation{
{
ResultsFile: "E2E_CLI_031_RESULT",
ResultsFormats: []string{"json", "sarif", "glsast", "html", "sonarqube", "junit", "cyclonedx", "asff"},
ResultsFormats: []string{"json", "sarif", "glsast", "html", "sonarqube", "junit", "cyclonedx", "asff", "csv"},
},
{
ResultsFile: "E2E_CLI_031_RESULT_CIS",
ResultsFormats: []string{"junit", "json-cis"},
ResultsFormats: []string{"junit", "json-cis", "csv-cis"},
},
},
UseMock: []bool{false, true},
Expand Down
92 changes: 92 additions & 0 deletions e2e/utils/csv.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
package utils

import (
"encoding/csv"
"encoding/json"
"os"
"path/filepath"
"strconv"
"testing"

"github.com/stretchr/testify/require"
)

// CSVToJSON - converts CSV to JSON Structure
func CSVToJSON(t *testing.T, filename string) []byte {
cwd, _ := os.Getwd()
filePath := filepath.Join("output", filename)
fullPath := filepath.Join(cwd, filePath)

csvFile, err := os.Open(fullPath)
require.NoError(t, err, "Error reading file: %s", fullPath)

reader := csv.NewReader(csvFile)
reader.FieldsPerRecord = -1
csvData, err := reader.ReadAll()
require.NoError(t, err, "Error reading CSV file: %s", fullPath)

err = csvFile.Close()
require.NoError(t, err, "Error when closing file: %s", fullPath)

var csvStruct csvSchema
var csvItems []csvSchema

for _, row := range csvData[1:] {
line, lineErr := strconv.Atoi(row[14])
require.NoError(t, lineErr, "Error when converting CSV: %s", fullPath)
searchLine, searchErr := strconv.Atoi(row[17])
require.NoError(t, searchErr, "Error when converting CSV: %s", fullPath)

csvStruct.QueryName = row[0]
csvStruct.QueryID = row[1]
csvStruct.QueryURI = row[2]
csvStruct.Severity = row[3]
csvStruct.Platform = row[4]
csvStruct.CloudProvider = row[5]
csvStruct.Category = row[6]
csvStruct.DescriptionID = row[7]
csvStruct.Description = row[8]
csvStruct.CISDescriptionIDFormatted = row[9]
csvStruct.CISDescriptionTitle = row[10]
csvStruct.CISDescriptionTextFormatted = row[11]
csvStruct.FileName = row[12]
csvStruct.SimilarityID = row[13]
csvStruct.Line = line
csvStruct.IssueType = row[15]
csvStruct.SearchKey = row[16]
csvStruct.SearchLine = searchLine
csvStruct.SearchValue = row[18]
csvStruct.ExpectedValue = row[19]
csvStruct.ActualValue = row[20]
csvItems = append(csvItems, csvStruct)
}

jsondata, err := json.Marshal(csvItems)
require.NoError(t, err, "Error marshaling file: %s", fullPath)

return jsondata
}

type csvSchema struct {
QueryName string
QueryID string
QueryURI string
Severity string
Platform string
CloudProvider string
Category string
DescriptionID string
Description string
CISDescriptionIDFormatted string
CISDescriptionTitle string
CISDescriptionTextFormatted string
FileName string
SimilarityID string
Line int
IssueType string
SearchKey string
SearchLine int
SearchValue string
ExpectedValue string
ActualValue string
}

0 comments on commit 428f243

Please sign in to comment.