Merge pull request Checkmarx#4339 from Checkmarx/tests/refactor-e2e-t…

…ests-structure

refactor: E2E Tests Structure

e2e/testcases/e2e-cli-001_help_text.go

@@ -0,0 +1,18 @@
+package testcases
+
+// E2E-CLI-001 - KICS command should display a help text in the CLI when provided with the
+// --help flag and it should describe the available commands plus the global flags
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should display the kics help text [E2E-CLI-001]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"--help"},
+			},
+			ExpectedOut: []string{"E2E_CLI_001"},
+		},
+		WantStatus: []int{0},
+	}
+
+	Tests = append(Tests, testSample)
+}

e2e/testcases/e2e-cli-002_help_scan.go

@@ -0,0 +1,18 @@
+package testcases
+
+// E2E-CLI-002 - KICS scan command should display a help text in the CLI when provided with the
+// --help flag and it should describe the options related with scan plus the global options
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should display the kics scan help text [E2E-CLI-002]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"scan", "--help"},
+			},
+			ExpectedOut: []string{"E2E_CLI_002"},
+		},
+		WantStatus: []int{0},
+	}
+
+	Tests = append(Tests, testSample)
+}

e2e/testcases/e2e-cli-003_scan_text.go

@@ -0,0 +1,19 @@
+package testcases
+
+// E2E-CLI-003 - KICS scan command had a mandatory flag -p the CLI should exhibit
+// an error message and return exit code 1
+
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should display an error regarding missing -p flag [E2E-CLI-003]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"scan"},
+			},
+			ExpectedOut: []string{"E2E_CLI_003"},
+		},
+		WantStatus: []int{126},
+	}
+
+	Tests = append(Tests, testSample)
+}

e2e/testcases/e2e-cli-004_invalid_combination.go

@@ -0,0 +1,23 @@
+package testcases
+
+// E2E-CLI-004 - KICS has an invalid flag combination
+// an error message and return exit code 1
+
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should display an error of invalid flag combination [E2E-CLI-004]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"scan", "--ci", "--verbose"},
+				[]string{"--ci", "scan", "--verbose"},
+			},
+			ExpectedOut: []string{
+				"E2E_CLI_004",
+				"E2E_CLI_004",
+			},
+		},
+		WantStatus: []int{126, 126},
+	}
+
+	Tests = append(Tests, testSample)
+}

e2e/testcases/e2e-cli-005_payload-path.go

@@ -0,0 +1,25 @@
+package testcases
+
+// E2E-CLI-005 - KICS scan with -- payload-path flag should create a file with the
+// passed name containing the payload of the files scanned
+
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should create a payload file [E2E-CLI-005]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"scan", "--silent", "-q", "../assets/queries", "-p", "fixtures/samples/terraform.tf",
+					"--payload-path", "output/E2E_CLI_005_PAYLOAD.json"},
+			},
+			ExpectedOut: []string{
+				"E2E_CLI_005",
+			},
+			ExpectedPayload: []string{
+				"E2E_CLI_005_PAYLOAD.json",
+			},
+		},
+		WantStatus: []int{50},
+	}
+
+	Tests = append(Tests, testSample)
+}

e2e/testcases/e2e-cli-006_generate-id.go

@@ -0,0 +1,24 @@
+package testcases
+
+import "regexp"
+
+// E2E-CLI-006 - KICS generate-id should exhibit
+// a valid UUID in the CLI and return exit code 0
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should generate a valid ID [E2E-CLI-006]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"generate-id"},
+			},
+		},
+		WantStatus: []int{0},
+		Validation: func(outputText string) bool {
+			uuidRegex := "[a-f0-9]{8}-[a-f0-9]{4}-4{1}[a-f0-9]{3}-[89ab]{1}[a-f0-9]{3}-[a-f0-9]{12}"
+			match, _ := regexp.MatchString(uuidRegex, outputText)
+			return match
+		},
+	}
+
+	Tests = append(Tests, testSample)
+}

e2e/testcases/e2e-cli-007_scan.go

@@ -0,0 +1,28 @@
+package testcases
+
+import "regexp"
+
+// E2E-CLI-007 - the default kics scan must show informations such as 'Files scanned',
+// 'Queries loaded', 'Scan Duration', '...' in the CLI
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should perform a simple scan [E2E-CLI-007]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"scan", "-q", "../assets/queries", "-p", "fixtures/samples/terraform.tf"},
+			},
+		},
+		WantStatus: []int{50},
+		Validation: func(outputText string) bool {
+			match1, _ := regexp.MatchString(`Files scanned: \d+`, outputText)
+			match2, _ := regexp.MatchString(`Parsed files: \d+`, outputText)
+			match3, _ := regexp.MatchString(`Queries loaded: \d+`, outputText)
+			match4, _ := regexp.MatchString(`Queries failed to execute: \d+`, outputText)
+			match5, _ := regexp.MatchString(`Results Summary:`, outputText)
+			match6, _ := regexp.MatchString(`Scan duration: \d+(m\d+)?(.\d+)?s`, outputText)
+			return match1 && match2 && match3 && match4 && match5 && match6
+		},
+	}
+
+	Tests = append(Tests, testSample)
+}

e2e/testcases/e2e-cli-008_scan_silent.go

@@ -0,0 +1,19 @@
+package testcases
+
+// E2E-CLI-008 - KICS  scan with --silent global flag
+// should hide all the output text in the CLI (empty output)
+
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should hide all output text in CLI [E2E-CLI-008]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"scan", "--silent", "-q", "../assets/queries", "-p", "fixtures/samples/terraform.tf"},
+			},
+			ExpectedOut: []string{"E2E_CLI_008"},
+		},
+		WantStatus: []int{50},
+	}
+
+	Tests = append(Tests, testSample)
+}

e2e/testcases/e2e-cli-009_scan_no-progress.go

@@ -0,0 +1,25 @@
+package testcases
+
+import "regexp"
+
+// E2E-CLI-009 - kics scan with no-progress flag
+// should perform a scan without showing progress bar in the CLI
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should hide the progress bar in the CLI [E2E-CLI-009]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"scan", "-q", "../assets/queries", "-p", "fixtures/samples/terraform.tf", "--no-progress"},
+			},
+		},
+		WantStatus: []int{50},
+		Validation: func(outputText string) bool {
+			getProgressRegex := "Executing queries:"
+			match, _ := regexp.MatchString(getProgressRegex, outputText)
+			// if not found -> the the test was successful
+			return !match
+		},
+	}
+
+	Tests = append(Tests, testSample)
+}

e2e/testcases/e2e-cli-010_scan_invalid_type.go

@@ -0,0 +1,24 @@
+package testcases
+
+import "regexp"
+
+// E2E-CLI-010 - KICS  scan with invalid --type flag
+// should exhibit an error message and return exit code 1
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should display an error message about unknown argument [E2E-CLI-010]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"scan", "-q", "../assets/queries", "-p", "fixtures/samples/terraform.tf", "-t", "xml", "--silent"},
+			},
+		},
+		Validation: func(outputText string) bool {
+			unknownArgRegex := regexp.MustCompile(`Error: unknown argument\(s\) for --type: xml`)
+			match := unknownArgRegex.MatchString(outputText)
+			return match
+		},
+		WantStatus: []int{126},
+	}
+
+	Tests = append(Tests, testSample)
+}

e2e/testcases/e2e-cli-011_scan_type.go

@@ -0,0 +1,21 @@
+package testcases
+
+// E2E-CLI-011 - KICS  scan with a valid case insensitive --type flag
+// should perform the scan successfully and return exit code 50
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should perform a valid scan with -t flag [E2E-CLI-011]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"scan", "-q", "../assets/queries", "-p", "fixtures/samples/terraform.tf",
+					"-t", "TeRraFOrM", "--silent", "--payload-path", "output/E2E_CLI_011_PAYLOAD.json"},
+			},
+			ExpectedPayload: []string{
+				"E2E_CLI_011_PAYLOAD.json",
+			},
+		},
+		WantStatus: []int{50},
+	}
+
+	Tests = append(Tests, testSample)
+}

e2e/testcases/e2e-cli-012_scan_minimal-ui.go

@@ -0,0 +1,25 @@
+package testcases
+
+import "regexp"
+
+// E2E-CLI-012 - kics scan with minimal-ui flag should perform a scan
+// without showing detailed results on each line of code
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should display minimal-ui [E2E-CLI-012]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"scan", "-q", "../assets/queries", "-p", "../test/fixtures/tc-sim01/positive1.tf", "--minimal-ui"},
+			},
+		},
+		WantStatus: []int{50},
+		Validation: func(outputText string) bool {
+			match1, _ := regexp.MatchString("Description:", outputText)
+			match2, _ := regexp.MatchString("Platform:", outputText)
+			// if not found -> the the test was successful
+			return !match1 && !match2
+		},
+	}
+
+	Tests = append(Tests, testSample)
+}

e2e/testcases/e2e-cli-013_list-platforms.go

@@ -0,0 +1,20 @@
+package testcases
+
+// E2E-CLI-013 - KICS root command list-platforms
+// should return all the supported platforms in the CLI
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should list all supported platforms [E2E-CLI-013]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"list-platforms"},
+			},
+			ExpectedOut: []string{
+				"E2E_CLI_013",
+			},
+		},
+		WantStatus: []int{0},
+	}
+
+	Tests = append(Tests, testSample)
+}

e2e/testcases/e2e-cli-014_scan_preview-lines.go

@@ -0,0 +1,26 @@
+package testcases
+
+import "regexp"
+
+// E2E-CLI-014 - KICS preview-lines command must delimit the number of
+// code lines that are displayed in each scan results code block.
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should modify the default preview-lines value [E2E-CLI-014]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"scan", "--preview-lines", "1", "--no-color", "--no-progress",
+					"-q", "../assets/queries", "-p", "fixtures/samples/terraform-single.tf"},
+			},
+		},
+		Validation: func(outputText string) bool {
+			// only the match1 must be true
+			match1, _ := regexp.MatchString(`001\: resource \"aws_redshift_cluster\" \"default1\" \{`, outputText)
+			match2, _ := regexp.MatchString(`002\:   publicly_accessible = false`, outputText)
+			return match1 && !match2
+		},
+		WantStatus: []int{40},
+	}
+
+	Tests = append(Tests, testSample)
+}

e2e/testcases/e2e-cli-015_scan_no-color.go

@@ -0,0 +1,26 @@
+package testcases
+
+import "regexp"
+
+// E2E-CLI-015 KICS scan with --no-color flag
+// should disable the colored outputs of kics in the CLI
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should disable colored output in the CLI [E2E-CLI-015]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"scan", "--no-color", "-q", "../assets/queries", "-p", "fixtures/samples/terraform.tf"},
+			},
+		},
+		Validation: func(outputText string) bool {
+			match1, _ := regexp.MatchString(`HIGH: \d+`, outputText)
+			match2, _ := regexp.MatchString(`MEDIUM: \d+`, outputText)
+			match3, _ := regexp.MatchString(`LOW: \d+`, outputText)
+			match4, _ := regexp.MatchString(`INFO: \d+`, outputText)
+			return match1 && match2 && match3 && match4
+		},
+		WantStatus: []int{50},
+	}
+
+	Tests = append(Tests, testSample)
+}

e2e/testcases/e2e-cli-016_scan_invalid_flag.go

@@ -0,0 +1,26 @@
+package testcases
+
+// E2E-CLI-016 - KICS has an invalid flag or invalid command
+// an error message and return exit code 1
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should throw error messages for kics' flags [E2E-CLI-016]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"scan", "--invalid-flag"},
+				[]string{"--invalid-flag"},
+				[]string{"invalid"},
+				[]string{"-i"},
+			},
+			ExpectedOut: []string{
+				"E2E_CLI_016_INVALID_SCAN_FLAG",
+				"E2E_CLI_016_INVALID_FLAG",
+				"E2E_CLI_016_INVALID_COMMAND",
+				"E2E_CLI_016_INVALID_SHOTHAND",
+			},
+		},
+		WantStatus: []int{126, 126, 126, 126},
+	}
+
+	Tests = append(Tests, testSample)
+}