forked from Checkmarx/kics
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Adding tests docs, fixing makefile tasks (Checkmarx#2907)
Signed-off-by: Rogério Peixoto <[email protected]>
- Loading branch information
1 parent
19d9872
commit 2d46173
Showing
10 changed files
with
301 additions
and
34 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,130 @@ | ||
| #!/usr/bin/env python3 | ||
| import itertools | ||
| import os | ||
| import re | ||
| import sys | ||
| import typing | ||
| from argparse import ArgumentParser | ||
|
|
||
|
|
||
| class Arguments: | ||
| coverage_file: str | ||
|
|
||
|
|
||
| class BaseItem: | ||
| def __init__(self, **kw): | ||
| """Inits the BaseItem object.""" | ||
| self.__dict__.update(kw) | ||
|
|
||
| def __repr__(self): | ||
| """Converts the BaseItem object to string.""" | ||
| return f"{self.__class__.__name__}(**{self.__dict__!r})" | ||
|
|
||
|
|
||
| class LineStats(BaseItem): | ||
| filepath: str | ||
| line_from: int | ||
| line_to: int | ||
| count_covered: int | ||
| covered: bool | ||
|
|
||
|
|
||
| class FileStats(BaseItem): | ||
| filepath: str | ||
| total_lines: int | ||
| covered_lines: int | ||
| uncovered_lines: int | ||
|
|
||
| @property | ||
| def coverage(self) -> float: | ||
| try: | ||
| return 100*self.covered_lines/self.total_lines | ||
| except ZeroDivisionError: | ||
| return 100. | ||
|
|
||
|
|
||
| def parse_args() -> Arguments: | ||
| args = ArgumentParser() | ||
| args.add_argument("coverage_file") | ||
| return typing.cast(Arguments, args.parse_args(sys.argv[1:])) | ||
|
|
||
|
|
||
| def load_coverage(args: Arguments) -> typing.List[LineStats]: | ||
| pat = re.compile( | ||
| r"([^:]*):" | ||
| r"(\d+)\.\d*,(\d+)\.\d* (\d+) (\d+)" | ||
| ) | ||
|
|
||
| out = [] | ||
|
|
||
| with open(args.coverage_file, "r") as fd: | ||
| next(fd) | ||
|
|
||
| for line in fd: | ||
| attrs = dict(zip( | ||
| ["filename", "line_from", "line_to", "count_covered", "covered"], | ||
| pat.findall(line)[0] | ||
| )) | ||
|
|
||
| for col in ["line_from", "line_to", "count_covered"]: | ||
| attrs[col] = int(attrs[col]) | ||
|
|
||
| attrs["covered"] = bool(int(attrs["covered"])) | ||
|
|
||
| el = LineStats(**attrs) | ||
| out.append(el) | ||
|
|
||
| return out | ||
|
|
||
|
|
||
| def calc_file_stats(lines: typing.List[LineStats]) -> typing.List[FileStats]: | ||
| lines = lines[:] | ||
| def key(i): return i.filename | ||
| lines.sort(key=key) | ||
|
|
||
| out = [] | ||
|
|
||
| for filename, group in itertools.groupby(lines, key=key): | ||
| group = list(group) | ||
| group.sort(key=lambda i: i.line_to) | ||
|
|
||
| f = FileStats(filepath=filename) | ||
| f.total_lines = sum([ | ||
| i.count_covered | ||
| for i in group | ||
| ]) | ||
| f.covered_lines = sum([ | ||
| i.count_covered | ||
| for i in group | ||
| if i.covered | ||
| ]) | ||
| f.uncovered_lines = sum([ | ||
| i.count_covered | ||
| for i in group | ||
| if not i.covered | ||
| ]) | ||
| out.append(f) | ||
| return out | ||
|
|
||
|
|
||
| def total_cov(data: typing.List[FileStats]) -> float: | ||
| covered = 0 | ||
| total = 0 | ||
| for item in data: | ||
| covered += item.covered_lines | ||
| total += item.total_lines | ||
| return 100*covered/total | ||
|
|
||
|
|
||
| def main(): | ||
| args = parse_args() | ||
| lines = load_coverage(args) | ||
| stats = calc_file_stats(lines) | ||
| total = total_cov(stats) | ||
| if os.environ.get('GITHUB_RUN_ID'): | ||
| print(f"::set-output name=coverage::{total}") | ||
| print(f"Total coverage: {total}") | ||
|
|
||
|
|
||
| if __name__ == '__main__': | ||
| main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| # E2E tests | ||
|
|
||
| The purpose of this docs is to describe KICS' E2E test suite | ||
|
|
||
| ## Getting Started | ||
|
|
||
| There are several ways to execute the E2E tests. | ||
|
|
||
| ### TLDR | ||
|
|
||
| This steps will build the kics and then run the E2E using the built binary (placed by default under `${PWD}/bin/kics`) | ||
|
|
||
| ```bash | ||
| make test-e2e | ||
| ``` | ||
|
|
||
| ### Step by Step | ||
|
|
||
| These steps will build the kics and then run the E2E using the built binary. | ||
|
|
||
| ```bash | ||
| go build -o ./bin/kics cmd/console/main.go | ||
| ``` | ||
|
|
||
| If you want to provide a version: | ||
| ```bash | ||
| go build -o ./bin/kics -ldflags "-X github.com/Checkmarx/kics/internal/constants.Version=$(git rev-parse --short HEAD) cmd/console/main.go | ||
| ``` | ||
| and then: | ||
| ```bash | ||
| E2E_KICS_BINARY=./bin/kics go test "github.com/Checkmarx/kics/e2e" -v | ||
| ``` | ||
| ## Test Scenarios | ||
| Test scenarios are defined as follows: | ||
| ```go | ||
| var tests = []struct { | ||
| name string | ||
| args args | ||
| wantStatus int | ||
| removePayload []string | ||
| }{ | ||
| // E2E-CLI-005 - KICS scan with -- payload-path flag should create a file with the | ||
| // passed name containing the payload of the files scanned | ||
| { | ||
| name: "E2E-CLI-005", | ||
| args: args{ | ||
| // These are CLI arguments passed down to the KICS binary | ||
| args: []cmdArgs{ | ||
| []string{"scan", "--silent", "-q", "../assets/queries", "-p", "fixtures/samples/terraform.tf", | ||
| "--payload-path", "fixtures/payload.json", "-q", "../assets/queries"}, | ||
| }, | ||
| // this is a reference to a fixture placed under e2e/fixtures that contains the expected stdout output | ||
| expectedOut: []string{ | ||
| "E2E_CLI_005", | ||
| }, | ||
| // this is a reference to a fixture placed under e2e/fixtures that contains the expected KICS' payload | ||
| expectedPayload: []string{ | ||
| "E2E_CLI_005_PAYLOAD", | ||
| }, | ||
| }, | ||
| wantStatus: 0, | ||
| // we should cleanup the payload after running this scenario | ||
| removePayload: []string{"payload.json"}, | ||
| }, | ||
| } | ||
| ``` | ||
| E2E tests are skiped in short mode: | ||
| ```go | ||
| func Test_E2E_CLI(t *testing.T) { | ||
| kicsPath := getKICSBinaryPath("") | ||
| if testing.Short() { | ||
| t.Skip("skipping E2E tests in short mode.") | ||
| } | ||
| //... | ||
| } | ||
| ``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.