feat(logs): added lines scanned and lines parsed (Checkmarx#5050)

e2e/fixtures/E2E_CLI_032_RESULT.json

@@ -1,7 +1,9 @@
 {
 	"kics_version": "development",
 	"files_scanned": 1,
+	"lines_scanned": 278,
 	"files_parsed": 1,
+	"lines_parsed": 278,
 	"files_failed_to_scan": 0,
 	"queries_total": 963,
 	"queries_failed_to_execute": 0,

e2e/fixtures/E2E_CLI_033_RESULT.json

@@ -1,7 +1,9 @@
 {
 	"kics_version": "development",
 	"files_scanned": 1,
+	"lines_scanned": 5,
 	"files_parsed": 1,
+	"lines_parsed": 5,
 	"files_failed_to_scan": 0,
 	"queries_total": 855,
 	"queries_failed_to_execute": 0,

e2e/fixtures/E2E_CLI_036_RESULT.json

@@ -1,7 +1,9 @@
 {
 	"kics_version": "development",
 	"files_scanned": 1,
+	"lines_scanned": 278,
 	"files_parsed": 1,
+	"lines_parsed": 278,
 	"files_failed_to_scan": 0,
 	"queries_total": 13,
 	"queries_failed_to_execute": 0,

e2e/fixtures/E2E_CLI_036_RESULT_2.json

@@ -1,7 +1,9 @@
 {
 	"kics_version": "development",
 	"files_scanned": 1,
+	"lines_scanned": 278,
 	"files_parsed": 1,
+	"lines_parsed": 278,
 	"files_failed_to_scan": 0,
 	"queries_total": 1,
 	"queries_failed_to_execute": 0,

e2e/fixtures/schemas/result.json

@@ -3,7 +3,9 @@
   "required": [
     "kics_version",
     "files_scanned",
+    "lines_scanned",
     "files_parsed",
+    "lines_parsed",
     "files_failed_to_scan",
     "queries_total",
     "queries_failed_to_execute",
@@ -24,10 +26,18 @@
       "type": "integer",
       "minimum": 0
     },
+    "lines_scanned": {
+      "type": "integer",
+      "minimum": 0
+    },
     "files_parsed": {
       "type": "integer",
       "minimum": 0
     },
+    "lines_parsed": {
+      "type": "integer",
+      "minimum": 0
+    },
     "files_failed_to_scan": {
       "type": "integer",
       "const": 0

e2e/utils/helper.go

@@ -28,7 +28,7 @@ func RunCommand(kicsDockerImage string, kicsArgs []string, useMock bool) (*CmdOu
 	baseDir := filepath.Dir(cwd)
 	dockerArgs := []string{"run", "-e", descriptionServer, "--add-host=host.docker.internal:host-gateway",
 		"-v", baseDir + ":/path", kicsDockerImage}
-	completeArgs := append(dockerArgs, kicsArgs...)
+	completeArgs := append(dockerArgs, kicsArgs...) //nolint
 
 	cmd := exec.Command("docker", completeArgs...) //nolint
 	cmd.Env = append(os.Environ(), descriptionServer)

internal/console/helpers/helpers.go

@@ -124,7 +124,9 @@ func PrintResult(summary *model.Summary, failedQueries map[string]error, printer
 	fmt.Printf("TOTAL: %d\n\n", summary.SeveritySummary.TotalCounter)
 
 	log.Info().Msgf("Files scanned: %d", summary.ScannedFiles)
+	log.Info().Msgf("Lines scanned: %d", summary.ScannedFilesLines)
 	log.Info().Msgf("Parsed files: %d", summary.ParsedFiles)
+	log.Info().Msgf("Lines parsed: %d", summary.ParsedFilesLines)
 	log.Info().Msgf("Queries loaded: %d", summary.TotalQueries)
 	log.Info().Msgf("Queries failed to execute: %d", summary.FailedToExecuteQueries)
 	log.Info().Msg("Inspector stopped")

internal/tracker/ci.go

@@ -25,6 +25,8 @@ type CITracker struct {
 	ScanSecrets        int
 	ScanPaths          int
 	lines              int
+	FoundCountLines    int
+	ParsedCountLines   int
 	Version            model.Version
 }
 
@@ -96,3 +98,13 @@ func (c *CITracker) TrackScanPath() {
 func (c *CITracker) TrackVersion(retrievedVersion model.Version) {
 	c.Version = retrievedVersion
 }
+
+// TrackFileFoundCountLines - information about the lines of the scanned files
+func (c *CITracker) TrackFileFoundCountLines(countLines int) {
+	c.FoundCountLines += countLines
+}
+
+// TrackFileParseCountLines - information about the lines of the parsed files
+func (c *CITracker) TrackFileParseCountLines(countLines int) {
+	c.ParsedCountLines += countLines
+}

pkg/kics/resolver_sink.go

@@ -1,6 +1,7 @@
 package kics
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -25,6 +26,9 @@ func (s *Service) resolverSink(ctx context.Context, filename, scanID string) ([]
 
 	for _, rfile := range resFiles.File {
 		s.Tracker.TrackFileFound()
+		countLines := bytes.Count(rfile.Content, []byte{'\n'}) + 1
+		s.Tracker.TrackFileFoundCountLines(countLines)
+
 		documents, err := s.Parser.Parse(rfile.FileName, rfile.Content)
 		if err != nil {
 			if documents.Kind == "break" {
@@ -66,6 +70,7 @@ func (s *Service) resolverSink(ctx context.Context, filename, scanID string) ([]
 			s.saveToFile(ctx, &file)
 		}
 		s.Tracker.TrackFileParse()
+		s.Tracker.TrackFileParseCountLines(documents.CountLines)
 	}
 	return resFiles.Excluded, nil
 }

pkg/kics/service.go

@@ -1,6 +1,7 @@
 package kics
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"io"
@@ -39,6 +40,8 @@ type Storage interface {
 type Tracker interface {
 	TrackFileFound()
 	TrackFileParse()
+	TrackFileFoundCountLines(countLines int)
+	TrackFileParseCountLines(countLines int)
 }
 
 // Service is a struct that contains a SourceProvider to receive sources, a storage to save and retrieve scanning informations
@@ -111,30 +114,47 @@ func (s *Service) StartScan(
 	}
 }
 
+// Content keeps the content of the file and the number of lines
+type Content struct {
+	Content    *[]byte
+	CountLines int
+}
+
 /*
    getContent will read the passed file 1MB at a time
    to prevent resource exhaustion and return its content
 */
-func getContent(rc io.Reader) (*[]byte, error) {
+func getContent(rc io.Reader) (*Content, error) {
 	maxSizeMB := 5 // Max size of file in MBs
 	var content []byte
+	countLines := 0
 	data := make([]byte, mbConst)
+
+	c := &Content{
+		Content:    &[]byte{},
+		CountLines: 0,
+	}
+
 	for {
 		if maxSizeMB < 0 {
-			return &[]byte{}, errors.New("file size limit exceeded")
+			return c, errors.New("file size limit exceeded")
 		}
 		data = data[:cap(data)]
 		n, err := rc.Read(data)
 		if err != nil {
 			if err == io.EOF {
 				break
 			}
-			return &[]byte{}, err
+			return c, err
 		}
+		countLines += bytes.Count(data[:n], []byte{'\n'}) + 1
 		content = append(content, data[:n]...)
 		maxSizeMB--
 	}
-	return &content, nil
+	c.Content = &content
+	c.CountLines = countLines
+
+	return c, nil
 }
 
 // GetVulnerabilities returns a list of scan detected vulnerabilities

pkg/kics/sink.go

@@ -27,7 +27,12 @@ var (
 func (s *Service) sink(ctx context.Context, filename, scanID string, rc io.Reader) error {
 	s.Tracker.TrackFileFound()
 
-	content, err := getContent(rc)
+	c, err := getContent(rc)
+
+	content := c.Content
+
+	s.Tracker.TrackFileFoundCountLines(c.CountLines)
+
 	if err != nil {
 		return errors.Wrapf(err, "failed to get file content: %s", filename)
 	}
@@ -71,6 +76,7 @@ func (s *Service) sink(ctx context.Context, filename, scanID string, rc io.Reade
 		s.saveToFile(ctx, &file)
 	}
 	s.Tracker.TrackFileParse()
+	s.Tracker.TrackFileParseCountLines(documents.CountLines)
 
 	return errors.Wrap(err, "failed to save file content")
 }

pkg/model/summary.go

@@ -63,7 +63,9 @@ type QueryResultSlice []QueryResult
 // and how many queries failed to execute
 type Counters struct {
 	ScannedFiles           int `json:"files_scanned"`
+	ScannedFilesLines      int `json:"lines_scanned"`
 	ParsedFiles            int `json:"files_parsed"`
+	ParsedFilesLines       int `json:"lines_parsed"`
 	FailedToScanFiles      int `json:"files_failed_to_scan"`
 	TotalQueries           int `json:"queries_total"`
 	FailedToExecuteQueries int `json:"queries_failed_to_execute"`

pkg/parser/parser.go

@@ -1,6 +1,7 @@
 package parser
 
 import (
+	"bytes"
 	"errors"
 	"os"
 	"path/filepath"
@@ -79,6 +80,7 @@ type ParsedDocument struct {
 	Kind        model.FileKind
 	Content     string
 	IgnoreLines []int
+	CountLines  int
 }
 
 // CommentsCommands gets commands on comments in the file beginning, before the code starts
@@ -138,6 +140,7 @@ func (c *Parser) Parse(filePath string, fileContent []byte) (ParsedDocument, err
 			Kind:        c.parsers.GetKind(),
 			Content:     cont,
 			IgnoreLines: igLines,
+			CountLines:  bytes.Count(*resolved, []byte{'\n'}) + 1,
 		}, nil
 	}
 	return ParsedDocument{

pkg/scan/post_scan.go

@@ -20,6 +20,8 @@ import (
 func (c *Client) getSummary(results []model.Vulnerability, end time.Time, pathParameters model.PathParameters) model.Summary {
 	counters := model.Counters{
 		ScannedFiles:           c.Tracker.FoundFiles,
+		ScannedFilesLines:      c.Tracker.FoundCountLines,
+		ParsedFilesLines:       c.Tracker.ParsedCountLines,
 		ParsedFiles:            c.Tracker.ParsedFiles,
 		TotalQueries:           c.Tracker.LoadedQueries,
 		FailedToExecuteQueries: c.Tracker.ExecutingQueries - c.Tracker.ExecutedQueries,