feat(core): implemented embedded libraries (Checkmarx#4115)

Co-authored-by: Felipe Avelar <[email protected]>

assets/assets.go

@@ -0,0 +1,15 @@
+package assets
+
+import "embed" // used for embedding KICS libraries
+
+
+//go:embed libraries/*.rego
+var embeddedLibraries embed.FS
+
+
+// GetEmbeddedLibrary returns the embedded library.rego for the platform passed in the argument
+func GetEmbeddedLibrary(platform string) (string, error){
+	content, err := embeddedLibraries.ReadFile("libraries/" + platform + ".rego")
+
+	return string(content), err
+}

docs/commands.md

@@ -74,7 +74,7 @@ Flags:
                                      can be provided multiple times or as a comma separated string
                                      example: 'e69890e6-fce5-461d-98ad-cb98318dfc96,4728cd65-a20c-49da-8b31-9c08b423e4db'
       --input-data string            path to query input data files
-  -b, --library string               path to directory with libraries (default "./assets/libraries")
+  -b, --libraries-path string        path to directory with libraries (default "./assets/libraries")
       --minimal-ui                   simplified version of CLI output
       --no-progress                  hides the progress bar
       --output-name string           name used on report creations (default "results")
@@ -105,8 +105,9 @@ The other commands have no further options.
 
 ## Library Flag Usage
 
-As mentioned above, the library flag (`-b`) refers to the directory with libraries. The functions need to be grouped by platform and the library file name should follow the format: `<platform>.rego` to be loaded by KICS. It doesn't matter your directory structure. In other words, for example, if you want to indicate a directory that contains a library for your terraform queries, you should group your functions (used in your terraform queries) in a file named `terraform.rego` wherever you want.
+As mentioned above, the library flag (`-b` or `--libraries-path`) refers to the directory with libraries. The functions need to be grouped by platform and the library file name should follow the format: `<platform>.rego` to be loaded by KICS. It doesn't matter your directory structure. In other words, for example, if you want to indicate a directory that contains a library for your terraform queries, you should group your functions (used in your terraform queries) in a file named `terraform.rego` wherever you want.
 
+This will merge the custom libraries found on the flag's path with KICS's default libraries. Note that any functions declared in a custom library with the same signature as an existing function in the [default libraries](https://github.com/Checkmarx/kics/tree/master/assets/libraries) will cause **the default library function to be overwritten by the custom definition provided**.
 
 ---
 

docs/configuration-file.md

@@ -57,6 +57,7 @@ KICS is able to infer the format without the need of file extension.
   "exclude-queries": "exclude queries by providing the query ID",
   "exclude-results": "exclude results by providing a list of similarity IDs of a result",
   "exclude-severities": "exclude results by providing the severity of a result",
+  "libraries-path": "path to directory with libraries (default \"./assets/libraries\")",
   "log-file": true,
   "log-level": "INFO",
   "log-path": "path to the log file",
@@ -87,6 +88,7 @@ exclude-paths: "exclude paths or files from scan"
 exclude-queries: "exclude queries by providing the query ID"
 exclude-results: "exclude results by providing a list of similarity IDs of a result"
 exclude-severities: "exclude results by providing the severity of a result"
+libraries-path: "path to directory with libraries (default "./assets/libraries")"
 log-file: true
 log-level: INFO
 log-path: path to the log file
@@ -116,6 +118,7 @@ exclude-paths = "exclude paths or files from scan"
 exclude-queries = "exclude queries by providing the query ID"
 exclude-results = "exclude results by providing a list of similarity IDs of a result"
 exclude-severities = "exclude results by providing the severity of a result"
+libraries-path = "path to directory with libraries (default \"./assets/libraries\")"
 log-file = true
 log-level = "INFO"
 log-path = "path to the log file"
@@ -145,6 +148,7 @@ disable-full-descriptions = "disable request for full descriptions and use defau
 "exclude-queries" = "exclude queries by providing the query ID"
 "exclude-results" = "exclude results by providing a list of similarity IDs of a result"
 "exclude-severities" = "exclude results by providing the severity of a result"
+"libraries-path" = "path to directory with libraries (default \"./assets/libraries\")"
 "log-file" = true
 "log-level" = "INFO"
 "log-path" = "path to the log file"

docs/creating-queries.md

@@ -445,5 +445,3 @@ With these simple steps, users will be able to overwrite the keys they want, els
 
 #### Query Dependencies
 If you want to use the functions defined in your own library, you should use the flag `-b` to indicate the directory where the libraries are placed. The functions need to be grouped by platform and the library name should follow the following format: `<platform>.rego`. It doesn't matter your directory structure. In other words, for example, if you want to indicate a directory that contains a library for your terraform queries, you should group your functions (used in your terraform queries) in a file named `terraform.rego` wherever you want.
-
-

docs/dockerhub.md

@@ -73,7 +73,7 @@ Flags:
                                      can be provided multiple times or as a comma separated string
                                      example: 'e69890e6-fce5-461d-98ad-cb98318dfc96,4728cd65-a20c-49da-8b31-9c08b423e4db'
       --input-data string            path to query input data files
-  -b, --library string               path to directory with libraries (default "./assets/libraries")
+  -b, --libraries-path string        path to directory with libraries (default "./assets/libraries")
       --minimal-ui                   simplified version of CLI output
       --no-progress                  hides the progress bar
       --output-name string           name used on report creations (default "results")

e2e/fixtures/assets/scan_help

@@ -34,7 +34,7 @@ Flags:
                                      can be provided multiple times or as a comma separated string
                                      example: 'e69890e6-fce5-461d-98ad-cb98318dfc96,4728cd65-a20c-49da-8b31-9c08b423e4db'
       --input-data string            path to query input data files
-  -b, --library string               path to directory with libraries (default "./assets/libraries")
+  -b, --libraries-path string        path to directory with libraries (default "./assets/libraries")
       --minimal-ui                   simplified version of CLI output
       --no-progress                  hides the progress bar
       --output-name string           name used on report creations (default "results")

internal/console/assets/scan-flags.json

@@ -88,6 +88,12 @@
     "defaultValue": "",
     "usage": "path to query input data files"
   },
+  "libraries-path": {
+    "flagType": "str",
+    "shorthandFlag": "b",
+    "defaultValue": "./assets/libraries",
+    "usage": "path to directory with libraries"
+  },
   "minimal-ui": {
     "flagType": "bool",
     "shorthandFlag": "",
@@ -118,12 +124,6 @@
     "defaultValue": null,
     "usage": "paths or directories to scan\nexample: \"./somepath,somefile.txt\""
   },
-  "library": {
-    "flagType": "str",
-    "shorthandFlag": "b",
-    "defaultValue": "./assets/libraries",
-    "usage": "path to directory with libraries"
-  },
   "payload-lines": {
     "flagType": "bool",
     "shorthandFlag": "",

internal/console/flags/scan_flags.go

@@ -23,7 +23,7 @@ const (
 	PayloadPathFlag       = "payload-path"
 	PreviewLinesFlag      = "preview-lines"
 	QueriesPath           = "queries-path"
-	LibraryPath           = "library"
+	LibrariesPath         = "libraries-path"
 	ReportFormatsFlag     = "report-formats"
 	TypeFlag              = "type"
 	QueryExecTimeoutFlag  = "timeout"

internal/console/scan.go

@@ -493,7 +493,7 @@ func scan(changedDefaultQueryPath bool) error {
 		flags.GetStrFlag(flags.QueriesPath),
 		flags.GetMultiStrFlag(flags.TypeFlag),
 		flags.GetMultiStrFlag(flags.CloudProviderFlag),
-		flags.GetStrFlag(flags.LibraryPath))
+		flags.GetStrFlag(flags.LibrariesPath))
 	store := storage.NewMemoryStorage()
 
 	inspector, err := createServiceAndStartScan(&startServiceParameters{

pkg/engine/inspector_test.go

@@ -6,10 +6,12 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
+	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/Checkmarx/kics/assets"
 	"github.com/Checkmarx/kics/internal/tracker"
 	"github.com/Checkmarx/kics/pkg/detector"
 	"github.com/Checkmarx/kics/pkg/detector/docker"
@@ -18,6 +20,7 @@ import (
 	"github.com/Checkmarx/kics/pkg/model"
 	"github.com/Checkmarx/kics/pkg/progress"
 	"github.com/Checkmarx/kics/test"
+	"github.com/rs/zerolog/log"
 	"github.com/stretchr/testify/require"
 
 	"github.com/open-policy-agent/opa/cover"
@@ -671,11 +674,18 @@ func (m *mockSource) GetQueries(queryFilter *source.QueryInspectorParameters) ([
 }
 
 func (m *mockSource) GetQueryLibrary(platform string) (string, error) {
-	pathToLib, err := source.GetPathToLibrary(platform, filepath.FromSlash("./assets/libraries"))
-	if err != nil {
-		return "", err
+	library := source.GetPathToCustomLibrary(platform, "./assets/libraries")
+
+	if library != "default" {
+		content, err := os.ReadFile(library)
+
+		return string(content), err
 	}
-	content, err := os.ReadFile(filepath.Clean(pathToLib))
 
-	return string(content), err
+	log.Warn().Msgf("Custom library not provided. Loading embedded library instead")
+
+	// getting embedded library
+	embeddedLibrary, errGettingEmbeddedLibrary := assets.GetEmbeddedLibrary(strings.ToLower(platform))
+
+	return embeddedLibrary, errGettingEmbeddedLibrary
 }

pkg/engine/source/filesystem.go

@@ -8,6 +8,7 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/Checkmarx/kics/assets"
 	"github.com/Checkmarx/kics/internal/constants"
 	"github.com/Checkmarx/kics/pkg/model"
 	"github.com/getsentry/sentry-go"
@@ -36,6 +37,8 @@ const (
 	emptyInputData = "{}"
 
 	common = "Common"
+
+	kicsDefault = "default"
 )
 
 // NewFilesystemSource initializes a NewFilesystemSource with source to queries and types of queries to load
@@ -96,80 +99,46 @@ func isDefaultLibrary(libraryPath string) bool {
 	return filepath.FromSlash(libraryPath) == filepath.FromSlash(LibrariesDefaultBasePath)
 }
 
-func getKicsDirPath() string {
-	kicsPath, err := os.Executable()
-	if err != nil {
-		log.Err(err)
-		return ""
-	}
-
-	return filepath.Dir(kicsPath)
-}
+// GetPathToCustomLibrary - returns the libraries path for a given platform
+func GetPathToCustomLibrary(platform, libraryPathFlag string) string {
+	var libraryFilePath string
 
-// GetPathToLibrary returns the libraries path for a given platform
-func GetPathToLibrary(platform, libraryPathFlag string) (string, error) {
-	var libraryFilePath, relativeBasePath string
-	// user uses the library path flag
 	if !isDefaultLibrary(libraryPathFlag) {
 		library := getLibraryInDir(platform, libraryPathFlag)
 		// found a library named according to the platform
 		if library != "" {
 			libraryFilePath = library
-		} else if library == "" && strings.EqualFold(common, platform) {
-			libraryFilePath = ""
-		}
-		// user did not use the library path flag
-	} else {
-		kicsDirPath := getKicsDirPath()
-		libraryFilePath = filepath.FromSlash(kicsDirPath + "/assets/libraries/common.rego")
-		// the system does not have kics binary accessible
-		_, err := os.Stat(libraryFilePath)
-		if err != nil && os.IsNotExist(err) {
-			currentDir, err := os.Getwd()
-			if err != nil {
-				log.Error().Msgf("Error getting wd: %s", err)
-				return "", err
-			}
-			if strings.LastIndex(currentDir, "kics") > -1 {
-				currentDir = currentDir[:strings.LastIndex(currentDir, "kics")] + "kics"
-			} else {
-				currentDir = filepath.Join(currentDir, "kics")
-			}
-			libraryFilePath = filepath.FromSlash(currentDir + "/assets/libraries/common.rego")
-			_, err = os.Stat(libraryFilePath)
-			if os.IsNotExist(err) {
-				log.Error().Msgf("Error getting wd: %s", err)
-				return "", err
-			}
-			relativeBasePath = currentDir
-			// the system has kics binary accessible
 		} else {
-			relativeBasePath = kicsDirPath
-		}
-		for _, supPlatform := range constants.AvailablePlatforms {
-			if strings.Contains(strings.ToUpper(platform), strings.ToUpper(supPlatform)) {
-				libraryFilePath = filepath.FromSlash(relativeBasePath + "/assets/libraries/" + strings.ToLower(supPlatform) + ".rego")
-				break
-			}
+			libraryFilePath = kicsDefault
 		}
+	} else {
+		libraryFilePath = kicsDefault
 	}
-	return libraryFilePath, nil
+
+	return libraryFilePath
 }
 
 // GetQueryLibrary returns the library.rego for the platform passed in the argument
 func (s *FilesystemSource) GetQueryLibrary(platform string) (string, error) {
-	pathToLib, err := GetPathToLibrary(platform, s.Library)
-	if err != nil {
-		return "", err
-	}
+	library := GetPathToCustomLibrary(platform, s.Library)
+	content := ""
 
-	content, err := os.ReadFile(filepath.Clean(pathToLib))
-	if err != nil {
-		log.Err(err).
-			Msgf("Failed to get filesystem source rego library %s", pathToLib)
+	if library != kicsDefault {
+		byteContent, err := os.ReadFile(library)
+		if err != nil {
+			return "", err
+		}
+		content = string(byteContent)
+	} else {
+		log.Warn().Msgf("Custom library not provided. Loading embedded library instead")
+	}
+	// getting embedded library
+	embeddedLibrary, errGettingEmbeddedLibrary := assets.GetEmbeddedLibrary(strings.ToLower(platform))
+	if errGettingEmbeddedLibrary != nil {
+		return "", errGettingEmbeddedLibrary
 	}
 
-	return string(content), err
+	return mergeLibraries(content, embeddedLibrary)
 }
 
 // CheckType checks if the queries have the type passed as an argument in '--type' flag to be loaded

pkg/engine/source/filesystem_test.go

@@ -376,8 +376,8 @@ func TestFilesystemSource_GetQueryLibrary(t *testing.T) { // nolint
 			args: args{
 				platform: "unknown",
 			},
-			contains: "generic.common",
-			wantErr:  false,
+			contains: "",
+			wantErr:  true,
 		},
 	}
 	for _, tt := range tests {

pkg/engine/source/source.go

@@ -1,7 +1,13 @@
 // Package source (go:generate go run -mod=mod github.com/golang/mock/mockgen -package mock -source=./$GOFILE -destination=../mock/$GOFILE)
 package source
 
-import "github.com/Checkmarx/kics/pkg/model"
+import (
+	"strings"
+
+	"github.com/Checkmarx/kics/pkg/model"
+	"github.com/open-policy-agent/opa/ast"
+	"github.com/rs/zerolog/log"
+)
 
 // QueryInspectorParameters is a struct that represents the optionn to select queries to be executed
 type QueryInspectorParameters struct {
@@ -29,3 +35,41 @@ type QueriesSource interface {
 	GetQueries(querySelection *QueryInspectorParameters) ([]model.QueryMetadata, error)
 	GetQueryLibrary(platform string) (string, error)
 }
+
+// mergeLibraries return custom library and embedded library merged, overwriting embedded library functions, if necessary
+func mergeLibraries(customLib, embeddedLib string) (string, error) {
+	if customLib == "" {
+		return embeddedLib, nil
+	}
+	statements, _, err := ast.NewParser().WithReader(strings.NewReader(customLib)).Parse()
+	if err != nil {
+		log.Err(err).Msg("Could not parse custom library")
+		return "", err
+	}
+	headers := make(map[string]string)
+	for _, st := range statements {
+		if rule, ok := st.(*ast.Rule); ok {
+			headers[string(rule.Head.Name)] = ""
+		}
+	}
+	statements, _, err = ast.NewParser().WithReader(strings.NewReader(embeddedLib)).Parse()
+	if err != nil {
+		log.Err(err).Msg("Could not parse default library")
+		return "", err
+	}
+	for _, st := range statements {
+		if rule, ok := st.(*ast.Rule); ok {
+			if _, remove := headers[string(rule.Head.Name)]; remove {
+				embeddedLib = strings.Replace(embeddedLib, string(rule.Location.Text), "", 1)
+			}
+		}
+		if regoPackage, ok := st.(*ast.Package); ok {
+			firstHalf := strings.Join(strings.Split(embeddedLib, "\n")[:regoPackage.Location.Row-1], "\n")
+			secondHalf := strings.Join(strings.Split(embeddedLib, "\n")[regoPackage.Location.Row+1:], "\n")
+			embeddedLib = firstHalf + "\n" + secondHalf
+		}
+	}
+	customLib += "\n" + embeddedLib
+
+	return customLib, nil
+}