From 20b3a54a16c93f64e01e8f8d962f9ae9f96ce304 Mon Sep 17 00:00:00 2001 From: Rafaela Soares Date: Mon, 30 Aug 2021 18:09:18 +0100 Subject: [PATCH] feat(core): implemented embedded libraries (#4115) Co-authored-by: Felipe Avelar --- assets/assets.go | 15 ++++ docs/commands.md | 5 +- docs/configuration-file.md | 4 + docs/creating-queries.md | 2 - docs/dockerhub.md | 2 +- e2e/fixtures/assets/scan_help | 2 +- internal/console/assets/scan-flags.json | 12 +-- internal/console/flags/scan_flags.go | 2 +- internal/console/scan.go | 2 +- pkg/engine/inspector_test.go | 20 +++-- pkg/engine/source/filesystem.go | 85 ++++++------------ pkg/engine/source/filesystem_test.go | 4 +- pkg/engine/source/source.go | 46 +++++++++- pkg/engine/source/source_test.go | 114 ++++++++++++++++++++++++ test/main_test.go | 21 +++-- 15 files changed, 248 insertions(+), 88 deletions(-) create mode 100644 assets/assets.go create mode 100644 pkg/engine/source/source_test.go diff --git a/assets/assets.go b/assets/assets.go new file mode 100644 index 00000000000..396ed6285f3 --- /dev/null +++ b/assets/assets.go @@ -0,0 +1,15 @@ +package assets + +import "embed" // used for embedding KICS libraries + + +//go:embed libraries/*.rego +var embeddedLibraries embed.FS + + +// GetEmbeddedLibrary returns the embedded library.rego for the platform passed in the argument +func GetEmbeddedLibrary(platform string) (string, error){ + content, err := embeddedLibraries.ReadFile("libraries/" + platform + ".rego") + + return string(content), err +} diff --git a/docs/commands.md b/docs/commands.md index 6c9ab0c86a7..d1f26941c20 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -74,7 +74,7 @@ Flags: can be provided multiple times or as a comma separated string example: 'e69890e6-fce5-461d-98ad-cb98318dfc96,4728cd65-a20c-49da-8b31-9c08b423e4db' --input-data string path to query input data files - -b, --library string path to directory with libraries (default "./assets/libraries") + -b, --libraries-path string path to directory with libraries (default "./assets/libraries") --minimal-ui simplified version of CLI output --no-progress hides the progress bar --output-name string name used on report creations (default "results") @@ -105,8 +105,9 @@ The other commands have no further options. ## Library Flag Usage -As mentioned above, the library flag (`-b`) refers to the directory with libraries. The functions need to be grouped by platform and the library file name should follow the format: `.rego` to be loaded by KICS. It doesn't matter your directory structure. In other words, for example, if you want to indicate a directory that contains a library for your terraform queries, you should group your functions (used in your terraform queries) in a file named `terraform.rego` wherever you want. +As mentioned above, the library flag (`-b` or `--libraries-path`) refers to the directory with libraries. The functions need to be grouped by platform and the library file name should follow the format: `.rego` to be loaded by KICS. It doesn't matter your directory structure. In other words, for example, if you want to indicate a directory that contains a library for your terraform queries, you should group your functions (used in your terraform queries) in a file named `terraform.rego` wherever you want. +This will merge the custom libraries found on the flag's path with KICS's default libraries. Note that any functions declared in a custom library with the same signature as an existing function in the [default libraries](https://github.com/Checkmarx/kics/tree/master/assets/libraries) will cause **the default library function to be overwritten by the custom definition provided**. --- diff --git a/docs/configuration-file.md b/docs/configuration-file.md index b0f0c58c9c7..af9ddbab276 100644 --- a/docs/configuration-file.md +++ b/docs/configuration-file.md @@ -57,6 +57,7 @@ KICS is able to infer the format without the need of file extension. "exclude-queries": "exclude queries by providing the query ID", "exclude-results": "exclude results by providing a list of similarity IDs of a result", "exclude-severities": "exclude results by providing the severity of a result", + "libraries-path": "path to directory with libraries (default \"./assets/libraries\")", "log-file": true, "log-level": "INFO", "log-path": "path to the log file", @@ -87,6 +88,7 @@ exclude-paths: "exclude paths or files from scan" exclude-queries: "exclude queries by providing the query ID" exclude-results: "exclude results by providing a list of similarity IDs of a result" exclude-severities: "exclude results by providing the severity of a result" +libraries-path: "path to directory with libraries (default "./assets/libraries")" log-file: true log-level: INFO log-path: path to the log file @@ -116,6 +118,7 @@ exclude-paths = "exclude paths or files from scan" exclude-queries = "exclude queries by providing the query ID" exclude-results = "exclude results by providing a list of similarity IDs of a result" exclude-severities = "exclude results by providing the severity of a result" +libraries-path = "path to directory with libraries (default \"./assets/libraries\")" log-file = true log-level = "INFO" log-path = "path to the log file" @@ -145,6 +148,7 @@ disable-full-descriptions = "disable request for full descriptions and use defau "exclude-queries" = "exclude queries by providing the query ID" "exclude-results" = "exclude results by providing a list of similarity IDs of a result" "exclude-severities" = "exclude results by providing the severity of a result" +"libraries-path" = "path to directory with libraries (default \"./assets/libraries\")" "log-file" = true "log-level" = "INFO" "log-path" = "path to the log file" diff --git a/docs/creating-queries.md b/docs/creating-queries.md index 30a0ca39cd1..420debe8635 100644 --- a/docs/creating-queries.md +++ b/docs/creating-queries.md @@ -445,5 +445,3 @@ With these simple steps, users will be able to overwrite the keys they want, els #### Query Dependencies If you want to use the functions defined in your own library, you should use the flag `-b` to indicate the directory where the libraries are placed. The functions need to be grouped by platform and the library name should follow the following format: `.rego`. It doesn't matter your directory structure. In other words, for example, if you want to indicate a directory that contains a library for your terraform queries, you should group your functions (used in your terraform queries) in a file named `terraform.rego` wherever you want. - - diff --git a/docs/dockerhub.md b/docs/dockerhub.md index 367f59b53dc..e0209f81ab4 100644 --- a/docs/dockerhub.md +++ b/docs/dockerhub.md @@ -73,7 +73,7 @@ Flags: can be provided multiple times or as a comma separated string example: 'e69890e6-fce5-461d-98ad-cb98318dfc96,4728cd65-a20c-49da-8b31-9c08b423e4db' --input-data string path to query input data files - -b, --library string path to directory with libraries (default "./assets/libraries") + -b, --libraries-path string path to directory with libraries (default "./assets/libraries") --minimal-ui simplified version of CLI output --no-progress hides the progress bar --output-name string name used on report creations (default "results") diff --git a/e2e/fixtures/assets/scan_help b/e2e/fixtures/assets/scan_help index a647d35c114..98344c0d6d0 100644 --- a/e2e/fixtures/assets/scan_help +++ b/e2e/fixtures/assets/scan_help @@ -34,7 +34,7 @@ Flags: can be provided multiple times or as a comma separated string example: 'e69890e6-fce5-461d-98ad-cb98318dfc96,4728cd65-a20c-49da-8b31-9c08b423e4db' --input-data string path to query input data files - -b, --library string path to directory with libraries (default "./assets/libraries") + -b, --libraries-path string path to directory with libraries (default "./assets/libraries") --minimal-ui simplified version of CLI output --no-progress hides the progress bar --output-name string name used on report creations (default "results") diff --git a/internal/console/assets/scan-flags.json b/internal/console/assets/scan-flags.json index 503ed484eb4..ea462f1bf13 100644 --- a/internal/console/assets/scan-flags.json +++ b/internal/console/assets/scan-flags.json @@ -88,6 +88,12 @@ "defaultValue": "", "usage": "path to query input data files" }, + "libraries-path": { + "flagType": "str", + "shorthandFlag": "b", + "defaultValue": "./assets/libraries", + "usage": "path to directory with libraries" + }, "minimal-ui": { "flagType": "bool", "shorthandFlag": "", @@ -118,12 +124,6 @@ "defaultValue": null, "usage": "paths or directories to scan\nexample: \"./somepath,somefile.txt\"" }, - "library": { - "flagType": "str", - "shorthandFlag": "b", - "defaultValue": "./assets/libraries", - "usage": "path to directory with libraries" - }, "payload-lines": { "flagType": "bool", "shorthandFlag": "", diff --git a/internal/console/flags/scan_flags.go b/internal/console/flags/scan_flags.go index cbaa911f347..b0edaabcb6e 100644 --- a/internal/console/flags/scan_flags.go +++ b/internal/console/flags/scan_flags.go @@ -23,7 +23,7 @@ const ( PayloadPathFlag = "payload-path" PreviewLinesFlag = "preview-lines" QueriesPath = "queries-path" - LibraryPath = "library" + LibrariesPath = "libraries-path" ReportFormatsFlag = "report-formats" TypeFlag = "type" QueryExecTimeoutFlag = "timeout" diff --git a/internal/console/scan.go b/internal/console/scan.go index 9efceda37ad..c357699bae6 100644 --- a/internal/console/scan.go +++ b/internal/console/scan.go @@ -493,7 +493,7 @@ func scan(changedDefaultQueryPath bool) error { flags.GetStrFlag(flags.QueriesPath), flags.GetMultiStrFlag(flags.TypeFlag), flags.GetMultiStrFlag(flags.CloudProviderFlag), - flags.GetStrFlag(flags.LibraryPath)) + flags.GetStrFlag(flags.LibrariesPath)) store := storage.NewMemoryStorage() inspector, err := createServiceAndStartScan(&startServiceParameters{ diff --git a/pkg/engine/inspector_test.go b/pkg/engine/inspector_test.go index 01548e3cf20..ec181a930fd 100644 --- a/pkg/engine/inspector_test.go +++ b/pkg/engine/inspector_test.go @@ -6,10 +6,12 @@ import ( "os" "path/filepath" "reflect" + "strings" "sync" "testing" "time" + "github.com/Checkmarx/kics/assets" "github.com/Checkmarx/kics/internal/tracker" "github.com/Checkmarx/kics/pkg/detector" "github.com/Checkmarx/kics/pkg/detector/docker" @@ -18,6 +20,7 @@ import ( "github.com/Checkmarx/kics/pkg/model" "github.com/Checkmarx/kics/pkg/progress" "github.com/Checkmarx/kics/test" + "github.com/rs/zerolog/log" "github.com/stretchr/testify/require" "github.com/open-policy-agent/opa/cover" @@ -671,11 +674,18 @@ func (m *mockSource) GetQueries(queryFilter *source.QueryInspectorParameters) ([ } func (m *mockSource) GetQueryLibrary(platform string) (string, error) { - pathToLib, err := source.GetPathToLibrary(platform, filepath.FromSlash("./assets/libraries")) - if err != nil { - return "", err + library := source.GetPathToCustomLibrary(platform, "./assets/libraries") + + if library != "default" { + content, err := os.ReadFile(library) + + return string(content), err } - content, err := os.ReadFile(filepath.Clean(pathToLib)) - return string(content), err + log.Warn().Msgf("Custom library not provided. Loading embedded library instead") + + // getting embedded library + embeddedLibrary, errGettingEmbeddedLibrary := assets.GetEmbeddedLibrary(strings.ToLower(platform)) + + return embeddedLibrary, errGettingEmbeddedLibrary } diff --git a/pkg/engine/source/filesystem.go b/pkg/engine/source/filesystem.go index d08f706a7a4..70e418a2b10 100644 --- a/pkg/engine/source/filesystem.go +++ b/pkg/engine/source/filesystem.go @@ -8,6 +8,7 @@ import ( "sort" "strings" + "github.com/Checkmarx/kics/assets" "github.com/Checkmarx/kics/internal/constants" "github.com/Checkmarx/kics/pkg/model" "github.com/getsentry/sentry-go" @@ -36,6 +37,8 @@ const ( emptyInputData = "{}" common = "Common" + + kicsDefault = "default" ) // NewFilesystemSource initializes a NewFilesystemSource with source to queries and types of queries to load @@ -96,80 +99,46 @@ func isDefaultLibrary(libraryPath string) bool { return filepath.FromSlash(libraryPath) == filepath.FromSlash(LibrariesDefaultBasePath) } -func getKicsDirPath() string { - kicsPath, err := os.Executable() - if err != nil { - log.Err(err) - return "" - } - - return filepath.Dir(kicsPath) -} +// GetPathToCustomLibrary - returns the libraries path for a given platform +func GetPathToCustomLibrary(platform, libraryPathFlag string) string { + var libraryFilePath string -// GetPathToLibrary returns the libraries path for a given platform -func GetPathToLibrary(platform, libraryPathFlag string) (string, error) { - var libraryFilePath, relativeBasePath string - // user uses the library path flag if !isDefaultLibrary(libraryPathFlag) { library := getLibraryInDir(platform, libraryPathFlag) // found a library named according to the platform if library != "" { libraryFilePath = library - } else if library == "" && strings.EqualFold(common, platform) { - libraryFilePath = "" - } - // user did not use the library path flag - } else { - kicsDirPath := getKicsDirPath() - libraryFilePath = filepath.FromSlash(kicsDirPath + "/assets/libraries/common.rego") - // the system does not have kics binary accessible - _, err := os.Stat(libraryFilePath) - if err != nil && os.IsNotExist(err) { - currentDir, err := os.Getwd() - if err != nil { - log.Error().Msgf("Error getting wd: %s", err) - return "", err - } - if strings.LastIndex(currentDir, "kics") > -1 { - currentDir = currentDir[:strings.LastIndex(currentDir, "kics")] + "kics" - } else { - currentDir = filepath.Join(currentDir, "kics") - } - libraryFilePath = filepath.FromSlash(currentDir + "/assets/libraries/common.rego") - _, err = os.Stat(libraryFilePath) - if os.IsNotExist(err) { - log.Error().Msgf("Error getting wd: %s", err) - return "", err - } - relativeBasePath = currentDir - // the system has kics binary accessible } else { - relativeBasePath = kicsDirPath - } - for _, supPlatform := range constants.AvailablePlatforms { - if strings.Contains(strings.ToUpper(platform), strings.ToUpper(supPlatform)) { - libraryFilePath = filepath.FromSlash(relativeBasePath + "/assets/libraries/" + strings.ToLower(supPlatform) + ".rego") - break - } + libraryFilePath = kicsDefault } + } else { + libraryFilePath = kicsDefault } - return libraryFilePath, nil + + return libraryFilePath } // GetQueryLibrary returns the library.rego for the platform passed in the argument func (s *FilesystemSource) GetQueryLibrary(platform string) (string, error) { - pathToLib, err := GetPathToLibrary(platform, s.Library) - if err != nil { - return "", err - } + library := GetPathToCustomLibrary(platform, s.Library) + content := "" - content, err := os.ReadFile(filepath.Clean(pathToLib)) - if err != nil { - log.Err(err). - Msgf("Failed to get filesystem source rego library %s", pathToLib) + if library != kicsDefault { + byteContent, err := os.ReadFile(library) + if err != nil { + return "", err + } + content = string(byteContent) + } else { + log.Warn().Msgf("Custom library not provided. Loading embedded library instead") + } + // getting embedded library + embeddedLibrary, errGettingEmbeddedLibrary := assets.GetEmbeddedLibrary(strings.ToLower(platform)) + if errGettingEmbeddedLibrary != nil { + return "", errGettingEmbeddedLibrary } - return string(content), err + return mergeLibraries(content, embeddedLibrary) } // CheckType checks if the queries have the type passed as an argument in '--type' flag to be loaded diff --git a/pkg/engine/source/filesystem_test.go b/pkg/engine/source/filesystem_test.go index 354ba44e1d1..6fea6326266 100644 --- a/pkg/engine/source/filesystem_test.go +++ b/pkg/engine/source/filesystem_test.go @@ -376,8 +376,8 @@ func TestFilesystemSource_GetQueryLibrary(t *testing.T) { // nolint args: args{ platform: "unknown", }, - contains: "generic.common", - wantErr: false, + contains: "", + wantErr: true, }, } for _, tt := range tests { diff --git a/pkg/engine/source/source.go b/pkg/engine/source/source.go index b69a25418d4..f7c5b0e30ee 100644 --- a/pkg/engine/source/source.go +++ b/pkg/engine/source/source.go @@ -1,7 +1,13 @@ // Package source (go:generate go run -mod=mod github.com/golang/mock/mockgen -package mock -source=./$GOFILE -destination=../mock/$GOFILE) package source -import "github.com/Checkmarx/kics/pkg/model" +import ( + "strings" + + "github.com/Checkmarx/kics/pkg/model" + "github.com/open-policy-agent/opa/ast" + "github.com/rs/zerolog/log" +) // QueryInspectorParameters is a struct that represents the optionn to select queries to be executed type QueryInspectorParameters struct { @@ -29,3 +35,41 @@ type QueriesSource interface { GetQueries(querySelection *QueryInspectorParameters) ([]model.QueryMetadata, error) GetQueryLibrary(platform string) (string, error) } + +// mergeLibraries return custom library and embedded library merged, overwriting embedded library functions, if necessary +func mergeLibraries(customLib, embeddedLib string) (string, error) { + if customLib == "" { + return embeddedLib, nil + } + statements, _, err := ast.NewParser().WithReader(strings.NewReader(customLib)).Parse() + if err != nil { + log.Err(err).Msg("Could not parse custom library") + return "", err + } + headers := make(map[string]string) + for _, st := range statements { + if rule, ok := st.(*ast.Rule); ok { + headers[string(rule.Head.Name)] = "" + } + } + statements, _, err = ast.NewParser().WithReader(strings.NewReader(embeddedLib)).Parse() + if err != nil { + log.Err(err).Msg("Could not parse default library") + return "", err + } + for _, st := range statements { + if rule, ok := st.(*ast.Rule); ok { + if _, remove := headers[string(rule.Head.Name)]; remove { + embeddedLib = strings.Replace(embeddedLib, string(rule.Location.Text), "", 1) + } + } + if regoPackage, ok := st.(*ast.Package); ok { + firstHalf := strings.Join(strings.Split(embeddedLib, "\n")[:regoPackage.Location.Row-1], "\n") + secondHalf := strings.Join(strings.Split(embeddedLib, "\n")[regoPackage.Location.Row+1:], "\n") + embeddedLib = firstHalf + "\n" + secondHalf + } + } + customLib += "\n" + embeddedLib + + return customLib, nil +} diff --git a/pkg/engine/source/source_test.go b/pkg/engine/source/source_test.go new file mode 100644 index 00000000000..71dca568ef5 --- /dev/null +++ b/pkg/engine/source/source_test.go @@ -0,0 +1,114 @@ +package source + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMergeLibraries(t *testing.T) { //nolint + tests := []struct { + name string + customLib string + embeddedLib string + expected string + errExpected bool + }{ + { + name: "Should return default library", + customLib: "", + embeddedLib: `dummy_test(a) { + a + }`, + expected: `dummy_test(a) { + a + }`, + errExpected: false, + }, + { + name: "Should return just custom library", + customLib: `dummy_test(b) { + b + }`, + embeddedLib: `dummy_test(a) { + a + }`, + expected: `dummy_test(b) { + b + } +`, + errExpected: false, + }, + { + name: "Should return merged libraries", + customLib: `other_dummy_test(b) { + b + }`, + embeddedLib: `dummy_test(a) { + a + }`, + expected: `other_dummy_test(b) { + b + } +dummy_test(a) { + a + }`, + errExpected: false, + }, + { + name: "Should return merged libraries with overwritten functions", + customLib: `other_dummy_test(b) { + b + } +`, + embeddedLib: `dummy_test(a) { + a + } +other_dummy_test(c) { + c + }`, + expected: `other_dummy_test(b) { + b + } + +dummy_test(a) { + a + } +`, + errExpected: false, + }, + { + name: "Should return error since custom lib is invalid", + customLib: `other_dummy_test(b) { + b + `, + embeddedLib: `dummy_test(a) { + a + }`, + expected: "", + errExpected: true, + }, + { + name: "Should return error since embedded lib is invalid", + customLib: `other_dummy_test(b) { + b + }`, + embeddedLib: `dummy_test(a) { + a + `, + expected: "", + errExpected: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := mergeLibraries(tt.customLib, tt.embeddedLib) + if tt.errExpected { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.Equal(t, tt.expected, got) + }) + } +} diff --git a/test/main_test.go b/test/main_test.go index f442cbea59c..fb2e7b26a29 100644 --- a/test/main_test.go +++ b/test/main_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + "github.com/Checkmarx/kics/assets" "github.com/Checkmarx/kics/pkg/engine/source" "github.com/Checkmarx/kics/pkg/kics" "github.com/Checkmarx/kics/pkg/model" @@ -219,16 +220,20 @@ func sliceContains(s []string, str string) bool { } func readLibrary(platform string) (string, error) { - pathToLib, err := source.GetPathToLibrary(platform, filepath.FromSlash("./assets/libraries")) - if err != nil { - return "", err - } - content, err := os.ReadFile(pathToLib) - if err != nil { - log.Err(err) + library := source.GetPathToCustomLibrary(platform, "./assets/libraries") + + if library != "default" { + content, err := os.ReadFile(library) + + return string(content), err } - return string(content), err + log.Warn().Msgf("Custom library not provided. Loading embedded library instead") + + // getting embedded library + embeddedLibrary, errGettingEmbeddedLibrary := assets.GetEmbeddedLibrary(strings.ToLower(platform)) + + return embeddedLibrary, errGettingEmbeddedLibrary } func isValidURL(toTest string) bool {