From f68de9fd9f9c2dafaa7be67229703e9c7a7c1801 Mon Sep 17 00:00:00 2001 From: Felipe Avelar Date: Mon, 12 Jul 2021 11:03:46 +0100 Subject: [PATCH] feat(query): add --input-data option (#3808) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Felipe Avelar Co-authored-by: Rogério Peixoto --- assets/libraries/common/library.rego | 157 ------------------ .../data.json | 148 +++++++++++++++++ .../query.rego | 12 +- docs/commands.md | 1 + docs/dockerhub.md | 1 + e2e/fixtures/assets/scan_help | 1 + internal/console/scan.go | 80 ++++++--- pkg/engine/inspector.go | 11 +- pkg/engine/inspector_test.go | 27 ++- pkg/engine/mock/source.go | 2 +- pkg/engine/source/filesystem.go | 68 +++++++- pkg/engine/source/filesystem_test.go | 110 +++++++++--- pkg/engine/source/source.go | 7 +- pkg/model/model.go | 9 +- pkg/scanner/scanner_test.go | 2 +- test/fixtures/input_data/merge.json | 1 + test/fixtures/input_data/test.json | 1 + test/main_test.go | 5 +- test/queries_content_test.go | 3 +- test/queries_test.go | 3 +- test/similarity_id_test.go | 12 +- 21 files changed, 417 insertions(+), 244 deletions(-) create mode 100644 assets/queries/common/passwords_and_secrets_in_infrastructure_code/data.json create mode 100644 test/fixtures/input_data/merge.json create mode 100644 test/fixtures/input_data/test.json diff --git a/assets/libraries/common/library.rego b/assets/libraries/common/library.rego index 9a7d4ce87ce..534c4fe2e71 100644 --- a/assets/libraries/common/library.rego +++ b/assets/libraries/common/library.rego @@ -81,163 +81,6 @@ containsOrInArrayContains(field, value) { contains(lower(field[i]), value) } -isDefaultPassword(p) { - ar = { - "!@", - "root", - "wubao", - "password", - "123456", - "admin", - "12345", - "1234", - "p@ssw0rd", - "123", - "1", - "jiamima", - "test", - "root123", - "!", - "!q@w", - "!qaz@wsx", - "idc!@", - "admin!@", - "", - "alpine", - "qwerty", - "12345678", - "111111", - "123456789", - "1q2w3e4r", - "123123", - "default", - "1234567", - "qwe123", - "1qaz2wsx", - "1234567890", - "abcd1234", - "000000", - "user", - "toor", - "qwer1234", - "1q2w3e", - "asdf1234", - "redhat", - "1234qwer", - "cisco", - "12qwaszx", - "test123", - "1q2w3e4r5t", - "admin123", - "changeme", - "1qazxsw2", - "123qweasd", - "q1w2e3r4", - "letmein", - "server", - "root1234", - "master", - "abc123", - "rootroot", - "a", - "system", - "pass", - "1qaz2wsx3edc", - "p@$$w0rd", - "112233", - "welcome", - "!QAZ2wsx", - "linux", - "123321", - "manager", - "1qazXSW@", - "q1w2e3r4t5", - "oracle", - "asd123", - "admin123456", - "ubnt", - "123qwe", - "qazwsxedc", - "administrator", - "superuser", - "zaq12wsx", - "121212", - "654321", - "ubuntu", - "0000", - "zxcvbnm", - "root@123", - "1111", - "vmware", - "q1w2e3", - "qwerty123", - "cisco123", - "11111111", - "pa55w0rd", - "asdfgh", - "11111", - "123abc", - "asdf", - "centos", - "888888", - "54321", - "password123", - "pa$$", - } - - ar[p] -} - -isCommonValue(p) { - bl = { - "RESOURCE", - "GROUP", - "SUBNET", - "S3", - "SERVICE", - "AZURE", - "BUCKET", - "VIRTUAL", - "NETWORK", - "POLICY", - "AWS", - "PROTOCOL", - "CLOUD", - "MINUTE", - "TLS", - "EC2", - "VPC", - "INTERNET", - "ROUTE", - "EFS", - "INSTANCE", - "VPN", - "MOUNT", - "MYSQL", - "APACHE", - "ETHERNET", - "TERRAFORM", - "TARGET", - "ENVIRONMENT", - "MEMORY", - "PACKAGE", - "STATEMENT", - "REGION", - "INGRESS", - "CHECKPOINT", - "MODULE", - "BASIC", - "NUMBER", - "MASLEN", - "VERSION", - "MAKE", - "ARCH", - } - - black := bl[_] - contains(upper(p), black) -} - isCommonKey(p) { bl = { "namespace", diff --git a/assets/queries/common/passwords_and_secrets_in_infrastructure_code/data.json b/assets/queries/common/passwords_and_secrets_in_infrastructure_code/data.json new file mode 100644 index 00000000000..49dec6435ee --- /dev/null +++ b/assets/queries/common/passwords_and_secrets_in_infrastructure_code/data.json @@ -0,0 +1,148 @@ +{ + "defaultPasswords": [ + "!@", + "root", + "wubao", + "password", + "123456", + "admin", + "12345", + "1234", + "p@ssw0rd", + "123", + "1", + "jiamima", + "test", + "root123", + "!", + "!q@w", + "!qaz@wsx", + "idc!@", + "admin!@", + "", + "alpine", + "qwerty", + "12345678", + "111111", + "123456789", + "1q2w3e4r", + "123123", + "default", + "1234567", + "qwe123", + "1qaz2wsx", + "1234567890", + "abcd1234", + "000000", + "user", + "toor", + "qwer1234", + "1q2w3e", + "asdf1234", + "redhat", + "1234qwer", + "cisco", + "12qwaszx", + "test123", + "1q2w3e4r5t", + "admin123", + "changeme", + "1qazxsw2", + "123qweasd", + "q1w2e3r4", + "letmein", + "server", + "root1234", + "master", + "abc123", + "rootroot", + "a", + "system", + "pass", + "1qaz2wsx3edc", + "p@$$w0rd", + "112233", + "welcome", + "!QAZ2wsx", + "linux", + "123321", + "manager", + "1qazXSW@", + "q1w2e3r4t5", + "oracle", + "asd123", + "admin123456", + "ubnt", + "123qwe", + "qazwsxedc", + "administrator", + "superuser", + "zaq12wsx", + "121212", + "654321", + "ubuntu", + "0000", + "zxcvbnm", + "root@123", + "1111", + "vmware", + "q1w2e3", + "qwerty123", + "cisco123", + "11111111", + "pa55w0rd", + "asdfgh", + "11111", + "123abc", + "asdf", + "centos", + "888888", + "54321", + "password123", + "pa$$" + ], + "blackList": [ + "RESOURCE", + "GROUP", + "SUBNET", + "S3", + "SERVICE", + "AZURE", + "BUCKET", + "VIRTUAL", + "NETWORK", + "POLICY", + "AWS", + "PROTOCOL", + "CLOUD", + "MINUTE", + "TLS", + "EC2", + "VPC", + "INTERNET", + "ROUTE", + "EFS", + "INSTANCE", + "VPN", + "MOUNT", + "MYSQL", + "APACHE", + "ETHERNET", + "TERRAFORM", + "TARGET", + "ENVIRONMENT", + "MEMORY", + "PACKAGE", + "STATEMENT", + "REGION", + "INGRESS", + "CHECKPOINT", + "MODULE", + "BASIC", + "NUMBER", + "MASLEN", + "VERSION", + "MAKE", + "ARCH" + ] +} diff --git a/assets/queries/common/passwords_and_secrets_in_infrastructure_code/query.rego b/assets/queries/common/passwords_and_secrets_in_infrastructure_code/query.rego index f1e728a5cf1..fa98894ccf1 100644 --- a/assets/queries/common/passwords_and_secrets_in_infrastructure_code/query.rego +++ b/assets/queries/common/passwords_and_secrets_in_infrastructure_code/query.rego @@ -111,7 +111,7 @@ is_under_secret_key(p) = res { #search for default passwords check_vulnerability(correctStrings) { - commonLib.isDefaultPassword(correctStrings.value) + isDefaultPassword(correctStrings.value) is_under_password_key(correctStrings.key) #remove common key and values @@ -190,7 +190,7 @@ check_vulnerability(correctStrings) { check_common(correctStrings) { #remove common values - not commonLib.isCommonValue(correctStrings.value) + not isCommonValue(correctStrings.value) #remove common keys not commonLib.isCommonKey(correctStrings.key) @@ -201,3 +201,11 @@ replace_unicode(allValues) = treatedValue { treatedValue_first := replace(allValues, "\\u003c", "<") treatedValue = replace(treatedValue_first, "\\u003e", ">") } + +isDefaultPassword(p) { + data.defaultPasswords[_] == p +} + +isCommonValue(p) { + contains(upper(p), data.blackList[_]) +} diff --git a/docs/commands.md b/docs/commands.md index 3a26fa8d6d4..8bc885f6233 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -68,6 +68,7 @@ Flags: cannot be provided with query exclusion flags can be provided multiple times or as a comma separated string example: 'e69890e6-fce5-461d-98ad-cb98318dfc96,4728cd65-a20c-49da-8b31-9c08b423e4db' + --input-data string path to query input data files --minimal-ui simplified version of CLI output --no-progress hides the progress bar --output-name string name used on report creations (default "results") diff --git a/docs/dockerhub.md b/docs/dockerhub.md index 67cb358b5b7..30b009db6ba 100644 --- a/docs/dockerhub.md +++ b/docs/dockerhub.md @@ -96,6 +96,7 @@ Flags: cannot be provided with query exclusion flags can be provided multiple times or as a comma separated string example: 'e69890e6-fce5-461d-98ad-cb98318dfc96,4728cd65-a20c-49da-8b31-9c08b423e4db' + --input-data string path to query input data files --minimal-ui simplified version of CLI output --no-progress hides the progress bar --output-name string name used on report creations (default "results") diff --git a/e2e/fixtures/assets/scan_help b/e2e/fixtures/assets/scan_help index 9e4598fa939..0d793506bec 100644 --- a/e2e/fixtures/assets/scan_help +++ b/e2e/fixtures/assets/scan_help @@ -28,6 +28,7 @@ Flags: cannot be provided with query exclusion flags can be provided multiple times or as a comma separated string example: 'e69890e6-fce5-461d-98ad-cb98318dfc96,4728cd65-a20c-49da-8b31-9c08b423e4db' + --input-data string path to query input data files --minimal-ui simplified version of CLI output --no-progress hides the progress bar --output-name string name used on report creations (default "results") diff --git a/internal/console/scan.go b/internal/console/scan.go index 08c742aa377..c9ebc3a7303 100644 --- a/internal/console/scan.go +++ b/internal/console/scan.go @@ -62,6 +62,7 @@ var ( reportFormats []string types []string queryExecTimeout int + inputData string ) const ( @@ -74,6 +75,7 @@ const ( excludeResutlsShorthand = "x" includeQueriesFlag = "include-queries" inlcudeQueriesShorthand = "i" + inputDataFlag = "input-data" failOnFlag = "fail-on" ignoreOnExitFlag = "ignore-on-exit" minimalUIFlag = "minimal-ui" @@ -296,18 +298,25 @@ func setBoundFlags(flagName string, val interface{}, cmd *cobra.Command) { } func initScanFlags(scanCmd *cobra.Command) { - scanCmd.Flags().StringSliceVarP(&path, - pathFlag, pathFlagShorthand, - []string{}, - "paths or directories to scan\nexample: \"./somepath,somefile.txt\"") scanCmd.Flags().StringVar(&cfgFile, configFlag, "", "path to configuration file") - scanCmd.Flags().StringVarP(&queryPath, - queriesPathCmdName, queriesPathShorthand, - "./assets/queries", - "path to directory with queries", - ) + scanCmd.Flags().IntVar(&queryExecTimeout, + queryExecTimeoutFlag, + 60, + "number of seconds the query has to execute before being canceled") + scanCmd.Flags().StringVar(&inputData, + inputDataFlag, + "", + "path to query input data files") + initPathsFlags(scanCmd) + initStdoutFlags(scanCmd) + initOutputFlags(scanCmd) + initInclusionFlags(scanCmd) + initExitStatusFlags(scanCmd) +} + +func initOutputFlags(scanCmd *cobra.Command) { scanCmd.Flags().StringVar(&outputName, outputNameFlag, "results", @@ -320,6 +329,15 @@ func initScanFlags(scanCmd *cobra.Command) { scanCmd.Flags().StringSliceVar(&reportFormats, reportFormatsFlag, []string{"json"}, "formats in which the results will be exported (all, json, sarif, html, glsast, pdf)", ) + + scanCmd.Flags().StringSliceVarP(&types, + typeFlag, typeShorthand, + []string{""}, + "case insensitive list of platform types to scan\n"+ + fmt.Sprintf("(%s)", strings.Join(source.ListSupportedPlatforms(), ", "))) +} + +func initStdoutFlags(scanCmd *cobra.Command) { scanCmd.Flags().IntVar(&previewLines, previewLinesFlag, 3, @@ -328,25 +346,35 @@ func initScanFlags(scanCmd *cobra.Command) { payloadPathFlag, payloadPathShorthand, "", "path to store internal representation JSON file") - scanCmd.Flags().StringSliceVarP(&excludePath, - excludePathsFlag, excludePathsShorthand, - []string{}, - "exclude paths from scan\nsupports glob and can be provided multiple times or as a quoted comma separated string"+ - "\nexample: './shouldNotScan/*,somefile.txt'", - ) scanCmd.Flags().BoolVar(&min, minimalUIFlag, false, "simplified version of CLI output") - scanCmd.Flags().StringSliceVarP(&types, - typeFlag, typeShorthand, - []string{""}, - "case insensitive list of platform types to scan\n"+ - fmt.Sprintf("(%s)", strings.Join(source.ListSupportedPlatforms(), ", "))) scanCmd.Flags().BoolVar(&noProgress, noProgressFlag, false, "hides the progress bar") +} + +func initPathsFlags(scanCmd *cobra.Command) { + scanCmd.Flags().StringSliceVarP(&path, + pathFlag, pathFlagShorthand, + []string{}, + "paths or directories to scan\nexample: \"./somepath,somefile.txt\"") + scanCmd.Flags().StringSliceVarP(&excludePath, + excludePathsFlag, excludePathsShorthand, + []string{}, + "exclude paths from scan\nsupports glob and can be provided multiple times or as a quoted comma separated string"+ + "\nexample: './shouldNotScan/*,somefile.txt'", + ) + scanCmd.Flags().StringVarP(&queryPath, + queriesPathCmdName, queriesPathShorthand, + "./assets/queries", + "path to directory with queries", + ) +} + +func initInclusionFlags(scanCmd *cobra.Command) { scanCmd.Flags().StringSliceVar(&excludeIDs, excludeQueriesFlag, []string{}, @@ -378,6 +406,9 @@ func initScanFlags(scanCmd *cobra.Command) { msg+ "example: 'Access control,Best practices'", ) +} + +func initExitStatusFlags(scanCmd *cobra.Command) { scanCmd.Flags().StringSliceVar(&failOn, failOnFlag, []string{"high", "medium", "low", "info"}, @@ -391,10 +422,6 @@ func initScanFlags(scanCmd *cobra.Command) { "defines which kind of non-zero exits code should be ignored\n"+"accepts: all, results, errors, none\n"+ "example: if 'results' is set, only engine errors will make KICS exit code different from 0", ) - scanCmd.Flags().IntVar(&queryExecTimeout, - queryExecTimeoutFlag, - 60, - "number of seconds the query has to execute before being canceled") } func initScanCmd(scanCmd *cobra.Command) { @@ -443,16 +470,17 @@ func createInspector(t engine.Tracker, querySource source.QueriesSource) (*engin ByIDs: includeIDs, } - queryFilter := source.QuerySelectionFilter{ + queryFilter := source.QueryInspectorParameters{ IncludeQueries: includeQueries, ExcludeQueries: excludeQueries, + InputDataPath: inputData, } inspector, err := engine.NewInspector(ctx, querySource, engine.DefaultVulnerabilityBuilder, t, - queryFilter, + &queryFilter, excludeResultsMap, queryExecTimeout) if err != nil { diff --git a/pkg/engine/inspector.go b/pkg/engine/inspector.go index ff0b6988bd7..a5b8ba311c3 100644 --- a/pkg/engine/inspector.go +++ b/pkg/engine/inspector.go @@ -1,6 +1,7 @@ package engine import ( + "bytes" "context" "encoding/json" "strings" @@ -16,6 +17,7 @@ import ( "github.com/open-policy-agent/opa/ast" "github.com/open-policy-agent/opa/cover" "github.com/open-policy-agent/opa/rego" + "github.com/open-policy-agent/opa/storage/inmem" "github.com/open-policy-agent/opa/topdown" "github.com/pkg/errors" "github.com/rs/zerolog/log" @@ -101,13 +103,13 @@ func NewInspector( queriesSource source.QueriesSource, vb VulnerabilityBuilder, tracker Tracker, - queryFilter source.QuerySelectionFilter, + queryParameters *source.QueryInspectorParameters, excludeResults map[string]bool, queryTimeout int) (*Inspector, error) { log.Debug().Msg("engine.NewInspector()") metrics.Metric.Start("get_queries") - queries, err := queriesSource.GetQueries(queryFilter) + queries, err := queriesSource.GetQueries(queryParameters) if err != nil { return nil, errors.Wrap(err, "failed to get queries") } @@ -133,11 +135,14 @@ func NewInspector( case <-ctx.Done(): return nil, nil default: - opaQuery, err := rego.New( + var opaQuery rego.PreparedEvalQuery + store := inmem.NewFromReader(bytes.NewBufferString(metadata.InputData)) + opaQuery, err = rego.New( rego.Query(regoQuery), rego.Module("Common", commonGeneralQuery), rego.Module("Generic", platformGeneralQuery), rego.Module(metadata.Query, metadata.Content), + rego.Store(store), rego.UnsafeBuiltins(unsafeRegoFunctions), ).PrepareForEval(ctx) if err != nil { diff --git a/pkg/engine/inspector_test.go b/pkg/engine/inspector_test.go index de5250192da..0c473e4d9ab 100644 --- a/pkg/engine/inspector_test.go +++ b/pkg/engine/inspector_test.go @@ -147,7 +147,8 @@ func TestInspect(t *testing.T) { //nolint opaQueries = append(opaQueries, &preparedQuery{ opaQuery: opaQuery, metadata: model.QueryMetadata{ - Query: "add_instead_of_copy", + Query: "add_instead_of_copy", + InputData: "{}", Content: `package Cx CxPolicy [ result ] { @@ -336,9 +337,10 @@ func TestNewInspector(t *testing.T) { // nolint opaQueries = append(opaQueries, &preparedQuery{ opaQuery: rego.PreparedEvalQuery{}, metadata: model.QueryMetadata{ - Query: "all_auth_users_get_read_access", - Content: string(contentByte), - Platform: "unknown", + Query: "all_auth_users_get_read_access", + Content: string(contentByte), + InputData: "{}", + Platform: "unknown", Metadata: map[string]interface{}{ "id": "57b9893d-33b1-4419-bcea-b828fb87e318", "queryName": "All Auth Users Get Read Access", @@ -356,7 +358,7 @@ func TestNewInspector(t *testing.T) { // nolint source source.QueriesSource vb VulnerabilityBuilder tracker Tracker - queryFilter source.QuerySelectionFilter + queryFilter source.QueryInspectorParameters excludeResults map[string]bool queryExecTimeout int } @@ -373,7 +375,7 @@ func TestNewInspector(t *testing.T) { // nolint vb: vbs, tracker: track, source: sources, - queryFilter: source.QuerySelectionFilter{ + queryFilter: source.QueryInspectorParameters{ IncludeQueries: source.IncludeQueries{ ByIDs: []string{}, }, @@ -399,7 +401,7 @@ func TestNewInspector(t *testing.T) { // nolint tt.args.source, tt.args.vb, tt.args.tracker, - tt.args.queryFilter, + &tt.args.queryFilter, tt.args.excludeResults, tt.args.queryExecTimeout) @@ -560,7 +562,14 @@ func newInspectorInstance(t *testing.T, queryPath string) *Inspector { detector *detector.DetectLine) (model.Vulnerability, error) { return model.Vulnerability{}, nil } - ins, err := NewInspector(context.Background(), querySource, vb, &tracker.CITracker{}, source.QuerySelectionFilter{}, map[string]bool{}, 60) + ins, err := NewInspector( + context.Background(), + querySource, + vb, + &tracker.CITracker{}, + &source.QueryInspectorParameters{}, + map[string]bool{}, 60, + ) require.NoError(t, err) return ins } @@ -570,7 +579,7 @@ type mockSource struct { Types []string } -func (m *mockSource) GetQueries(queryFilter source.QuerySelectionFilter) ([]model.QueryMetadata, error) { +func (m *mockSource) GetQueries(queryFilter *source.QueryInspectorParameters) ([]model.QueryMetadata, error) { sources := source.NewFilesystemSource(m.Source, []string{""}) return sources.GetQueries(queryFilter) diff --git a/pkg/engine/mock/source.go b/pkg/engine/mock/source.go index 26888c8897b..4b5c132ab34 100644 --- a/pkg/engine/mock/source.go +++ b/pkg/engine/mock/source.go @@ -36,7 +36,7 @@ func (m *MockQueriesSource) EXPECT() *MockQueriesSourceMockRecorder { } // GetQueries mocks base method. -func (m *MockQueriesSource) GetQueries(querySelection source.QuerySelectionFilter) ([]model.QueryMetadata, error) { +func (m *MockQueriesSource) GetQueries(querySelection *source.QueryInspectorParameters) ([]model.QueryMetadata, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetQueries", querySelection) ret0, _ := ret[0].([]model.QueryMetadata) diff --git a/pkg/engine/source/filesystem.go b/pkg/engine/source/filesystem.go index faf4d5430c2..4789ac24abf 100644 --- a/pkg/engine/source/filesystem.go +++ b/pkg/engine/source/filesystem.go @@ -31,6 +31,8 @@ const ( LibraryFileName = "library.rego" // LibrariesDefaultBasePath the path to rego libraries LibrariesDefaultBasePath = "./assets/libraries/" + + emptyInputData = "{}" ) var ( @@ -146,7 +148,7 @@ func checkQueryExclude(id interface{}, excludeQueries []string) bool { // GetQueries walks a given filesource path returns all queries found in an array of // QueryMetadata struct -func (s *FilesystemSource) GetQueries(queryFilter QuerySelectionFilter) ([]model.QueryMetadata, error) { +func (s *FilesystemSource) GetQueries(queryParameters *QueryInspectorParameters) ([]model.QueryMetadata, error) { queryDirs := make([]string, 0) err := filepath.Walk(s.Source, func(p string, f os.FileInfo, err error) error { @@ -172,7 +174,6 @@ func (s *FilesystemSource) GetQueries(queryFilter QuerySelectionFilter) ([]model sentry.CaptureException(errRQ) log.Err(errRQ). Msgf("Query provider failed to read query, query=%s", path.Base(queryDir)) - continue } @@ -180,13 +181,28 @@ func (s *FilesystemSource) GetQueries(queryFilter QuerySelectionFilter) ([]model continue } - if len(queryFilter.IncludeQueries.ByIDs) > 0 { - if checkQueryInclude(query.Metadata["id"], queryFilter.IncludeQueries.ByIDs) { + customInputData, readInputErr := readInputData(filepath.Join(queryParameters.InputDataPath, query.Metadata["id"].(string)+".json")) + if readInputErr != nil { + log.Err(errRQ). + Msgf("failed to read input data, query=%s", path.Base(queryDir)) + continue + } + + inputData, mergeError := mergeInputData(query.InputData, customInputData) + if mergeError != nil { + log.Err(mergeError). + Msgf("failed to merge input data, query=%s", path.Base(queryDir)) + continue + } + query.InputData = inputData + + if len(queryParameters.IncludeQueries.ByIDs) > 0 { + if checkQueryInclude(query.Metadata["id"], queryParameters.IncludeQueries.ByIDs) { queries = append(queries, query) } } else { - if checkQueryExclude(query.Metadata["id"], queryFilter.ExcludeQueries.ByIDs) || - checkQueryExclude(query.Metadata["category"], queryFilter.ExcludeQueries.ByCategories) { + if checkQueryExclude(query.Metadata["id"], queryParameters.ExcludeQueries.ByIDs) || + checkQueryExclude(query.Metadata["category"], queryParameters.ExcludeQueries.ByCategories) { log.Debug(). Msgf("Excluding query ID: %s category: %s", query.Metadata["id"], query.Metadata["category"]) continue @@ -209,6 +225,11 @@ func ReadQuery(queryDir string) (model.QueryMetadata, error) { metadata := ReadMetadata(queryDir) platform := getPlatform(queryDir) + inputData, errInputData := readInputData(filepath.Join(queryDir, "data.json")) + if errInputData != nil { + log.Err(errInputData). + Msgf("Query provider failed to read input data, query=%s", path.Base(queryDir)) + } aggregation := 1 if agg, ok := metadata["aggregation"]; ok { @@ -220,6 +241,7 @@ func ReadQuery(queryDir string) (model.QueryMetadata, error) { Content: string(queryContent), Metadata: metadata, Platform: platform, + InputData: inputData, Aggregation: aggregation, }, nil } @@ -279,3 +301,37 @@ func getPlatform(queryPath string) string { return "unknown" } + +func readInputData(inputDataPath string) (string, error) { + inputData, err := os.ReadFile(filepath.Clean(inputDataPath)) + if err != nil { + if os.IsNotExist(err) { + return emptyInputData, nil + } + return emptyInputData, errors.Wrapf(err, "failed to read query input data %s", path.Base(inputDataPath)) + } + return string(inputData), nil +} + +func mergeInputData(queryInputData, customInputData string) (string, error) { + if customInputData == emptyInputData || customInputData == "" { + return queryInputData, nil + } + dataJSON := map[string]interface{}{} + customDataJSON := map[string]interface{}{} + if unmarshalError := json.Unmarshal([]byte(queryInputData), &dataJSON); unmarshalError != nil { + return "", errors.Wrapf(unmarshalError, "failed to merge query input data") + } + if unmarshalError := json.Unmarshal([]byte(customInputData), &customDataJSON); unmarshalError != nil { + return "", errors.Wrapf(unmarshalError, "failed to merge query input data") + } + + for key, value := range customDataJSON { + dataJSON[key] = value + } + mergedJSON, mergeErr := json.Marshal(dataJSON) + if mergeErr != nil { + return "", errors.Wrapf(mergeErr, "failed to merge query input data") + } + return string(mergedJSON), nil +} diff --git a/pkg/engine/source/filesystem_test.go b/pkg/engine/source/filesystem_test.go index e78924c4e81..cbb810033da 100644 --- a/pkg/engine/source/filesystem_test.go +++ b/pkg/engine/source/filesystem_test.go @@ -1,6 +1,7 @@ package source import ( + "encoding/json" "os" "path/filepath" "reflect" @@ -38,8 +39,12 @@ func BenchmarkFilesystemSource_GetQueries(b *testing.B) { b.Run(tt.name, func(b *testing.B) { s := NewFilesystemSource(tt.fields.Source, tt.fields.Types) for n := 0; n < b.N; n++ { - filter := QuerySelectionFilter{IncludeQueries{ByIDs: []string{}}, ExcludeQueries{ByIDs: []string{}, ByCategories: []string{}}} - if _, err := s.GetQueries(filter); err != nil { + filter := QueryInspectorParameters{ + IncludeQueries: IncludeQueries{ByIDs: []string{}}, + ExcludeQueries: ExcludeQueries{ByIDs: []string{}, ByCategories: []string{}}, + InputDataPath: "", + } + if _, err := s.GetQueries(&filter); err != nil { b.Errorf("Error: %s", err) } } @@ -76,8 +81,9 @@ func TestFilesystemSource_GetQueriesWithExclude(t *testing.T) { excludeIDs: []string{"57b9893d-33b1-4419-bcea-a717ea87e4449"}, want: []model.QueryMetadata{ { - Query: "all_auth_users_get_read_access", - Content: string(contentByte), + Query: "all_auth_users_get_read_access", + Content: string(contentByte), + InputData: "{}", Metadata: map[string]interface{}{ "category": "Access Control", "descriptionText": "Misconfigured S3 buckets can leak private information to the entire internet or allow unauthorized data tampering / deletion", //nolint @@ -129,14 +135,12 @@ func TestFilesystemSource_GetQueriesWithExclude(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := NewFilesystemSource(tt.fields.Source, []string{""}) - filter := QuerySelectionFilter{ - IncludeQueries{ByIDs: []string{}}, - ExcludeQueries{ - ByIDs: tt.excludeIDs, - ByCategories: tt.excludeCategory, - }, + filter := QueryInspectorParameters{ + IncludeQueries: IncludeQueries{ByIDs: []string{}}, + ExcludeQueries: ExcludeQueries{ByIDs: tt.excludeIDs, ByCategories: tt.excludeCategory}, + InputDataPath: "", } - got, err := s.GetQueries(filter) + got, err := s.GetQueries(&filter) if (err != nil) != tt.wantErr { t.Errorf("FilesystemSource.GetQueries() error = %v, wantErr %v", err, tt.wantErr) return @@ -179,8 +183,9 @@ func TestFilesystemSource_GetQueriesWithInclude(t *testing.T) { includeIDs: []string{"57b9893d-33b1-4419-bcea-b828fb87e318"}, want: []model.QueryMetadata{ { - Query: "all_auth_users_get_read_access", - Content: string(contentByte), + Query: "all_auth_users_get_read_access", + Content: string(contentByte), + InputData: "{}", Metadata: map[string]interface{}{ "category": "Access Control", "descriptionText": "Misconfigured S3 buckets can leak private information to the entire internet or allow unauthorized data tampering / deletion", //nolint @@ -219,16 +224,17 @@ func TestFilesystemSource_GetQueriesWithInclude(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := NewFilesystemSource(tt.fields.Source, []string{""}) - filter := QuerySelectionFilter{ - IncludeQueries{ + filter := QueryInspectorParameters{ + IncludeQueries: IncludeQueries{ ByIDs: tt.includeIDs, }, - ExcludeQueries{ + ExcludeQueries: ExcludeQueries{ ByIDs: []string{}, ByCategories: []string{}, }, + InputDataPath: "", } - got, err := s.GetQueries(filter) + got, err := s.GetQueries(&filter) if (err != nil) != tt.wantErr { t.Errorf("FilesystemSource.GetQueries() error = %v, wantErr %v", err, tt.wantErr) return @@ -382,8 +388,9 @@ func TestFilesystemSource_GetQueries(t *testing.T) { }, want: []model.QueryMetadata{ { - Query: "all_auth_users_get_read_access", - Content: string(contentByte), + Query: "all_auth_users_get_read_access", + Content: string(contentByte), + InputData: "{}", Metadata: map[string]interface{}{ "category": "Access Control", "descriptionText": "Misconfigured S3 buckets can leak private information to the entire internet or allow unauthorized data tampering / deletion", //nolint @@ -411,15 +418,16 @@ func TestFilesystemSource_GetQueries(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := NewFilesystemSource(tt.fields.Source, []string{""}) - filter := QuerySelectionFilter{ - IncludeQueries{ + filter := QueryInspectorParameters{ + IncludeQueries: IncludeQueries{ ByIDs: []string{}}, - ExcludeQueries{ + ExcludeQueries: ExcludeQueries{ ByIDs: []string{}, ByCategories: []string{}, }, + InputDataPath: "", } - got, err := s.GetQueries(filter) + got, err := s.GetQueries(&filter) if (err != nil) != tt.wantErr { t.Errorf("FilesystemSource.GetQueries() error = %v, wantErr %v", err, tt.wantErr) return @@ -566,3 +574,59 @@ func TestListSupportedPlatforms(t *testing.T) { actual := ListSupportedPlatforms() require.Equal(t, expected, actual, "expected=%s\ngot=%s", expected, actual) } + +// TestReadInputData tests readInputData function +func TestReadInputData(t *testing.T) { + tests := []struct { + name string + path string + want string + }{ + { + name: "Should generate input data string", + path: filepath.FromSlash("./test/fixtures/input_data/test.json"), + want: `{"test": "success"}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := readInputData(tt.path) + require.NoError(t, err) + require.Equal(t, tt.want, got) + }) + } +} + +// TestMergeInputData tests mergeInputData function +func TestMergeInputData(t *testing.T) { + tests := []struct { + name string + pathData string + pathMerge string + want string + }{ + { + name: "Should merge input data strings", + pathData: filepath.FromSlash("./test/fixtures/input_data/test.json"), + pathMerge: filepath.FromSlash("./test/fixtures/input_data/merge.json"), + want: `{"test": "merge","merge": "success"}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := readInputData(tt.pathData) + require.NoError(t, err) + customData, err := readInputData(tt.pathMerge) + require.NoError(t, err) + got, err := mergeInputData(data, customData) + require.NoError(t, err) + wantJSON := map[string]interface{}{} + gotJSON := map[string]interface{}{} + err = json.Unmarshal([]byte(tt.want), &wantJSON) + require.NoError(t, err) + err = json.Unmarshal([]byte(got), &gotJSON) + require.NoError(t, err) + require.Equal(t, wantJSON, gotJSON) + }) + } +} diff --git a/pkg/engine/source/source.go b/pkg/engine/source/source.go index 72e4a8bb8dd..dd24fda841a 100644 --- a/pkg/engine/source/source.go +++ b/pkg/engine/source/source.go @@ -3,10 +3,11 @@ package source import "github.com/Checkmarx/kics/pkg/model" -// QuerySelectionFilter is a struct that represents the optionn to select queries to be executed -type QuerySelectionFilter struct { +// QueryInspectorParameters is a struct that represents the optionn to select queries to be executed +type QueryInspectorParameters struct { IncludeQueries IncludeQueries ExcludeQueries ExcludeQueries + InputDataPath string } // ExcludeQueries is a struct that represents the option to exclude queries by ids or by categories @@ -24,6 +25,6 @@ type IncludeQueries struct { // GetQueries gets all queries from a QueryMetadata list // GetQueryLibrary gets a library of rego functions given a plataform's name type QueriesSource interface { - GetQueries(querySelection QuerySelectionFilter) ([]model.QueryMetadata, error) + GetQueries(querySelection *QueryInspectorParameters) ([]model.QueryMetadata, error) GetQueryLibrary(platform string) (string, error) } diff --git a/pkg/model/model.go b/pkg/model/model.go index 72fb464759a..e6177b9fec1 100644 --- a/pkg/model/model.go +++ b/pkg/model/model.go @@ -86,10 +86,11 @@ type FileMetadata struct { // QueryMetadata is a representation of general information about a query type QueryMetadata struct { - Query string - Content string - Metadata map[string]interface{} - Platform string + InputData string + Query string + Content string + Metadata map[string]interface{} + Platform string // special field for generic queries // represents how many queries are aggregated into a single rego file Aggregation int diff --git a/pkg/scanner/scanner_test.go b/pkg/scanner/scanner_test.go index 77039384f38..05c579d8918 100644 --- a/pkg/scanner/scanner_test.go +++ b/pkg/scanner/scanner_test.go @@ -69,7 +69,7 @@ func createServices(types []string) (serviceSlice, *storage.MemoryStorage, error inspector, err := engine.NewInspector(context.Background(), querySource, engine.DefaultVulnerabilityBuilder, - t, source.QuerySelectionFilter{}, map[string]bool{}, 60) + t, &source.QueryInspectorParameters{}, map[string]bool{}, 60) if err != nil { return nil, nil, err } diff --git a/test/fixtures/input_data/merge.json b/test/fixtures/input_data/merge.json new file mode 100644 index 00000000000..52720e67d5a --- /dev/null +++ b/test/fixtures/input_data/merge.json @@ -0,0 +1 @@ +{"test": "merge", "merge": "success"} \ No newline at end of file diff --git a/test/fixtures/input_data/test.json b/test/fixtures/input_data/test.json new file mode 100644 index 00000000000..172e73fd7ed --- /dev/null +++ b/test/fixtures/input_data/test.json @@ -0,0 +1 @@ +{"test": "success"} \ No newline at end of file diff --git a/test/main_test.go b/test/main_test.go index e104d71fff5..17af52adc71 100644 --- a/test/main_test.go +++ b/test/main_test.go @@ -235,9 +235,10 @@ func isValidURL(toTest string) bool { return err == nil && u.Scheme != "" && u.Host != "" } -func getQueryFilter() source.QuerySelectionFilter { - return source.QuerySelectionFilter{ +func getQueryFilter() *source.QueryInspectorParameters { + return &source.QueryInspectorParameters{ IncludeQueries: source.IncludeQueries{ByIDs: []string{}}, ExcludeQueries: source.ExcludeQueries{ByIDs: []string{}, ByCategories: []string{}}, + InputDataPath: "", } } diff --git a/test/queries_content_test.go b/test/queries_content_test.go index 2a442ceac74..85897ebeef2 100644 --- a/test/queries_content_test.go +++ b/test/queries_content_test.go @@ -240,9 +240,10 @@ func testQueryHasGoodReturnParams(t *testing.T, entry queryEntry) { return model.Vulnerability{}, nil }, trk, - source.QuerySelectionFilter{ + &source.QueryInspectorParameters{ IncludeQueries: source.IncludeQueries{ByIDs: []string{}}, ExcludeQueries: source.ExcludeQueries{ByIDs: []string{}, ByCategories: []string{}}, + InputDataPath: "", }, map[string]bool{}, 60, diff --git a/test/queries_test.go b/test/queries_test.go index f59e6ee6003..3c7fda3be12 100644 --- a/test/queries_test.go +++ b/test/queries_test.go @@ -138,9 +138,10 @@ func testQuery(tb testing.TB, entry queryEntry, filesPath []string, expectedVuln queriesSource, engine.DefaultVulnerabilityBuilder, &tracker.CITracker{}, - source.QuerySelectionFilter{ + &source.QueryInspectorParameters{ IncludeQueries: source.IncludeQueries{ByIDs: []string{}}, ExcludeQueries: source.ExcludeQueries{ByIDs: []string{}, ByCategories: []string{}}, + InputDataPath: "", }, map[string]bool{}, 60) diff --git a/test/similarity_id_test.go b/test/similarity_id_test.go index 54e28437ae3..6d9be1975c2 100644 --- a/test/similarity_id_test.go +++ b/test/similarity_id_test.go @@ -265,10 +265,11 @@ func createInspectorAndGetVulnerabilities(ctx context.Context, t testing.TB, } q := model.QueryMetadata{ - Query: testParams.queryID(), - Content: testParams.queryContent(t), - Metadata: metadata, - Platform: testParams.platform, + Query: testParams.queryID(), + Content: testParams.queryContent(t), + InputData: "{}", + Metadata: metadata, + Platform: testParams.platform, } return []model.QueryMetadata{q}, nil }) @@ -291,9 +292,10 @@ func createInspectorAndGetVulnerabilities(ctx context.Context, t testing.TB, queriesSource, engine.DefaultVulnerabilityBuilder, &tracker.CITracker{}, - source.QuerySelectionFilter{ + &source.QueryInspectorParameters{ IncludeQueries: source.IncludeQueries{ByIDs: []string{}}, ExcludeQueries: source.ExcludeQueries{ByIDs: []string{}, ByCategories: []string{}}, + InputDataPath: "", }, map[string]bool{}, 60)