From 40018eef7f102e85ce4788a3540b07806d70c2c1 Mon Sep 17 00:00:00 2001 From: Theo Guidoux Date: Mon, 16 Aug 2021 10:44:01 +0200 Subject: [PATCH 1/4] edit help + case where 'select=' --- tools/sigma/backends/sql.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/sigma/backends/sql.py b/tools/sigma/backends/sql.py index cd0b86474ec..18f8ab30956 100644 --- a/tools/sigma/backends/sql.py +++ b/tools/sigma/backends/sql.py @@ -45,19 +45,21 @@ class SQLBackend(SingleTextQueryBackend): mapLength = "(%s %s)" options = SingleTextQueryBackend.options + ( - ("table", False, "Use this option to specify table name, default is \"eventlog\"", None), + ("table", "eventlog", "Use this option to specify table name.", None), + ("select", "*", "Use this option to specify fields you want to select. Example: \"--backend-option select=xxx,yyy\"", None), ) def __init__(self, sigmaconfig, options): super().__init__(sigmaconfig) + if "table" in options: self.table = options["table"] else: self.table = "eventlog" - if "select" in options: + if "select" in options and options["select"]: self.select_fields = options["select"].split(',') else: self.select_fields = list() From 16269c0d6388b15f72cf5ca416496a66199a22e5 Mon Sep 17 00:00:00 2001 From: Theo Guidoux Date: Mon, 16 Aug 2021 10:47:05 +0200 Subject: [PATCH 2/4] cleaner default value handling --- tools/sigma/backends/sql.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tools/sigma/backends/sql.py b/tools/sigma/backends/sql.py index 18f8ab30956..31afbb58d5d 100644 --- a/tools/sigma/backends/sql.py +++ b/tools/sigma/backends/sql.py @@ -48,7 +48,6 @@ class SQLBackend(SingleTextQueryBackend): ("table", "eventlog", "Use this option to specify table name.", None), ("select", "*", "Use this option to specify fields you want to select. Example: \"--backend-option select=xxx,yyy\"", None), ) - def __init__(self, sigmaconfig, options): @@ -62,7 +61,7 @@ def __init__(self, sigmaconfig, options): if "select" in options and options["select"]: self.select_fields = options["select"].split(',') else: - self.select_fields = list() + self.select_fields = list("*") def generateANDNode(self, node): generated = [ self.generateNode(val) for val in node ] @@ -197,10 +196,7 @@ def generateQuery(self, parsed): if self._recursiveFtsSearch(parsed.parsedSearch): raise NotImplementedError("FullTextSearch not implemented for SQL Backend.") result = self.generateNode(parsed.parsedSearch) - select = "*" - - if self.select_fields: - select = ", ".join(self.select_fields) + select = ", ".join(self.select_fields) if parsed.parsedAgg: #Handle aggregation From c1876b9ff68af739f55889f09e1cadcedf982c1b Mon Sep 17 00:00:00 2001 From: Theo Guidoux Date: Mon, 16 Aug 2021 13:33:43 +0200 Subject: [PATCH 3/4] add fields from rules to query + sqlite --- tools/sigma/backends/sql.py | 60 +++++++++++++++++++++++++++++++--- tools/sigma/backends/sqlite.py | 15 +++------ 2 files changed, 61 insertions(+), 14 deletions(-) diff --git a/tools/sigma/backends/sql.py b/tools/sigma/backends/sql.py index 31afbb58d5d..f18cac2fd22 100644 --- a/tools/sigma/backends/sql.py +++ b/tools/sigma/backends/sql.py @@ -21,7 +21,6 @@ from sigma.backends.base import SingleTextQueryBackend from sigma.parser.condition import SigmaAggregationParser, NodeSubexpression, ConditionAND, ConditionOR, ConditionNOT from sigma.parser.exceptions import SigmaParseError - class SQLBackend(SingleTextQueryBackend): """Converts Sigma rule into SQL query""" identifier = "sql" @@ -61,7 +60,7 @@ def __init__(self, sigmaconfig, options): if "select" in options and options["select"]: self.select_fields = options["select"].split(',') else: - self.select_fields = list("*") + self.select_fields = list() def generateANDNode(self, node): generated = [ self.generateNode(val) for val in node ] @@ -142,6 +141,47 @@ def fieldNameMapping(self, fieldname, value): """ return fieldname + def generate(self, sigmaparser): + """Method is called for each sigma rule and receives the parsed rule (SigmaParser)""" + fields = list() + + # First add fields specified in the rule + try: + for field in sigmaparser.parsedyaml["fields"]: + mapped = sigmaparser.config.get_fieldmapping(field).resolve_fieldname(field, sigmaparser) + if type(mapped) == str: + fields.append(mapped) + elif type(mapped) == list: + fields.extend(mapped) + else: + raise TypeError("Field mapping must return string or list") + + except KeyError: # no 'fields' attribute + pass + + # Then add fields specified in the backend configuration + fields.extend(self.select_fields) + + # Finally, in case fields is empty, add the default value + if not fields: + fields = list("*") + + for parsed in sigmaparser.condparsed: + #query = self.generateQuery(parsed) + query = self._generateQueryWithFields(parsed, fields) + before = self.generateBefore(parsed) + after = self.generateAfter(parsed) + + result = "" + if before is not None: + result = before + if query is not None: + result += query + if after is not None: + result += after + + return result + def cleanValue(self, val): if not isinstance(val, str): return str(val) @@ -191,12 +231,24 @@ def generateAggregation(self, agg, where_clausel): return temp_table, agg_condition raise NotImplementedError("{} aggregation not implemented in SQL Backend".format(agg.aggfunc_notrans)) - + def generateQuery(self, parsed): + return self._generateQueryWithFields(parsed, list("*")) + + def checkFTS(self, parsed, result): if self._recursiveFtsSearch(parsed.parsedSearch): raise NotImplementedError("FullTextSearch not implemented for SQL Backend.") + + def _generateQueryWithFields(self, parsed, fields): + """ + Return a SQL query with fields specified. + """ + result = self.generateNode(parsed.parsedSearch) - select = ", ".join(self.select_fields) + + self.checkFTS(parsed, result) + + select = ", ".join(fields) if parsed.parsedAgg: #Handle aggregation diff --git a/tools/sigma/backends/sqlite.py b/tools/sigma/backends/sqlite.py index 8eec13ea73d..1f7e4e7ecce 100644 --- a/tools/sigma/backends/sqlite.py +++ b/tools/sigma/backends/sqlite.py @@ -18,7 +18,6 @@ from sigma.parser.condition import NodeSubexpression, ConditionAND, ConditionOR, ConditionNOT import re - class SQLiteBackend(SQLBackend): """Converts Sigma rule into SQL query for SQLite""" identifier = "sqlite" @@ -26,6 +25,8 @@ class SQLiteBackend(SQLBackend): mapFullTextSearch = "%s MATCH ('\"%s\"')" + countFTS = 0 + def __init__(self, sigmaconfig, table): super().__init__(sigmaconfig, table) self.mappingItem = False @@ -108,16 +109,10 @@ def generateValueNode(self, node): return self.generateFTS(self.cleanValue(str(node))) def generateQuery(self, parsed): - self.countFTS = 0 - result = self.generateNode(parsed.parsedSearch) + return self._generateQueryWithFields(parsed, list("*")) + + def checkFTS(self, parsed, result): if self.countFTS > 1: raise NotImplementedError( "Match operator ({}) is allowed only once in SQLite, parse rule in a different way:\n{}".format(self.countFTS, result)) self.countFTS = 0 - - if parsed.parsedAgg: - # Handle aggregation - fro, whe = self.generateAggregation(parsed.parsedAgg, result) - return "SELECT * FROM {} WHERE {}".format(fro, whe) - - return "SELECT * FROM {} WHERE {}".format(self.table, result) From 2a3acd7d119f561bc9f17bac2de6bd4fc2f3ca16 Mon Sep 17 00:00:00 2001 From: Theo Guidoux Date: Mon, 16 Aug 2021 19:32:54 +0200 Subject: [PATCH 4/4] add selection flag for backward compatibility --- tools/sigma/backends/sql.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tools/sigma/backends/sql.py b/tools/sigma/backends/sql.py index f18cac2fd22..b31fba2b84c 100644 --- a/tools/sigma/backends/sql.py +++ b/tools/sigma/backends/sql.py @@ -46,7 +46,10 @@ class SQLBackend(SingleTextQueryBackend): options = SingleTextQueryBackend.options + ( ("table", "eventlog", "Use this option to specify table name.", None), ("select", "*", "Use this option to specify fields you want to select. Example: \"--backend-option select=xxx,yyy\"", None), + ("selection", False, "Use this option to enable fields selection from Sigma rules.", None), ) + + selection_enabled = False def __init__(self, sigmaconfig, options): @@ -62,6 +65,9 @@ def __init__(self, sigmaconfig, options): else: self.select_fields = list() + if "selection" in options: + self.selection_enabled = True + def generateANDNode(self, node): generated = [ self.generateNode(val) for val in node ] filtered = [ g for g in generated if g is not None ] @@ -162,13 +168,19 @@ def generate(self, sigmaparser): # Then add fields specified in the backend configuration fields.extend(self.select_fields) + # In case select is specified in backend option, we want to enable selection + if len(self.select_fields) > 0: + self.selection_enabled = True + # Finally, in case fields is empty, add the default value if not fields: fields = list("*") for parsed in sigmaparser.condparsed: - #query = self.generateQuery(parsed) - query = self._generateQueryWithFields(parsed, fields) + if self.selection_enabled: + query = self._generateQueryWithFields(parsed, fields) + else: + query = self.generateQuery(parsed) before = self.generateBefore(parsed) after = self.generateAfter(parsed)