See readme/CHANGELOG.md for details

bin/TA-dmarc/mail-o365.py

@@ -47,7 +47,7 @@
 from classes import splunk_info as si
 from classes import custom_logger as c_logger
 
-__version__ = "1.1.0"
+__version__ = "1.2.0"
 __author__ = 'Arnold Holzel'
 __license__ = 'Apache License 2.0'
 
@@ -86,6 +86,10 @@
 options.add_argument('-a', '--action', help='The action to take when a mail is processed; move/delete/mark_read', default='mark_read')
 options.add_argument('-m', '--move_to', help='The folder to move emails to the following variables can be used \
     [year] [month] [day] [ week] Example: Inbox/done/[year]/week_[week] will become Inbox/done/2023/week_03', default='Inbox/done/[year]/week_[week]')
+options.add_argument('--proxy', action='store_true', help='Use a proxy server to connect to the internet' )
+options.add_argument('-x', '--proxy_server', help='The proxy server + port to use. IE: https://10.20.30.40:5060')
+options.add_argument('-y', '--proxy_user', help='The user to authenticate to the proxy to if needed', default='default_None')
+options.add_argument('-z', '--proxy_pwd', help='The password for the proxy user if needed', default='default_None')
 options.add_argument('-v', '--verbose', action='store_true', help='enable verbose logging on the CLI')
 options.add_argument('--sessionKey', help='The splunk session key to use')
 args = options.parse_args()
@@ -144,6 +148,11 @@
     mailfolder = splunk_info.get_config(custom_conf_file, 'main', 'mailserver_mailboxfolder')
     action = splunk_info.get_config(custom_conf_file, 'main', 'mailserver_action')
     move_to_folder = splunk_info.get_config(custom_conf_file, 'main', 'mailserver_moveto')
+
+    proxy_use = splunk_info.get_config(custom_conf_file, 'main', 'proxy_use')
+    proxy_server = splunk_info.get_config(custom_conf_file, 'main', 'proxy_server')
+    proxy_username = splunk_info.get_config(custom_conf_file, 'main', 'proxy_username')
+    proxy_pwd = splunk_info.get_credentials(proxy_username)
 else:
     client_id = args.client_id
     tenant_id = args.tenant_id
@@ -153,6 +162,36 @@
     action = args.action
     move_to_folder = args.move_to
 
+    if args.proxy:
+        proxy_use = True
+        proxy_server = args.proxy_server
+        proxy_username = args.proxy_user
+        proxy_pwd = args.proxy_pwd
+
+if proxy_use or proxy_use == 1 or proxy_use.lower() == 't' or proxy_use.lower() == 'true':
+    script_logger.debug(f"A proxy needs to be used to connect to internet. The following will be used: {proxy_server}")
+    proxy_use = True
+    proxy_regex = re.search(r"(?:^([htps]*)(?=[:]+)(?:\:\/\/)|^)(.*)", proxy_server)
+
+    if not proxy_regex.group(1):
+        script_logger.warning(f"No schema provided with the proxy_server:{proxy_server} will assume HTTP, please adjust the config via the setup page!")
+
+    if proxy_username != 'default_None' and proxy_username is not None and proxy_pwd != 'default_None' and proxy_pwd is not None:
+        # there is a proxy user name and password configured
+        if proxy_regex.group(1):
+            # the url has a schema provided rebuild the proxy_server variable with the username and password
+            proxy_server = f"{proxy_regex.group(1)}://{proxy_username}:{proxy_pwd}@{proxy_regex.group(2)}"
+        else:
+            proxy_server = f"http://{proxy_username}:{proxy_pwd}@{proxy_regex.group(2)}"
+    elif proxy_username != 'default_None' and proxy_username is not None:
+        if proxy_regex.group(1):
+            # the url has a schema provided rebuild the proxy_server variable with the username and password
+            proxy_server = f"{proxy_regex.group(1)}://{proxy_username}@{proxy_regex.group(2)}"
+        else:
+            proxy_server = f"http://{proxy_username}@{proxy_regex.group(2)}"
+
+    proxies = { 'http': proxy_server, 'https': proxy_server }
+
 if client_id is None or tenant_id is None or client_secret is None or user is None:
     script_logger.error("Not all the needed o365 fields are configured or accessable.")
     exit(1)
@@ -180,7 +219,15 @@ def get_request(endpoint, token):
         return None
 
     headers = { 'Authorization': f'Bearer {token}' }
-    response = requests.get(endpoint, headers=headers)
+
+    try:
+        if proxy_use:
+            response = requests.get(endpoint, headers=headers, proxies=proxies)
+        else:
+            response = requests.get(endpoint, headers=headers)
+    except Exception as exception:
+        script_logger.exception(f"Connection error {type(exception).__name__}; ")
+
 
     if response.status_code != 200:
         return None
@@ -294,8 +341,13 @@ def create_folder(folder_name, token, parent_folder_id=None, root_folder=1):
         else:
             # this folder needs to be created below an already existing folder
             F_ENDPOINT = f"{FOLDER_ENDPOINT}{parent_folder_id}/childFolders"
-
-        create_request = requests.post(F_ENDPOINT, data=content, headers=headers)
+        try:
+            if proxy_use:
+                create_request = requests.post(F_ENDPOINT, data=content, headers=headers, proxies=proxies)
+            else:
+                create_request = requests.post(F_ENDPOINT, data=content, headers=headers)
+        except Exception as exception:
+            script_logger.exception(f"Connection error {type(exception).__name__}; ")
 
         if create_request.status_code != 201:
             script_logger.error(f"Something went wrong creating the folder: {json.loads(create_request.content)['error']['message']}")
@@ -382,15 +434,27 @@ def create_folder(folder_name, token, parent_folder_id=None, root_folder=1):
                             move_content = f"{{ 'destinationId' : '{move_folder_id.lstrip('/')}' }}"
                             move_headers = { 'Authorization' : f"Bearer {result['access_token']}", 'Accept' : 'application/json', 'Content-Type' : 'application/json' }
 
-                            move_request = requests.post(f"{MESSAGE_ENDPOINT}/{message['id']}/move", data=move_content, headers=move_headers)
+                            try:
+                                if proxy_use:
+                                    move_request = requests.post(f"{MESSAGE_ENDPOINT}/{message['id']}/move", data=move_content, headers=move_headers, proxies=proxies)
+                                else:
+                                    move_request = requests.post(f"{MESSAGE_ENDPOINT}/{message['id']}/move", data=move_content, headers=move_headers)
+                            except Exception as exception:
+                                script_logger.exception(f"Connection error {type(exception).__name__}; ")
 
                             if move_request.status_code != 201:
                                 script_logger.error(f"HTTP {move_request.status_code} recieved. Error message: {json.loads(move_request.content)['error']['message']}")
                         elif action.lower() == "delete":
                             # delete the message
                             delete_header = { 'Authorization': f"Bearer {result['access_token']}" }
-
-                            delete_request = requests.delete(f"{MESSAGE_ENDPOINT}/{message['id']}", headers=delete_header)
+
+                            try:
+                                if proxy_use:
+                                    delete_request = requests.delete(f"{MESSAGE_ENDPOINT}/{message['id']}", headers=delete_header, proxies=proxies)
+                                else:
+                                    delete_request = requests.delete(f"{MESSAGE_ENDPOINT}/{message['id']}", headers=delete_header)
+                            except Exception as exception:
+                                script_logger.exception(f"Connection error {type(exception).__name__}; ")
 
                             if delete_request.status_code != 204:
                                 script_logger.error(f"HTTP {delete_request.status_code} recieved. Error message: {json.loads(delete_request.content)['error']['message']}")
@@ -402,7 +466,13 @@ def create_folder(folder_name, token, parent_folder_id=None, root_folder=1):
                             mark_content = f"{{ 'isRead' : 'True' }}"
                             mark_headers = { 'Authorization' : f"Bearer {result['access_token']}", 'Accept' : 'application/json', 'Content-Type' : 'application/json' }
 
-                            mark_request = requests.patch(f"{MESSAGE_ENDPOINT}/{message['id']}", data=mark_content, headers=mark_headers)
+                            try:
+                                if proxy_use:
+                                    mark_request = requests.patch(f"{MESSAGE_ENDPOINT}/{message['id']}", data=mark_content, headers=mark_headers, proxies=proxies)
+                                else:
+                                    mark_request = requests.patch(f"{MESSAGE_ENDPOINT}/{message['id']}", data=mark_content, headers=mark_headers)
+                            except Exception as exception:
+                                script_logger.exception(f"Connection error {type(exception).__name__}; ")
 
                             if mark_request.status_code != 201:
                                 script_logger.error(f"HTTP {mark_request.status_code} recieved. Error message: {json.loads(mark_request.content)['error']['message']}")

bin/ta-dmarc_setup.py

@@ -32,11 +32,11 @@
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "lib"))
 
 from splunk import admin as admin
-from  splunklib import client as client
+from splunklib import client as client
 
 from classes import custom_logger as c_logger
 
-__version__ = '2.0.0'
+__version__ = '2.1.0'
 __author__ = 'Arnold Holzel'
 __license__ = 'Apache License 2.0'
 
@@ -60,11 +60,12 @@ def setup(self):
             for arg in [
                 'mailserver_host', 'mailserver_port', 'mailserver_protocol', 'mailserver_mailboxfolder', 
                 'mailserver_user', 'mailserver_pwd', 'o365_client_id', 'o365_tenant_id', 'o365_client_secret', 
-                'mailserver_action', 'mailserver_moveto', 'skip_mail_download', 'log_level',  'output', 'resolve_ips'
+                'mailserver_action', 'mailserver_moveto', 'skip_mail_download', 'log_level',  'output', 'resolve_ips',
+                'proxy_use', 'proxy_server', 'proxy_username', 'proxy_pwd'
             ]:
                 self.supportedArgs.addOptArg(arg)
 
-    # Read the inital values of the options from the file ta-dmarc.conf and place them in the setup page.
+    # Read the inital values of the options from the file <<app_name>>.conf and place them in the setup page.
     # If no setup has been done before read from the app default file
     # If the setup has been done before read from the app local file first and if a field has no value there
     # fallback to the default values
@@ -86,10 +87,10 @@ def handleList(self, confInfo):
         if confDict is not None:
             for stanza, settings in confDict.items():
                 for key, val in settings.items():
-                    if (key in ['mailserver_host', 'mailserver_port', 'mailserver_protocol', 'mailserver_mailboxfolder', 'mailserver_user', 'mailserver_pwd', 'output', 'o365_client_id', 'o365_tenant_id', 'o365_client_secret', 'mailserver_action', 'mailserver_moveto'] and val in [None, '', 'configured']):
+                    if (key in ['mailserver_host', 'mailserver_port', 'mailserver_protocol', 'mailserver_mailboxfolder', 'mailserver_user', 'mailserver_pwd', 'output', 'o365_client_id', 'o365_tenant_id', 'o365_client_secret', 'mailserver_action', 'mailserver_moveto', 'proxy_server', 'proxy_username', 'proxy_pwd'] and val in [None, '', 'configured']):
                         val = ''
 
-                    if key in ['skip_mail_download', 'resolve_ips']:
+                    if key in ['skip_mail_download', 'resolve_ips', 'proxy_use']:
                         if int(val) == 1 or str(val.lower()) == "true" :
                             val = '1'
                         else:
@@ -126,10 +127,10 @@ def handleEdit(self, confInfo):
                 script_logger.exception("An error occurred connecting to splunkd")
 
         # Make two lists with field names, 
-        # one with the text fields, minus the username, password and o365 fields, one with the boolean fields. 
+        # one with the text fields, minus the usernames, passwords and o365 fields, one with the boolean fields. 
         # this is so we don't have to make a if else loop for each field.
-        text_fields_list = [ 'mailserver_host', 'mailserver_port', 'mailserver_protocol', 'mailserver_mailboxfolder', 'output', 'mailserver_action', 'mailserver_moveto' ]
-        boolean_fields_list = [ 'skip_mail_download', 'resolve_ips' ]
+        text_fields_list = [ 'mailserver_host', 'mailserver_port', 'mailserver_protocol', 'mailserver_mailboxfolder', 'output', 'mailserver_action', 'mailserver_moveto', 'proxy_server' ]
+        boolean_fields_list = [ 'skip_mail_download', 'resolve_ips', 'proxy_use' ]
 
         args_data = self.callerArgs.data
 
@@ -148,6 +149,7 @@ def handleEdit(self, confInfo):
         # Check the username, password and o365 fields if they are None or empty
         # set a variable so we know we don't have to store them in the Splunk
         # credential store later.
+        ############ Mailbox credentials ############
         if args_data['mailserver_user'][0] in [None, '']:
             mailserver_user = 0
             args_data['mailserver_user'][0] = 'None'
@@ -161,6 +163,21 @@ def handleEdit(self, confInfo):
             mailserver_pwd = args_data['mailserver_pwd'][0]
             args_data['mailserver_pwd'][0] = 'configured'
 
+        ############ Proxy credentials ############
+        if args_data['proxy_username'][0] in [None, '']:
+            proxy_username = 0
+            args_data['proxy_username'][0] = 'None'
+        else:
+            proxy_username = args_data['proxy_username'][0]
+
+        if args_data['proxy_pwd'][0] in [None, '', 'configured']:
+            proxy_pwd = 0
+            args_data['proxy_pwd'][0] = ''
+        else:
+            proxy_pwd = args_data['proxy_pwd'][0]
+            args_data['proxy_pwd'][0] = 'configured'
+
+        ############ o365 credentials ############
         if args_data['o365_client_id'][0] in [None, '']:
             o365_client_id = 0
             args_data['o365_client_id'][0] = 'None'
@@ -196,7 +213,7 @@ def handleEdit(self, confInfo):
 
         # write everything to the custom config file
         self.writeConf(app_name.lower(), 'main', args_data)
-          
+
         # Store the username and password if they are not empty
         if mailserver_user != 0 and mailserver_pwd != 0:
             try:
@@ -210,8 +227,9 @@ def handleEdit(self, confInfo):
                 service.storage_passwords.create(mailserver_pwd, mailserver_user)
 
             except Exception:
-                script_logger.exception("An error occurred updating user credentials. Please ensure your user account has admin_all_objects and/or list_storage_passwords capabilities.")
+                script_logger.exception("An error occurred updating mailbox credentials. Please ensure your user account has admin_all_objects and/or list_storage_passwords capabilities.")
 
+        # Store the o365 credentials if they are not empty
         if o365_client_id != 0 and o365_client_secret != 0:
             try:
                 for storage_password in service.storage_passwords:
@@ -222,6 +240,21 @@ def handleEdit(self, confInfo):
                 service.storage_passwords.create(o365_client_secret, o365_client_id)
             except Exception:
                 script_logger.exception("An error occurred updating o365 credentials. Please ensure your user account has admin_all_objects and/or list_storage_passwords capabilities.")
+
+        # Store the proxy credentials if they are not empty
+        if proxy_username != 0 and proxy_pwd != 0:
+            try:
+                # If the credential already exists, delete it.
+                for storage_password in service.storage_passwords:
+                    if storage_password.username == proxy_username:
+                        service.storage_passwords.delete(username=storage_password.username)
+                        break
+
+                # Create the credentials 
+                service.storage_passwords.create(proxy_pwd, proxy_username)
+
+            except Exception:
+                script_logger.exception("An error occurred updating mailbox credentials. Please ensure your user account has admin_all_objects and/or list_storage_passwords capabilities.")
 
 admin.init(ConfigApp, admin.CONTEXT_NONE)
 

default/app.conf

@@ -4,14 +4,14 @@
 [launcher]
 author = Arnold
 description = App to collect dmarc reports from a POP3/POP3S/IMAP/IMAPS/o365 mailbox and process the attachements
-version = 5.0.3
+version = 5.1.0
 
 [package]
 check_for_updates = 0
 
 [install]
 is_configured = 0
-build = 20230405
+build = 20230414
 
 [ui]
 is_visible = true

default/setup.xml

@@ -78,6 +78,29 @@
         <text>Folder to move the mail to. The following variabels can be used: "[YEAR]", "[MONTH]", "[DAY]", "[WEEK]" (without the quotes and in &lt;b&gt;upper case!&lt;/b&gt;) to create dynamic folders. Example: Inbox/done/[YEAR]/week_[WEEK] </text>
     </block>
 
+    <block title="Proxy options" endpoint="ta-dmarc/ta-dmarc_configure/" entity="main">
+        <input field="proxy_use">
+            <label>Use a proxy to connect to the internet (o365)</label>
+            <type>bool</type>
+        </input>
+
+        <input field="proxy_server">
+            <label>Proxy server</label>
+            <type>text</type>
+        </input>
+        <text>Please also provide the port to connect on in the url. Example: http(s)://192.168.123.123:8080 OR http(s)://proxy.domain.tld:8080</text>
+
+        <input field="proxy_username">
+            <label>Proxy user (optional)</label>
+            <type>text</type>
+        </input>
+
+        <input field="proxy_pwd">
+            <label>Proxy password (optional)</label>
+            <type>password</type>
+        </input>
+    </block>
+
     <block title="Output options" endpoint="ta-dmarc/ta-dmarc_configure/" entity="main">
         <text>&lt;b&gt;Set the output format for the dmarc log&lt;/b&gt;</text>
         <input field="output">

default/ta-dmarc.conf

@@ -34,3 +34,9 @@ output = json
 
 # resolve the PTR of the given source_ip at the time of ingestion.
 resolve_ips = 1
+
+# proxy config 
+proxy_use = 0
+proxy_server =
+proxy_username = 
+proxy_pwd =