Respect scan_date at import time for all findings imported (DefectD…

…ojo#5547)

* Respect `scan_date` at import time for all findings imported

* Update scan_date set logic

* Add more unit tests

* Remove first attempt at unit test

* Add documentation and set default import date to be timezone considerate

* Make scan_date optional field

* Update test_importers_importer.py

* Update api scan_date validation

* Fix required false issue

* Update docs title

* Remove default date from unit tests

docs/content/en/integrations/importing.md

@@ -118,4 +118,23 @@ A classic way of reimporting a scan is by specifying the ID of the test instead:
     "scan_type": 'ZAP Scan',
     "test": 123,
 }
-```
+```
+
+## Using the Scan Completion Date (API: `scan_date`) field
+
+DefectDojo offers a plethora of supported scanner reports, but not all of them contain the 
+information most important to a user. The `scan_date` field is a flexible smart feature that
+allows users to set the completion date of the a given scan report, and have it propagate
+down to all the findings imported. This field is **not** mandatory, but the default value for 
+this field is the date of import (whenever the request is processed and a successful response is returned). 
+
+Here are the following use cases for using this field:
+
+1. The report **does not** set the date, and `scan_date` is **not** set at import
+    - Finding date will be the default value of `scan_date`
+2. The report **sets** the date, and the `scan_date` is **not** set at import
+    - Finding date will be whatever the report sets
+3. The report **does not** set the date, and the `scan_date` is **set** at import
+    - Finding date will be whatever the user set for `scan_date`
+4. The report **sets** the date, and the `scan_date` is **set** at import
+    - Finding date will be whatever the user set for `scan_date`

dojo/api_v2/serializers.py

@@ -1202,7 +1202,7 @@ def get_findings_list(self, obj) -> List[int]:
 
 
 class ImportScanSerializer(serializers.Serializer):
-    scan_date = serializers.DateField(default=datetime.date.today)
+    scan_date = serializers.DateField(required=False)
 
     minimum_severity = serializers.ChoiceField(
         choices=SEVERITY_CHOICES,
@@ -1258,7 +1258,7 @@ def save(self, push_to_jira=False):
         verified = data['verified']
         minimum_severity = data['minimum_severity']
         endpoint_to_add = data['endpoint_to_add']
-        scan_date = data['scan_date']
+        scan_date = data.get('scan_date', None)
         # Will save in the provided environment or in the `Development` one if absent
         version = data.get('version', None)
         build_id = data.get('build_id', None)
@@ -1326,7 +1326,10 @@ def validate(self, data):
         return data
 
     def validate_scan_data(self, value):
-        if value.date() > datetime.today().date():
+        # scan_date is no longer deafulted to "today" at import time, so set it here if necessary
+        if not value.date:
+            return None
+        if value.date() > timezone.localtime(timezone.now()).date():
             raise serializers.ValidationError(
                 'The date cannot be in the future!')
         return value

dojo/forms.py

@@ -6,7 +6,6 @@
 from crispy_forms.helper import FormHelper
 from crispy_forms.layout import Layout
 from django.db.models import Count, Q
-
 from dateutil.relativedelta import relativedelta
 from django import forms
 from django.contrib.auth.password_validation import validate_password
@@ -373,10 +372,9 @@ class Meta:
 
 class ImportScanForm(forms.Form):
     scan_date = forms.DateTimeField(
-        required=True,
+        required=False,
         label="Scan Completion Date",
         help_text="Scan completion date will be used on all findings.",
-        initial=datetime.now().strftime("%Y-%m-%d"),
         widget=forms.TextInput(attrs={'class': 'datepicker'}))
     minimum_severity = forms.ChoiceField(help_text='Minimum severity level to be imported',
                                          required=True,
@@ -445,6 +443,9 @@ def clean(self):
     # date can only be today or in the past, not the future
     def clean_scan_date(self):
         date = self.cleaned_data['scan_date']
+        # scan_date is no longer deafulted to "today" at import time, so set it here if necessary
+        if not date:
+            return None
         if date.date() > datetime.today().date():
             raise forms.ValidationError("The date cannot be in the future!")
         return date
@@ -514,7 +515,7 @@ def clean(self):
     # date can only be today or in the past, not the future
     def clean_scan_date(self):
         date = self.cleaned_data['scan_date']
-        if date.date() > datetime.today().date():
+        if date.date() > timezone.localtime(timezone.now()).date():
             raise forms.ValidationError("The date cannot be in the future!")
         return date
 

dojo/importers/importer/importer.py

@@ -62,7 +62,7 @@ def create_test(self, scan_type, test_type_name, engagement, lead, environment,
     @dojo_async_task
     @app.task(ignore_result=False)
     def process_parsed_findings(self, test, parsed_findings, scan_type, user, active, verified, minimum_severity=None,
-                                endpoints_to_add=None, push_to_jira=None, group_by=None, now=timezone.now(), service=None, **kwargs):
+                                endpoints_to_add=None, push_to_jira=None, group_by=None, now=timezone.now(), service=None, scan_date=None, **kwargs):
         logger.debug('endpoints_to_add: %s', endpoints_to_add)
         new_findings = []
         items = parsed_findings
@@ -100,6 +100,13 @@ def process_parsed_findings(self, test, parsed_findings, scan_type, user, active
                 item.active = active
             if item.verified:
                 item.verified = verified
+            # Set the date if the parser does not set it
+            if not item.date:
+                item.date = scan_date
+
+            # Indicates the scan_date is not the default, overwrite everything
+            if (scan_date.date() if isinstance(scan_date, datetime.datetime) else scan_date) != now.date():
+                item.date = scan_date
 
             item.created = now
             item.updated = now
@@ -256,6 +263,11 @@ def import_scan(self, scan, scan_type, engagement, lead, environment, active, ve
         user = user or get_current_user()
 
         now = timezone.now()
+        # scan_date is no longer deafulted to "today" at import time, so set it here if necessary
+        finding_scan_date = scan_date
+        if not scan_date:
+            scan_date = now
+            finding_scan_date = now
         # retain weird existing logic to use current time for provided scan date
         scan_date_time = datetime.datetime.combine(scan_date, timezone.now().time())
         if settings.USE_TZ:
@@ -328,7 +340,7 @@ def import_scan(self, scan, scan_type, engagement, lead, environment, active, ve
                 result = self.process_parsed_findings(test, findings_list, scan_type, user, active,
                                                             verified, minimum_severity=minimum_severity,
                                                             endpoints_to_add=endpoints_to_add, push_to_jira=push_to_jira,
-                                                            group_by=group_by, now=now, service=service, sync=False)
+                                                            group_by=group_by, now=now, service=service, scan_date=finding_scan_date, sync=False)
                 # Since I dont want to wait until the task is done right now, save the id
                 # So I can check on the task later
                 results_list += [result]
@@ -346,7 +358,7 @@ def import_scan(self, scan, scan_type, engagement, lead, environment, active, ve
             new_findings = self.process_parsed_findings(test, parsed_findings, scan_type, user, active,
                                                             verified, minimum_severity=minimum_severity,
                                                             endpoints_to_add=endpoints_to_add, push_to_jira=push_to_jira,
-                                                            group_by=group_by, now=now, service=service, sync=True)
+                                                            group_by=group_by, now=now, service=service, scan_date=finding_scan_date, sync=True)
 
         closed_findings = []
         if close_old_findings:

unittests/dojo_test_case.py

@@ -424,9 +424,8 @@ def get_test_api(self, test_id):
 
     def import_scan_with_params(self, filename, scan_type='ZAP Scan', engagement=1, minimum_severity='Low', active=True, verified=True,
                                 push_to_jira=None, endpoint_to_add=None, tags=None, close_old_findings=False, group_by=None, engagement_name=None,
-                                product_name=None, product=None, product_type_name=None, auto_create_context=None, expected_http_status_code=201, test_title=None):
+                                product_name=None, product=None, product_type_name=None, auto_create_context=None, expected_http_status_code=201, test_title=None, scan_date=None):
         payload = {
-                "scan_date": '2020-06-04',
                 "minimum_severity": minimum_severity,
                 "active": active,
                 "verified": verified,
@@ -469,6 +468,9 @@ def import_scan_with_params(self, filename, scan_type='ZAP Scan', engagement=1,
         if test_title is not None:
             payload['test_title'] = test_title
 
+        if scan_date is not None:
+            payload['scan_date'] = scan_date
+
         return self.import_scan(payload, expected_http_status_code)
 
     def reimport_scan_with_params(self, test_id, filename, scan_type='ZAP Scan', engagement=1, minimum_severity='Low', active=True, verified=True, push_to_jira=None,

unittests/test_import_reimport.py

@@ -3,6 +3,7 @@
 from rest_framework.authtoken.models import Token
 from rest_framework.test import APIClient
 from django.test.client import Client
+from django.utils import timezone
 from .dojo_test_case import DojoAPITestCase, get_unit_tests_path
 from .test_utils import assertTestImportModelsCreated
 from django.test import override_settings
@@ -55,6 +56,9 @@ def __init__(self, *args, **kwargs):
         self.anchore_file_name = self.scans_path + 'anchore/one_vuln_many_files.json'
         self.scan_type_anchore = 'Anchore Engine Scan'
 
+        self.acunetix_file_name = self.scans_path + 'acunetix/one_finding.xml'
+        self.scan_type_acunetix = 'Acunetix Scan'
+
         self.gitlab_dep_scan_components_filename = self.scans_path + 'gitlab_dep_scan/gl-dependency-scanning-report-many-vuln.json'
         self.scan_type_gtlab_dep_scan = 'GitLab Dependency Scanning Report'
 
@@ -156,6 +160,78 @@ def test_zap_scan_base_not_active_not_verified(self):
 
         return test_id
 
+    # import zap scan, testing:
+    # - import
+    # - deafult scan_date (today) overrides date not set by parser
+    def test_import_default_scan_date_parser_not_sets_date(self):
+        logger.debug('importing zap xml report with date set by parser')
+        with assertTestImportModelsCreated(self, imports=1, affected_findings=4, created=4):
+            import0 = self.import_scan_with_params(self.zap_sample0_filename, active=False, verified=False)
+
+        test_id = import0['test']
+        findings = self.get_test_findings_api(test_id, active=False, verified=False)
+        self.log_finding_summary_json_api(findings)
+
+        # Get the date
+        date = findings['results'][0]['date']
+        self.assertEqual(date, str(timezone.localtime(timezone.now()).date()))
+
+        return test_id
+
+    # import acunetix scan, testing:
+    # - import
+    # - deafult scan_date (today) does not overrides date set by parser
+    def test_import_default_scan_date_parser_sets_date(self):
+        logger.debug('importing original acunetix xml report')
+        with assertTestImportModelsCreated(self, imports=1, affected_findings=1, created=1):
+            import0 = self.import_scan_with_params(self.acunetix_file_name, scan_type=self.scan_type_acunetix, active=False, verified=False)
+
+        test_id = import0['test']
+        findings = self.get_test_findings_api(test_id, active=False, verified=False)
+        self.log_finding_summary_json_api(findings)
+
+        # Get the date
+        date = findings['results'][0]['date']
+        self.assertEqual(date, '2018-09-24')
+
+        return test_id
+
+    # import acunetix scan, testing:
+    # - import
+    # - set scan_date overrides date not set by parser
+    def test_import_set_scan_date_parser_not_sets_date(self):
+        logger.debug('importing original zap xml report')
+        with assertTestImportModelsCreated(self, imports=1, affected_findings=4, created=4):
+            import0 = self.import_scan_with_params(self.zap_sample0_filename, active=False, verified=False, scan_date='2006-12-26')
+
+        test_id = import0['test']
+        findings = self.get_test_findings_api(test_id, active=False, verified=False)
+        self.log_finding_summary_json_api(findings)
+
+        # Get the date
+        date = findings['results'][0]['date']
+        self.assertEqual(date, '2006-12-26')
+
+        return test_id
+
+    # import acunetix scan, testing:
+    # - import
+    # - set scan_date overrides date set by parser
+    def test_import_set_scan_date_parser_sets_date(self):
+        logger.debug('importing acunetix xml report with date set by parser')
+        with assertTestImportModelsCreated(self, imports=1, affected_findings=1, created=1):
+            import0 = self.import_scan_with_params(self.acunetix_file_name, scan_type=self.scan_type_acunetix, active=False, verified=False, scan_date='2006-12-26')
+
+        test_id = import0['test']
+        findings = self.get_test_findings_api(test_id, active=False, verified=False)
+        self.log_finding_summary_json_api(findings)
+
+        # Get the date
+        date = findings['results'][0]['date']
+        self.assertEqual(date, '2006-12-26')
+
+        return test_id
+
     # import checkmarx scan. ZAP parser will never create a finding with active/verified false
     # checkmarx will (for false positive for example)
     # the goal of this test is to verify the final active/verified status depending on the parser status vs the options choosen during import
@@ -1135,9 +1211,8 @@ def reimport_scan_ui(self, test, payload):
         test = Test.objects.get(id=response.url.split('/')[-1])
         return {'test': test.id}
 
-    def import_scan_with_params_ui(self, filename, scan_type='ZAP Scan', engagement=1, minimum_severity='Low', active=True, verified=True, push_to_jira=None, endpoint_to_add=None, tags=None, close_old_findings=False):
+    def import_scan_with_params_ui(self, filename, scan_type='ZAP Scan', engagement=1, minimum_severity='Low', active=True, verified=True, push_to_jira=None, endpoint_to_add=None, tags=None, close_old_findings=False, scan_date=None):
         payload = {
-                "scan_date": '2020-06-04',
                 "minimum_severity": minimum_severity,
                 "active": active,
                 "verified": verified,
@@ -1157,6 +1232,9 @@ def import_scan_with_params_ui(self, filename, scan_type='ZAP Scan', engagement=
         if tags is not None:
             payload['tags'] = tags
 
+        if scan_date is not None:
+            payload['scan_date'] = scan_date
+
         return self.import_scan_ui(engagement, payload)
 
     def reimport_scan_with_params_ui(self, test_id, filename, scan_type='ZAP Scan', minimum_severity='Low', active=True, verified=True, push_to_jira=None, tags=None, close_old_findings=True):

unittests/test_importers_importer.py

@@ -78,6 +78,7 @@ def test_parse_findings(self):
         minimum_severity = "Info"
         active = True
         verified = True
+        scan_date = timezone.localtime(timezone.now()).date()
         new_findings = importer.process_parsed_findings(
             test,
             parsed_findings,
@@ -86,6 +87,7 @@ def test_parse_findings(self):
             active,
             verified,
             minimum_severity=minimum_severity,
+            scan_date=scan_date,
             sync=True
         )