Skip to content

Commit

Permalink
add scan_date fix also for reimport, fix validation (DefectDojo#5574)
Browse files Browse the repository at this point in the history 
* simplify scan_date handling

* simplify scan_date handling

* simplify scan_date handling

* simplify scan_date handling

* simplify scan_date handling

* simplify scan_date handling

* simplify scan_date handling

* api: fix scan_date validation

* typos

* fixes

* add tests

* add tests

* deutsche Grundlichkeit
  • Loading branch information
@valentijnscholten
valentijnscholten authored Dec 28, 2021
1 parent 0999ccd commit a70de90
Show file tree
Hide file tree
Showing 12 changed files with 255 additions and 131 deletions.
1 change: 0 additions & 1 deletion docs/content/en/integrations/api-v2-docs.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,6 @@ Example for importing a scan result:
active:true
lead:1
tags:test
scan_date:2019-04-30
scan_type:ZAP Scan
minimum_severity:Info
skip_duplicates:true
Expand Down
10 changes: 3 additions & 7 deletions docs/content/en/integrations/importing.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ An import can be performed by specifying the names of these entities in the API

```JSON
{
"scan_date": '2020-06-04',
"minimum_severity": 'Info',
"active": True,
"verified": Trued,
Expand All @@ -69,7 +68,6 @@ A classic way of importing a scan is by specifying the ID of the engagement inst

```JSON
{
"scan_date": '2020-06-04',
"minimum_severity": 'Info',
"active": True,
"verified": Trued,
Expand All @@ -88,7 +86,6 @@ An reimport can be performed by specifying the names of these entities in the AP

```JSON
{
"scan_date": '2020-06-04',
"minimum_severity": 'Info',
"active": True,
"verified": Trued,
Expand All @@ -111,7 +108,6 @@ A classic way of reimporting a scan is by specifying the ID of the test instead:

```JSON
{
"scan_date": '2020-06-04',
"minimum_severity": 'Info',
"active": True,
"verified": Trued,
Expand All @@ -122,11 +118,11 @@ A classic way of reimporting a scan is by specifying the ID of the test instead:

## Using the Scan Completion Date (API: `scan_date`) field

DefectDojo offers a plethora of supported scanner reports, but not all of them contain the
DefectDojo offers a plethora of supported scanner reports, but not all of them contain the
information most important to a user. The `scan_date` field is a flexible smart feature that
allows users to set the completion date of the a given scan report, and have it propagate
down to all the findings imported. This field is **not** mandatory, but the default value for
this field is the date of import (whenever the request is processed and a successful response is returned).
down to all the findings imported. This field is **not** mandatory, but the default value for
this field is the date of import (whenever the request is processed and a successful response is returned).

Here are the following use cases for using this field:

Expand Down
32 changes: 16 additions & 16 deletions dojo/api_v2/serializers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from drf_yasg.utils import swagger_serializer_method
from rest_framework.exceptions import NotFound
from rest_framework.fields import DictField, MultipleChoiceField

from datetime import datetime
from dojo.endpoint.utils import endpoint_filter
from dojo.importers.reimporter.utils import get_or_create_engagement, get_target_engagement_if_exists, get_target_product_by_id_if_exists, \
get_target_product_if_exists, get_target_test_if_exists
Expand All @@ -27,7 +27,6 @@
from django.core.exceptions import ValidationError, PermissionDenied
from django.contrib.auth.password_validation import validate_password
from django.utils import timezone
import datetime
import six
from django.utils.translation import ugettext_lazy as _
import json
Expand Down Expand Up @@ -1280,13 +1279,15 @@ def save(self, push_to_jira=False):
_, test_title, scan_type, engagement_id, engagement_name, product_name, product_type_name, auto_create_context = get_import_meta_data_from_dict(data)
engagement = get_or_create_engagement(engagement_id, engagement_name, product_name, product_type_name, auto_create_context)

# have to make the scan_date_time timezone aware otherwise uploads via the API would fail (but unit tests for api upload would pass...)
scan_date_time = timezone.make_aware(datetime.combine(scan_date, datetime.min.time())) if scan_date else None
importer = Importer()
try:
test, finding_count, closed_finding_count = importer.import_scan(scan, scan_type, engagement, lead, environment,
active=active, verified=verified, tags=tags,
minimum_severity=minimum_severity,
endpoints_to_add=endpoints_to_add,
scan_date=scan_date, version=version,
scan_date=scan_date_time, version=version,
branch_tag=branch_tag, build_id=build_id,
commit_hash=commit_hash,
push_to_jira=push_to_jira,
Expand Down Expand Up @@ -1325,18 +1326,15 @@ def validate(self, data):
raise serializers.ValidationError(f'API scan configuration must be of tool type {tool_type}')
return data

def validate_scan_data(self, value):
# scan_date is no longer deafulted to "today" at import time, so set it here if necessary
if not value.date:
return None
if value.date() > timezone.localtime(timezone.now()).date():
def validate_scan_date(self, value):
if value and value > timezone.localdate():
raise serializers.ValidationError(
'The date cannot be in the future!')
'The scan_date cannot be in the future!')
return value


class ReImportScanSerializer(TaggitSerializer, serializers.Serializer):
scan_date = serializers.DateField()
scan_date = serializers.DateField(required=False)
minimum_severity = serializers.ChoiceField(
choices=SEVERITY_CHOICES,
default='Info')
Expand Down Expand Up @@ -1390,7 +1388,7 @@ def save(self, push_to_jira=False):
scan_type = data['scan_type']
endpoint_to_add = data['endpoint_to_add']
minimum_severity = data['minimum_severity']
scan_date = data['scan_date']
scan_date = data.get('scan_date', None)
close_old_findings = data['close_old_findings']
verified = data['verified']
active = data['active']
Expand All @@ -1416,14 +1414,16 @@ def save(self, push_to_jira=False):
engagement = get_target_engagement_if_exists(None, engagement_name, product)
test = get_target_test_if_exists(test_id, test_title, scan_type, engagement)

# have to make the scan_date_time timezone aware otherwise uploads via the API would fail (but unit tests for api upload would pass...)
scan_date_time = timezone.make_aware(datetime.combine(scan_date, datetime.min.time())) if scan_date else None
try:
if test:
# reimport into provided / latest test
reimporter = ReImporter()
test, finding_count, new_finding_count, closed_finding_count, reactivated_finding_count, untouched_finding_count = \
reimporter.reimport_scan(scan, scan_type, test, active=active, verified=verified,
tags=None, minimum_severity=minimum_severity,
endpoints_to_add=endpoints_to_add, scan_date=scan_date,
endpoints_to_add=endpoints_to_add, scan_date=scan_date_time,
version=version, branch_tag=branch_tag, build_id=build_id,
commit_hash=commit_hash, push_to_jira=push_to_jira,
close_old_findings=close_old_findings,
Expand All @@ -1439,7 +1439,7 @@ def save(self, push_to_jira=False):
active=active, verified=verified, tags=tags,
minimum_severity=minimum_severity,
endpoints_to_add=endpoints_to_add,
scan_date=scan_date, version=version,
scan_date=scan_date_time, version=version,
branch_tag=branch_tag, build_id=build_id,
commit_hash=commit_hash,
push_to_jira=push_to_jira,
Expand Down Expand Up @@ -1481,10 +1481,10 @@ def validate(self, data):
raise serializers.ValidationError(f'API scan configuration must be of tool type {tool_type}')
return data

def validate_scan_data(self, value):
if value.date() > datetime.today().date():
def validate_scan_date(self, value):
if value and value > timezone.localdate():
raise serializers.ValidationError(
'The date cannot be in the future!')
'The scan_date cannot be in the future!')
return value


Expand Down
14 changes: 5 additions & 9 deletions dojo/forms.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -443,11 +443,8 @@ def clean(self):

# date can only be today or in the past, not the future
def clean_scan_date(self):
date = self.cleaned_data['scan_date']
# scan_date is no longer deafulted to "today" at import time, so set it here if necessary
if not date:
return None
if date.date() > datetime.today().date():
date = self.cleaned_data.get('scan_date', None)
if date and date.date() > datetime.today().date():
raise forms.ValidationError("The date cannot be in the future!")
return date

Expand All @@ -458,10 +455,9 @@ def get_scan_type(self):

class ReImportScanForm(forms.Form):
scan_date = forms.DateTimeField(
required=True,
required=False,
label="Scan Completion Date",
help_text="Scan completion date will be used on all findings.",
initial=datetime.now().strftime("%m/%d/%Y"),
widget=forms.TextInput(attrs={'class': 'datepicker'}))
minimum_severity = forms.ChoiceField(help_text='Minimum severity level to be imported',
required=True,
Expand Down Expand Up @@ -515,8 +511,8 @@ def clean(self):

# date can only be today or in the past, not the future
def clean_scan_date(self):
date = self.cleaned_data['scan_date']
if date.date() > timezone.localtime(timezone.now()).date():
date = self.cleaned_data.get('scan_date', None)
if date and date.date() > timezone.localtime(timezone.now()).date():
raise forms.ValidationError("The date cannot be in the future!")
return date

Expand Down
54 changes: 7 additions & 47 deletions dojo/importers/importer/importer.py
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
import base64
import datetime
from dojo.importers import utils as importer_utils
from dojo.decorators import dojo_async_task
from dojo.utils import get_current_user, max_safe
from dojo.utils import get_current_user
from dojo.celery import app
from django.core.exceptions import ValidationError
from django.core import serializers
Expand Down Expand Up @@ -100,17 +99,11 @@ def process_parsed_findings(self, test, parsed_findings, scan_type, user, active
item.active = active
if item.verified:
item.verified = verified
# Set the date if the parser does not set it
if not item.date:
item.date = scan_date

# Indicates the scan_date is not the default, overwrite everything
if (scan_date.date() if isinstance(scan_date, datetime.datetime) else scan_date) != now.date():
# if scan_date was provided, override value from parser
if scan_date:
item.date = scan_date

item.created = now
item.updated = now

item.service = service

item.save(dedupe_option=False)
Expand Down Expand Up @@ -229,30 +222,6 @@ def close_old_findings(self, test, scan_date_time, user, push_to_jira=None):

return old_findings

def update_timestamps(self, test, scan_date, version, branch_tag, build_id, commit_hash, now, scan_date_time):
test.engagement.updated = now
if test.engagement.engagement_type == 'CI/CD':
test.engagement.target_end = max_safe([scan_date, test.engagement.target_end])

test.updated = now
test.target_end = max_safe([scan_date_time, test.target_end])

if version:
test.version = version

if branch_tag:
test.branch_tag = branch_tag
test.engagement.version = version

if build_id:
test.build_id = build_id

if branch_tag:
test.commit_hash = commit_hash

test.save()
test.engagement.save()

def import_scan(self, scan, scan_type, engagement, lead, environment, active, verified, tags=None, minimum_severity=None,
user=None, endpoints_to_add=None, scan_date=None, version=None, branch_tag=None, build_id=None,
commit_hash=None, push_to_jira=None, close_old_findings=False, group_by=None, api_scan_configuration=None,
Expand All @@ -263,15 +232,6 @@ def import_scan(self, scan, scan_type, engagement, lead, environment, active, ve
user = user or get_current_user()

now = timezone.now()
# scan_date is no longer deafulted to "today" at import time, so set it here if necessary
finding_scan_date = scan_date
if not scan_date:
scan_date = now
finding_scan_date = now
# retain weird existing logic to use current time for provided scan date
scan_date_time = datetime.datetime.combine(scan_date, timezone.now().time())
if settings.USE_TZ:
scan_date_time = timezone.make_aware(scan_date_time, timezone.get_default_timezone())

if api_scan_configuration and api_scan_configuration.product != engagement.product:
raise ValidationError('API Scan Configuration has to be from same product as the Engagement')
Expand Down Expand Up @@ -340,7 +300,7 @@ def import_scan(self, scan, scan_type, engagement, lead, environment, active, ve
result = self.process_parsed_findings(test, findings_list, scan_type, user, active,
verified, minimum_severity=minimum_severity,
endpoints_to_add=endpoints_to_add, push_to_jira=push_to_jira,
group_by=group_by, now=now, service=service, scan_date=finding_scan_date, sync=False)
group_by=group_by, now=now, service=service, scan_date=scan_date, sync=False)
# Since I dont want to wait until the task is done right now, save the id
# So I can check on the task later
results_list += [result]
Expand All @@ -358,15 +318,15 @@ def import_scan(self, scan, scan_type, engagement, lead, environment, active, ve
new_findings = self.process_parsed_findings(test, parsed_findings, scan_type, user, active,
verified, minimum_severity=minimum_severity,
endpoints_to_add=endpoints_to_add, push_to_jira=push_to_jira,
group_by=group_by, now=now, service=service, scan_date=finding_scan_date, sync=True)
group_by=group_by, now=now, service=service, scan_date=scan_date, sync=True)

closed_findings = []
if close_old_findings:
logger.debug('IMPORT_SCAN: Closing findings no longer present in scan report')
closed_findings = self.close_old_findings(test, scan_date_time, user=user, push_to_jira=push_to_jira)
closed_findings = self.close_old_findings(test, scan_date, user=user, push_to_jira=push_to_jira)

logger.debug('IMPORT_SCAN: Updating test/engagement timestamps')
importer_utils.update_timestamps(test, scan_date, version, branch_tag, build_id, commit_hash, now, scan_date_time)
importer_utils.update_timestamps(test, version, branch_tag, build_id, commit_hash, now, scan_date)

if settings.TRACK_IMPORT_HISTORY:
logger.debug('IMPORT_SCAN: Updating Import History')
Expand Down
Loading

0 comments on commit a70de90

Please sign in to comment.