Merge pull request rackerlabs#229 from rackerlabs/issue46-scan-engine…

…-pooling

Adding scan engine pooling

ansible-playbooks/roles/console/tasks/main.yml

@@ -256,7 +256,7 @@
   shell: echo "from django.contrib.auth.models import User; User.objects.create_superuser('{{ scantron_secrets["django_super_user"] }}', '{{ scantron_secrets["django_super_user_email"] }}', '{{ scantron_secrets["django_super_user_password"] }}')" | {{ venv_python }} {{ scantron_dir }}/manage.py shell
   args:
     chdir: "{{ scantron_dir }}"
-  ignore_errors: yes  # Only applicable if playbook has already been run.  Use 'python manage.py change changepassword admin'.
+  ignore_errors: yes  # Only applicable if playbook has already been run.  Use 'python manage.py changepassword admin'.
 
 - name: Create "{{ django_user }}" user.
   shell: echo "from django.contrib.auth.models import User; User.objects.create_user('{{ scantron_secrets["django_user"] }}', '{{ scantron_secrets["django_user_email"] }}', '{{ scantron_secrets['django_user_password']  }}')" | {{ venv_python }} {{ scantron_dir }}/manage.py shell
@@ -400,13 +400,15 @@
     owner: nobody
     group: nogroup
     recurse: yes
+  tags: update_code
 
 - name: Change ownership of scan_results
   file:
     path: "/home/{{ non_root_user }}/console/scan_results"
     owner: nobody
     group: nogroup
     recurse: yes
+  tags: update_code
 
 - name: Restart NFS service.
   service:
@@ -465,6 +467,7 @@
 - name: Add crontab entry for scan_scheduler.sh
   cron:
     name: Schedule any new scans every minute.
+    disabled: false
     minute: "*"
     hour: "*"
     day: "*"
@@ -476,6 +479,7 @@
 - name: Add crontab entry for nmap_to_csv.sh
   cron:
     name: Convert nmap scan files for big data analytics platform ingestion every minute.
+    disabled: false
     minute: "*"
     hour: "*"
     day: "*"
@@ -487,6 +491,7 @@
 - name: Add crontab entry for masscan_json_to_csv.sh
   cron:
     name: Convert masscan json scan files for big data analytics platform ingestion every minute.
+    disabled: false
     minute: "*"
     hour: "*"
     day: "*"

console/django_scantron/__init__.py

@@ -1 +1 @@
-__version__ = "1.36"
+__version__ = "1.37"

console/django_scantron/admin.py

@@ -25,6 +25,11 @@ class EngineAdmin(admin.ModelAdmin):
     readonly_fields = ("id", "scan_engine", "api_token", "last_checkin")
 
 
+class EnginePoolAdmin(admin.ModelAdmin):
+
+    list_display = ("id", "engine_pool_name")
+
+
 class GloballyExcludedTargetAdmin(admin.ModelAdmin):
 
     list_display = ("id", "globally_excluded_targets", "note", "last_updated")
@@ -51,6 +56,7 @@ class SiteAdmin(admin.ModelAdmin):
         "excluded_targets",
         "scan_command",
         "scan_engine",
+        "scan_engine_pool",
         "email_scan_alerts",
         "email_alert_addresses",
     )
@@ -84,6 +90,7 @@ def _register(model, admin_class):
 _register(Session, SessionAdmin)
 
 _register(models.Engine, EngineAdmin)
+_register(models.EnginePool, EnginePoolAdmin)
 _register(models.GloballyExcludedTarget, GloballyExcludedTargetAdmin)
 _register(models.ScanCommand, ScanCommandAdmin)
 _register(models.Scan, ScanAdmin)

console/django_scantron/api/serializers.py

@@ -2,6 +2,7 @@
 
 from django_scantron.models import (
     Engine,
+    EnginePool,
     GloballyExcludedTarget,
     ScanCommand,
     Scan,
@@ -21,6 +22,12 @@ class Meta:
         fields = ("id", "scan_engine", "description", "api_token", "last_checkin")
 
 
+class EnginePoolSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = EnginePool
+        fields = ("id", "engine_pool_name", "scan_engines")
+
+
 class GloballyExcludedTargetSerializer(serializers.ModelSerializer):
 
     # Separate validation needed for DRF; doesn't use model's clean() function anymore.
@@ -67,6 +74,8 @@ class SiteSerializer(serializers.ModelSerializer):
     def validate(self, attrs):
         """Checks for any invalid IPs, IP subnets, or FQDNs in the targets or excluded_targets fields."""
 
+        # TODO add scan engine / scan engine pool checks.
+
         # Targets
         if "targets" in attrs:
             targets = attrs["targets"]
@@ -123,6 +132,7 @@ class Meta:
             "excluded_targets",
             "scan_command",
             "scan_engine",
+            "scan_engine_pool",
             "email_scan_alerts",
             "email_alert_addresses",
         )

console/django_scantron/api/urls.py

@@ -9,6 +9,7 @@
 # base_name requirement: http://www.django-rest-framework.org/api-guide/routers/#usage
 router = routers.DefaultRouter(trailing_slash=False)
 router.register(r"engines", views.EngineViewSet, base_name="engines")
+router.register(r"engine_pools", views.EnginePoolViewSet, base_name="engine_pools")
 router.register(
     r"globally_excluded_targets", views.GloballyExcludedTargetViewSet, base_name="globally_excluded_targets"
 )

console/django_scantron/api/views.py

@@ -13,6 +13,7 @@
 # Custom Python libraries.
 from django_scantron.api.serializers import (
     EngineSerializer,
+    EnginePoolSerializer,
     GloballyExcludedTargetSerializer,
     ScanCommandSerializer,
     ScanSerializer,
@@ -21,6 +22,7 @@
 )
 from django_scantron.models import (
     Engine,
+    EnginePool,
     GloballyExcludedTarget,
     ScanCommand,
     Scan,
@@ -69,6 +71,15 @@ class EngineViewSet(DefaultsMixin, viewsets.ModelViewSet):
     permission_classes = (IsAuthenticated, IsAdminUser)
 
 
+class EnginePoolViewSet(DefaultsMixin, viewsets.ModelViewSet):
+    """API CRUD operations for EnginePool Model."""
+
+    model = EnginePool
+    serializer_class = EnginePoolSerializer
+    queryset = EnginePool.objects.all()
+    permission_classes = (IsAuthenticated, IsAdminUser)
+
+
 class GloballyExcludedTargetViewSet(DefaultsMixin, viewsets.ModelViewSet):
     """API CRUD operations for GloballyExcludedTarget Model."""
 

console/django_scantron/models.py

@@ -52,6 +52,20 @@ class Meta:
         verbose_name_plural = "Engines"
 
 
+class EnginePool(models.Model):
+    """Model for an Engine Pool"""
+
+    id = models.AutoField(primary_key=True, verbose_name="Engine Pool ID")
+    engine_pool_name = models.CharField(unique=True, max_length=255, verbose_name="Engine Pool Name")
+    scan_engines = models.ManyToManyField(Engine, verbose_name="Scan engines in pool",)
+
+    def __str__(self):
+        return str(self.engine_pool_name)
+
+    class Meta:
+        verbose_name_plural = "Engine Pools"
+
+
 class GloballyExcludedTarget(models.Model):
     """Model for globally excluded targets."""
 
@@ -163,7 +177,10 @@ class Site(models.Model):
         verbose_name="Excluded targets",
     )
     scan_command = models.ForeignKey(ScanCommand, on_delete=models.CASCADE, verbose_name="Scan binary and name")
-    scan_engine = models.ForeignKey(Engine, on_delete=models.CASCADE, verbose_name="Scan Engine")
+    scan_engine = models.ForeignKey(Engine, blank=True, null=True, on_delete=models.CASCADE, verbose_name="Scan Engine")
+    scan_engine_pool = models.ForeignKey(
+        EnginePool, blank=True, null=True, on_delete=models.CASCADE, verbose_name="Scan Engine Pool"
+    )
     email_scan_alerts = models.BooleanField(verbose_name="Email scan alerts?")
     email_alert_addresses = models.CharField(
         unique=False, blank=True, max_length=4096, verbose_name="Email alert addresses, comma separated"
@@ -172,6 +189,16 @@ class Site(models.Model):
     def clean(self):
         """Checks for any invalid IPs, IP subnets, or FQDNs in the targets and excluded_targets fields."""
 
+        # Ensure only 1 scan engine / scan engine pool is selected.
+        if self.scan_engine and self.scan_engine_pool:
+            raise ValidationError("Only select a single scan engine or scan engine pool.")
+
+        # Ensure a scan engine or scan engine pool is selected.  Can't enforce within models.ForeignKey using
+        # blank=False and null=False, because they could be blank/empty if the other scan engine or scan engine pool is
+        # selected.
+        if not self.scan_engine and not self.scan_engine_pool:
+            raise ValidationError("Select a single scan engine or scan engine pool.")
+
         # Targets
         target_extractor = extract_targets.TargetExtractor(
             targets_string=self.targets, private_ips_allowed=True, sort_targets=True