[Filebeat] Update PanOS parsing and ingest pipeline (elastic#24799)

PanOS Updates - prevent duplicates in event.* - add virtual system field Closes elastic#22413 Closes elastic#22748 Co-authored-by: Lee E. Hinman <[email protected]>
narph · Mar 30, 2021 · c94a8f8 · c94a8f8
1 parent 5960d7f
commit c94a8f8
Show file tree

Hide file tree

Showing 11 changed files with 406 additions and 2 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -382,6 +382,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix date parsing in GSuite/login fileset. {issue}24694[24694]
 - Improve Cisco ASA/FTD parsing of messages - better support for identity FW messages. Change network.bytes, source.bytes, and destination.bytes to long from integer since value can exceed integer capacity.  Add descriptions for various processors for easier pipeline editing in Kibana UI. {pull}23766[23766]
 - Updating Oauth2 flow for m365_defender fileset. {pull}24829[24829]
+- Improve PanOS parsing and ingest pipeline. {issue}22413[22413] {issue}22748[22748] {pull}24799[24799]
 
 *Heartbeat*
 

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -112583,6 +112583,16 @@ Specifies the sub type of the log
 
 --
 
+*`panw.panos.virtual_sys`*::
++
+--
+Virtual system instance
+
+
+type: keyword
+
+--
+
 [[exported-fields-pensando]]
 == Pensando fields
 

diff --git a/x-pack/filebeat/module/panw/fields.go b/x-pack/filebeat/module/panw/fields.go
diff --git a/x-pack/filebeat/module/panw/panos/_meta/fields.yml b/x-pack/filebeat/module/panw/panos/_meta/fields.yml
@@ -142,3 +142,8 @@
     - name: sub_type
       description: >-
         Specifies the sub type of the log
+
+    - name: virtual_sys
+      type: keyword
+      description: >
+        Virtual system instance
diff --git a/x-pack/filebeat/module/panw/panos/config/input.yml b/x-pack/filebeat/module/panw/panos/config/input.yml
@@ -68,6 +68,7 @@ processors:
         server.user.name: 13
         destination.user.name: 13
         network.application: 14
+        panw.panos.virtual_sys: 15
         panw.panos.source.zone: 16
         observer.ingress.zone: 16
         panw.panos.destination.zone: 17
@@ -130,6 +131,7 @@ processors:
         server.user.name: 13
         destination.user.name: 13
         network.application: 14
+        panw.panos.virtual_sys: 15
         panw.panos.source.zone: 16
         observer.ingress.zone: 16
         panw.panos.destination.zone: 17

diff --git a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml
@@ -223,6 +223,7 @@ processors:
      if: 'ctx?.panw?.panos?.type == "TRAFFIC"'
  - append:
      field: event.category
+     allow_duplicates: false
      value:
        - network_traffic
        - network
@@ -233,17 +234,20 @@ processors:
      if: 'ctx?.panw?.panos?.type == "THREAT"'
  - append:
      field: event.category
+     allow_duplicates: false
      value:
        - security_threat
        - intrusion_detection
        - network
      if: 'ctx?.panw?.panos?.type == "THREAT"'
  - append:
      field: event.type
+     allow_duplicates: false
      value: allowed
      if: "ctx?.panw?.panos?.action != null && ['alert', 'allow', 'continue'].contains(ctx.panw.panos.action)"
  - append:
      field: event.type
+     allow_duplicates: false
      value: denied
      if: "ctx?.panw?.panos?.action != null && ['deny', 'drop', 'reset-client', 'reset-server', 'reset-both', 'block-url', 'block-ip', 'random-drop', 'sinkhole', 'block'].contains(ctx.panw.panos.action)"
  - set:
@@ -258,6 +262,7 @@ processors:
      if: 'ctx?.panw?.panos?.sub_type == "start"'
  - append:
      field: event.type
+     allow_duplicates: false
      value:
        - start
        - connection
@@ -268,6 +273,7 @@ processors:
      if: 'ctx?.panw?.panos?.sub_type == "end"'
  - append:
      field: event.type
+     allow_duplicates: false
      value:
        - end
        - connection
@@ -278,6 +284,7 @@ processors:
      if: 'ctx?.panw?.panos?.sub_type == "drop"'
  - append:
      field: event.type
+     allow_duplicates: false
      value:
        - denied
        - connection
@@ -288,6 +295,7 @@ processors:
      if: 'ctx?.panw?.panos?.sub_type == "deny"'
  - append:
      field: event.type
+     allow_duplicates: false
      value:
        - denied
        - connection
@@ -467,8 +475,9 @@ processors:
 
 # Append NAT community_id to network.community_id
  - append:
-     if: 'ctx?.panw?.panos?.network?.nat?.community_id != null && ctx.panw.panos.network.nat.community_id != ctx?.network?.community_id'
+     if: 'ctx?.panw?.panos?.network?.nat?.community_id != null'
      field: network.community_id
+     allow_duplicates: false
      value:
       - '{{panw.panos.network.nat.community_id}}'
 

diff --git a/x-pack/filebeat/module/panw/panos/test/pan_inc_other.log-expected.json b/x-pack/filebeat/module/panw/panos/test/pan_inc_other.log-expected.json
@@ -800,6 +800,7 @@
         "panw.panos.sub_type": "start",
         "panw.panos.type": "TRAFFIC",
         "panw.panos.url.category": "any",
+        "panw.panos.virtual_sys": "vsys1",
         "related.ip": [
             "192.168.0.2",
             "204.232.231.46",