enable additional sflow fields

tank0226 · Aug 30, 2018 · 507a9cf · 507a9cf
1 parent b3554d3
commit 507a9cf
Show file tree

Hide file tree

Showing 5 changed files with 143 additions and 10 deletions.
diff --git a/README.md b/README.md
@@ -1,3 +1,6 @@
+[![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.me/robcowart)
+---
+
 # ElastiFlow&trade;
 ElastiFlow&trade; provides network flow data collection and visualization using the Elastic Stack. It supports Netflow v5/v9, sFlow and IPFIX flow types (1.x versions support only Netflow v5/v9).
 > Release 3.x is designed for use with the Elastic Stack 6.2 and higher. If you are using an older version of the Elastic Stack, please use version 2.1 or 1.2.

diff --git a/logstash/elastiflow/conf.d/10_input_sflow_ipv4.logstash.conf b/logstash/elastiflow/conf.d/10_input_sflow_ipv4.logstash.conf
@@ -24,7 +24,9 @@ input {
     workers => "${ELASTIFLOW_SFLOW_UDP_WORKERS:4}"
     queue_size => "${ELASTIFLOW_SFLOW_UDP_QUEUE_SIZE:2048}"
     receive_buffer_bytes => "${ELASTIFLOW_SFLOW_UDP_RCV_BUFF:33554432}"
-    codec => sflow { }
+    codec => sflow {
+      optional_removed_field => [ "sflow_version" ]
+    }
     type => "sflow"
   }
 }
diff --git a/logstash/elastiflow/conf.d/10_input_sflow_ipv6.logstash.conf.disabled b/logstash/elastiflow/conf.d/10_input_sflow_ipv6.logstash.conf.disabled
@@ -24,7 +24,9 @@ input {
     workers => "${ELASTIFLOW_SFLOW_UDP_WORKERS:4}"
     queue_size => "${ELASTIFLOW_SFLOW_UDP_QUEUE_SIZE:2048}"
     receive_buffer_bytes => "${ELASTIFLOW_SFLOW_UDP_RCV_BUFF:33554432}"
-    codec => sflow { }
+    codec => sflow {
+      optional_removed_field => [ "sflow_version" ]
+    }
     type => "sflow"
   }
 }
diff --git a/logstash/elastiflow/conf.d/20_filter_40_sflow.logstash.conf b/logstash/elastiflow/conf.d/20_filter_40_sflow.logstash.conf
@@ -51,13 +51,16 @@ filter {
       mutate {
         id => "sflow_rename_sflow_fields"
         rename => {
-          "ip_version" => "[flow][ip_version]"
-          "sample_seq_number" => "[sflow][sample_seq_number]"
-          "sflow_type" => "[sflow][sflow_type]"
-          "source_id_index" => "[sflow][source_id_index]"
-          "source_id_type" => "[sflow][source_id_type]"
-          "sub_agent_id" => "[sflow][sub_agent_id]"
-          "uptime_in_ms" => "[sflow][uptime_in_ms]"
+          #"agent_ip" => "[sflow][agent_ip]"
+          #"tcp_is_ack" => "[sflow][tcp_is_ack]"
+          #"tcp_is_cwr" => "[sflow][tcp_is_cwr]"
+          #"tcp_is_ecn_echo" => "[sflow][tcp_is_ecn_echo]"
+          #"tcp_is_fin" => "[sflow][tcp_is_fin]"
+          #"tcp_is_nonce" => "[sflow][tcp_is_nonce]"
+          #"tcp_is_push" => "[sflow][tcp_is_push]"
+          #"tcp_is_reset" => "[sflow][tcp_is_reset]"
+          #"tcp_is_syn" => "[sflow][tcp_is_syn]"
+          #"tcp_is_urgent" => "[sflow][tcp_is_urgent]"
           "drops" => "[sflow][drops]"
           "dst_ip" => "[flow][dst_addr]"
           "dst_mac" => "[sflow][dst_mac]"
@@ -84,12 +87,14 @@ filter {
           "ip_header_length" => "[sflow][ip_header_length]"
           "ip_identification" => "[sflow][ip_identification]"
           "ip_next_header" => "[sflow][ip_next_header]"
+          "ip_options" => "[sflow][ip_options]"
           "ip_packet_length" => "[sflow][ip_packet_length]"
           "ip_priority" => "[sflow][ip_priority]"
           "ip_protocol" => "[flow][ip_protocol]"
           "ip_total_length" => "[sflow][ip_total_length]"
           "ip_ttl" => "[sflow][ip_ttl]"
           "ip_type" => "[sflow][ip_type]"
+          "ip_version" => "[flow][ip_version]"
           "output_interface" => "[flow][output_snmp]"
           "output_interface_format" => "[sflow][output_interface_format]"
           "output_interface_value" => "[sflow][output_interface_value]"
@@ -98,20 +103,26 @@ filter {
           "protocol" => "[sflow][protocol]"
           "sample_length" => "[sflow][sample_length]"
           "sample_pool" => "[sflow][sample_pool]"
+          "sample_seq_number" => "[sflow][sample_seq_number]"
           "sampling_rate" => "[flow][sampling_interval]"
+          "sequence_number" => "[sflow][sequence_number]"
+          "sflow_type" => "[sflow][sflow_type]"
           "sflow_version" => "[sflow][sflow_version]"
           "size_header" => "[sflow][size_header]"
+          "source_id_index" => "[sflow][source_id_index]"
+          "source_id_type" => "[sflow][source_id_type]"
           "src_ip" => "[flow][src_addr]"
           "src_mac" => "[sflow][src_mac]"
           "src_mask_len" => "[flow][src_mask_len]"
           "src_port" => "[flow][src_port]"
           "src_priority" => "[sflow][src_priority]"
           "src_vlan" => "[sflow][src_vlan]"
           "stripped" => "[sflow][stripped]"
+          "sub_agent_id" => "[sflow][sub_agent_id]"
           "tcp_ack_number" => "[sflow][tcp_ack_number]"
           "tcp_checksum" => "[sflow][tcp_checksum]"
-          "tcp_flags" => "[flow][tcp_flags]"
           "tcp_header_length" => "[sflow][tcp_header_length]"
+          "tcp_options" => "[sflow][tcp_options]"
           "tcp_reserved" => "[sflow][tcp_reserved]"
           "tcp_seq_number" => "[sflow][tcp_seq_number]"
           "tcp_urgent_pointer" => "[sflow][tcp_urgent_pointer]"
@@ -122,8 +133,18 @@ filter {
           "vlan_id" => "[sflow][vlan_id]"
           "vlan_priority" => "[sflow][vlan_priority]"
           "vlan_type" => "[sflow][vlan_type]"
+          "uptime_in_ms" => "[sflow][uptime_in_ms]"
         }
       }
+
+      # Create array of TCP flag tags.
+      ruby {
+        id => "sflow_ruby_construct_tcp_flags"
+        code => "
+          event.set('[flow][tcp_flags]', event.get('[tcp_is_fin]').to_i + (event.get('[tcp_is_syn]').to_i * 2) + (event.get('[tcp_is_reset]').to_i * 4) + (event.get('[tcp_is_push]').to_i * 8) + (event.get('[tcp_is_ack]').to_i * 16) + (event.get('[tcp_is_urgent]').to_i * 32) + (event.get('[tcp_is_ecn_echo]').to_i * 64) + (event.get('[tcp_is_cwr]').to_i * 128));
+        "
+      }
+
       mutate {
         id => "sflow_convert_sflow_fields"
         convert => {
@@ -134,6 +155,7 @@ filter {
           "[flow][sampling_interval]" => "integer"
           "[flow][tcp_flags]" => "integer"
         }
+        remove_field => [ "[tcp_is_fin]", "[tcp_is_syn]", "[tcp_is_reset]", "[tcp_is_push]", "[tcp_is_ack]", "[tcp_is_urgent]", "[tcp_is_ecn_echo]", "[tcp_is_cwr]", "[tcp_is_nonce]" ]
       }
 
       # lookup source ID type to find the a value for source_id_type.

diff --git a/logstash/elastiflow/templates/elastiflow.template.json b/logstash/elastiflow/templates/elastiflow.template.json
@@ -11778,6 +11778,14 @@
             }
           }
         },
+        {
+          "sflow.ip_options": {
+            "path_match": "sflow.ip_options",
+            "mapping": {
+              "type": "long"
+            }
+          }
+        },
         {
           "sflow.ip_packet_length": {
             "path_match": "sflow.ip_packet_length",
@@ -11882,6 +11890,14 @@
             }
           }
         },
+        {
+          "sflow.protocol_name": {
+            "path_match": "sflow.protocol_name",
+            "mapping": {
+              "type": "keyword"
+            }
+          }
+        },
         {
           "sflow.sample_length": {
             "path_match": "sflow.sample_length",
@@ -11914,6 +11930,14 @@
             }
           }
         },
+        {
+          "sflow.sequence_number": {
+            "path_match": "sflow.sequence_number",
+            "mapping": {
+              "type": "long"
+            }
+          }
+        },
         {
           "sflow.sflow_type": {
             "path_match": "sflow.sflow_type",
@@ -12058,6 +12082,86 @@
             }
           }
         },
+        {
+          "sflow.tcp_is_ack": {
+            "path_match": "sflow.tcp_is_ack",
+            "mapping": {
+              "type": "integer"
+            }
+          }
+        },
+        {
+          "sflow.tcp_is_cwr": {
+            "path_match": "sflow.tcp_is_cwr",
+            "mapping": {
+              "type": "integer"
+            }
+          }
+        },
+        {
+          "sflow.tcp_is_ecn_echo": {
+            "path_match": "sflow.tcp_is_ecn_echo",
+            "mapping": {
+              "type": "integer"
+            }
+          }
+        },
+        {
+          "sflow.tcp_is_fin": {
+            "path_match": "sflow.tcp_is_fin",
+            "mapping": {
+              "type": "integer"
+            }
+          }
+        },
+        {
+          "sflow.tcp_is_nonce": {
+            "path_match": "sflow.tcp_is_nonce",
+            "mapping": {
+              "type": "integer"
+            }
+          }
+        },
+        {
+          "sflow.tcp_is_push": {
+            "path_match": "sflow.tcp_is_push",
+            "mapping": {
+              "type": "integer"
+            }
+          }
+        },
+        {
+          "sflow.tcp_is_reset": {
+            "path_match": "sflow.tcp_is_reset",
+            "mapping": {
+              "type": "integer"
+            }
+          }
+        },
+        {
+          "sflow.tcp_is_syn": {
+            "path_match": "sflow.tcp_is_syn",
+            "mapping": {
+              "type": "integer"
+            }
+          }
+        },
+        {
+          "sflow.tcp_is_urgent": {
+            "path_match": "sflow.tcp_is_urgent",
+            "mapping": {
+              "type": "integer"
+            }
+          }
+        },        
+        {
+          "sflow.tcp_options": {
+            "path_match": "sflow.tcp_options",
+            "mapping": {
+              "type": "long"
+            }
+          }
+        },
         {
           "sflow.tcp_reserved": {
             "path_match": "sflow.tcp_reserved",