Merge pull request #144 from weslambert/master

configure output/index creation for Elastic Beats

configfiles/0006_input_beats.conf

@@ -1,10 +1,11 @@
 # Author: Justin Henderson
 # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
 # Email: justin@hasecuritysolutions.com
-# Last Update: 11/15/2017
+# Last Update: 12/11/2017
 
 input {
   beats {
     port => "5044"
+    tags => [ "beat" ]
   }
 }

configfiles/9500_output_beats.conf

@@ -0,0 +1,18 @@
+# Author: Wes Lambert
+# Last Update: 12/11/2017
+filter {
+  if "beat" in [tags] {
+    mutate {
+          ##add_tag => [ "conf_file_9000"]
+        }
+  }
+}
+output {
+  if "beat" in [tags] {
+    elasticsearch {
+      hosts => elasticsearch
+      index => "logstash-beats-%{+YYYY.MM.dd}"
+      template => "/beats-template.json"
+    }
+  }
+}

etc/logstash/beats-template.json

@@ -0,0 +1,182 @@
+{
+  "template" : "beats-*",
+  "version" : 50001,
+  "settings" : {
+    "number_of_replicas": 0,
+    "number_of_shards": 1,
+    "index.refresh_interval" : "5s"
+  },
+  "mappings": {
+    "_default_": {
+      "_meta": {
+        "version": "5.6.4"
+      },
+      "date_detection": false,
+      "dynamic_templates": [
+        {
+          "strings_as_keyword": {
+            "mapping": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "match_mapping_type": "string"
+          }
+        }
+      ],
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "activity_id": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "beat": {
+          "properties": {
+            "hostname": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "name": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "version": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
+        },
+        "computer_name": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "event_id": {
+          "type": "long"
+        },
+        "keywords": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "level": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "log_name": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "message": {
+          "norms": false,
+          "type": "text"
+        },
+        "message_error": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "meta": {
+          "properties": {
+            "cloud": {
+              "properties": {
+                "availability_zone": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "instance_id": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "machine_type": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "project_id": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "provider": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "region": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+        },
+        "opcode": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "process_id": {
+          "type": "long"
+        },
+        "provider_guid": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "record_number": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "related_activity_id": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "source_name": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "tags": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "task": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "thread_id": {
+          "type": "long"
+        },
+        "type": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "user": {
+          "properties": {
+            "domain": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "identifier": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "name": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "type": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
+        },
+        "version": {
+          "type": "long"
+        },
+        "xml": {
+          "norms": false,
+          "type": "text"
+        }
+      }
+    }
+  },
+  "order": 0,
+  "settings": {
+    "index.mapping.total_fields.limit": 10000,
+    "index.refresh_interval": "5s"
+  }
+}

usr/sbin/so-elastic-start-logstash

@@ -37,6 +37,7 @@ if [ "$LOGSTASH_ENABLED" = "yes" ]; then
 			--volume /etc/logstash/log4j2.properties:/usr/share/logstash/config/log4j2.properties:ro \
                         --volume /etc/logstash/logstash.yml:/usr/share/logstash/config/logstash.yml:ro \
                         --volume /etc/logstash/logstash-template.json:/logstash-template.json:ro \
+			--volume /etc/logstash/beats-template.json:/beats-template.json:ro \
                         --volume /etc/logstash/conf.d:/usr/share/logstash/pipeline/:ro \
                         --volume /etc/nsm/rules:/etc/nsm/rules:ro \
                         --volume /lib/dictionaries:/lib/dictionaries:ro \