Merge pull request #109 from rocknsm/dcode/pull-102/fsf

Updates towards merging in FSF
LGTM

ansible/files/logrotate-fsf.conf

@@ -0,0 +1,7 @@
+/data/fsf/*.log
+{
+    rotate 3
+    missingok
+    compress
+    create 0644 fsf fsf
+}

ansible/files/logstash-fsf-es.conf

@@ -0,0 +1,27 @@
+input {
+  file {
+    codec => "json"
+    path => "/data/fsf/scan.log"
+    add_field => { "[@metadata][stage]" => "fsf" }
+  }
+}
+
+filter {
+  if "_jsonparsefailure" in [tags] {
+    drop { }
+  }
+  if [@metadata][stage] == "fsf" {
+    mutate { remove_field => ["path"] }
+  } 
+}
+
+output {
+  if [@metadata][stage] == "fsf" {
+    #stdout { codec => rubydebug }
+    elasticsearch {
+      hosts => ["127.0.0.1"]
+      index => "fsf-%{+YYYY.MM.dd}"
+      document_type => "fsf"
+    }
+  }
+}

ansible/files/rock_start

@@ -1,43 +1,71 @@
 #!/bin/bash
 
-echo "Starting Zookeeper..."
-systemctl start zookeeper
-sleep 5
-systemctl status zookeeper | egrep "^\s*Active"
-
-echo "Starting Elasticsearch..."
-systemctl start elasticsearch
-sleep 5
-systemctl status elasticsearch | egrep "^\s*Active"
-
-echo "Starting Kafka..."
-systemctl start kafka
-sleep 5
-systemctl status kafka | egrep "^\s*Active"
-
-echo "Starting Logstash..."
-systemctl start logstash
-sleep 5
-systemctl status logstash | egrep "^\s*Active"
-
-echo "Starting Kibana..."
-systemctl start kibana
-sleep 5 
-systemctl status kibana | egrep "^\s*Active"
-
-echo "Starting Suricata..."
-systemctl start suricata
-sleep 5
-systemctl status suricata | egrep "^\s*Active"
-
-echo "Starting Bro..."
-systemctl start broctl
-sleep 5
-systemctl status broctl | egrep "^\s*Active"
-
-#echo "Starting Stenographer..."
-#systemctl start stenographer
-#sleep 5
-#systemctl status stenographer | egrep "^\s*Active"
+function feature_enabled() {
+  grep -qiE "^enable_$1: (true|yes)" /etc/rocknsm/config.yml;
+  return $?
+}
+
+if feature_enabled zookeeper; then
+  echo "Starting Zookeeper..."
+  systemctl start zookeeper
+  sleep 5
+  systemctl status zookeeper | egrep "^\s*Active"
+fi
+
+if feature_enabled elasticsearch; then
+  echo "Starting Elasticsearch..."
+  systemctl start elasticsearch
+  sleep 5
+  systemctl status elasticsearch | egrep "^\s*Active"
+fi
+
+if feature_enabled kafka; then
+  echo "Starting Kafka..."
+  systemctl start kafka
+  sleep 5
+  systemctl status kafka | egrep "^\s*Active"
+fi
+
+if feature_enabled logstash; then
+  echo "Starting Logstash..."
+  systemctl start logstash
+  sleep 5
+  systemctl status logstash | egrep "^\s*Active"
+fi
+
+if feature_enabled kibana; then
+  echo "Starting Kibana..."
+  systemctl start kibana
+  sleep 5
+  systemctl status kibana | egrep "^\s*Active"
+fi
+
+if feature_enabled suricata; then
+  echo "Starting Suricata..."
+  systemctl start suricata
+  sleep 5
+  systemctl status suricata | egrep "^\s*Active"
+fi
+
+if feature_enabled bro; then
+  echo "Starting Bro..."
+  systemctl start broctl
+  sleep 5
+  systemctl status broctl | egrep "^\s*Active"
+fi
+
+if feature_enabled stenographer; then
+  echo "Starting Stenographer..."
+  systemctl start stenographer
+  sleep 5
+  systemctl status stenographer | egrep "^\s*Active"
+fi
+
+if feature_enabled fsf; then
+  echo "Starting FSF..."
+  systemctl start fsf
+  sleep 5
+  systemctl status fsf | egrep "^\s*Active"
+fi
 
 exit 0

ansible/files/rock_status

@@ -84,6 +84,13 @@ export MON_IFS=$(cat /opt/bro/etc/node.cfg | grep interface | \
   systemctl status suricata
 }
 
+#----------------------------------------------------------------------------
+##                               FSF
+##----------------------------------------------------------------------------
+@test "Check that FSF is running" {
+  systemctl status fsf
+}
+
 #----------------------------------------------------------------------------
 #                               ZOOKEEPER
 #----------------------------------------------------------------------------

ansible/files/rock_stop

@@ -1,27 +1,59 @@
 #!/bin/bash
 
-#echo "Stopping Stenographer..."
-#systemctl stop stenographer
 
-echo "Stopping Suricata..."
-systemctl stop suricata
-
-echo "Stopping Bro..."
-systemctl stop broctl
-
-echo "Stopping Logstash..."
-systemctl stop logstash
-
-echo "Stopping Kibana..."
-systemctl stop kibana
-
-echo "Stopping Elasticsearch..."
-systemctl stop elasticsearch
-
-echo "Stopping Kafka..."
-systemctl stop kafka
-
-echo "Stopping Zookeeper..."
-systemctl stop zookeeper
+function feature_enabled() {
+  grep -qiE "^enable_$1: (true|yes)" /etc/rocknsm/config.yml;
+  return $?
+}
+
+if feature_enabled fsf; then
+  echo "Stopping FSF..."
+  systemctl stop fsf
+fi
+
+if feature_enabled suricata; then
+  echo "Stopping Stenographer..."
+  systemctl stop stenographer
+fi
+
+if feature_enabled suricata; then
+  echo "Stopping Suricata..."
+  systemctl stop suricata
+fi
+
+if feature_enabled snort; then
+  echo "Stopping Snort..."
+  systemctl stop snortd
+fi
+
+if feature_enabled bro; then
+  echo "Stopping Bro..."
+  systemctl stop broctl
+fi
+
+if feature_enabled logstash; then
+  echo "Stopping Logstash..."
+  systemctl stop logstash
+fi
+
+if feature_enabled kibana; then
+  echo "Stopping Kibana..."
+  systemctl stop kibana
+fi
+
+if feature_enabled elasticsearch; then
+  echo "Stopping Elasticsearch..."
+  systemctl stop elasticsearch
+fi
+
+if feature_enabled kafka; then
+  echo "Stopping Kafka..."
+  systemctl stop kafka
+fi
+
+if feature_enabled zookeeper; then
+  echo "Stopping Zookeeper..."
+  systemctl stop zookeeper
+fi
 
 exit 0

ansible/files/rocknsm_config.dist.yml

@@ -28,6 +28,21 @@ with_kibana: true
 with_zookeeper: true
 with_kafka: true
 with_nginx: true
+with_fsf: true
+
+# Feature options - Don't flip these unless you know what you're doing
+enable_stenographer: false
+enable_bro: true
+enable_suricata: true
+enable_snort: false
+enable_pulledpork: true
+enable_logstash: true
+enable_elasticsearch: true
+enable_kibana: true
+enable_zookeeper: true
+enable_kafka: true
+enable_nginx: true
+enable_fsf: false
 
 rocknsm_package_list:
   - java-1.8.0-openjdk-headless
@@ -84,6 +99,11 @@ suricata_group: suricata
 suricata_data_dir: "{{ rock_data_dir }}/suricata"
 pulled_pork_repo: https://github.com/shirkdog/pulledpork.git
 pulled_pork_oinkcode: 796f26a2188c4c953ced38ff3ec899d8ae543350
+fsf_user: fsf
+fsf_group: fsf
+fsf_data_dir: "{{ rock_data_dir }}/fsf"
+fsf_archive_dir: "{{ fsf_data_dir }}/archive"
+fsf_client_logfile: "{{ fsf_data_dir }}/client.log"
 kafka_user: kafka
 kafka_group: kafka
 kafka_data_dir: "{{ rock_data_dir }}/kafka"

ansible/rock.yml

@@ -172,6 +172,7 @@
       - { pkg: zookeeper, test: "{{with_zookeeper}}", state: installed }
       - { pkg: kafka, test: "{{with_kafka}}", state: installed }
       - { pkg: kafkacat, test: "{{with_kafka}}", state: installed }
+      - { pkg: fsf, test: "{{with_fsf}}", state: installed }
       - { pkg: chrony, state: installed }
       - { pkg: firewalld, state: installed }
       - { pkg: postfix, state: absent }
@@ -381,7 +382,6 @@
     git:
       repo: "{{ bro_rockscripts_repo }}"
       dest: /opt/bro/share/bro/site/scripts/rock
-      depth: 1
       version: "{{ bro_rockscripts_branch }}"
     when: rock_online_install
 
@@ -403,6 +403,17 @@
       force: yes
     when: not rock_online_install
 
+  - name: Update owner for ROCK NSM Bro scripts
+    file:
+      path: /opt/bro/share/bro/site/scripts/rock
+      owner: "{{ bro_user }}"
+      group: "{{ bro_group }}"
+      state: directory
+      recurse: yes
+      follow: yes
+    tags:
+      - bro_scripts
+
   - name: Add ROCK scripts to local.bro
     lineinfile:
       dest: /opt/bro/share/bro/site/local.bro
@@ -665,6 +676,59 @@
         {{ "/usr/bin/systemctl restart snortd;" if with_snort else None }}
     when: with_pulledpork
 
+    #######################################################
+    ######################## FSF ##########################
+    #######################################################
+
+  - name: Create FSF data dir
+    file:
+      path: "{{ fsf_data_dir }}"
+      mode: 0755
+      owner: "{{ fsf_user }}"
+      group: "{{ fsf_group }}"
+      state: directory
+    when: with_fsf
+
+  - name: Create FSF archive dir
+    file:
+      path: "{{ fsf_archive_dir }}"
+      mode: 0755
+      owner: "{{ fsf_user }}"
+      group: "{{ fsf_group }}"
+      state: directory
+    when: with_fsf
+
+  - name: Configure logrotate for FSF logs
+    copy:
+      src: files/logrotate-fsf.conf
+      dest: /etc/logrotate.d/fsf.conf
+      mode: 0644
+      owner: root
+      group: root
+    when: with_fsf
+
+  - name: Configure fsf-server
+    template:
+      src: templates/fsf-server-config.j2
+      dest: /opt/fsf/fsf-server/conf/config.py
+      owner: "{{ fsf_user }}"
+      group: "{{ fsf_group }}"
+      mode: 0644
+    when: with_fsf
+
+  - name: Configure fsf-client
+    template:
+      src: templates/fsf-client-config.j2
+      dest: /opt/fsf/fsf-client/conf/config.py
+      owner: "{{ fsf_user }}"
+      group: "{{ fsf_group }}"
+      mode: 0644
+    when: with_fsf
+
+  - name: Enable and start FSF
+    service: name=fsf state=started enabled=yes
+    when: with_fsf and enable_fsf
+
     ######################################################
     ################### Setup Zookeeper ##################
     ######################################################
@@ -831,6 +895,15 @@
       group: root
     when: with_logstash and with_suricata
 
+  - name: Configure Logstash to pickup the FSF scan.log
+    copy:
+      src: "files/logstash-fsf-es.conf"
+      dest: "/etc/logstash/conf.d/fsf-es.conf"
+      mode: 0644
+      owner: root
+      group: root
+    when: with_logstash and with_fsf
+
   - name: Enable and start Logstash
     service: name=logstash state=started enabled=yes
     when: with_logstash