fluent.conf

# In v1 configuration, type and id are @ prefix parameters.
# @type and @id are recommended. type and id are still available for backward compatibility

## built-in TCP input
## $ echo <json> | fluent-cat <tag>
<source>
  @type forward
  @id forward_input
</source>

## built-in UNIX socket input
#<source>
#  @type unix
#</source>

## HTTP input
## http://localhost:8888/<tag>?json=<json>
<source>
  @type http
  @id http_input
  port 8888
</source>

## File input
## read apache logs with tag=apache.access
#<source>
#  @type tail
#  format apache
#  path /var/log/httpd-access.log
#  tag apache.access
#</source>

<source>
  @type tail
# This will need to be configurable from some interface: the path and the file name
  path /tmp/log.HITtoRDO
  tag log
  read_from_head true
  pos_file /var/log/fluentd/tail.log.pos
  <parse>
    @type none
    message_key line
  </parse>
</source>

<match log>
  @type rewrite_tag_filter
  <rule>
    key line
    pattern /(?<logtime>[^\s]*)\s+(?<service>[^\s]+\w)(.*)\s+(?<level>INFO|CRITICAL|WARNING|VERBOSE|ERROR|DEBUG|FATAL|CATASTROPHE)\s+(?<message>.*)/
    tag log.matched
  </rule>
  <rule>
    key line
    pattern /(?<logtime>[^\s]*)\s+(?<service>[^\s]+\w)(.*)\s+(?<level>INFO|CRITICAL|WARNING|VERBOSE|ERROR|DEBUG|FATAL|CATASTROPHE)\s+(?<message>.*)/
    tag log.unmatched
    invert true
  </rule>
</match>

<filter log.matched>
  @type parser
  key_name line
  <parse>
    @type regexp
    expression /(?<logtime>[^\s]*)\s+(?<service>[^\s]+\w)(.*)\s+(?<level>INFO|CRITICAL|WARNING|VERBOSE|ERROR|DEBUG|FATAL|CATASTROPHE)\s+(?<message>.*)/
    #time_key logtime
  </parse>
</filter>

## Mutating event filter
## Add hostname and tag fields to apache.access tag events
#<filter apache.access>
#  @type record_transformer
#  <record>
#    hostname ${hostname}
#    tag ${tag}
#  </record>
#</filter>

## Selecting event filter
## Remove unnecessary events from apache prefixed tag events
#<filter apache.**>
#  @type grep
#  include1 method GET # pass only GET in 'method' field
#  exclude1 message debug # remove debug event
#</filter>

# Listen HTTP for monitoring
# http://localhost:24220/api/plugins
# http://localhost:24220/api/plugins?type=TYPE
# http://localhost:24220/api/plugins?tag=MYTAG
<source>
  @type monitor_agent
  @id monitor_agent_input

  port 24220
</source>

# Listen DRb for debug
<source>
  @type debug_agent
  @id debug_agent_input

  bind 127.0.0.1
  port 24230
</source>

## match tag=apache.access and write to file
#<match apache.access>
#  @type file
#  path /var/log/fluent/access
#</match>

## match tag=debug.** and dump to console
<match debug.**>
  @type stdout
  @id stdout_output
</match>

<match log.*>
  @type elasticsearch
  host 127.0.0.1
  port 9200
  index_name fluentd.${tag}
#  <buffer tag, time>
  <buffer tag>
    @type file
    path /etc/elasticsearch/buffers
#    timekey 300s
#    timekey_use_utc true
#    timekey_wait 0m
  </buffer>
  #flush_interval 10s
</match>

## match tag=myapp.** and forward and write to file
#<match myapp.**>
#  @type copy
#  <store>
#    @type forward
#    buffer_type file
#    buffer_path /var/log/fluent/myapp-forward
#    retry_limit 50
#    flush_interval 10s
#    <server>
#      host 192.168.0.13
#    </server>
#  </store>
#  <store>
#    @type file
#    path /var/log/fluent/myapp
#  </store>
#</match>

## match fluent's internal events
#<match fluent.**>
#  @type null
#</match>

## match not matched logs and write to file
#<match **>
#  @type file
#  path /var/log/fluent/else
#  compress gz
#</match>

## Label: For handling complex event routing
#<label @STAGING>
#  <match system.**>
#    @type forward
#    @id staging_forward_output
#    <server>
#      host 192.168.0.101
#    </server>
#  </match>
#</label>