omelasticsearch: Elasticsearch Output Module¶

Rainer Gerhards <rgerhards@adiscon.com>

Examples:
     server="localhost:9200"
     server=["elasticsearch1", "elasticsearch2"]

$template JSONDefault, "{\"message\":\"%msg:::json%\",\"fromhost\":\"%HOSTNAME:::json%\",\"facility\":\"%syslogfacility-text%\",\"priority\":\"%syslogpriority-text%\",\"timereported\":\"%timereported:::date-rfc3339%\",\"timegenerated\":\"%timegenerated:::date-rfc3339%\"}"

{
    "message": " this is a test message",
    "fromhost": "test-host",
    "facility": "user",
    "priority": "info",
    "timereported": "2013-03-12T18:05:01.344864+02:00",
    "timegenerated": "2013-03-12T18:05:01.344864+02:00"
}

{"omes":
  {"writeoperation": "create",
   "_index": "rsyslog_testbench",
   "_type": "test-type",
   "_id": "92BE7AF79CD44305914C7658AF846A08",
   "status": 400,
   "error":
     {"type": "mapper_parsing_exception",
      "reason": "failed to parse [msgnum]",
      "caused_by":
        {"type": "number_format_exception",
         "reason": "For input string: \"x00000025\""}}}}

template(name="es_output_template"
         type="list"
         option.json="on") {
           constant(value="{")
             constant(value="\"timestamp\":\"")      property(name="timereported" dateFormat="rfc3339")
             constant(value="\",\"message\":\"")     property(name="msg")
             constant(value="\",\"host\":\"")        property(name="hostname")
             constant(value="\",\"severity\":\"")    property(name="syslogseverity-text")
             constant(value="\",\"facility\":\"")    property(name="syslogfacility-text")
             constant(value="\",\"syslogtag\":\"")   property(name="syslogtag")
           constant(value="\"}")
         }

template(name="id-template" type="string" string="%$.es_msg_id%")
if strlen($.omes!_id) > 0 then {
    set $.es_msg_id = $.omes!_id;
} else {
    # NOTE: depends on rsyslog being compiled with --enable-uuid
    set $.es_msg_id = $uuid;
}
action(type="omelasticsearch" bulkid="id-template" ...)

module(load="omelasticsearch")
*.*     action(type="omelasticsearch")

module(load="omelasticsearch")
template(name="testTemplate"
         type="list"
         option.json="on") {
           constant(value="{")
             constant(value="\"timestamp\":\"")      property(name="timereported" dateFormat="rfc3339")
             constant(value="\",\"message\":\"")     property(name="msg")
             constant(value="\",\"host\":\"")        property(name="hostname")
             constant(value="\",\"severity\":\"")    property(name="syslogseverity-text")
             constant(value="\",\"facility\":\"")    property(name="syslogfacility-text")
             constant(value="\",\"syslogtag\":\"")   property(name="syslogtag")
           constant(value="\"}")
         }
action(type="omelasticsearch"
       server="myserver.local"
       serverport="9200"
       template="testTemplate"
       searchIndex="test-index"
       searchType="test-type"
       bulkmode="on"
       maxbytes="100m"
       queue.type="linkedlist"
       queue.size="5000"
       queue.dequeuebatchsize="300"
       action.resumeretrycount="-1")

module(load="omelasticsearch")
set $!es_record_id = $uuid;
template(name="bulkid-template" type="list") { property(name="$!es_record_id") }
action(type="omelasticsearch"
       ...
       bulkmode="on"
       bulkid="bulkid-template"
       dynbulkid="on"
       writeoperation="create")

module(load="omelasticsearch")
module(load="omfile")
template(name="bulkid-template" type="list") { property(name="$.es_record_id") }

ruleset(name="error_es") {
        action(type="omfile" template="RSYSLOG_DebugFormat" file="es-bulk-errors.log")
}

ruleset(name="try_es") {
    if strlen($.omes!status) > 0 then {
        # retry case
        if ($.omes!status == 200) or ($.omes!status == 201) or (($.omes!status == 409) and ($.omes!writeoperation == "create")) then {
            stop # successful
        }
        if ($.omes!writeoperation == "unknown") or (strlen($.omes!error!type) == 0) or (strlen($.omes!error!reason) == 0) then {
            call error_es
            stop
        }
        if ($.omes!status == 400) or ($.omes!status < 200) then {
            call error_es
            stop
        }
        # else fall through to retry operation
    }
    if strlen($.omes!_id) > 0 then {
        set $.es_record_id = $.omes!_id;
    } else {
        # NOTE: depends on rsyslog being compiled with --enable-uuid
        set $.es_record_id = $uuid;
    }
    action(type="omelasticsearch"
              ...
              bulkmode="on"
              bulkid="bulkid-template"
              dynbulkid="on"
              writeoperation="create"
              retryfailures="on"
              retryruleset="try_es")
}
call try_es

Module Name:	omelasticsearch
Author:	Rainer Gerhards <rgerhards@adiscon.com>

omelasticsearch: Elasticsearch Output Module¶

Purpose¶

Notable Features¶

Configuration Parameters¶

Action Parameters¶

Server¶

Serverport¶

HealthCheckTimeout¶

searchIndex¶

dynSearchIndex¶

searchType¶

dynSearchType¶

pipelineName¶

dynPipelineName¶

asyncrepl¶

usehttps¶

timeout¶

template¶

bulkmode¶

maxbytes¶

parent¶

dynParent¶

uid¶

pwd¶

errorFile¶

tls.cacert¶

tls.mycert¶

tls.myprivkey¶

allowunsignedcerts¶

skipverifyhost¶

bulkid¶

dynbulkid¶

writeoperation¶

retryfailures¶

retryruleset¶

ratelimit.interval¶

ratelimit.burst¶

rebindinterval¶

Statistic Counter¶

How Retries Are Handled¶

Examples¶

Example 1¶

Example 2¶

Example 3¶

Example 4¶