fluentbit.conf

[SERVICE]
    flush        1
    log_level    info
    parsers_file parsers.conf
    parsers_file /home/vcap/deps/0/apt/etc/fluent-bit/parsers.conf
    parsers_file project_conf/parsers.conf
    plugins_File plugins.conf

[INPUT]
    name      dummy
    dummy     {"message":"A simple test message", "temp": "0.74", "extra": "false"}
    samples   1

[INPUT]
    name tcp
    port 8888
    format none

# Ship to s3:
[OUTPUT]
    Name s3
    Match *
    bucket ${BUCKET_NAME}
    region ${AWS_DEFAULT_REGION}
    json_date_key time
    json_date_format iso8601
    store_dir /tmp/fluent-bit/s3
    total_file_size 50M
    upload_timeout 10m
    # TODO: modify s3_key_format -- by app?
    s3_key_format /fluent-bit-logs/%Y/%m/%d/%H/%M/%S
    retry_limit 5
    # TODO: do we care about data ordering?
    preserve_data_ordering On

# Ship to New Relic:
[OUTPUT]
    Name newrelic
    Match *
    licenseKey ${NEW_RELIC_LICENSE_KEY}
    endpoint ${NEW_RELIC_LOGS_ENDPOINT}

### Filters run in order of appearance ###

# Initial pass at parsing the body of the request.
[FILTER]
    name parser
    match tcp.*
    key_name log
    parser post-with-syslog
    reserve_data On
    preserve_key On

[FILTER]
    name modify
    match tcp.*
    Rename log raw_message

# Further filter of the already-extracted fields
[FILTER]
    name parser
    match tcp.*
    key_name message
    parser extract-gauge
    reserve_data On
    preserve_key On

# And so on ... (multiple passes let us capture fields in varying order)
[FILTER]
    name parser
    match tcp.*
    key_name message
    parser extract-tags
    reserve_data On
    preserve_key On

[FILTER]
    name parser
    match tcp.*
    key_name message
    parser extract-remainder
    reserve_data On
    preserve_key Off

# TODO: does this do anything?
[FILTER]
    name parser
    match tcp.*
    key_name message
    parser extract-json-object-from-message
    reserve_data On
    preserve_key On

# Process selected keys (tags, gauge) into (stringified) JSON
[FILTER]
    name lua
    match tcp.*
    time_as_table On
    script scripts/parse_keys_with_eq_pairs.lua
    call parse_keys_with_eq_pairs

@INCLUDE project_conf/fluentbit.conf