Skip to content

Commit

Permalink
Merge pull request #1492 from cosmo0920/support-rfc5424-syslog
Browse files Browse the repository at this point in the history 
Support rfc5424 syslog format
  • Loading branch information
@repeatedly
repeatedly authored Mar 8, 2017
2 parents 9618ca1 + 866b2ca commit 8c70616
Show file tree
Hide file tree
Showing 2 changed files with 203 additions and 1 deletion.
42 changes: 41 additions & 1 deletion lib/fluent/plugin/parser_syslog.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,13 @@ class SyslogParser < Parser
REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
# From in_syslog default pattern
REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
REGEXP_RFC5424 = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/

config_set_default :time_format, "%b %d %H:%M:%S"
config_param :with_priority, :bool, default: false
config_param :message_format, :enum, list: [:rfc3164, :rfc5424, :auto], default: :rfc3164
config_param :rfc5424_time_format, :string, default: "%Y-%m-%dT%H:%M:%S.%L%z"

def initialize
super
Expand All @@ -39,7 +43,26 @@ def initialize
def configure(conf)
super

@regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
@time_parser_rfc3164 = @time_parser_rfc5424 = nil
@regexp = case @message_format
when :rfc3164
class << self
alias_method :parse, :parse_plain
end
@with_priority ? REGEXP_WITH_PRI : REGEXP
when :rfc5424
class << self
alias_method :parse, :parse_plain
end
REGEXP_RFC5424
when :auto
class << self
alias_method :parse, :parse_auto
end
@time_parser_rfc3164 = time_parser_create(format: @time_format)
@time_parser_rfc5424 = time_parser_create(format: @rfc5424_time_format)
nil
end
@time_parser = time_parser_create
end

Expand All @@ -48,6 +71,23 @@ def patterns
end

def parse(text)
# This is overwritten in configure
end

def parse_auto(text, &block)
if @message_format == :auto
if REGEXP_DETECT_RFC5424.match(text)
@regexp = REGEXP_RFC5424
@time_parser = @time_parser_rfc5424
else
@regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
@time_parser = @time_parser_rfc3164
end
end
parse_plain(text, &block)
end

def parse_plain(text, &block)
m = @regexp.match(text)
unless m
yield nil, nil
Expand Down
162 changes: 162 additions & 0 deletions test/plugin/test_parser_syslog.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,4 +63,166 @@ def test_parse_with_keep_time_key
assert_equal "Feb 28 00:00:12", record['time']
end
end

class TestRFC5424Regexp < self
def test_parse_with_rfc5424_message
@parser.configure(
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
'message_format' => 'rfc5424',
)
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "-", record["pid"]
assert_equal "-", record["msgid"]
assert_equal "-", record["extradata"]
assert_equal "Hi, from Fluentd!", record["message"]
end
end

def test_parse_with_rfc5424_structured_message
@parser.configure(
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
'message_format' => 'rfc5424',
)
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "11111", record["pid"]
assert_equal "ID24224", record["msgid"]
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
record["extradata"]
assert_equal "Hi, from Fluentd!", record["message"]
end
end
end

class TestAutoRegexp < self
def test_auto_with_legacy_syslog_message
@parser.configure(
'time_format' => '%b %d %M:%S:%H',
'mseeage_format' => 'auto',
)
text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("Feb 28 00:00:12", format: '%b %d %M:%S:%H'), time)
assert_equal(@expected, record)
end
end

def test_auto_with_legacy_syslog_priority_message
@parser.configure(
'time_format' => '%b %d %M:%S:%H',
'with_priority' => true,
'mseeage_format' => 'auto',
)
text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
assert_equal(@expected.merge('pri' => 6), record)
end
end

def test_parse_with_rfc5424_message
@parser.configure(
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
'message_format' => 'auto',
)
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "-", record["pid"]
assert_equal "-", record["msgid"]
assert_equal "-", record["extradata"]
assert_equal "Hi, from Fluentd!", record["message"]
end
end

def test_parse_with_rfc5424_structured_message
@parser.configure(
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
'message_format' => 'auto',
)
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "11111", record["pid"]
assert_equal "ID24224", record["msgid"]
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
record["extradata"]
assert_equal "Hi, from Fluentd!", record["message"]
end
end

def test_parse_with_both_message_type
@parser.configure(
'time_format' => '%b %d %M:%S:%H',
'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
'message_format' => 'auto',
)
text = 'Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
assert_equal(@expected, record)
end
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "11111", record["pid"]
assert_equal "ID24224", record["msgid"]
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
record["extradata"]
assert_equal "Hi, from Fluentd!", record["message"]
end
text = 'Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
assert_equal(@expected, record)
end
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "-", record["pid"]
assert_equal "-", record["msgid"]
assert_equal "-", record["extradata"]
assert_equal "Hi, from Fluentd!", record["message"]
end
end

def test_parse_with_both_message_type_and_priority
@parser.configure(
'time_format' => '%b %d %M:%S:%H',
'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
'with_priority' => true,
'message_format' => 'auto',
)
text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
assert_equal(@expected.merge('pri' => 6), record)
end
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "11111", record["pid"]
assert_equal "ID24224", record["msgid"]
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
record["extradata"]
assert_equal "Hi, from Fluentd!", record["message"]
end
text = '<16>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
assert_equal(@expected.merge('pri' => 16), record)
end
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "-", record["pid"]
assert_equal "-", record["msgid"]
assert_equal "-", record["extradata"]
assert_equal "Hi, from Fluentd!", record["message"]
end
end
end
end

0 comments on commit 8c70616

Please sign in to comment.