-
Notifications
You must be signed in to change notification settings - Fork 0
/
log2pg.conf
325 lines (320 loc) · 10.5 KB
/
log2pg.conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
# ==================================================================
# LOG2PG CONFIGURATION FILE
# https://www.generacio.com/log2pg.html
#
# There are three comment formats:
# - a hash (#) as in the UNIX shells,
# - the double slash (//)
# - the C-style /* ... */.
#
# Strings are surrounded by quotes. The escape sequences '\\',
# '\f', '\n', '\r', and '\t' are recognized, and have the usual meaning.
# The '\x' followed by exactly two hexadecimal digits represents an 8-bit
# ASCII value. For example, ‘\xFF’ represents the character with ASCII
# code 0xFF.
# ==================================================================
# ==================================================================
# SYSLOG CONFIGURATION
#
# https://en.wikipedia.org/wiki/Syslog
# https://tools.ietf.org/html/rfc3164
# This section is optional.
#
# facility:
# Facility of syslog messages, as defined in RFC 3164.
# Accepted values are:
# - kern, user, mail, daemon, auth, intern, lpr,
# - news, uucp, clock, authpriv, ftp, ntp, audit,
# - alert, cron, local0..local7.
# Default value is "local7".
#
# level:
# Level of logging, as defined in RFC 3164 (see severity).
# Accepted values are:
# - debug, info, notice, warn, error, crit, alert, emerg.
# Log levels above are listed in the order of increasing severity.
# Setting a certain log level will cause all messages of the specified
# and more severe log levels to be logged. For example, the default level
# error will cause error, crit, alert, and emerg messages to be logged.
# Default value is "info".
#
# tag:
# Tag used by log2pg syslog traces.
# Default value is "log2pg".
# ==================================================================
syslog = {
facility = "local7";
level = "info";
tag = "log2pg";
};
# ==================================================================
# DATABASE
#
# Currently, only Postgresql database is supported.
# https://www.postgresql.org/
# Only one database connection is created.
#
# connection-url:
# Connection string to database.
# https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
# postgresql://[user[:password]@][netloc][:port][/dbname][?param1=value1&...]
#
# retry-interval:
# Elapsed time (in milliseconds) to reconnect if the connection is lost.
# This value is optional. Default value is 30000 (30 seconds).
#
# max-failed-connections:
# Can occur errors just after a succeeded reconnection, for example:
# - an invalid prepared statement
# - error re-executing the pending inserts (non-committed inserts on the failed connection).
# In this case, the connection is closed and re-attempted after retry-interval millis.
# This parameter indicates the maximum number of failed re-connection allowed.
# This value is optional. default value is 3.
#
# transaction.max-inserts:
# Maximum number of rows per commit.
# This value is optional. Default value is 1000.
#
# transaction.max-duration:
# Maximum elapsed time (in milliseconds) between insert and commit.
# This value is optional. Default value is 10000 (10 seconds).
#
# transaction.idle-timeout:
# Maximum idle time -no rows inserted- (in milliseconds) before commiting.
# This value would be less or equal than max-duration.
# This value is optional. Default value is 10000 (1 second).
# ==================================================================
database = {
connection-url = "postgresql://log2pg:log2pg@localhost:5432/log2pg";
retry-interval = 10000; // 10 seconds
max-failed-reconnections = 3;
transaction = {
max-inserts = 10000;
max-duration = 2000; // 2 seconds
idle-timeout = 500; // 0.5 seconds
};
};
# ==================================================================
# FILES TO MONITOR
#
# List of files to monitor. When a change is detected, the new
# information is parsed according to file format and inserted into
# the database according to the destination table.
# The directories where files reside are also monitored in order to
# catch the creation of new files fulfilling the conditions. In this
# case, the new files become monitored too.
#
# Example:
# {
# path = "/home/g*/{,temp/,projects/}/*.txt";
# format = "httpd_access";
# table = "httpd";
# discard = "$DIRNAME/$BASENAME.l2p";
# }
#
# path:
# Path to file or files to monitor.
# - Can exists or not yet
# - Can be absolute (eg. /etc/passwd)
# - Can be relative to $PWD (eg. passwd)
# - File patterns supported (see 'man 7 glob', eg. /home/*/*.txt)
# - Supports expand csh(1) style brace expressions of the form {a,b}
#
# format:
# Format of files (see formats section).
#
# table:
# Table where info will be saved (see tables section).
#
# discard:
# File to which the discarded data will be appended. Because the filename
# may not be known in advance (eg. declared as pattern) the following variables
# are available:
# - $REALPATH: realpath(/etc/log2pg.conf) = /etc/log2pg.conf
# - $EXTENSION: extension(/etc/log2pg.conf) = conf
# - $BASENAME: basename(/etc/log2pg.conf) = log2pg
# - $FILENAME: filename(/etc/log2pg.conf) = log2pg.conf
# - $DIRNAME: dirname(/etc/log2pg.conf) = /etc
# If not set then discarded content is not preserved.
# This value is optional. Default value is empty (no discard file).
# ==================================================================
files = (
{
path = "/var/log/httpd/access_log";
format = "httpd_access";
table = "httpd_access";
discard = "$BASENAME.l2p";
}/*,
{
path = "/var/log/httpd/*error_log";
format = "httpd_error";
table = "httpd";
},
{
path = "/var/log/messages";
format = "syslog";
table = "syslog";
},
{
path = "/var/log/maillog";
format = "syslog";
table = "syslog";
},
{
path = "/home/* / *.txt";
format = "plain";
table = "events";
},
{
path = "/home/gerard/coco1.txt";
format = "plain";
table = "events";
},
{
path = "/home/gerard/coco2.txt";
format = "plain2";
table = "events";
},
{
path = "/home/gerard/coco3.txt";
format = "plain3";
table = "events";
discard = "$DIRNAME/$BASENAME.l2p";
},
{
path = "/home/gerard/messages.log";
format = "syslog";
table = "events_syslog";
discard = "$DIRNAME/$BASENAME.l2p";
}*/
);
# ==================================================================
# FILE FORMATS
#
# name:
# Format unique identifier
#
# maxlength:
# Size of the buffer used to read and parse the file content.
# Is required to have a value great than line/chunk maximum size, otherwise,
# starts-ends type line/chunk cannot be identified and content is discarded.
# This parameter is optional. By default it is 10000.
#
# starts:
# Regular expression to identify the start of an event.
# https://www.pcre.org/current/doc/html/pcre2syntax.html
# This value is optional.
#
# ends:
# Regular expression to identify the end of an event.
# https://www.pcre.org/current/doc/html/pcre2syntax.html
# This value is optional. Default value is "\\n".
#
# values:
# Regular expression to select event values.
# https://www.pcre.org/current/doc/html/pcre2syntax.html
# Use named capturing groups to declare the parameters that will be
# inserted in the database (see tables). These parameter identifiers
# consist of up to 32 alphanumeric characters and underscore, but
# must start with a non-digit.
# ==================================================================
formats = (
{
name = "httpd_access";
// https://httpd.apache.org/docs/2.4/logs.html#accesslog
values = "^(?<hostname>[^ \\t]+)[ \\t]+"
"(?<identd>[^ \\t]+)[ \\t]+"
"(?<userid>[^ \\t]+)[ \\t]+"
"\[(?<rtime>[^\]]+)\][ \\t]+"
"\"(?<request>[^\"]*)\"[ \\t]+"
"(?<status>[0-9-]+)[ \\t]+"
"(?<bytes>[0-9-]+)[ \\t]+"
"\"(?<referer>[^\"]*)\"[ \\t]+"
"\"(?<useragent>[^\"]*)\".*$"
},
{
name = "httpd_error";
// https://httpd.apache.org/docs/2.4/mod/core.html#errorlogformat
values = "^\[(?<rtime>[^\]]+)\][ \\t]+"
"\[(?<loglevel>[^\]]+)\][ \\t]+"
"\[(?<pid>[^\]]+)\][ \\t]+"
"\[(?<ipaddress>[^\]]+)\][ \\t]+"
"(?<msg>.*)$";
},
{
name = "syslog";
// May 20 09:43:00 fobos NetworkManager[687]: <info> [1526802180.5062]
values = "^(?<timestamp>[[:alpha:]]+ [0-9]{1,2} [0-9:]+)\s+"
"(?<loghost>[^ ]+)\s+"
"(?<app>[^:\[\s]+)[^:]*:\s+"
"(?<msg>.*)$";
maxlength = 10000;
},
{
name = "properties";
values = "(?<key>[[:alpha:]]\w*)\s*=\s*(?<value>.*)";
starts = "^[[:alpha:]]";
ends = "[^\\\\]\\n";
},
{
name = "plain";
values = "(?<line>.*)";
ends = "\\n";
maxlength = 200;
},
{
name = "plain2";
values = "^(?<line>.*)$";
starts = "^line";
maxlength = 200;
},
{
name = "plain3";
values = "line(?<line>.*)";
//starts = "^.*\\n";
ends = "\\n";
maxlength = 200;
}
);
# ==================================================================
# DATABASE TABLES
#
# SQL commands are converted to prepared
# statements. You can use database function and variables (eg. upper,
# to_timestamp, current_timestamp, etc.) and parameters parsed by
# format regex (eg. :timestamp, :host, :error).
#
# name:
# Table unique identifier
#
# sql:
# SQL command to insert values.
# Parameters linked to regex parameters are indicated as '$param'.
# Parameters identifiers consist of up to 32 alphanumeric characters and
# underscores but must start with a non-digit.
# ==================================================================
tables = (
{
name = "httpd_access";
sql = "insert into t_httpd_access(hostname, identd, userid, rtime, request, status, bytes, referer, useragent) "
"values($hostname, $identd, $userid, to_timestamp($rtime, 'DD/Mon/YYYY:HH24:MI:SS'), $request, "
"NULLIF($status, '-')::integer, NULLIF($bytes, '-')::integer, $referer, $useragent)";
}/*,
{
name = "splunk";
sql = "insert into events(timestamp, host, source, type, content) "
"values(:timestamp, :host, :source, :type, :content)";
},
{
name = "syslog";
sql = "insert into syslog(timestamp, host, app, msg) values(:timestamp, :loghost, :app, :msg)";
},
{
name = "events";
sql = "insert into events(time_stamp, msg) values(current_timestamp, :line)";
},
{
name = "events_syslog";
sql = "insert into events(time_stamp, msg) values(current_timestamp, :msg)";
}*/
);