ds-pretty-log

#!/usr/bin/env python

from __future__ import print_function
import sys
import re

log_line_re = re.compile('^(?P<date>(\w{3} [0-9 ]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}|[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3})) '
                         '((?P<hostname>[a-zA-Z0-9_.-]+) '
                         '(?P<service>[a-zA-Z0-9_.@-]*)(: )?'
                         '\[(?P<pid>\d+)\]:? )?'
                         '(?P<file>\[(http://|/)[^\]]*\] )?(?P<log>.*)$')

levels = ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'FATAL')

http_methods = ('GET', 'PUT', 'POST', 'DELETE')

http_request_re = re.compile('(?P<request>"?(%s) /[^ )"]*( HTTP/\d\.\d" [1-5]\d\d)?)' % ("|".join(http_methods)))

http_status_re = re.compile('status: (?P<code>[1-5]\d\d)')
http_response_re = re.compile('HTTP/[12]\.[0-2]"? (?P<code>[1-5]\d\d)')

error_re = re.compile('\W[Ee]rror')

highlight_re = re.compile('\WGTh\W')

states = ('SUCCESS', 'RUNNING', 'PENDING')

colors = {
    'DEBUG': None,
    'INFO': None,
    'WARNING': 33,
    'ERROR': 31,
    'FATAL': 31,
}

styles = {
    'DEBUG': None,
    'INFO': None,
    'WARNING': None,
    'ERROR': None,
    'FATAL': 1,
}

while True:
    try:
        l = sys.stdin.readline()
    except KeyboardInterrupt:
        break

    if l == '':
        break

    m = log_line_re.match(l)
    if m:
        d = m.groupdict(m)

        log = d['log']

        level = 'INFO'

        for l in levels:
            if log.startswith('%s ' % (l)):
                level = l
                break

        hm = http_status_re.search(log)
        if not hm:
            hm = http_response_re.search(log)
        if hm is not None:
            http = hm.groupdict()
            code = int(http['code'])

            if code < 400:
                pass
            else:
                if level != 'FATAL':
                    level = 'ERROR'

        if level not in ('FATAL', 'ERROR'):
            em = error_re.search(log)
            if em:
                level = 'ERROR'

        line_style = ''
        hl = highlight_re.search(log)
        if hl:
            line_style = '\033[1m'

        color_code = ''
        color = colors[level]
        if color is not None:
            style = styles[level]
            if style:
                style_code = '%d;' % (style)
            elif len(line_style) == 0:
                style_code = '0;'
            else:
                style_code = ''

            color_code = '\033[%s%dm' % (style_code, color)
        elif len(line_style) == 0:
            color_code = '\033[0m'

        if hm:
            s = hm.start()
            e = hm.end()
            log = "%s\033[34;1m%s%s%s" % (log[0:s], log[s:e], color_code, log[e:])

        for st in states:
            log = re.sub("'%s'" % (st), "'\033[35m%s\033[0m'" % (st), log)

        rm = http_request_re.search(log)
        if rm:
            s = rm.start()
            e = rm.end()
            log = "%s\033[34;1m%s%s%s" % (log[0:s], log[s:e], color_code, log[e:])

        if isinstance(d['hostname'], str):
            line = '%s %s %s[%s]: %s' % (d['date'], d['hostname'], d['service'], d['pid'], log)
        else:
            line = '%s %s' % (d['date'], log)
        line = '%s%s%s%s' % (line_style, color_code, line, '\033[0m')

        print(line)

    else:
        print(l, end='')
    sys.stdout.flush()