check-mk-server 1.2.6p12-1 / usr / share / check_mk / checks / logwatch

#!/usr/bin/python
# -*- encoding: utf-8; py-indent-offset: 4 -*-
# +------------------------------------------------------------------+
# |             ____ _               _        __  __ _  __           |
# |            / ___| |__   ___  ___| | __   |  \/  | |/ /           |
# |           | |   | '_ \ / _ \/ __| |/ /   | |\/| | ' /            |
# |           | |___| | | |  __/ (__|   <    | |  | | . \            |
# |            \____|_| |_|\___|\___|_|\_\___|_|  |_|_|\_\           |
# |                                                                  |
# | Copyright Mathias Kettner 2014             mk@mathias-kettner.de |
# +------------------------------------------------------------------+
#
# This file is part of Check_MK.
# The official homepage is at http://mathias-kettner.de/check_mk.
#
# check_mk is free software;  you can redistribute it and/or modify it
# under the  terms of the  GNU General Public License  as published by
# the Free Software Foundation in version 2.  check_mk is  distributed
# in the hope that it will be useful, but WITHOUT ANY WARRANTY;  with-
# out even the implied warranty of  MERCHANTABILITY  or  FITNESS FOR A
# PARTICULAR PURPOSE. See the  GNU General Public License for more de-
# ails.  You should have  received  a copy of the  GNU  General Public
# License along with GNU Make; see the file  COPYING.  If  not,  write
# to the Free Software Foundation, Inc., 51 Franklin St,  Fifth Floor,
# Boston, MA 02110-1301 USA.

# Configuration variables in main.mk needed during the actual check
logwatch_dir = var_dir + '/logwatch'
logwatch_patterns = { }
logwatch_rules = []
logwatch_max_filesize = 500000 # do not save more than 500k of message (configurable)
logwatch_service_output = "default"
logwatch_groups = []

# Variables embedded in precompiled checks
check_config_variables += [ "logwatch_dir", "logwatch_max_filesize", "logwatch_service_output" ]

def logwatch_ec_forwarding_enabled(params, item):
    if 'restrict_logfiles' not in params:
        return True # matches all logs on this host
    else:
        # only logs which match the specified patterns
        for pattern in params['restrict_logfiles']:
            if re.match(pattern, item):
                return True

    return False

# Splits the number of existing logfiles into
# forwarded (to ec) and not forwarded. Returns a
# pair of forwarded and not forwarded logs.
def logwatch_select_forwarded(info):
    forwarded_logs = []
    not_forwarded_logs = []

    forward_settings = host_extra_conf(g_hostname, checkgroup_parameters.get('logwatch_ec', []))

    for l in info:
        line = " ".join(l)
        if len(line) > 6 and line[0:3] == "[[[" and line[-3:] == "]]]" \
           and ':missing' not in line and ':cannotopen' not in line:
            logfile_name = line[3:-3]

            # Is forwarding enabled in general?
            if forward_settings and forward_settings[0] != None:
                if logwatch_ec_forwarding_enabled(forward_settings[0], logfile_name):
                    forwarded_logs.append(logfile_name)
                else:
                    not_forwarded_logs.append(logfile_name)

            # No forwarding rule configured
            else:
                not_forwarded_logs.append(logfile_name)

    return forwarded_logs, not_forwarded_logs


def inventory_logwatch(info):
    forwarded_logs, not_forwarded_logs = logwatch_select_forwarded(info)
    inventory = []
    for logfile in not_forwarded_logs:
        groups = logwatch_groups_of_logfile(logfile)
        if groups:
            continue
        else:
            inventory.append((logfile, None))
    return inventory


# logwatch_patterns = {
#    'System': [
#    ( 'W', 'sshd' ),
#    ( ['host1', 'host2'],        'C', 'ssh' ), # only applies to certain hosts
#    ( ['lnx', 'dmz'], ALL_HOSTS, 'C', 'ssh' ), # only applies to host having certain tags
#    ( ALL_HOSTS, (10, 20), 'x' ), # at 10 messages per interval warn, at 20 crit
#    ( 'I', '0' )
#    ],
#    'Application': [
#    ( 'W', 'crash.exe' ),
#    ( 'E', 'ssh' )
#    ]
#    }

# New rule-stule logwatch_rules in WATO friendly consistent rule notation:
#
# logwatch_rules = [
#   ( [ PATTERNS ], ALL_HOSTS, [ "Application", "System" ] ),
# ]
# All [ PATTERNS ] of matching rules will be concatenated in order of
# appearance.
#
# PATTERN is a list like:
# [ ( 'O',      ".*ssh.*" ),          # Make informational (OK) messages from these
#   ( (10, 20), "login"   ),          # Warning at 10 messages, Critical at 20
#   ( 'C',      "bad"     ),          # Always critical
#   ( 'W',      "not entirely bad" ), # Always warning
# ]
#


def logwatch_state(state):
    if state == 1:
        return "WARN"
    elif state != 0:
        return "CRIT"
    else:
        return "OK"

def logwatch_level_name(level):
    if   level == 'O': return 'OK'
    elif level == 'W': return 'WARN'
    elif level == 'u': return 'WARN' # undefined states are treated as warning
    elif level == 'C': return 'CRIT'
    else: return 'IGN'

def logwatch_level_worst(worst, level):
    if   level == 'O': return max(worst, 0)
    elif level == 'W': return max(worst, 1)
    elif level == 'u': return max(worst, 1)
    elif level == 'C': return max(worst, 2)
    else: return worst

# Extracts patterns that are relevant for the current host and item.
# Constructs simple list of pairs: [ ('W', 'crash.exe'), ('C', 'sshd.*test') ]
def logwatch_precompile(hostname, item, _unused):
    # Initialize the patterns list with the logwatch_rules
    params = []
    description = check_info['logwatch']['service_description'] % item

    # This is the new (-> WATO controlled) variable
    rules = service_extra_conf(hostname, item, logwatch_rules)
    for rule in rules:
        for pattern in rule:
            params.append((pattern[0], pattern[1]))

    # Now load the old logwatch_patterns var
    patterns = logwatch_patterns.get(item)
    if patterns:
        for entry in patterns:
            hostlist = None
            tags = []

            pattern = entry[-1]
            level = entry[-2]

            if len(entry) >= 3:    # found optional host list
                hostlist = entry[-3]
            if len(entry) >= 4:    # found optional host tags
                tags = entry[-4]

            if hostlist and not \
                   (hosttags_match_taglist(tags_of_host(hostname), tags) and \
                    in_extraconf_hostlist(hostlist, hostname)):
                continue

            params.append((level, pattern))

    return params


def logwatch_reclassify(counts, patterns, text):
    for level, pattern in patterns:
        reg = compiled_regexes.get(pattern)
        if not reg:
            reg = re.compile(pattern)
            compiled_regexes[pattern] = reg
        if reg.search(text):
            # If the level is not fixed like 'C' or 'W' but a pair like (10, 20),
            # then we count how many times this pattern has already matched and
            # assign the levels according to the number of matches of this pattern.
            if type(level) == tuple:
                warn, crit = level
                newcount = counts.setdefault(id(pattern), 0) + 1
                counts[id(pattern)] = newcount
                if newcount >= crit:
                    return 'C'
                elif newcount >= warn:
                    return 'W'
                else:
                    return 'I'
            else:
                return level
    return None

def logwatch_parse_line(line):
    parts = line.split(None, 1)
    level = parts[0]
    if len(parts) > 1:
        text = parts[1]
    else:
        text = ""
    return level, text

# In case of a precompiled check, params contains the precompiled
# logwatch_patterns for the logfile we deal with. If using check_mk
# without precompiled checks, the params must be None an will be
# ignored.
def check_logwatch(item, params, info):
    if len(info) == 1:
        line = " ".join(info[0])
        if line.startswith("CANNOT READ CONFIG FILE"):
            return 3, "Error in agent configuration: %s" % " ".join(info[0][4:])

    found = False
    loglines = []
    for l in info:
        line = " ".join(l)
        if line == "[[[%s]]]" % item:
            found = True
        elif len(line) > 6 and line[0:3] == "[[[" and line[-3:] == "]]]":
            if found:
                break
            found = False
        elif found:
            loglines.append(line)
    return check_logwatch_generic(item, params, loglines, found)


check_info['logwatch'] = {
    'check_function':      check_logwatch,
    'inventory_function':  inventory_logwatch,
    'service_description': "Log %s",
    'group':               'logwatch',
}

precompile_params['logwatch'] = logwatch_precompile

#   .--lw.groups-----------------------------------------------------------.
#   |              _                                                       |
#   |             | |_      ____ _ _ __ ___  _   _ _ __  ___               |
#   |             | \ \ /\ / / _` | '__/ _ \| | | | '_ \/ __|              |
#   |             | |\ V  V / (_| | | | (_) | |_| | |_) \__ \              |
#   |             |_| \_/\_(_)__, |_|  \___/ \__,_| .__/|___/              |
#   |                        |___/                |_|                      |
#   +----------------------------------------------------------------------+
#   |                                                                      |
#   '----------------------------------------------------------------------'

def logwatch_group_precompile(hostname, item, _unused):
    return logwatch_precompile(hostname, item, None), host_extra_conf(hostname, logwatch_groups)

def logwatch_groups_of_logfile(filename, params=False):
    import fnmatch
    groups = []
    if not params:
        params = host_extra_conf(g_hostname, logwatch_groups)
    else:
        params = params[1]
    for line in params:
        for group_name, pattern in line:
            inclusion, exclusion = pattern
            if fnmatch.fnmatch(filename, inclusion) \
                    and not fnmatch.fnmatch(filename, exclusion):
                groups.append(group_name)
    return groups

def inventory_logwatch_groups(info):
    forwarded_logs, not_forwarded_logs = logwatch_select_forwarded(info)
    added_groups = []
    inventory = []
    for logfile in not_forwarded_logs:
        groups = logwatch_groups_of_logfile(logfile)
        for group in groups:
            if group not in added_groups:
                added_groups.append(group)
                inventory.append((group, None))
    return inventory

def check_logwatch_groups(item, params, info):
    if len(info) == 1:
        line = " ".join(info[0])
        if line.startswith("CANNOT READ CONFIG FILE"):
            return 3, "Error in agent configuration: %s" % " ".join(info[0][4:])

    found = False
    logfile_found = False
    loglines = []
    for l in info:
        line = " ".join(l)
        if logfile_found == True and not line.startswith('[[['):
            loglines.append(line)
        if line.startswith('[[['):
            logfile = line[3:-3]
            if item in logwatch_groups_of_logfile(logfile, params):
                found = True
                logfile_found = True
            else:
                logfile_found = False
            continue
    return check_logwatch_generic(item, params, loglines, found, True)


check_info['logwatch.groups'] = {
    'check_function':      check_logwatch_groups,
    'inventory_function':  inventory_logwatch_groups,
    'service_description': "LOG %s",
    'group':               'logwatch',
}
precompile_params['logwatch.groups'] = logwatch_group_precompile
#.

def check_logwatch_generic(item, params, loglines, found, groups=False):
    # Create directories, if neccessary
    try:
        logdir = logwatch_dir + "/" + g_hostname
        if not os.path.exists(logwatch_dir):
            os.mkdir(logwatch_dir)
        if not os.path.exists(logdir):
            os.mkdir(logdir)
            if www_group != None:
                try:
                    if i_am_root():
                        to_user = nagios_user
                    else:
                        to_user = "" # keep user unchanged
                    os.system("chown %s:%s %s" % (to_user, www_group, quote_shell_string(logdir)))
                    os.chmod(logdir, 0775)
                except Exception, e:
                    os.rmdir(logdir)
                    raise MKGeneralException(("User %s cannot chown directory to group id %d: %s. Please make sure "+
                                             "that %s is a member of that group.") %
                                             (username(), www_group, e, username()))

    except MKGeneralException:
        raise
    except Exception, e:
        raise MKGeneralException("User %s cannot create logwatch directory: %s" % \
                                 (username(), e) )

    logfile = logdir + "/" + item.replace("/", "\\")

    # Logfile (=item) section not found and no local file found. This usually
    # means, that the corresponding logfile also vanished on the target host.
    if found == False and not os.path.exists(logfile):
        return (3, "log not present anymore")

    # Write out new log lines (no reclassify here. It is done later in general for all logs)
    if len(loglines) > 0:
        worst = -1
        for line in loglines:
            worst = logwatch_level_worst(worst, logwatch_parse_line(line)[0])

        # Ignore log lines which result in an "I" -> "Ignore" state
        if worst > -1:
            try:
                logarch = file(logfile, "a+")
                logarch.write(time.strftime("<<<%Y-%m-%d %H:%M:%S " + logwatch_state(worst) + ">>>\n"))
                logarch.write("\n".join(loglines) + "\n")
                logarch.close()
            except Exception, e:
                raise MKGeneralException("User %s cannot create logfile: %s" % \
                                         (username(), e))

    # Get the patterns (either compile or reuse the precompiled ones)
    # Check_MK creates an empty string if the precompile function has
    # not been executed yet. The precompile function creates an empty
    # list when no ruless/patterns are defined. In case of the logwatch.groups
    # checks, params are a tuple with the normal logwatch parameters on the first
    # and the grouping patterns on the second position
    if params not in ('', None):
        if groups:
            patterns = params[0]
        else:
            patterns = params # patterns already precompiled
    else:
        patterns = logwatch_precompile(g_hostname, item, None)

    def reclassify_line(counts, patterns, text, level):
        if patterns:
            newlevel = logwatch_reclassify(counts, patterns, text)
            if newlevel != None:
                level = newlevel
        return level

    # Read current log messages, reclassify all messages and write out the
    # whole file again if at least one line has been reclassified
    worst = 0
    last_worst_line = ''
    reclassified_lines = []
    reclassified = False
    counts = {} # for counting number of matches of a certain pattern
    state_counts = {} # for counting number of blocks with a certain state
    block_header = None
    block_worst = -1
    block_lines = []
    block_last_worst_line = ''

    def finish_block(header, block_worst, block_lines):
        start = ' '.join(header.split(' ')[0:2])
        return [ start + ' %s>>>' % logwatch_state(block_worst) ] + block_lines

    if os.path.exists(logfile):
        for line in file(logfile):
            line = line.rstrip('\n')
            # Skip empty lines
            if not line:
                continue
            if line.startswith('<<<') and line.endswith('>>>'):
                if block_header:
                    # The section is finished here. Add it to the list of reclassified lines if the
                    # state of the block is not "I" -> "ignore"
                    if block_worst > -1:
                        reclassified_lines += finish_block(block_header, block_worst, block_lines)

                        if block_worst >= worst:
                            worst = block_worst
                            last_worst_line = block_last_worst_line

                    block_lines = []
                    block_worst = -1
                block_header = line
                counts = {}
            else:
                old_level, text = logwatch_parse_line(line)
                level = reclassify_line(counts, patterns, text, old_level)

                if old_level != level or level == "I":
                    reclassified = True

                old_block_worst = block_worst
                block_worst = logwatch_level_worst(block_worst, level)

                # Save the last worst line of this block
                if old_block_worst != block_worst or level == block_worst:
                    block_last_worst_line = text

                # Count the number of lines by state
                if level != '.':
                    try:
                        state_counts[level] += 1
                    except KeyError:
                        state_counts[level] = 1

                if level == "I":
                    continue

                block_lines.append(level + ' ' + text)

        # The last section is finished here. Add it to the list of reclassified lines if the
        # state of the block is not "I" -> "ignore"
        if block_header and block_worst > -1:
            reclassified_lines += finish_block(block_header, block_worst, block_lines)

            if block_worst >= worst:
                worst = block_worst
                last_worst_line = block_last_worst_line


        # Only rewrite the lines if at least one line has been reclassified
        if reclassified:
            if reclassified_lines:
                file(logfile, 'w').write('\n'.join(reclassified_lines) + '\n')
            else:
                os.unlink(logfile)

    # if logfile has reached maximum size, abort with critical state
    if os.path.exists(logfile) and os.path.getsize(logfile) > logwatch_max_filesize:
        return (2, "unacknowledged messages have exceeded max size (%d Bytes)" % logwatch_max_filesize)

    #
    # Render output
    #

    if len(reclassified_lines) == 0:
        return (0, "no error messages")
    else:
        count_txt = []
        for level, num in state_counts.iteritems():
            count_txt.append('%d %s' % (num, logwatch_level_name(level)))
        if logwatch_service_output == 'default':
            return (worst, "%s messages (Last worst: \"%s\")" %
                                           (', '.join(count_txt), last_worst_line))
        else:
            return (worst, "%s messages" % ', '.join(count_txt))

#   .----------------------------------------------------------------------.
#   |      _____ ____   _____ ___  ______        ___    ____  ____         |
#   |     | ____/ ___| |  ___/ _ \|  _ \ \      / / \  |  _ \|  _ \        |
#   |     |  _|| |     | |_ | | | | |_) \ \ /\ / / _ \ | |_) | | | |       |
#   |     | |__| |___  |  _|| |_| |  _ < \ V  V / ___ \|  _ <| |_| |       |
#   |     |_____\____| |_|   \___/|_| \_\ \_/\_/_/   \_\_| \_\____/        |
#   |                                                                      |
#   +----------------------------------------------------------------------+
#   | Forwarding logwatch messages to event console                        |
#   '----------------------------------------------------------------------'

import socket, time

# OK      -> priority 5 (notice)
# WARN    -> priority 4 (warning)
# CRIT    -> priority 2 (crit)
# context -> priority 6 (info)
# u = Uknown
def logwatch_to_prio(level):
    if level == 'W':
        return 4
    elif level == 'C':
        return 2
    elif level == 'O':
        return 5
    elif level == '.':
        return 6
    else:
        return 4

def inventory_logwatch_ec(info):
    forwarded_logs, not_forwarded_logs = logwatch_select_forwarded(info)
    if forwarded_logs:
        return [ (None, { "expected_logfiles": forwarded_logs } ) ]

def check_logwatch_ec(item, params, info):
    if len(info) == 1:
        line = " ".join(info[0])
        if line.startswith("CANNOT READ CONFIG FILE"):
            return 3, "Error in agent configuration: %s" % " ".join(info[0][4:])

    # 1. Parse lines in info and separate by logfile
    logs = {}
    logfile = None
    for l in info:
        line = " ".join(l)
        if len(line) > 6 and line[0:3] == "[[[" and line[-3:] == "]]]":
            # new logfile, extract name
            logfile = line[3:-3]
            logs.setdefault(logfile, [])

        elif logfile and line:
            # new regular line, skip context lines and ignore lines
            if line[0] not in ['.', 'I']:
                logs[logfile].append(line)

    # 2. Maybe filter logfiles if some should be excluded
    if 'restrict_logfiles' in params:
        for logfile in logs.keys():
            if not logwatch_ec_forwarding_enabled(params, logfile):
                del logs[logfile]

    # Check if the number of expected files matches the actual one
    status = 0
    infotexts = []
    if params.get('monitor_logfilelist'):
        if 'expected_logfiles' not in params:
            infotexts.append("You enabled monitoring the list of forwarded logfiles. You need to re-inventorize the check once.")
            status = 1
        else:
            expected = params['expected_logfiles']
            missing = []
            for f in expected:
                if f not in logs:
                    missing.append(f)
            if missing:
                infotexts.append("Missing logfiles: %s" % (", ".join(missing)))
                status = 1

            exceeding = []
            for f in logs:
                if f not in expected:
                    exceeding.append(f)
            if exceeding:
                infotexts.append("Newly appeared logfiles: %s" % (", ".join(exceeding)))
                status = 1

    # 3. create syslog message of each line
    # <128> Oct 24 10:44:27 Klappspaten /var/log/syslog: Oct 24 10:44:27 Klappspaten logger: asdasdad as
    # <facility+priority> timestamp hostname logfile: message
    facility = params.get('facility', 17) << 3 # default to "local1"
    messages = []
    cur_time = int(time.time())

    forwarded_logfiles = set([])
    for logfile, lines in logs.items():
        for line in lines:
            msg = '<%d>' % (facility + logwatch_to_prio(line[0]),)
            msg += '@%s %s %s: %s' % (cur_time, g_hostname, logfile, line[2:])
            messages.append(msg)
            forwarded_logfiles.add(logfile)

    # 4. send lines to event console
    # a) local in same omd site
    # b) local pipe
    # c) remote via udp
    # d) remote via tcp
    method = params.get('method')
    if not method:
        method = os.getenv('OMD_ROOT') + "/tmp/run/mkeventd/eventsocket"
    elif method == 'spool:':
        method += os.getenv('OMD_ROOT') + "/var/mkeventd/spool"

    num_messages = len(messages)
    try:
        if messages:
            if isinstance(method, tuple):
                # connect either via tcp or udp
                if method[0] == 'udp':
                    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
                else:
                    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                sock.connect((method[1], method[2]))
                for message in messages:
                    sock.send(message + "\n")
                sock.close()

            elif not method.startswith('spool:'):
                # write into local event pipe
                # Important: When the event daemon is stopped, then the pipe
                # is *not* existing! This prevents us from hanging in such
                # situations. So we must make sure that we do not create a file
                # instead of the pipe!
                sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
                sock.connect(method)
                sock.send('\n'.join(messages) + '\n')
                sock.close()

            else:
                # Spool the log messages to given spool directory.
                # First write a file which is not read into ec, then
                # perform the move to make the file visible for ec
                spool_path = method[6:]
                file_name  = '.%s_%s%d' % (g_hostname, item and item.replace('/', '\\') + '_' or '', time.time())
                if not os.path.exists(spool_path):
                    os.makedirs(spool_path)
                file('%s/%s' % (spool_path, file_name), 'w').write('\n'.join(messages) + '\n')
                os.rename('%s/%s' % (spool_path, file_name), '%s/%s' % (spool_path, file_name[1:]))

        if forwarded_logfiles:
            logfile_info = " from " + ",".join(list(forwarded_logfiles))
        else:
            logfile_info = ""

        infotexts.append('Forwarded %d messages%s to event console' % (num_messages, logfile_info))
        return (status, ", ".join(infotexts), [('messages', num_messages)])

    except Exception, e:
        return (2, 'Unable to forward messages to event console (%s). Lost %d messages.' %
                (e, num_messages))

check_info['logwatch.ec'] = {
    'check_function':      check_logwatch_ec,
    'inventory_function':  inventory_logwatch_ec,
    'service_description': "Log Forwarding",
    'group':               'logwatch_ec',
}