check-mk-server 1.2.8p16-1ubuntu0.1 / usr / share / check_mk / checks / logwatch

#!/usr/bin/python
# -*- encoding: utf-8; py-indent-offset: 4 -*-
# +------------------------------------------------------------------+
# |             ____ _               _        __  __ _  __           |
# |            / ___| |__   ___  ___| | __   |  \/  | |/ /           |
# |           | |   | '_ \ / _ \/ __| |/ /   | |\/| | ' /            |
# |           | |___| | | |  __/ (__|   <    | |  | | . \            |
# |            \____|_| |_|\___|\___|_|\_\___|_|  |_|_|\_\           |
# |                                                                  |
# | Copyright Mathias Kettner 2014             mk@mathias-kettner.de |
# +------------------------------------------------------------------+
#
# This file is part of Check_MK.
# The official homepage is at http://mathias-kettner.de/check_mk.
#
# check_mk is free software;  you can redistribute it and/or modify it
# under the  terms of the  GNU General Public License  as published by
# the Free Software Foundation in version 2.  check_mk is  distributed
# in the hope that it will be useful, but WITHOUT ANY WARRANTY;  with-
# out even the implied warranty of  MERCHANTABILITY  or  FITNESS FOR A
# PARTICULAR PURPOSE. See the  GNU General Public License for more de-
# tails. You should have  received  a copy of the  GNU  General Public
# License along with GNU Make; see the file  COPYING.  If  not,  write
# to the Free Software Foundation, Inc., 51 Franklin St,  Fifth Floor,
# Boston, MA 02110-1301 USA.

# Configuration variables in main.mk needed during the actual check
logwatch_dir = var_dir + '/logwatch'
logwatch_patterns = { }
logwatch_rules = []
logwatch_max_filesize = 500000 # do not save more than 500k of message (configurable)
logwatch_service_output = "default"
logwatch_groups = []

# Variables embedded in precompiled checks
check_config_variables += [ "logwatch_dir", "logwatch_max_filesize", "logwatch_service_output" ]

#   .--General-------------------------------------------------------------.
#   |                   ____                           _                   |
#   |                  / ___| ___ _ __   ___ _ __ __ _| |                  |
#   |                 | |  _ / _ \ '_ \ / _ \ '__/ _` | |                  |
#   |                 | |_| |  __/ | | |  __/ | | (_| | |                  |
#   |                  \____|\___|_| |_|\___|_|  \__,_|_|                  |
#   |                                                                      |
#   +----------------------------------------------------------------------+
#   |  General functions, for normal and forwarding logwatch check         |
#   '----------------------------------------------------------------------'

def logwatch_ec_forwarding_enabled(params, item):
    if 'restrict_logfiles' not in params:
        return True # matches all logs on this host
    else:
        # only logs which match the specified patterns
        for pattern in params['restrict_logfiles']:
            if re.match(pattern, item):
                return True

    return False

# Splits the number of existing logfiles into
# forwarded (to ec) and not forwarded. Returns a
# pair of forwarded and not forwarded logs.
def logwatch_select_forwarded(info):
    forwarded_logs = []
    not_forwarded_logs = []

    forward_settings = host_extra_conf(g_hostname, checkgroup_parameters.get('logwatch_ec', []))

    for l in info:
        line = " ".join(l)
        if len(line) > 6 and line[0:3] == "[[[" and line[-3:] == "]]]" \
           and ':missing' not in line and ':cannotopen' not in line:
            logfile_name = line[3:-3]

            # Is forwarding enabled in general?
            if forward_settings and forward_settings[0] != None:
                if logwatch_ec_forwarding_enabled(forward_settings[0], logfile_name):
                    forwarded_logs.append(logfile_name)
                else:
                    not_forwarded_logs.append(logfile_name)

            # No forwarding rule configured
            else:
                not_forwarded_logs.append(logfile_name)

    return forwarded_logs, not_forwarded_logs


def inventory_logwatch(info):
    forwarded_logs, not_forwarded_logs = logwatch_select_forwarded(info)
    inventory = []
    for logfile in not_forwarded_logs:
        groups = logwatch_groups_of_logfile(logfile)
        if groups:
            continue
        else:
            inventory.append((logfile, None))
    return inventory


# logwatch_patterns = {
#    'System': [
#    ( 'W', 'sshd' ),
#    ( ['host1', 'host2'],        'C', 'ssh' ), # only applies to certain hosts
#    ( ['lnx', 'dmz'], ALL_HOSTS, 'C', 'ssh' ), # only applies to host having certain tags
#    ( ALL_HOSTS, (10, 20), 'x' ), # at 10 messages per interval warn, at 20 crit
#    ( 'I', '0' )
#    ],
#    'Application': [
#    ( 'W', 'crash.exe' ),
#    ( 'E', 'ssh' )
#    ]
#    }

# New rule-stule logwatch_rules in WATO friendly consistent rule notation:
#
# logwatch_rules = [
#   ( [ PATTERNS ], ALL_HOSTS, [ "Application", "System" ] ),
# ]
# All [ PATTERNS ] of matching rules will be concatenated in order of
# appearance.
#
# PATTERN is a list like:
# [ ( 'O',      ".*ssh.*" ),          # Make informational (OK) messages from these
#   ( (10, 20), "login"   ),          # Warning at 10 messages, Critical at 20
#   ( 'C',      "bad"     ),          # Always critical
#   ( 'W',      "not entirely bad" ), # Always warning
# ]
#


def logwatch_state(state):
    if state == 1:
        return "WARN"
    elif state != 0:
        return "CRIT"
    else:
        return "OK"

def logwatch_level_name(level):
    if   level == 'O': return 'OK'
    elif level == 'W': return 'WARN'
    elif level == 'u': return 'WARN' # undefined states are treated as warning
    elif level == 'C': return 'CRIT'
    else: return 'IGN'

def logwatch_level_worst(worst, level):
    if   level == 'O': return max(worst, 0)
    elif level == 'W': return max(worst, 1)
    elif level == 'u': return max(worst, 1)
    elif level == 'C': return max(worst, 2)
    else: return worst

# Extracts patterns that are relevant for the current host and item.
# Constructs simple list of pairs: [ ('W', 'crash.exe'), ('C', 'sshd.*test') ]
def logwatch_precompile(hostname, item, _unused):
    # Initialize the patterns list with the logwatch_rules
    params = []
    description = check_info['logwatch']['service_description'] % item

    # This is the new (-> WATO controlled) variable
    rules = service_extra_conf(hostname, item, logwatch_rules)
    for rule in rules:
        for pattern in rule:
            params.append((pattern[0], pattern[1]))

    # Now load the old logwatch_patterns var
    patterns = logwatch_patterns.get(item)
    if patterns:
        for entry in patterns:
            hostlist = None
            tags = []

            pattern = entry[-1]
            level = entry[-2]

            if len(entry) >= 3:    # found optional host list
                hostlist = entry[-3]
            if len(entry) >= 4:    # found optional host tags
                tags = entry[-4]

            if hostlist and not \
                   (hosttags_match_taglist(tags_of_host(hostname), tags) and \
                    in_extraconf_hostlist(hostlist, hostname)):
                continue

            params.append((level, pattern))

    return params


def logwatch_reclassify(counts, patterns, text):
    for level, pattern in patterns:
        reg = regex(pattern)
        if reg.search(text):
            # If the level is not fixed like 'C' or 'W' but a pair like (10, 20),
            # then we count how many times this pattern has already matched and
            # assign the levels according to the number of matches of this pattern.
            if type(level) == tuple:
                warn, crit = level
                newcount = counts.setdefault(id(pattern), 0) + 1
                counts[id(pattern)] = newcount
                if newcount >= crit:
                    return 'C'
                elif newcount >= warn:
                    return 'W'
                else:
                    return 'I'
            else:
                return level
    return None

def logwatch_parse_line(line):
    parts = line.split(None, 1)
    level = parts[0]
    if len(parts) > 1:
        text = parts[1]
    else:
        text = ""
    return level, text

#.
#   .--Logwatch------------------------------------------------------------.
#   |              _                           _       _                   |
#   |             | | ___   __ ___      ____ _| |_ ___| |__                |
#   |             | |/ _ \ / _` \ \ /\ / / _` | __/ __| '_ \               |
#   |             | | (_) | (_| |\ V  V / (_| | || (__| | | |              |
#   |             |_|\___/ \__, | \_/\_/ \__,_|\__\___|_| |_|              |
#   |                      |___/                                           |
#   +----------------------------------------------------------------------+
#   |  Normal logwatch check                                               |
#   '----------------------------------------------------------------------'

# In case of a precompiled check, params contains the precompiled
# logwatch_patterns for the logfile we deal with. If using check_mk
# without precompiled checks, the params must be None an will be
# ignored.
def check_logwatch(item, params, info):
    if len(info) == 1:
        line = " ".join(info[0])
        if line.startswith("CANNOT READ CONFIG FILE"):
            return 3, "Error in agent configuration: %s" % " ".join(info[0][4:])

    found = False
    loglines = []
    for l in info:
        line = " ".join(l)
        if line == "[[[%s]]]" % item:
            found = True
        elif len(line) > 6 and line[0:3] == "[[[" and line[-3:] == "]]]":
            if found:
                break
            found = False
        elif found:
            loglines.append(line)
    return check_logwatch_generic(item, params, loglines, found)


check_info['logwatch'] = {
    'check_function':      check_logwatch,
    'inventory_function':  inventory_logwatch,
    'service_description': "Log %s",
    'group':               'logwatch',
}

precompile_params['logwatch'] = logwatch_precompile

#   .--lw.groups-----------------------------------------------------------.
#   |              _                                                       |
#   |             | |_      ____ _ _ __ ___  _   _ _ __  ___               |
#   |             | \ \ /\ / / _` | '__/ _ \| | | | '_ \/ __|              |
#   |             | |\ V  V / (_| | | | (_) | |_| | |_) \__ \              |
#   |             |_| \_/\_(_)__, |_|  \___/ \__,_| .__/|___/              |
#   |                        |___/                |_|                      |
#   +----------------------------------------------------------------------+
#   |                                                                      |
#   '----------------------------------------------------------------------'

def logwatch_group_precompile(hostname, item, _unused):
    return logwatch_precompile(hostname, item, None), host_extra_conf(hostname, logwatch_groups)

def logwatch_groups_of_logfile(filename, params=False):
    groups = []
    if not params:
        params = host_extra_conf(g_hostname, logwatch_groups)
    else:
        params = params[1]
    for line in params:
        for group_name, pattern in line:
            inclusion, exclusion = pattern
            if fnmatch.fnmatch(filename, inclusion) \
                    and not fnmatch.fnmatch(filename, exclusion):
                groups.append(group_name)
    return groups

def inventory_logwatch_groups(info):
    forwarded_logs, not_forwarded_logs = logwatch_select_forwarded(info)
    added_groups = []
    inventory = []
    for logfile in not_forwarded_logs:
        groups = logwatch_groups_of_logfile(logfile)
        for group in groups:
            if group not in added_groups:
                added_groups.append(group)
                inventory.append((group, None))
    return inventory

def check_logwatch_groups(item, params, info):
    if len(info) == 1:
        line = " ".join(info[0])
        if line.startswith("CANNOT READ CONFIG FILE"):
            return 3, "Error in agent configuration: %s" % " ".join(info[0][4:])

    found = False
    logfile_found = False
    loglines = []
    for l in info:
        line = " ".join(l)
        if logfile_found == True and not line.startswith('[[['):
            loglines.append(line)
        if line.startswith('[[['):
            logfile = line[3:-3]
            if item in logwatch_groups_of_logfile(logfile, params):
                found = True
                logfile_found = True
            else:
                logfile_found = False
            continue
    return check_logwatch_generic(item, params, loglines, found, True)


check_info['logwatch.groups'] = {
    'check_function':      check_logwatch_groups,
    'inventory_function':  inventory_logwatch_groups,
    'service_description': "LOG %s",
    'group':               'logwatch',
}
precompile_params['logwatch.groups'] = logwatch_group_precompile
#.


# truncate a file near the specified offset while keeping lines intact
def truncate_by_line(filename, offset):
    f = open(filename, 'r+')
    f.seek(offset)
    f.readline()  # ensures we don't cut inside a line
    f.truncate()
    f.close()


def logwatch_username():
    import getpass
    return getpass.getuser()


def check_logwatch_generic(item, params, loglines, found, groups=False):
    # Create directories, if neccessary
    try:
        logdir = logwatch_dir + "/" + g_hostname
        if not os.path.exists(logwatch_dir):
            os.mkdir(logwatch_dir)
        if not os.path.exists(logdir):
            os.mkdir(logdir)
            if www_group != None:
                try:
                    if i_am_root():
                        to_user = nagios_user
                    else:
                        to_user = "" # keep user unchanged
                    os.system("chown %s:%s %s" % (to_user, www_group, quote_shell_string(logdir)))
                    os.chmod(logdir, 0775)
                except Exception, e:
                    os.rmdir(logdir)
                    raise MKGeneralException(("User %s cannot chown directory to group id %d: %s. Please make sure "+
                                             "that %s is a member of that group.") %
                                             (logwatch_username(), www_group, e, logwatch_username()))

    except MKGeneralException:
        raise
    except Exception, e:
        raise MKGeneralException("User %s cannot create logwatch directory: %s" % \
                                 (logwatch_username(), e))

    logfile = logdir + "/" + item.replace("/", "\\")

    # Logfile (=item) section not found and no local file found. This usually
    # means, that the corresponding logfile also vanished on the target host.
    if type(logfile) == unicode:
        logfile = logfile.encode("utf-8")

    if not found and not os.path.exists(logfile):
        return (3, "log not present anymore")

    # Get the patterns (either compile or reuse the precompiled ones)
    # Check_MK creates an empty string if the precompile function has
    # not been executed yet. The precompile function creates an empty
    # list when no ruless/patterns are defined. In case of the logwatch.groups
    # checks, params are a tuple with the normal logwatch parameters on the first
    # and the grouping patterns on the second position
    if params not in ('', None):
        if groups:
            patterns = params[0]
        else:
            patterns = params # patterns already precompiled
    else:
        patterns = logwatch_precompile(g_hostname, item, None)


    state_counts = {} # for counting number of blocks with a certain state

    class LogwatchBlock(object):
        def __init__(self, header):
            self.worst  = -1
            self.header = header
            self.lines  = []
            self.last_worst_line = ''
            self.counts = {} # for counting number of matches of a certain pattern

        def __reclassify_line(self, patterns, text, level):
            if patterns:
                newlevel = logwatch_reclassify(self.counts, patterns, text)
                if newlevel != None:
                    level = newlevel
            return level

        def finalize(self):
            start = " ".join(self.header.split(' ')[0:2])
            return [ start + " %s>>>\n" % logwatch_state(self.worst) ] + self.lines

        def add_line(self, line, skip_reclassification):
            old_level, text = logwatch_parse_line(line)

            if skip_reclassification:
                level = old_level
            else:
                level = self.__reclassify_line(patterns, text, old_level)

            self.worst = logwatch_level_worst(self.worst, level)

            # Save the last worst line of this block
            if logwatch_level_worst(0, level) == self.worst:
                self.last_worst_line = text

            # Count the number of lines by state
            if level != '.':
                state_counts[level] = state_counts.get(level, 0) + 1

            if not skip_reclassification and level != "I":
                self.lines.append(level + " " + text + "\n")

    def collect_block(block):
        if block and block.worst > -1:
            collect_block.output_lines += block.finalize()

            if block.worst >= collect_block.worst:
                collect_block.worst = block.worst
                collect_block.last_worst_line = block.last_worst_line

    collect_block.worst = 0
    collect_block.last_worst_line = ''
    collect_block.output_lines = []

    current_block = None

    log_exists = os.path.exists(logfile)
    try:
        if log_exists:
            logwatch_file = open(logfile, 'r+')
        else:
            logwatch_file = open(logfile, 'w')
    except Exception, e:
        raise MKGeneralException("User %s cannot open file for writing: %s" % (logwatch_username(), e))

    pattern_hash = hash(tuple(patterns))

    net_lines = 0

    # parse cached log lines
    if log_exists:
        # new format contains hash of patterns on the first line so we only reclassify if they
        # changed
        initline = logwatch_file.readline().rstrip('\n')
        if initline.startswith('[[[') and initline.endswith(']]]'):
            old_pattern_hash = int(initline[3:-3])
            skip_reclassification = old_pattern_hash == pattern_hash
        else:
            logwatch_file.seek(0)
            skip_reclassification = False

        logfile_size = os.path.getsize(logfile)
        if skip_reclassification and logfile_size > logwatch_max_filesize:
            # early out: without reclassification the file wont shrink and if it is already at
            # the maximum size, all input is dropped anyway
            if logfile_size > logwatch_max_filesize * 2:
                # if the file is far too large, truncate it
                truncate_by_line(logfile, logwatch_max_filesize)
            return (2, "unacknowledged messages have exceeded max size, "
                    "new messages are dropped (limit %d Bytes)" % logwatch_max_filesize)

        for line in logwatch_file:
            line = line.rstrip('\n')
            # Skip empty lines
            if not line:
                continue
            elif line.startswith('<<<') and line.endswith('>>>'):
                # The section is finished here. Add it to the list of reclassified lines if the
                # state of the block is not "I" -> "ignore"
                collect_block(current_block)
                current_block = LogwatchBlock(line)
            elif current_block is not None:
                current_block.add_line(line, skip_reclassification)
                net_lines += 1

        # The last section is finished here. Add it to the list of reclassified lines if the
        # state of the block is not "I" -> "ignore"
        collect_block(current_block)

        if skip_reclassification:
            output_size = logwatch_file.tell()
            # when skipping reclassification, output lines contains only headers anyway
            collect_block.output_lines = []
        else:
            output_size = sum([len(line) for line in collect_block.output_lines])
    else:
        output_size = 0
        skip_reclassification = False

    header = time.strftime("<<<%Y-%m-%d %H:%M:%S UNKNOWN>>>\n")
    output_size += len(header)

    # process new input lines - but only when there is some room left in the file
    if output_size < logwatch_max_filesize:
        current_block = LogwatchBlock(header)
        for line in loglines:
            current_block.add_line(line.encode("utf-8"), False)
            net_lines += 1
            output_size += len(line)
            if output_size >= logwatch_max_filesize:
                break
        collect_block(current_block)

    # when reclassifying, rewrite the whole file, outherwise append
    if not skip_reclassification and collect_block.output_lines:
        logwatch_file.seek(0)
        logwatch_file.truncate()
        logwatch_file.write("[[[%d]]]\n" % pattern_hash)

    if collect_block.output_lines:
        logwatch_file.writelines(collect_block.output_lines)
    # correct output size
    logwatch_file.close()
    if net_lines == 0 and log_exists:
        os.unlink(logfile)

    # if logfile has reached maximum size, abort with critical state
    if os.path.exists(logfile) and os.path.getsize(logfile) > logwatch_max_filesize:
        return (2, "unacknowledged messages have exceeded max size, "
                   "new messages are lost (limit %d Bytes)" % logwatch_max_filesize)

    #
    # Render output
    #

    if collect_block.worst <= 0:
        return (0, "no error messages")
    else:
        count_txt = []
        for level, num in state_counts.iteritems():
            count_txt.append('%d %s' % (num, logwatch_level_name(level)))
        if logwatch_service_output == 'default':
            return (collect_block.worst, "%s messages (Last worst: \"%s\")" %
                                           (', '.join(count_txt), collect_block.last_worst_line))
        else:
            return (collect_block.worst, "%s messages" % ', '.join(count_txt))

#.
#   .--Event Console Forwarding--------------------------------------------.
#   |        _                           _       _                         |
#   |       | | ___   __ ___      ____ _| |_ ___| |__    ___  ___          |
#   |       | |/ _ \ / _` \ \ /\ / / _` | __/ __| '_ \  / _ \/ __|         |
#   |       | | (_) | (_| |\ V  V / (_| | || (__| | | ||  __/ (__          |
#   |       |_|\___/ \__, | \_/\_/ \__,_|\__\___|_| |_(_)___|\___|         |
#   |                |___/                                                 |
#   +----------------------------------------------------------------------+
#   | Forwarding logwatch messages to event console                        |
#   '----------------------------------------------------------------------'

# OK      -> priority 5 (notice)
# WARN    -> priority 4 (warning)
# CRIT    -> priority 2 (crit)
# context -> priority 6 (info)
# u = Uknown
def logwatch_to_prio(level):
    if level == 'W':
        return 4
    elif level == 'C':
        return 2
    elif level == 'O':
        return 5
    elif level == '.':
        return 6
    else:
        return 4

def inventory_logwatch_ec(info, one_svc_per_log):
    forwarded_logs, not_forwarded_logs = logwatch_select_forwarded(info)

    if forwarded_logs:
        forward_settings = host_extra_conf(g_hostname, checkgroup_parameters.get('logwatch_ec', []))

        merged_rules = {}
        for rule in forward_settings[-1::-1]:
            if type(rule) == dict:
                for key, value in rule.items():
                    merged_rules[key] = value
            elif type(rule) == str:
                return # Configured "no forwarding"

        separate_checks = merged_rules.get("separate_checks", False)
        if separate_checks != one_svc_per_log:
            return

        if separate_checks:
            single_log_params = {}
            for key in [ "method", "facility", "monitor_logfilelist", "logwatch_reclassify" ]:
                if key in merged_rules:
                    single_log_params[key] = merged_rules[key]
            for log in forwarded_logs:
                single_log_params["expected_logfiles"] = [log]
                yield log, single_log_params.copy()
        else:
            yield None, { "expected_logfiles": forwarded_logs }

def check_logwatch_ec(item, params, info):
    if len(info) == 1:
        line = " ".join(info[0])
        if line.startswith("CANNOT READ CONFIG FILE"):
            return 3, "Error in agent configuration: %s" % " ".join(info[0][4:])

    # 1. Parse lines in info and separate by logfile
    logs = {}
    logfile = None
    for l in info:
        line = " ".join(l)
        if len(line) > 6 and line[0:3] == "[[[" and line[-3:] == "]]]":
            # new logfile, extract name
            logfile = line[3:-3]
            logs.setdefault(logfile, [])

        elif logfile and line:
            # new regular line, skip context lines and ignore lines
            if line[0] not in ['.', 'I']:
                logs[logfile].append(line)

    # 2. Maybe filter logfiles if some should be excluded
    if 'restrict_logfiles' in params:
        for logfile in logs.keys():
            if not logwatch_ec_forwarding_enabled(params, logfile):
                del logs[logfile]

    # 3. If this check has an item (logwatch.ec_single), only forward the information from this log
    if item:
        if item not in logs.keys():
            return
        else:
            single_logs = {}
            single_logs[item] = logs[item]
            logs = single_logs

    # Check if the number of expected files matches the actual one
    status = 0
    infotexts = []
    if params.get('monitor_logfilelist'):
        if 'expected_logfiles' not in params:
            infotexts.append("You enabled monitoring the list of forwarded logfiles. You need to redo service discovery.")
            status = 1
        else:
            expected = params['expected_logfiles']
            missing = []
            for f in expected:
                if f not in logs:
                    missing.append(f)
            if missing:
                infotexts.append("Missing logfiles: %s" % (", ".join(missing)))
                status = 1

            exceeding = []
            for f in logs:
                if f not in expected:
                    exceeding.append(f)
            if exceeding:
                infotexts.append("Newly appeared logfiles: %s" % (", ".join(exceeding)))
                status = 1

    # 3. create syslog message of each line
    # <128> Oct 24 10:44:27 Klappspaten /var/log/syslog: Oct 24 10:44:27 Klappspaten logger: asdasdad as
    # <facility+priority> timestamp hostname logfile: message
    facility = params.get('facility', 17) << 3 # default to "local1"
    messages = []
    cur_time = int(time.time())

    forwarded_logfiles = set([])

    # Get the logwatch patterns if they are not already precompiled
    # The precompile feature is not required when using the cmc as core
    # HACK: opt_keepalive is used to detect cmc
    if not opt_keepalive and "logwatch_patterns" not in params:
        params = logwatch_ec_precompile(g_hostname, item, params)

    # Keep track of reclassifed lines
    rclfd_total     = 0
    rclfd_to_ignore = 0

    for logfile, lines in logs.items():
        # Determine logwatch patterns specifically for this logfile
        log_reclassify_patterns = []
        # HACK: opt_keepalive is used to detect cmc
        if opt_keepalive:
            logfile_patterns = service_extra_conf(g_hostname, logfile, logwatch_rules)
            if logfile_patterns:
                log_reclassify_patterns.extend(map(lambda x: x[:2], [z for y in logfile_patterns for z in y]))
        else:
            for entry in params["logwatch_patterns"]:
                patterns, log_items = entry
                for log_item in log_items:
                    reg = regex(log_item)
                    if reg.search(logfile):
                        log_reclassify_patterns.extend(patterns)

        for line in lines:
            rclfd_level = None
            if log_reclassify_patterns:
                counts = {} # unused...
                old_level, text = line.split(" ", 1)
                level = logwatch_reclassify(counts, log_reclassify_patterns, line[2:]) or old_level
                if level != old_level:
                    rclfd_total += 1
                    rclfd_level = level
                    if level == "I": # Ignored lines are not forwarded
                        rclfd_to_ignore += 1
                        continue

            msg = '<%d>' % (facility + logwatch_to_prio(rclfd_level or line[0]),)
            msg += '@%s;%d;; %s %s: "%s"' % (cur_time, params.get("service_level", 0), g_hostname, logfile, line[2:])

            messages.append(msg)
            forwarded_logfiles.add(logfile)

    # 4. send lines to event console
    # a) local in same omd site
    # b) local pipe
    # c) remote via udp
    # d) remote via tcp
    method = params.get('method')
    if not method:
        method = os.getenv('OMD_ROOT') + "/tmp/run/mkeventd/eventsocket"
    elif method == 'spool:':
        method += os.getenv('OMD_ROOT') + "/var/mkeventd/spool"

    num_messages = len(messages)
    try:
        if messages:
            if isinstance(method, tuple):
                # connect either via tcp or udp
                if method[0] == 'udp':
                    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
                else:
                    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                sock.connect((method[1], method[2]))
                for message in messages:
                    sock.send(message.encode("utf-8") + "\n")
                sock.close()

            elif not method.startswith('spool:'):
                # write into local event pipe
                # Important: When the event daemon is stopped, then the pipe
                # is *not* existing! This prevents us from hanging in such
                # situations. So we must make sure that we do not create a file
                # instead of the pipe!
                sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
                sock.connect(method)
                sock.send(('\n'.join(messages)).encode("utf-8") + '\n')
                sock.close()

            else:
                # Spool the log messages to given spool directory.
                # First write a file which is not read into ec, then
                # perform the move to make the file visible for ec
                spool_path = method[6:]
                file_name  = '.%s_%s%d' % (g_hostname, item and item.replace('/', '\\') + '_' or '', time.time())
                if not os.path.exists(spool_path):
                    os.makedirs(spool_path)
                file('%s/%s' % (spool_path, file_name), 'w').write(('\n'.join(messages)).encode("utf-8") + '\n')
                os.rename('%s/%s' % (spool_path, file_name), '%s/%s' % (spool_path, file_name[1:]))

        if forwarded_logfiles:
            logfile_info = " from " + ",".join(list(forwarded_logfiles))
        else:
            logfile_info = ""

        infotexts.append('Forwarded %d messages%s to event console' % (num_messages, logfile_info))
        if rclfd_total:
            infotexts.append('Reclassified %d messages through logwatch patterns (%d to IGNORE)' %
                             (rclfd_total, rclfd_to_ignore))
        return (status, ", ".join(infotexts), [('messages', num_messages)])

    except Exception, e:
        return (2, 'Unable to forward messages to event console (%s). Lost %d messages.' %
                (e, num_messages))

def logwatch_ec_precompile(hostname, item, params):
    params = params.copy()

    params["service_level"] = get_effective_service_level()

    # HACK: opt_keepalive is used to detect cmc
    if (opt_keepalive or not params.get("logwatch_reclassify")):
        params.update({"logwatch_patterns": []})
        return params

    # The following code is used for creating the params of precompiled checks
    # and when Check_MK is called on the command line.
    # Check_MK in keepalive mode (in a running cmc) should never reach this code...
    ruleset = []
    for rule in logwatch_rules:
        rule, rule_options = get_rule_options(rule)
        if rule_options.get("disabled"):
            continue

        if len(rule) == 3:
            item, hostlist, servlist = rule
            tags = []
        elif len(rule) == 4:
            item, tags, hostlist, servlist = rule
        else:
            continue

        # Directly compute set of all matching hosts here, this
        # will avoid recomputation later
        hosts = all_matching_hosts(tags, hostlist, with_foreign_hosts=False)
        ruleset.append((item, hosts, servlist))

    # Filter out any logwatch_rules which do apply to to this host
    # 1st filter: Do not use rules where the hostname does not match
    # 2nd filter: Do not use rules with configured items where no item matches the
    #             "restrict_logfiles" condition (if applicable)
    logwatch_patterns = []
    for rule in ruleset:
        patterns, hosts, rule_items = rule
        if hostname in hosts:
            if params.get("restrict_logfiles"):
                for rule_item in rule_items:
                    if rule_item == "" or logwatch_ec_forwarding_enabled(params, rule_item):
                        logwatch_patterns.append( (map(lambda x: x[:2], patterns), rule_items) )
                        break
            else:
                logwatch_patterns.append( (map(lambda x: x[:2], patterns), rule_items) )
    params.update({"logwatch_patterns": logwatch_patterns})
    return params

precompile_params['logwatch.ec'] = logwatch_ec_precompile
precompile_params['logwatch.ec_single'] = logwatch_ec_precompile

check_info['logwatch.ec'] = {
    'check_function':      check_logwatch_ec,
    'inventory_function':  lambda info: inventory_logwatch_ec(info, False),
    'service_description': "Log Forwarding",
    'group':               'logwatch_ec',
    'has_perfdata':        True,
}

check_info['logwatch.ec_single'] = {
    'check_function':      check_logwatch_ec,
    'inventory_function':  lambda info: inventory_logwatch_ec(info, True),
    'service_description': "Log %s",
    'has_perfdata':        True,
}