undertaker 1.6-2 / usr / bin / undertaker-calc-coverage

#! /usr/bin/python
#
#   undertaker-calc-coverage - configuration coverage analysis on Linux
#
# Copyright (C) 2011-2012 Reinhard Tartler <tartler@informatik.uni-erlangen.de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import sys
from os.path import join, dirname

sys.path = [join(dirname(sys.path[0]), 'lib', 'python%d.%d' % \
                (sys.version_info[0], sys.version_info[1]), 'site-packages')] \
                + sys.path

from vamos.golem.kbuild import *
from vamos.model import parse_model
from vamos.tools import setup_logging, execute
from vamos.vampyr.utils import get_conditional_blocks, get_loc_coverage
from vamos.vampyr.utils import ExpansionError, ExpansionSanityCheckError

import shutil

from optparse import OptionParser

model = None

std_configs = ["allyesconfig"]


def switch_config(config):
    """
    switches the current tree to the given config. throws an exception in case make fails
    """
    execute("find include -name autoconf.h -delete", failok=False)
    (output, _) = call_linux_makefile(config, failok=False)

    # sanity check
    apply_configuration()

    return output


def switch_config_path(filename):
    """
    similar to switch_config, but takes a filename instead of a config target

    As sideffect, the current .config file is overwritten
    """

    assert(os.path.exists(filename))

    # now replace the old .config with our 'expanded' one
    shutil.copyfile(filename, '.config')
    try:
        call_linux_makefile('oldconfig', filename=filename, failok=False)

    except CommandFailed as e:
        raise ExpansionError(e)

    apply_configuration(filename=filename)


def call_sparse(filename):
    """
    run sparse with current configuration

    @return output of sparse
    """

    (stdout, _) = call_linux_makefile(filename.replace(".c", ".o"),
                                      filename=filename,
                                      failok=True,
                                      extra_variables="C=2")

    return stdout


def run_sparse_on_config(filename, config):
    """
    the output of sparse is redirected to 'filename.configN.sparse'

    @return The number of sparse errors found.
    """

    sparse_output = call_sparse(filename)
    outfn = config + '.sparse'

    logging.debug("Creating " + outfn)
    with open (outfn, 'w') as f:
        for l in sparse_output:
            f.write(l + '\n')

    warnings = 0
    errors = 0
    for line in sparse_output.__str__().rsplit('\n'):
        if ' warning: ' in line:
            warnings = warnings + 1
        if ' error: ' in line:
            errors = errors + 1

    return (warnings, errors)


def get_covered_lines(filename):
    (sparse_coverage, _) = call_linux_makefile(filename.replace(".c", ".o"),
                                               filename=filename,
                                               extra_variables="C=2 CHECK=sparse_coverage",
                                               failok=False)

    lines = []
    for line in sparse_coverage:
        tokens = line.split(" ")
        if tokens[0] != "" and tokens[0] in filename:
            lines.append(int(tokens[1]))
    return lines


def default_configs(filename, results):

    if not filename in results:
        results[filename] = {}

    for config in std_configs:
        logging.info("Switching to configuration preset " + config)
        switch_config(config)
        results[filename][config] = len(get_conditional_blocks(filename, find_autoconf(),
                all_cpp_blocks=False))
    return results

# e.g. taken from the makefiles
extra_dependencies = {
    './drivers/cpuidle/cpuidle.c': ["CONFIG_CPU_IDLE=y", "CONFIG_ARCH_HAS_DEFAULT_IDLE=y"],
}


def expand_by_copy(config):
    logging.debug("Trying to expand configuration " + config)

    if not os.path.exists(config):
        raise RuntimeError("Partial configuration %s does not exist" % config)

    shutil.copy(config, config + '.config')
    return config + '.config'


def expand_with_defconfig(config):
    """
    Expands a partial configuration using the kbuild 'defconfig'
    target. As a side-effect, the current configuration is switched to
    that expanded config. A copy of the expanded configration is stored
    at " config + '.config'"

    @return the path to the expanded configuration
    """

    logging.debug("Trying to expand configuration '%s' with defconfig", config)

    if not os.path.exists(config):
        raise RuntimeError("Partial configuration %s does not exist" % config)

    try:
        extra_variables='KCONFIG_ALLCONFIG="%s"' % config

        call_linux_makefile("alldefconfig",
                            filename=config,
                            failok=False,
                            extra_variables=extra_variables)
    except CommandFailed as e:
        msg="Config '%s' failed to expand. Command used: '%s' (returncode: %d)" % \
            (config, e.command, e.returncode)
        raise ExpansionError(msg)

    expanded_config = config + '.config'
    shutil.copy('.config', expanded_config)

    return expanded_config


def expand_by_filtering(config):
    """
    configuration 'config' is a partial configuration, i.e., it
    contains an incomplete set of config options. The strategy
    here is to start with a default config ("allnoconfig"), and
    then filter out configs for which we have an entry in the
    partial selection. Finally, we verify and complete the
    "hacked" selection with "make oldconfig"

    @return the path to the expanded configuration
    """

    logging.debug("Trying to expand configuration " + config)

    if not os.path.exists(config):
        raise RuntimeError("Partial configuration %s does not exist" % config)

    switch_config('allnoconfig')

    with open(config + '.config', 'w+') as newconfigf:
        with open('.config', 'r') as oldconfigf:
            with open(config) as configf:
                found_items = []
                for l in configf:
                    if l.startswith('CONFIG_'):
                        item = l.split('=')[0]
                        found_items.append(item)

                for l in oldconfigf:
                    item = l.split('=')[0]
                    if item not in found_items:
                        newconfigf.write(l)

#            if filename in extra_dependencies.keys():
#                newconfigf.write("\n".join(extra_dependencies[filename]))

    return config + '.config'


def verify_config(partial_config):
    """
    verifies that the current .config file satisfies the constraints of the
    given partial configuration.

    @return a dictionary with items that violate the partial selection
    """

    value_errors = {}
    # magic values induced by our models
    config_whitelist = ('CONFIG_n', 'CONFIG_y', 'CONFIG_m')

    with open(partial_config) as partial_configf:
        for l in partial_configf:
            if l.startswith('CONFIG_'):
                (item, value) = l.split('=')
                if model and item in model.always_on_items:
                    continue
                if item in config_whitelist:
                    continue
                value_errors[item] = value.rstrip()

    with open('.config') as configf:
        found_items = {}
        for l in configf:
            if l.startswith('CONFIG_'):
                (item, value) = l.split('=')
                found_items[item] = value.rstrip()
                if model and item in model.always_on_items:
                    continue
                if (item in value_errors):
                    if (value_errors[item] == value.rstrip()):
                        del value_errors[item]

        for (item, value) in value_errors.items():
            if value == 'n':
                if not item in found_items:
                    del value_errors[item]

    return value_errors

def selected_items(partial_config):
    """
    counts how many items got selected by the given partial config
    """
    selected = 0

    with open(partial_config) as partial_configf:
        for l in partial_configf:
            # a configuration either starts with a comment '^#'
            # or looks like:
            # CONFIG_FOO=y
            if l.startswith('CONFIG_'):
                items = l.split('=')
                if items[1] != 'n':
                    selected += 1

    return selected


def vamos_coverage(filename, results, options):
    covered = set()
    found_configs = configurations(filename)
    logging.info("found %d configurations for %s",
                 len(found_configs), filename)

    if not filename in results:
        results[filename] = {}

    for c in found_configs:
        logging.debug(c)

    results[filename]['fail_count'] = 0

    for config in found_configs:
        results[filename][config] = {}

        try:
            expand_with_defconfig(config)
        except ExpansionSanityCheckError as error:
            results[filename]['fail_count'] = 1 + results[filename]['fail_count']
            logging.warning("sanity check failed, but continuing anyways")
            logging.warning(error)
        except ExpansionError as error:
            logging.error("Config %s failed to apply, skipping", config)
            results[filename]['fail_count'] = 1 + results[filename]['fail_count']
            logging.error(error)
            continue

        value_errors = "#error"
        try:
            value_errors = verify_config(config)
            if len(value_errors) > 0:
                logging.info("Failed to set %d items from partial config %s",
                             len(value_errors), config)
                mismatches = list()
                for c in value_errors.keys():
                    mismatches.append("%s != %s" % (c, value_errors[c]))
                logging.debug(", ".join(mismatches))
        except ValueError:
            logging.warning("Failure to analyze partial config expansion, continuing anyways")

        results[filename][config]['value_errors'] = value_errors
        results[filename][config]['selected_items'] = selected_items(config)
        results[filename][config]['loc_coverage'] = get_loc_coverage(filename, autoconf_h=find_autoconf())

        if options.run_sparse:
            (warnings, errors) = run_sparse_on_config(filename, config)
            results[filename][config]['sparse_warnings'] = warnings
            results[filename][config]['sparse_errors']   = errors

        # writeback the expanded configuration
        shutil.copy('.config', config + '.config')

        # now check what blocks are actually set
        old_len = len(covered)
        covered_blocks = set(get_conditional_blocks(filename, find_autoconf(),
                 all_cpp_blocks=False))
        covered |= covered_blocks
        logging.info("Config %s added %d additional blocks",
                     config, len(covered) - old_len)
        results[filename][config]['covered_blocks'] = covered_blocks


    total_blocks = results[filename]['total_blocks']
    results[filename]['uncovered_blocks'] = ", ".join(total_blocks - covered)
    results[filename]['vamos'] = len(covered)
    return results


def configurations(filename):
    l  = glob(filename + ".config[0-9]")
    l += glob(filename + ".config[0-9][0-9]")
    return sorted(l)


def parse_options():
    parser = OptionParser()
    parser.add_option("-s", "--skip-configs", dest="skip_configs",
                      action="store_true", default=False,
                      help="Skip analyzing configurations (defaut: only general, config agnostic analysis)")
    parser.add_option('-v', '--verbose', dest='verbose', action='count',
                      help="Increase verbosity (specify multiple times for more)")
    parser.add_option('-m', '--model', dest='model', action='store',
                      help="load the model and configuration was generated with model")
    parser.add_option('-F', '--fast', dest='min_strategy', action='store_false', default=True,
                      help="Use a faster (but maybe more naive) strategy for coverage analysis, (default: no)")

    parser.add_option("", '--strategy', dest='strategy', action='store', type="choice",
                      choices=["simple", "minimal"], default="minimal",
                      help="Configuration was generated by $strategy. This will only effect the direct output")
    parser.add_option("", '--expanded-config', dest='expanded_config', action='store_true', default=False,
                      help="Configuration was generated on expanded source file. This will only effect the direct output")
    parser.add_option("", '--expanded-source', dest='expanded_source', action='store_true', default=False,
                      help="Configuration is tested on expaned source file. This will only effect the direct output")
    parser.add_option("", '--run-sparse', dest='run_sparse', action='store_true', default=False,
                      help="call sparse on each configuration")

    (options, args) = parser.parse_args()
    setup_logging(options.verbose)

    if options.model and os.path.exists(options.model):
        global model
        model = parse_model(options.model)
        logging.info("Loaded %d options from Model %s", len(model), options.model)
        logging.info("%d items are always on", len(model.always_on_items))

    if len(args) == 0:
        logging.critical("please specify a worklist")
        sys.exit(1)

    return parser.parse_args()

def do_work(filenames, options):
    results = {}

    for filename in filenames:
        if not os.path.exists(filename):
            if filename.endswith('.pi'):
                # we're working on partially preprocessed files, replace
                # them here with the actual files
                base = filename[:-3]
                logging.debug("working on %s instead of %s",
                              base + '.c', filename)
                filename = base + '.c'
            else:
                logging.debug("%s does not exist, skipping", filename)
                continue

        filename = os.path.normpath(filename)
        logging.info("processing %s", filename)

        default_configs(filename, results)
        results[filename]['total_blocks'] = set(get_conditional_blocks(filename,
                 all_cpp_blocks=False))

        print "\nRESULT"
        print "filename: %s" % filename
        print "expanded: %s" % options.expanded_config
        print "expanded_source: %s" % options.expanded_source
        print "kconfig: %s"  % (options.model is not None)
        print "min_strategy: %s" % (options.strategy == "minimal")
        print "loc: %d" % get_loc_coverage(filename, autoconf_h=find_autoconf())
        print "total_blocks: %d"     % len(results[filename]['total_blocks'])
        print "allyes_coverage: %d" % results[filename]['allyesconfig']

        try:
            if options.skip_configs:
                continue

            if len(results[filename]['total_blocks']) == 0:
                continue

            vamos_coverage(filename, results, options)
            print "covered_blocks: %d"   % results[filename]['vamos']
            print "uncovered_blocks: %s" % results[filename]['uncovered_blocks']
            print "expansion_errors: %d" % results[filename]['fail_count']

            for config in results[filename].keys():
                if config.startswith(filename):
                    print "\nCONFIG"
                    print "config_path: %s" % config
                    try:
                        print "item_mismatches: %d" % len(results[filename][config]['value_errors'])
                        print "wrong_items: %s"     % ", ".join(results[filename][config]['value_errors'])
                        print "selected_items: %d"  % results[filename][config]['selected_items']
                        print "runtime: 0.0"
                        print "covered_blocks: %d"  % len(results[filename][config]['covered_blocks'])
                        print "loc_coverage: %d" % results[filename][config]['loc_coverage']

                        if options.run_sparse:
                            print "sparse_warnings: %d" % results[filename][config]['sparse_warnings']
                            print "sparse_errors: %d"   % results[filename][config]['sparse_errors']

                    except KeyError:
                        pass


        except RuntimeError as error:
            logging.error(error.__str__())
            logging.error("Skipping file " + filename)
            continue

def main():
    (options, args) = parse_options()
    setup_logging(options.verbose)

    if len(args) == 0:
        sys.exit("please specify a worklist")

    worklist = args[0]
    try:
        with open(worklist) as f:
            filenames = [x.strip() for x in f.readlines()]
    except IOError as error:
        sys.exit("Failed to open worklist: '%s'" % error.__str__())

    if len(filenames) > 0:
        logging.debug("Worklist contains %d items", len(filenames))
        do_work(filenames, options)
        sys.exit(0)
    else:
        sys.exit("Worklist contains no work items!")

if __name__ == "__main__":
    main()