undertaker 1.6-2 / usr / bin / undertaker-linux-tree

#!/bin/bash
#
# Applies the Undertaker tool to a Linux source tree
#
# Copyright (C) 2009-2012 Reinhard Tartler <tartler@informatik.uni-erlangen.de>
# Copyright (C) 2011 Christian Dietrich <christian.dietrich@informatik.uni-erlangen.de>
# Copyright (C) 2014 Stefan Hengelein <stefan.hengelein@fau.de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

# This script is indented to run the undertaker on a whole linux
# tree. It will determine which files to be processed and how many
# threads can be started according to the count of processors your
# machine has. It assumes, that you have run undertaker-kconfigdump
# before, in order to create the models.
#

# scan for deads by default
MODE="scan-deads"

while getopts :t:m:a:csvh OPT; do
    case $OPT in
        m)
            MODELS="$OPTARG"
            ;;
        a)
            DEFAULT_ARCH="$OPTARG"
            ;;
        t)
            PROCESSORS="$OPTARG"
            ;;
        c)
            MODE="calc-coverage"
            ;;
        s)
            MODE="feature-statistics"
            ;;
        v)
            echo "undertaker-linux-tree"
            exit
            ;;
        h)
            echo "\`undertaker-linux-tree' drives the undertaker over a whole linux-tree"
            echo
            echo "Usage: ${0##*/} [-m DIR] [-a ARCH] [-t PROCS] [-c|-s]"
            echo " -m <modeldir>  Specify the directory for the models"
            echo "           (default: models)"
            echo " -a <arch>  Default architecture to check for"
            echo "        (default: x86)"
            echo " -t <count>   Number of analyzing processes"
            echo "        (default: _NPROCESSORS_ONLN)"
            echo " -c  Do coverage analysis instead of dead block search"
            echo " -s  Do feature statistics instead of dead block search"
            exit
            ;;
    esac
done
shift $(( OPTIND - 1 ))
OPTIND=1

MODELS=${MODELS:-models}
DEFAULT_ARCH=${DEFAULT_ARCH:-x86}
PROCESSORS=${PROCESSORS:-$(getconf _NPROCESSORS_ONLN)}

if [ ! -f arch/"$DEFAULT_ARCH"/Kconfig ]; then
    echo "Not run in an linux tree. Please run inside an linux tree without arguments"
    exit 1
else
    echo "Running on Linux Version $(git describe || echo '(no git)')"
fi

if ! which  undertaker > /dev/null; then
    echo "No undertaker binary found."
    exit 1
fi

if ! ls "$MODELS"/*.model >/dev/null 2>&1 && ! ls "$MODELS"/*.cnf >/dev/null 2>&1; then
    echo "No models found, please call undertaker-kconfigdump"
    exit
fi

#################################################################################
# calculate coverage                                                            #
#################################################################################

if [ "$MODE" = "calc-coverage" ]; then
    find -type f -name '*.c' \
        ! -regex '^./tools.*' ! -regex '^./Documentation.*' ! -regex '^./scripts.*' \
        -exec grep -q -E '^#else' {} \; -print | shuf > undertaker-coverage-worklist

    files=$(wc -l < undertaker-coverage-worklist)
    echo "Calculating partial configurations (greedy variant) on $files files"

    undertaker -v -j coverage -C min -t "$PROCESSORS" -b undertaker-coverage-worklist \
        -m "$MODELS" -M "$DEFAULT_ARCH" 2>&1 | grep '^I: ./' > coverage.txt

    if [ ! -s coverage.txt ]; then
        echo "Coverage analysis failed!"
        exit 1
    fi

    echo "TOP 50 variable files:"
    awk -F'I: ' '/^I: / { print $2 }' < coverage.txt |
            awk -F, '/Found Solutions/ { printf "%s %s\n", $2, $1 }' |
            sort -n -r | head -n 50 | tee coverage.stats

    awk '/c$/ { print $4 }' coverage.stats > undertaker-calc-coverage-worklist
    if ! undertaker-calc-coverage -m models/"$DEFAULT_ARCH".model \
              undertaker-calc-coverage-worklist --run-sparse \
              2>undertaker-calc-coverage.error >undertaker-calc-coverage.output; then
        echo "undertaker-calc-coverage failed, check 'undertaker-calc-coverage.error' for details"
        exit 1
    fi
    if ! test -s undertaker-calc-coverage.error; then
        rm -f undertaker-calc-coverage.error
    fi
fi

#################################################################################
# global dead analysis                                                          #
#################################################################################

if [ "$MODE" = "scan-deads" ]; then
    find -type f -name "*.[hcS]" \
        ! -regex '^./tools.*' ! -regex '^./Documentation.*' ! -regex '^./scripts.*' \
        -exec grep -q -E '^#[[:space:]]*if' {} \; -print | shuf > undertaker-worklist

    # delete potentially confusing .dead files first
    find . -type f -name '*dead' -delete

    echo "Analyzing $(wc -l < undertaker-worklist) files with $PROCESSORS threads."
    undertaker -t "$PROCESSORS" -b undertaker-worklist -m "$MODELS" -M "$DEFAULT_ARCH"
    printf "\n\nFound %s global defects\n" "$(find . -name '*dead'| grep globally | grep -v no_kconfig | wc -l)"
    exit 0
fi

#################################################################################
# global feature statistics analysis                                            #
#################################################################################

do_archstat () {
    CONFIG=$1
    ARCH=$2
    CROSS_COMPILE=
    # normalize x86
    if [ "$ARCH" = "x86" ]; then
        if env | grep -q ^VAMOS_PREFER_64BIT= ; then
            ARCH="x86_64"
        else
            ARCH="i386"
        fi
    fi

    export ARCH CROSS_COMPILE

    if ! make $CONFIG >/dev/null 2>&1; then
        # some strange architectures (e.g., h8300) are just utterly broken. skip them
        echo "Skipping $ARCH ($CONFIG)"
        return
    fi
    if ! golem -l >undertaker-stat/$CONFIG-$ARCH.list  2>undertaker-stat/$CONFIG-$ARCH.failed; then
        echo "golem -l failed on $ARCH with $CONFIG, skipping"
        return
    fi
    echo -n "Architecture $ARCH uses $(wc -l < undertaker-stat/$CONFIG-$ARCH.list) source files for $CONFIG"
    echo " (plus possibly $(grep -c 'Failed to guess' undertaker-stat/$CONFIG-$ARCH.failed) additional files)"

# FIXME since we don't do anything with the results below, we can simply skip it
#    undertaker -j cppsym -b undertaker-stat/$CONFIG-$ARCH.list -m models/$ARCH.model \
#        2> undertaker-stat/$CONFIG-$ARCH.cppsym-errors | \
#        grep -v -E '(MISSING|NON_KCONFIG)' | sort -u > undertaker-stat/$CONFIG-$ARCH.cppsym
#
#    if ! test -s undertaker-stat/$CONFIG-$ARCH.cppsym-errors; then
#        rm -f undertaker-stat/$CONFIG-$ARCH.cppsym-errors
#    else
#        echo "extraction of CPP symbols for $ARCH with $CONFIG had errors, inspect 'undertaker-stat/$CONFIG-$ARCH.cppsym-errors' for details"
#    fi
}

do_archspecific_cppsym() {
    # TODO add architecture specific cppsym analysis here, when ghoul is merged.
    echo ""
}

if [ "$MODE" = "feature-statistics" ]; then
    mkdir -p undertaker-kbuild
    for f in `ls -1 arch/ | grep -v Kconfig`; do
        ARCH=$f golem -o | sort > undertaker-kbuild/kbuild-$f
    done
    cat undertaker-kbuild/* | sort -u > undertaker-kbuild-variables

    echo "Found $(wc -l < undertaker-kbuild-variables) configuration variables mentioned in Makefiles"

    # create a worklist for cpp-sym analysis
    find -type f -name "*.[hcS]" \
        ! -regex '^./tools.*' ! -regex '^./Documentation.*' ! -regex '^./scripts.*' \
        -exec grep -q -E '^#[[:space:]]*if' {} \; -print | shuf > undertaker-worklist

    undertaker -j cppsym -b undertaker-worklist 2>undertaker-cppsym.errors >undertaker-all-cppsym.raw
    awk '
BEGIN { FS="," }

{
    references[$1] = references[$1] + $2;
    rewrites[$1] = rewrites[$1] + $3;
}

END {
    for (item in references) {
        printf "%s, %d, %d\n", item, references[item], rewrites[item]
    }
}' < undertaker-all-cppsym.raw > undertaker-all-cppsym
    echo "Found $(wc -l < undertaker-all-cppsym) distinct CPP symbols."

    if ! test -s undertaker-cppsym.errors; then
        rm -f undertaker-cppsym.errors
    else
        echo "checking for cppsym had errors, inspect 'undertaker-cppsym.errors' for details"
    fi

    modelarches="$(ls models/ | sed 's/.rsf//g;s/.model//g;s/.cnf//g;s/.golem-errors//g;s/.inferences//g' | sort -u)"
    mkdir -p undertaker-stat
    for m in $modelarches; do
        do_archstat allnoconfig $m
        do_archstat allyesconfig $m
        do_archstat allmodconfig $m
        do_archspecific_cppsym $m
    done
    exit 0
fi