undertaker 1.6-2 / usr / bin / undertaker-kconfigdump

#!/bin/bash -e
#
#   undertaker-kconfigdump - generates models in Linux source trees
#
# Copyright (C) 2009-2011 Christian Dietrich <christian.dietrich@informatik.uni-erlangen.de>
# Copyright (C) 2009-2012 Reinhard Tartler <tartler@informatik.uni-erlangen.de>
# Copyright (C) 2012-2013 Manuel Zerpies <manuel.f.zerpies@ww.stud,uni-erlangen.de>
# Copyright (C) 2012 Stefan Hengelein <stefan.hengelein@informatik.stud.uni-erlangen.de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# This script is used for precalculating the configuration models of a
# linux tree. Therefore the config is first dumped with dumpconf to an
# rsf file. This rsf file is piped to rsf2model which calculates the
# model which is used by the undertaker afterwards.
#
# dumpconf and rsf2model can be placed in $PREFIX/lib/undertaker or
# /usr/lib/undertaker or /usr/local/lib/undertaker, because they won't
# be needed anywhere else than here.
#
# Enviroment variables:
# - MODELS: the directory where the models are placed (default:
#           models)
# - DEBUG: give some debug informations
#
# Arguments:
# - Architecture models to build


MODELS=${MODELS:-models}
CNFMODELS=${MODELS:-cnfmodels}
PROCESSORS=${PROCESSORS:-$(getconf _NPROCESSORS_ONLN)}

DO_INFERENCES="no"
MODE="rsf"
USE_BUSYFIX="no"
CALC_FM="no"

# ensure the last (rightmost) non-zero exitcode is used when piping results into
# another program
set -o pipefail

trap "kill 0" SIGINT SIGTERM

while getopts ircmbfh OPT; do
    case $OPT in
        i)
            DO_INFERENCES="yes"
            ;;
        r)
            MODE="rsf"
            ;;
        c)
            MODE="cnf"
            ;;
        m)
            MODE="intermediateRSF"
            ;;
        b)
            USE_BUSYFIX="yes"
            ;;
        f)
            CALC_FM="yes"
            ;;
        h)
            echo "\`undertaker-kconfigdump' generates models for a linux, busybox"
            echo "and coreboot"
            echo "Usage: ${0##*/} [-r] [-c] [-i] [-h] [-f] [-b]"
            echo " -r  generate Format 1.0 (RSF) Models (default)"
            echo " -c  generate Format 2.0 (CNF) Models"
            echo " -i  calculate inferences"
            echo " -h  displays this message"
            echo " -f  create .fm files for all arches"
            echo " -b  use busyfix to transform busybox trees"
            exit
            ;;
    esac
done

shift $(($OPTIND - 1))

function debug() {
    if [ -n "$DEBUG" ]; then
        echo -e "$@"
    fi
}

if ! which undertaker > /dev/null; then
    echo "No undertaker programm found, please run 'make install' first."
    exit 1
fi

function find_undertaker_basepath () {
    # ensure to follow symbolic links
    local ud=$(which undertaker)
    local undertaker_path=$(readlink -f $ud)
    local b=$(dirname $undertaker_path)
    echo "${b}/.."
}

# This adds the correct path for dumpconf
PATH="$(find_undertaker_basepath)/lib/undertaker:$PATH"
debug "PATH=$PATH\n"

if [ "$CALC_FM" = "yes" ]; then
    if ! which undertaker-kconfigpp > /dev/null; then
        echo "No undertaker-kconfigpp program found."
        exit 1
    fi
    echo "creating .fm files for all arches in directory $MODELS"
elif [ "$MODE" = "rsf" ] || [ "$MODE" = "intermediateRSF" ]; then
    if ! which dumpconf > /dev/null; then
        echo "No dumpconf binary found."
        exit 1
    fi

    if ! which rsf2model > /dev/null; then
        echo "No rsf2model binary found."
        exit 1
    fi
    echo "Generating Format 1.0 (rsf) models"
else
    if ! which satyr > /dev/null; then
        echo "No satyr binary found."
        exit 1
    fi
    echo "Generating Format 2.0 (cnf) models"
fi

debug "undertaker: $(which undertaker)"
debug "rsf2model: $(which rsf2model)"
debug "dumpconf: $(which dumpconf)"
debug "satyr: $(which satyr)"
debug "undertaker-kconfigpp: $(which undertaker-kconfigpp)"

function all_linux_archs () {
    echo "$(ls arch/*/Kconfig* | cut -d '/' -f 2 | sort -u)"
}

mkdir -p "$MODELS"

function pwait() {
    # This functions blocks until less than $1 subprocesses are running
    jobs="$1"
    [ -z "$jobs" ] && jobs=5

    while [ $(jobs -r | wc -l) -ge "$jobs" ]; do
        sleep 0.5
    done
}

# This function removes stale, zero-sized intermediate files
# First paramter is the architecture for which to cleanup
cleanup_empty_intermediates_for_arch() {
    for suffix in "inferences" "whitelist" "blacklist" "model" "cnf"; do
        f="$MODELS/$1.$suffix"
        if [ ! -s $f ]; then
            rm -f $f
        fi
    done
}

function do_convert() {
    ARCH=$1

    # special case for user mode linux
    # Note that upstream commit 5c48b108ecbf6505d929e64d50dace13ac2bdf34 renamed
    # arch/{um/Kconfig.x86 => x86/um/Kconfig}
    # get the right file even in historic linux trees.
    SUBARCH=$ARCH
    ARCHSTR=$ARCH
    KCONFIG=Kconfig
    if [ "$ARCH" = "um" ]; then
        SUBARCH=x86
        if [ -r arch/x86/um/Kconfig ]; then
            ARCHSTR=x86/um
        else
            KCONFIG=Kconfig.x86
        fi
    fi

    test -f arch/$ARCHSTR/$KCONFIG || {
       echo "ERROR: Architecture $ARCH not found";
       return
    }

    if [ $MODE = "rsf" ]; then
        echo "Calculating RSF model for $ARCH"
        # remove all the comments from dumpconf because they will confuse rsf2model
        SUBARCH=$SUBARCH ARCH=$ARCH dumpconf arch/$ARCHSTR/$KCONFIG | sort -u \
            | grep -v '^#' > "$MODELS/$ARCH.rsf"

        do_rsf2model_conversion $ARCH
    fi

    if [ $MODE = "cnf" ]; then
        # run satyr
        echo "Calculating CNF model for $ARCH"
        SUBARCH=$SUBARCH SRCARCH=$ARCH ARCH=$ARCH satyr arch/$ARCHSTR/$KCONFIG -c "$MODELS/$ARCH.cnf"
    fi
}

# Parameters {Arch} {,intermediate}, $2 is only required if "-m" mode is used
function do_rsf2model_conversion() {
    ARCH=$1
    UPCASE_ARCH=$(echo $ARCH | tr 'a-z' 'A-Z')

    # if there is a 'CONFIG_$UPCASE_ARCH' without conditions, it will be filtered out
    rsf2model "$MODELS/$ARCH.rsf" | grep -v '^CONFIG_'"$UPCASE_ARCH"'$' | sort -u > "$MODELS/$ARCH.model"

    if [ "$2" = "intermediate" ]; then
        ALL_ARCHS=$(ls $MODELS/*.rsf | sed 's,models/,,g' | sed 's/.rsf//g')
    else
        ALL_ARCHS=$(all_linux_archs)
    fi

    # if there is already a 'CONFIG_$UPCASE_ARCH' rule in the model, conditions
    # that no other architecture can be selected at the same time will
    # be appended and if CONFIG_$UPCASE_ARCH is not present, a new rule is created.
    # E.g. X86 -> !ARM && !S390
    ALL_NOT_CONFIG_ARCH=$(echo $ALL_ARCHS | sed "s/ *$ARCH */ /" | tr 'a-z' 'A-Z' \
        | sed 's/ *$//g; s/^ *//g; s/\</!CONFIG_/g;')
    if grep -q "^CONFIG_$UPCASE_ARCH " $MODELS/$ARCH.model; then
        sed -i '/^CONFIG_'"$UPCASE_ARCH"' / s/"$/ \&\& '"$(echo $ALL_NOT_CONFIG_ARCH | \
            sed 's/ / \\\&\\\& /g')"'"/' $MODELS/$ARCH.model
    else
        echo $ALL_NOT_CONFIG_ARCH | sed 's/ / \&\& /g; s/$/"/; s/^/'"CONFIG_$UPCASE_ARCH "'"/;' \
            >> "$MODELS/$ARCH.model"
    fi

    # append CONFIG_n / CONFIG_y / CONFIG_m to models
    echo "CONFIG_n" >> "$MODELS/$ARCH.model"
    echo "CONFIG_y" >> "$MODELS/$ARCH.model"
    echo "CONFIG_m" >> "$MODELS/$ARCH.model"
}

# Parameters {Project/Arch} {Subarch}
function do_inference() {
    if [ $DO_INFERENCES = "yes" ]; then
        echo "Calculating makefile inferences for $1"
        if SUBARCH=$2 ARCH=$1 golem -i 2> $MODELS/$1.golem-errors | sort -u > \
            "$MODELS/$1.inferences"; then
            # Filter out annoying "AttributeError" messages
            # This is a known upstream python bug, cf:
            # http://bugs.python.org/issue14308
            sed -i "/'_DummyThread' object has no attribute '_Thread__block'/d"\
                $MODELS/$1.golem-errors
        else
            echo "Failed to extract inferences for $1"
        fi
    else
        touch "$MODELS/$1.inferences"
    fi
}

#Parameters {Project/Arch} Configpath ALWAYS_OFF_ITEMS ALWAYS_ON_ITEMS
function do_rsf_or_satyr_call() {

    if [ "$MODE" = "rsf" ]; then
        dumpconf $2 | sort -u | grep -v '^#' > "$MODELS/$1.rsf"

        # make model
        rsf2model "$MODELS/$1.rsf" > "$MODELS/$1.model"

        # append ALWAYS_OFF items to the model
        echo "UNDERTAKER_SET ALWAYS_OFF $3" >> "$MODELS/$1.model"

        # append ALWAYS_ON items to the model
        for i in $4; do
            sed -i "/^UNDERTAKER_SET ALWAYS_ON/s|$| \"$i\"|" "$MODELS/$1.model"
        done

        # create inferences (or at least an empty file)
        do_inference $1
        cat "$MODELS/$1.inferences" >> "$MODELS/$1.model"

        # kill warnings and errors from golem and rsf2model
        sed -i "/^[EWI]: /d" "$MODELS/$1.model"

        # append CONFIG_n / CONFIG_y / CONFIG_m to models
        echo "CONFIG_n" >> "$MODELS/$1.model"
        echo "CONFIG_y" >> "$MODELS/$1.model"
        echo "CONFIG_m" >> "$MODELS/$1.model"
    fi

    if [ "$MODE" = "cnf" ]; then
        # create empty blacklist file and append ALWAYS_OFF items
        :> "$MODELS/$1.blacklist"
        for i in $3; do
            echo $i >> "$MODELS/$1.blacklist"
        done

        # create empty whitelist file and append ALWAYS_ON items
        :> "$MODELS/$1.whitelist"
        for i in $4; do
            echo $i >> "$MODELS/$1.whitelist"
        done

        # call satyr
        echo "Calculating cnf model for '$1'"
        satyr $2 -c "$MODELS/$1.cnf"

        # create inferences (or at least an empty file)
        do_inference $1

        echo "Calling rsf2cnf..."
        rsf2cnf \
            -m "$MODELS/$1.inferences" \
            -c "$MODELS/$1.cnf" \
            -B "$MODELS/$1.blacklist" \
            -W "$MODELS/$1.whitelist" \
            > "$MODELS/$1.cnf2"
        mv $MODELS/$1.cnf2 $MODELS/$1.cnf
    fi

    cleanup_empty_intermediates_for_arch $1
}

if [ -f scripts/gen_build_files.sh ]; then
    echo "Detected a busybox source tree"

    # disable CONFIG_WERROR
    ALWAYS_OFF_ITEMS="CONFIG_WERROR CONFIG_STATIC"

    if [ $USE_BUSYFIX = "yes" ]; then
        # busyfix damages the excluded files below
        find -type f -name '*.[cS]' \
            ! -regex '^./applets_sh.*' ! -regex '^./docs.*' ! -regex '^./examples.*' \
            ! -regex '^./testsuite.*' ! -regex '^./scripts.*' \
            -print > busyfix-worklist
    fi

    # generate config files, execute dumpconf
    # remove all the comments because they will confuse rsf2model
    make gen_build_files

    do_rsf_or_satyr_call busybox Config.in "$ALWAYS_OFF_ITEMS"

    exit 0
fi

if test -r Makefile && grep -q 'This file is part of the coreboot project.' Makefile; then
    echo "Detected a coreboot source tree"
    CBMODEL=coreboot

    # disable bad CONFIG_ Symbols
    #
    # be careful here, if the desired CONFIG_ Symbol doesn't have a prompt within
    # the specific Kconfig file (eg. "config WARNINGS_ARE_ERRORS\n\tbool\n") then
    # it CANNOT be asured to be always off or on!

    ALWAYS_OFF_ITEMS="CONFIG_CCACHE CONFIG_SCANBUILD_ENABLE CONFIG_WARNINGS_ARE_ERRORS CONFIG_USE_BLOBS CONFIG_PAYLOAD_SEABIOS"
    ALWAYS_ON_ITEMS="CONFIG_PAYLOAD_NONE"

    #make the Kconfig Symbol WARNINGS_ARE_ERRORS changeable
    sed -i ':a;N;$!ba;s,config WARNINGS_ARE_ERRORS\n\tbool\n,config WARNINGS_ARE_ERRORS\n\tbool "DUMMY"\n,' src/Kconfig

    #delete default usage of PAYLOAD_SEABIOS (would mess with 'make allyesconfig')
    sed -i '/default PAYLOAD_SEABIOS/d' src/Kconfig
    #make printall target in Makefile .PHONY
    if test -r Makefile && ! grep '.PHONY' Makefile | grep -q printall ; then
        sed -i "/^\.PHONY:/s|$| printall|" Makefile
    fi
    #add optional CFLAGS to Makefile.inc
    if test -r Makefile.inc && ! grep -q 'KCFLAGS' Makefile.inc ; then
        sed -i "/^CFLAGS =/s|$|\nifneq (\$(KCFLAGS),)\n\tCFLAGS += (\$KCFLAGS)\nendif|" Makefile.inc
    fi

    do_rsf_or_satyr_call $CBMODEL ./src/Kconfig "$ALWAYS_OFF_ITEMS" "$ALWAYS_ON_ITEMS"

    exit 0
fi

if [ "$MODE" = "intermediateRSF" ] && [ "$2" = "intermediate" ]; then
    echo "Calling intermediate_rsf2model for arch $1 ..."
    do_rsf2model_conversion $1 $2
    exit 0
elif ! [ -f arch/x86/Kconfig -o -f arch/i386/Kconfig ]; then
    echo "This version supports Linux, busybox and coreboot"
    echo "Please run this programm inside an source tree without arguments"
    exit 1
fi

if [ "$CALC_FM" = "yes" ]; then
    for i in $(all_linux_archs); do
        ARCHSTR=$i
        KCONFIG=Kconfig
        if [ "$i" = "um" ]; then
            if [ -r arch/x86/um/Kconfig ]; then
                ARCHSTR=x86/um
            else
                KCONFIG=Kconfig.x86
            fi
        fi
        ARCH=$i undertaker-kconfigpp arch/$ARCHSTR/$KCONFIG > $MODELS/$i.fm
    done
    exit 0
fi

if [ -z "$1" ]; then
    ARCHS=$(all_linux_archs)
else
    ARCHS="$@"
fi

for ARCH in $ARCHS; do
    do_convert $ARCH &
    # run many converting processes in parallel
    pwait ${PROCESSORS}
done

while [ $(jobs -r | wc -l) -gt 0 ]; do
    sleep 0.5
done

for ARCH in $ARCHS; do
    do_inference $ARCH $ARCH

    if [ $MODE = "rsf" ]; then
        cat "$MODELS/$ARCH.inferences" >> "$MODELS/$ARCH.model"
        # kill warnings and errors from golem
        sed -i "/^[EWI]: /d" "$MODELS/$ARCH.model"
    fi

    if [ $MODE = "cnf" ]; then
        echo "Calling rsf2cnf for arch $ARCH..."
        touch "$MODELS/$ARCH.whitelist" "$MODELS/$ARCH.blacklist"
        rsf2cnf \
            -m "$MODELS/$ARCH.inferences" \
            -c "$MODELS/$ARCH.cnf" \
            -B "$MODELS/$ARCH.blacklist" \
            -W "$MODELS/$ARCH.whitelist" \
            > "$MODELS/$ARCH.cnf2"
        mv $MODELS/$ARCH.cnf2 $MODELS/$ARCH.cnf
    fi

    cleanup_empty_intermediates_for_arch $ARCH
done

exit 0