pcp-testsuite 4.0.1-1 / var / lib / pcp / testsuite / 169

#! /bin/sh
# PCP QA Test No. 169
#
# Test pmcd timeouts with the pmcd PMDA's pmcd.control.timeout
#
# Test setting the app's timeouts < pmcd's timeout and vice versa
#
# Copyright (c) 1995-2002 Silicon Graphics, Inc.  All Rights Reserved.
#

seq=`basename $0`
echo "QA output created by $seq"

# get standard environment, filters and checks
. ./common.product
. ./common.check
. ./common.filter

echo ""
echo "Test pmcd timeouts with the pmcd PMDA's pmcd.control.timeout"
echo ""

LOCALHOST=`hostname`
signal=$PCP_BINADM_DIR/pmsignal

_filter_dots()
{
    sed -e 's/\.\.\.\.\.*/[dots]/'
}

_cleanup()
{
    _service pcp stop >$tmp.stop
    # primary pmlogger exiting during the test is racey depending on timing
    # of fetches and stopping/starting pmcd (this is also not under test).
    _filter_pcp_stop <$tmp.stop | _filter_dots | sed -e '/logger not running/d'
    echo >>$seq.full
    cat $PCP_LOG_DIR/pmcd/pmcd.log >>$seq.full
    echo >>$seq.full
    cat $PCP_LOG_DIR/pmcd/dumb_pmda.log >>$seq.full
    _restore_config $PCP_PMCDCONF_PATH
    _service pcp start >$tmp.start
    _restore_auto_restart pmcd
    _filter_pcp_start <$tmp.start | _filter_dots
    _wait_for_pmcd
    _wait_for_pmlogger

    rm -f $tmp.*
}

status=1	# failure is the default!
trap "_cleanup; exit \$status" 0 1 2 3 15
rm -f $seq.full

_stop_auto_restart pmcd

# real QA test starts here

# This value is passed to processes this script creates on the command line.
# It's unlikely to occur anywhere in a ps output and is used as a tag to
# terminate or check for the existence of the processes later.
#
TAG=000000660066

# [Sat Jul 30 17:33:55] pminfo(26317) Error: __pmCloseChannel: fd=3 context=1: Timeout waiting for a response from PMCD
_filter()
{
    sed \
	-e 's/pminfo([0-9][0-9]*)/pminfo(PID)/' \
	-e 's/\[[A-Z][a-z][a-z] [A-Z][a-z][a-z]  *[0-9][0-9]* [0-9][0-9]:[0-9][0-9]:[0-9][0-9]]/[DATE]/' \
    # end
}

_check_pmcd_procs()
{
    n_pmcd=`ps $PCP_PS_ALL_FLAGS \
	    | tee $tmp.pmcd \
	    | fgrep "$PCP_PMCD_PATH" \
	    | grep -v pmcd_wait \
	    | grep -v grep \
	    | wc -l \
	    | tr -d ' '`
    if [ "X$n_pmcd" != "X$1" ]
    then
	echo "Expected $1 pmcd processes but found $n_pmcd:"
	cat $tmp.pmcd
	cat $PCP_PMCDLOG_PATH
	file $PCP_LOG_DIR/core* $PCP_LOG_DIR/pmcd/core*
	return 1
    else
	return 0
    fi
}

_set_timeouts()
{
    pmcd_timeout=$1
    app_timeout=$2

    echo "app timeout = $app_timeout"
    echo "pmcd timeout = $pmcd_timeout"
    PMCD_REQUEST_TIMEOUT=$app_timeout
    export PMCD_REQUEST_TIMEOUT

    pmstore pmcd.control.timeout $pmcd_timeout

}

_echo()
{
    echo "$*"
    echo "$*" >>$seq.full
}

rm -rf $tmp.*
pmns=$tmp.pmns

# use a dummy pmns for pminfo
dummy_domain=160 # choose non-existent domain
cat >$pmns <<EOF
root {
    dummyproc   $dummy_domain:0:0
}
EOF

# pmcd will run a dumb_pmda, masquerading as the IRIX agent.  This will
# not respond to PDUs from PMCD, which should then time it out.
# The pmcd PMDA is required so that pmcd.control.timeout can be modified.
# We don't need any other PMDAs and in particular all pmlogger and pmie
# processes are stopped before pmcd is reconfigured.
# Save the current pmcd.conf so it can be restored afterwards.

_save_config $PCP_PMCDCONF_PATH
cat >$tmp.tmp <<End-Of-File
# QA test 169 dummy pmcd.conf (blame Jonathan if you find this!)
#
dummyproc	$dummy_domain	pipe	binary 	$here/src/dumb_pmda -d $dummy_domain $TAG
End-Of-File

$PCP_AWK_PROG <$PCP_PMCDCONF_PATH >>$tmp.tmp '$1 == "pmcd" { print }'

pmcdlog=$PCP_PMCDLOG_PATH

_service pmlogger stop >>$here/$seq.full 2>&1
_service pmie stop >>$here/$seq.full 2>&1

$sudo cp $tmp.tmp $PCP_PMCDCONF_PATH
# Note: need to use $PCP_RC_DIR/pmcd here, not _service pmcd so that
# systemd (if it is available) does not notice any pmcd exiting
#
$sudo $PCP_RC_DIR/pmcd restart >$tmp.start
_filter_pcp_start <$tmp.start | sed -e 's/\.\.\.\.\.*/[dots]/'
_wait_for_pmcd
if _check_pmcd_procs 1
then
    :
else
    _echo "Error: pmcd didn't start!"
    cat $pmcdlog >>$seq.full
    exit
fi

ps $PCP_PS_ALL_FLAGS >$tmp.ps
egrep '[P]ID|[p]m|[d]umb' $tmp.ps >>$seq.full
old_dumb_pid=`$PCP_AWK_PROG <$tmp.ps '/dumb_pmda /{print $2}'`
echo "old_dumb_pid=$old_dumb_pid" >>$seq.full
old_pmcd_pid=`$PCP_AWK_PROG <$tmp.ps '/pmcd /{print $2}'`
echo "old_pmcd_pid=$old_pmcd_pid" >>$seq.full
if [ -z "$old_dumb_pid" ]
then
    _echo "Error: pmcd didn't create the dumb_pmda agent"
    cat $pmcdlog >>$seq.full
    exit
fi

# force to a known state
#
pmstore pmcd.control.timeout 5 >/dev/null

# for debugging
#
#debug# pmstore pmcd.control.tracenobuf 1 >/dev/null

#
# pmcd agent timeout < application time out
#
_echo ""
_echo "*** pmcd agent timeout < app timeout ***"
_echo "*** pmcd will killoff agent ***" 
_echo ""
_set_timeouts 5 10

# Fetch from dummy domain.  This should cause the dumb_pmda agent to be
# timed-out by pmcd, resulting in its termination.
#
_echo ""
_echo "Expect dummyproc to fail (IPC protocol failure):"
pminfo -n $pmns -d dummyproc
_echo ""
# make sure pmcd has had a chance to wait() on the pmda process
#
sleep 2
pminfo -n $pmns -d dummyproc >/dev/null 2>&1

ps $PCP_PS_ALL_FLAGS >$tmp.ps
egrep '[P]ID|[p]m|[d]umb' $tmp.ps >>$seq.full
new_dumb_pid=`$PCP_AWK_PROG <$tmp.ps '/dumb_pmda /{print $2}'`
echo "new_dumb_pid=$new_dumb_pid" >>$seq.full
new_pmcd_pid=`$PCP_AWK_PROG <$tmp.ps '/pmcd /{print $2}'`
echo "new_pmcd_pid=$new_pmcd_pid" >>$seq.full
if [ -z "$new_pmcd_pid" ]
then
    _echo "Error: pmcd died!"
    cat $pmcdlog >>$seq.full
    exit 
fi
if [ "$new_pmcd_pid" != "$old_pmcd_pid" ]
then
    _echo "Error: pmcd restarted!"
    cat $pmcdlog >>$seq.full
    exit 
fi
if [ -n "$new_dumb_pid" -a "$new_dumb_pid" = "$old_dumb_pid" ]
then
    _echo "Error: pmcd didn't terminate the dumb_pmda agent"
    cat $pmcdlog >>$seq.full
    exit
else
    _echo "pmcd terminated dummy the dumb_pmda agent as desired"
fi

echo >>$seq.full
cat $PCP_LOG_DIR/pmcd/pmcd.log >>$seq.full
echo >>$seq.full
cat $PCP_LOG_DIR/pmcd/dumb_pmda.log >>$seq.full

# reset (not restart) pmcd and restart any dead PMDAs
#
_echo ""
$sudo $signal -a -s HUP pmcd
sleep 1
_wait_for_pmcd
if _check_pmcd_procs 1
then
    :
else
    _echo "Error: pmcd was sent SIGHUP and died"
    cat $pmcdlog >>$seq.full
    exit
fi

ps $PCP_PS_ALL_FLAGS >$tmp.ps
egrep '[P]ID|[p]m|[d]umb' $tmp.ps >>$seq.full
new_dumb_pid=`$PCP_AWK_PROG <$tmp.ps '/dumb_pmda /{print $2}'`
echo "new_dumb_pid=$new_dumb_pid" >>$seq.full
new_pmcd_pid=`$PCP_AWK_PROG <$tmp.ps '/pmcd /{print $2}'`
echo "new_pmcd_pid=$new_pmcd_pid" >>$seq.full
if [ -z "$new_pmcd_pid" ]
then
    _echo "Error: pmcd died!"
    cat $pmcdlog >>$seq.full
    exit 
fi
if [ "$new_pmcd_pid" != "$old_pmcd_pid" ]
then
    _echo "Error: pmcd restarted!"
    cat $pmcdlog >>$seq.full
    exit 
fi
if [ -n "$new_dumb_pid" -a "$new_dumb_pid" != "$old_dumb_pid" ]
then
    old_dumb_pid=$new_dumb_pid
elif [ -z "$new_dumb_pid" ]
then
    _echo "Error: pmcd SIGHUP didn't restart the dumb_pmda agent"
    cat $pmcdlog >>$seq.full
    exit
else
    _echo "Error: dumb_pmda agent with same pid has arisen from the dead!"
    cat $pmcdlog >>$seq.full
    exit
fi

#
# pmcd agent timeout > application time out
#
_echo ""
_echo "*** pmcd agent timeout > app timeout ***"
_echo "*** pmcd will not killoff agent until later ***" 
_echo ""
_set_timeouts 10 5

# Fetch from dummy domain.  This should cause the dumb_pmda agent to be
# timed-out by pmcd, resulting in its termination.
#
_echo ""
_echo "Expect dummyproc to fail (timeout for pmcd failure):"
#DEBUG# date
pminfo -n $pmns -d dummyproc 2>&1 \
| _filter
#DEBUG# date
_echo ""

ps $PCP_PS_ALL_FLAGS >$tmp.ps
egrep '[P]ID|[p]m|[d]umb' $tmp.ps >>$seq.full
new_dumb_pid=`$PCP_AWK_PROG <$tmp.ps '/dumb_pmda /{print $2}'`
echo "new_dumb_pid=$new_dumb_pid" >>$seq.full
new_pmcd_pid=`$PCP_AWK_PROG <$tmp.ps '/pmcd /{print $2}'`
echo "new_pmcd_pid=$new_pmcd_pid" >>$seq.full
if [ -z "$new_pmcd_pid" ]
then
    _echo "Error: pmcd died!"
    cat $pmcdlog >>$seq.full
    exit 
fi
if [ "$new_pmcd_pid" != "$old_pmcd_pid" ]
then
    _echo "Error: pmcd restarted!"
    cat $pmcdlog >>$seq.full
    exit 
fi
if [ -n "$new_dumb_pid" ]
then 
    _echo "pmcd did NOT terminate the dumb_pmda agent as desired"

    if [ "$old_dumb_pid" != "$new_dumb_pid" ]
    then
	_echo "Error: dumb_pmda proc has been recreated"
	_echo "pid mismatch: $old_dumb_pid versus $new_dumb_pid"
	exit
    fi
else
    _echo "Error: pmcd terminated the dumb_pmda agent"
    cat $pmcdlog >>$seq.full
    exit
fi

_echo ""
_echo "Now wait for pmcd to timeout..."
_echo ""
sleep 8

ps $PCP_PS_ALL_FLAGS >$tmp.ps
egrep '[P]ID|[p]m|[d]umb' $tmp.ps >>$seq.full
new_dumb_pid=`$PCP_AWK_PROG <$tmp.ps '/dumb_pmda /{print $2}'`
echo "new_dumb_pid=$new_dumb_pid" >>$seq.full
new_pmcd_pid=`$PCP_AWK_PROG <$tmp.ps '/pmcd /{print $2}'`
echo "new_pmcd_pid=$new_pmcd_pid" >>$seq.full
if [ -z "$new_pmcd_pid" ]
then
    _echo "Error: pmcd died!"
    cat $pmcdlog >>$seq.full
    exit 
fi
if [ "$new_pmcd_pid" != "$old_pmcd_pid" ]
then
    _echo "Error: pmcd restarted!"
    cat $pmcdlog >>$seq.full
    exit 
fi
if [ -n "$new_dumb_pid" ]
then
    if [ "$new_dumb_pid" = "$old_dumb_pid" ]
    then
	_echo "Error: pmcd didn't terminate the dumb_pmda agent"
	cat $pmcdlog >>$seq.full
	exit
    else
	_echo "Error: dumb_pmda proc has been recreated"
	_echo "pid mismatch: $old_dumb_pid versus $new_dumb_pid"
	exit
    fi
else
    _echo "pmcd terminated the dumb_pmda agent as desired"
fi

_echo ""
status=0