heartbeat 1:3.0.6-7 / usr / share / heartbeat / cts / CTSlab.py

#!/usr/bin/python

'''CTS: Cluster Testing System: Lab environment module


 '''

__copyright__='''
Copyright (C) 2001,2005 Alan Robertson <alanr@unix.sh>
Licensed under the GNU GPL.
'''

#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.

from UserDict import UserDict
import sys, time, types, string, syslog, random, os, string, signal, traceback
from CTS  import ClusterManager
from CM_hb import HeartbeatCM
from CTStests import BSC_AddResource
from socket import gethostbyname_ex

tests = None
cm = None
old_handler = None
DefaultFacility = "daemon"

def sig_handler(signum, frame) :
    if cm != None:
        cm.log("Interrupted by signal %d"%signum)
    if signum == 10 and tests != None :
        tests.summarize()
    if signum == 15 :
        sys.exit(1)
        
class ResetMechanism:
    def reset(self, node):
        raise ValueError("Abstract class member (reset)")

class Stonith(ResetMechanism):
    def __init__(self, sttype="external/ssh", pName=None, pValue=None
        ,        path="/usr/sbin/stonith"):
        self.pathname=path
        self.configName=pName
        self.configValue=pValue
        self.stonithtype=sttype

    def reset(self, node):
        if self.configValue == None :
            config=node
        else:
            config=self.configValue
        cmdstring = "%s -t '%s' -p '%s' '%s' 2>/dev/null" % (self.pathname
        ,        self.stonithtype, config, node)
        return (os.system(cmdstring) == 0)
    
class Stonithd(ResetMechanism):
    def __init__(self, nodes, sttype = 'external/ssh'):
        self.sttype = sttype
        self.nodes = nodes

        self.query_cmd_pat   = '/usr/lib/heartbeat/stonithdtest/apitest 0 %s 60000 0'
        self.reset_cmd_pat   = '/usr/lib/heartbeat/stonithdtest/apitest 1 %s 60000 0'
        self.poweron_cmd_pat = '/usr/lib/heartbeat/stonithdtest/apitest 2 %s 60000 0'
        self.poweroff_cmd_pat= '/usr/lib/heartbeat/stonithdtest/apitest 3 %s 60000 0'

        self.lrmd_add_pat    = '/usr/sbin/lrmadmin -A %s stonith ' + sttype + ' NULL hostlist=%s livedangerously=yes'
        self.lrmd_start_pat  = '/usr/sbin/lrmadmin -E %s start 0 0 EVERYTIME'
        self.lrmd_stop_pat   = '/usr/sbin/lrmadmin -E %s stop 0 0 EVERYTIME'
        self.lrmd_del_pat    = '/usr/sbin/lrmadmin -D %s'

        self.rsc_id          = 'my_stonithd_id'

        self.command = "/usr/bin/ssh -l root -n -x"
        self.command_noblock = "/usr/bin/ssh -f -l root -n -x"

        self.stonithd_started_nodes = []
        self.fail_reason = ''

    def _remote_exec(self, node, cmnd):
        return (os.system("%s %s %s > /dev/null" % (self.command, node, cmnd)) == 0)

    def _remote_readlines(self, node, cmnd):
        f = os.popen("%s %s %s" % (self.command, node, cmnd))
        return f.readlines()

    def _stonithd_started(self, node):
        return node in self.stonithd_started_nodes

    def _start_stonithd(self, node,  hosts):
        hostlist = string.join(hosts, ',')

        lrmd_add_cmd = self.lrmd_add_pat % (self.rsc_id,  hostlist)
        ret = self._remote_exec(node, lrmd_add_cmd)
        if not ret:return ret

        lrmd_start_cmd = self.lrmd_start_pat % self.rsc_id

        ret = self._remote_exec(node, lrmd_start_cmd)

        if not ret:return ret

        self.stonithd_started_nodes.append(node)
        return 1

    def _stop_stonithd(self, node):
        lrmd_stop_cmd = self.lrmd_stop_pat % self.rsc_id
        ret = self._remote_exec(node, lrmd_stop_cmd)
        if not ret:return ret

        lrmd_del_cmd = self.lrmd_del_pat % self.rsc_id

        ret = self._remote_exec(node, lrmd_del_cmd)
        if not ret:return ret

        self.stonithd_started_nodes.remove(node)

        return 1

    def _do_stonith(self, init_node, target_node, action):

        stonithd_started = self._stonithd_started(init_node)

        if not stonithd_started:
            ret = self._start_stonithd(init_node, [target_node])
            if not ret:
                self.fail_reason = "failed to start stonithd on node %s" % init_node
                return ret

        command = ""

        if action == "RESET":
            command = self.reset_cmd_pat % target_node
        elif action == "POWEROFF":
            command = self.poweroff_cmd_pat % target_node
        elif action == "POWERON":
            command = self.poweron_cmd_pat % target_node
        else:
            self.fail_reason = "unknown operation type %s" % action
            return 0

        lines = self._remote_readlines(init_node, command)
        result = "".join(lines)

        if not stonithd_started:
            self._stop_stonithd(init_node)

        index = result.find("result=0")
        if index == -1:
            self.fail_reason = "unexpected stonithd status: %s" % result
            return 0

        return 1 

    # Should we randomly choose a node as init_node here if init_node not specified?
    def reset(self, init_node, target_node):
        return self._do_stonith(init_node, target_node, "RESET")

    def poweron(self, init_node, target_node):
        return self._do_stonith(init_node, target_node, "POWERON")

    def poweroff(self, init_node, target_node):
        return self._do_stonith(init_node, target_node, "POWEROFF")

class Logger:
    TimeFormat = "%b %d %H:%M:%S\t"

    def __call__(self, lines):
        raise ValueError("Abstract class member (__call__)")
    def write(self, line):
        return self(line.rstrip())
    def writelines(self, lines):
        for s in lines:
            self.write(s)
        return 1
    def flush(self):
        return 1
    def isatty(self):
        return None

class SysLog(Logger):
    # http://docs.python.org/lib/module-syslog.html
    defaultsource="CTS"
    map = {
            "kernel":   syslog.LOG_KERN,
            "user":     syslog.LOG_USER,
            "mail":     syslog.LOG_MAIL,
            "daemon":   syslog.LOG_DAEMON,
            "auth":     syslog.LOG_AUTH,
            "lpr":      syslog.LOG_LPR,
            "news":     syslog.LOG_NEWS,
            "uucp":     syslog.LOG_UUCP,
            "cron":     syslog.LOG_CRON,
            "local0":   syslog.LOG_LOCAL0,
            "local1":   syslog.LOG_LOCAL1,
            "local2":   syslog.LOG_LOCAL2,
            "local3":   syslog.LOG_LOCAL3,
            "local4":   syslog.LOG_LOCAL4,
            "local5":   syslog.LOG_LOCAL5,
            "local6":   syslog.LOG_LOCAL6,
            "local7":   syslog.LOG_LOCAL7,
    }
    def __init__(self, labinfo):

        if labinfo.has_key("syslogsource"):
            self.source=labinfo["syslogsource"]
        else:
            self.source=SysLog.defaultsource

        if labinfo.has_key("SyslogFacility"):
            self.facility=labinfo["SyslogFacility"]
        else:
            self.facility=DefaultFacility

        if SysLog.map.has_key(self.facility):
            self.facility=SysLog.map[self.facility]

        syslog.openlog(self.source, 0, self.facility)

    def setfacility(self, facility):
        self.facility = facility
        if SysLog.map.has_key(self.facility):
          self.facility=SysLog.map[self.facility]
        syslog.closelog()
        syslog.openlog(self.source, 0, self.facility)
        

    def __call__(self, lines):
        if isinstance(lines, types.StringType):
            syslog.syslog(lines)
        else:
            for line in lines:
                syslog.syslog(line)

    def name(self):
        return "Syslog"

class StdErrLog(Logger):

    def __init__(self, labinfo):
        pass

    def __call__(self, lines):
        t = time.strftime(Logger.TimeFormat, time.localtime(time.time()))  
        if isinstance(lines, types.StringType):
            sys.__stderr__.writelines([t, lines, "\n"])
        else:
            for line in lines:
                sys.__stderr__.writelines([t, line, "\n"])
        sys.__stderr__.flush()

    def name(self):
        return "StdErrLog"

class FileLog(Logger):
    def __init__(self, labinfo, filename=None):

        if filename == None:
            filename=labinfo["LogFileName"]
        
        self.logfile=filename
        import os
        self.hostname = os.uname()[1]+" "
        self.source = "CTS: "
    def __call__(self, lines):

        fd = open(self.logfile, "a")
        t = time.strftime(Logger.TimeFormat, time.localtime(time.time()))  

        if isinstance(lines, types.StringType):
            fd.writelines([t, self.hostname, self.source, lines, "\n"])
        else:
            for line in lines:
                fd.writelines([t, self.hostname, self.source, line, "\n"])
        fd.close()

    def name(self):
        return "FileLog"

class CtsLab(UserDict):
    '''This class defines the Lab Environment for the Cluster Test System.
    It defines those things which are expected to change from test
    environment to test environment for the same cluster manager.

    It is where you define the set of nodes that are in your test lab
    what kind of reset mechanism you use, etc.

    This class is derived from a UserDict because we hold many
    different parameters of different kinds, and this provides
    provide a uniform and extensible interface useful for any kind of
    communication between the user/administrator/tester and CTS.

    At this point in time, it is the intent of this class to model static
    configuration and/or environmental data about the environment which
    doesn't change as the tests proceed.

    Well-known names (keys) are an important concept in this class.
    The HasMinimalKeys member function knows the minimal set of
    well-known names for the class.

    The following names are standard (well-known) at this time:

        nodes           An array of the nodes in the cluster
        reset           A ResetMechanism object
        logger          An array of objects that log strings...
        CMclass         The type of ClusterManager we are running
                        (This is a class object, not a class instance)
        RandSeed        Random seed.  It is a triple of bytes. (optional)
        HAdir           Base directory for HA installation

    The CTS code ignores names it doesn't know about/need.
    The individual tests have access to this information, and it is
    perfectly acceptable to provide hints, tweaks, fine-tuning
    directions or other information to the tests through this mechanism.
    '''

    def __init__(self, nodes):
        self.data = {}
        self["nodes"] = nodes
        self.MinimalKeys=["nodes", "reset", "logger", "CMclass", "HAdir"]

    def HasMinimalKeys(self):
        'Return TRUE if our object has the minimal set of keys/values in it'
        result = 1
        for key in self.MinimalKeys:
            if not self.has_key(key):
                result = None
        return result

    def SupplyDefaults(self): 
        if not self.has_key("logger"):
            self["logger"] = (SysLog(self), StdErrLog(self))
        if not self.has_key("reset"):
            self["reset"] = Stonith()
        if not self.has_key("CMclass"):
            self["CMclass"] = HeartbeatCM
        if not self.has_key("LogFileName"):
            self["LogFileName"] = "@HA_VARLOGDIR@/ha-log"
        if not self.has_key("logrestartcmd"):
            self["logrestartcmd"] = "/etc/init.d/syslog restart"
        if not self.has_key("logfacility"):
            LogFacility = DefaultFacility

        #
        #  Now set up our random number generator...
        #
        self.RandomGen = random.Random()

        #  Get a random seed for the random number generator.

        if self.has_key("RandSeed"):
            randseed = self["RandSeed"]
            self.log("Random seed is: " + str(randseed))
            self.RandomGen.seed(str(randseed)) 
        else:
            randseed = int(time.time())
            self.log("Random seed is: " + str(randseed))
            self.RandomGen.seed(str(randseed))

    def log(self, args):
        "Log using each of the supplied logging methods"
        for logfcn in self._logfunctions:
            logfcn(string.strip(args))

    def debug(self, args):
        "Log using each of the supplied logging methods"
        for logfcn in self._logfunctions:
            if logfcn.name() != "StdErrLog":
                logfcn("debug: %s" % string.strip(args))

    def __setitem__(self, key, value):
        '''Since this function gets called whenever we modify the
        dictionary (object), we can (and do) validate those keys that we
        know how to validate.  For the most part, we know how to validate
        the "MinimalKeys" elements.
        '''

        #
        #        List of nodes in the system
        #
        if key == "nodes":
            self.Nodes = {}
            for node in value:
                # I don't think I need the IP address, etc. but this validates
                # the node name against /etc/hosts and/or DNS, so it's a
                # GoodThing(tm).
                try:
                    self.Nodes[node] = gethostbyname_ex(node)
                except:
                    print node+" not found in DNS... aborting"
                    raise
        #
        #        Reset Mechanism
        #
        elif key == "reset":
            if not issubclass(value.__class__, ResetMechanism):
                raise ValueError("'reset' Value must be a subclass"
                " of ResetMechanism") 
        #
        #        List of Logging Mechanism(s)
        #
        elif key == "logger":
            if len(value) < 1:
                raise ValueError("Must have at least one logging mechanism")
            for logger in value:
                if not callable(logger):
                    raise ValueError("'logger' elements must be callable")
            self._logfunctions = value
        #
        #        Cluster Manager Class
        #
        elif key == "CMclass":
            if not issubclass(value, ClusterManager):
                raise ValueError("'CMclass' must be a subclass of"
                " ClusterManager")
        #
        #        Initial Random seed...
        #
        #elif key == "RandSeed":
        #    if len(value) != 3:
        #        raise ValueError("'Randseed' must be a 3-element list/tuple")
        #    for elem in value:
        #        if not isinstance(elem, types.IntType):
        #            raise ValueError("'Randseed' list must all be ints")
              
        self.data[key] = value

    def IsValidNode(self, node):
        'Return TRUE if the given node is valid'
        return self.Nodes.has_key(node)

    def __CheckNode(self, node):
        "Raise a ValueError if the given node isn't valid"

        if not self.IsValidNode(node):
            raise ValueError("Invalid node [%s] in CheckNode" % node)

    def RandomNode(self):
        '''Choose a random node from the cluster'''
        return self.RandomGen.choice(self["nodes"])

    def ResetNode(self, node):
        "Reset a node, (normally) using a hardware mechanism"
        self.__CheckNode(node)
        return self["reset"].reset(node)
    
    def ResetNode2(self, init_node, target_node, reasons):
        self.__CheckNode(target_node)
        stonithd = Stonithd(self["nodes"])
        ret = stonithd.reset(init_node, target_node)
        if not ret: 
            reasons.append(stonithd.fail_reason)
        return ret

def usage(arg):
    print "Illegal argument " + arg
    print "usage: " + sys.argv[0]  \
    +  " --directory config-directory" \
    +  " -D config-directory" \
    +  " --logfile system-logfile-name" \
    +  " --trunc (truncate logfile before starting)" \
    +  " -L system-logfile-name" \
    +  " --limit-nodes maxnumnodes" \
    +  " --xmit-loss lost-rate(0.0-1.0)" \
    +  " --recv-loss lost-rate(0.0-1.0)" \
    +  " --suppressmonitoring" \
    +  " --syslog-facility syslog-facility" \
    +  " --facility syslog-facility" \
    +  " --choose testcase-name" \
    +  " --test-ip-base ip" \
    +  " --oprofile \"whitespace separated list of nodes to oprofile\"" \
    +  " (-2 |"\
    +  " -v2 |"\
    +  " --crm |"\
    +  " --classic)"\
    +  " (--populate-resources | -r)" \
    +  " --resource-can-stop" \
    +  " --stonith (1 | 0 | yes | no)" \
    +  " --stonith-type type" \
    +  " --stonith-args name=value" \
    +  " --standby (1 | 0 | yes | no)" \
    +  " --fencing (1 | 0 | yes | no)" \
    +  " --suppress_cib_writes (1 | 0 | yes | no)" \
    +  " -lstests" \
    +  " --seed" \
    +  "  [number-of-iterations]"
    sys.exit(1)

    
#
#   A little test code...
#
if __name__ == '__main__': 

    from CTSaudits import AuditList
    from CTStests import TestList,RandomTests
    from CTS import Scenario, InitClusterManager, PingFest, PacketLoss, BasicSanityCheck
    import CM_hb

    HAdir = "/etc/ha.d"
    LogFile = "@HA_VARLOGDIR@/ha-log-"+DefaultFacility
    DoStonith = 1
    DoStandby = 1
    DoFencing = 1
    NumIter = 500
    SuppressMonitoring = None
    Version = 1
    CIBfilename = None
    CIBResource = 0
    ClobberCIB = 0
    LimitNodes = 0
    TestCase = None
    LogFacility = None
    TruncateLog = 0
    ResCanStop = 0
    XmitLoss = "0.0"
    RecvLoss = "0.0"
    IPBase = "127.0.0.10"
    SuppressCib = 1
    DoBSC = 0
    ListTests = 0
    HaveSeed = 0
    oprofile = None
    StonithType = "external/ssh"
    StonithParams = None
    StonithParams = "hostlist=dynamic".split('=')

    #
    # The values of the rest of the parameters are now properly derived from
    # the configuration files.
    #
    # Stonith is configurable because it's slow, I have a few machines which
    # don't reboot very reliably, and it can mild damage to your machine if
    # you're using a real power switch.
    # 
    # Standby is configurable because the test is very heartbeat specific
    # and I haven't written the code to set it properly yet.  Patches are
    # being accepted...

    # Set the signal handler
    signal.signal(15, sig_handler)
    signal.signal(10, sig_handler)
    
    # Process arguments...

    skipthis=None
    args=sys.argv[1:]
    for i in range(0, len(args)):
       if skipthis:
           skipthis=None
           continue
       elif args[i] == "-D" or args[i] == "--directory":
           skipthis=1
           HAdir = args[i+1]
       elif args[i] == "-l" or args[i] == "--limit-nodes":
           skipthis=1
           LimitNodes = int(args[i+1])
       elif args[i] == "-r" or args[i] == "--populate-resources":
           CIBResource = 1
       elif args[i] == "-L" or args[i] == "--logfile":
           skipthis=1
           LogFile = args[i+1]
       elif args[i] == "--test-ip-base":
           skipthis=1
           IPBase = args[i+1]
       elif args[i] == "--oprofile":
           skipthis=1
           oprofile = args[i+1].split(' ')
       elif args[i] == "--trunc":
           TruncateLog=1
       elif args[i] == "-v2":
           Version=2
       elif args[i] == "-lstests":
           ListTests=1
       elif args[i] == "--stonith":
           skipthis=1
           if args[i+1] == "1" or args[i+1] == "yes":
               DoStonith=1
           elif args[i+1] == "0" or args[i+1] == "no":
               DoStonith=0
           else:
               usage(args[i+1])
       elif args[i] == "--stonith-type":
           StonithType = args[i+1]
           skipthis=1
       elif args[i] == "--stonith-args":
           StonithParams = args[i+1].split('=')
           skipthis=1
       elif args[i] == "--suppress-cib-writes":
           skipthis=1
           if args[i+1] == "1" or args[i+1] == "yes":
               SuppressCib=1
           elif args[i+1] == "0" or args[i+1] == "no":
               SuppressCib=0
           else:
               usage(args[i+1])
       elif args[i] == "--bsc":
           DoBSC=1
       elif args[i] == "--standby":
           skipthis=1
           if args[i+1] == "1" or args[i+1] == "yes":
               DoStandby=1
           elif args[i+1] == "0" or args[i+1] == "no":
               DoStandby=0
           else:
               usage(args[i+1])
       elif args[i] == "--fencing":
           skipthis=1
           if args[i+1] == "1" or args[i+1] == "yes":
               DoFencing=1
           elif args[i+1] == "0" or args[i+1] == "no":
               DoFencing=0
           else:
               usage(args[i+1])
       elif args[i] == "--suppressmonitoring":
           SuppressMonitoring = 1
       elif args[i] == "--resource-can-stop":
           ResCanStop = 1
       elif args[i] == "-2" or args[i] == "--crm":
           Version = 2
       elif args[i] == "-1" or args[i] == "--classic":
           Version = 1
       elif args[i] == "--clobber-cib" or args[i] == "-c":
           ClobberCIB = 1
       elif args[i] == "--cib-filename":
           skipthis=1
           CIBfilename = args[i+1]
       elif args[i] == "--xmit-loss":
           try:
               float(args[i+1])
           except ValueError:
               print ("--xmit-loss parameter should be float")
               usage(args[i+1])
           skipthis=1
           XmitLoss = args[i+1]
       elif args[i] == "--recv-loss":
           try:
               float(args[i+1])
           except ValueError:
               print ("--recv-loss parameter should be float")
               usage(args[i+1])
           skipthis=1
           RecvLoss = args[i+1]
       elif args[i] == "--choose":
           skipthis=1
           TestCase = args[i+1]
       elif args[i] == "--syslog-facility" or args[i] == "--facility":
           skipthis=1
           LogFacility = args[i+1]
       elif args[i] == "--seed":
           skipthis=1
           Seed=args[i+1]
           HaveSeed = 1
       else:
           NumIter=int(args[i])
     
    if not oprofile:
        oprofile = []

    #
    # This reading of HBconfig here is ugly, and I suppose ought to
    # be done by the Cluster manager.  This would probably mean moving the
    # list of cluster nodes into the ClusterManager class.  A good thought
    # for our Copious Spare Time in the future...
    #
    config = CM_hb.HBConfig(HAdir)
    node_list = config.Parameters["node"]

    if DoBSC:
        NumIter = 2
        Version = 2
        while len(node_list) > 1:
            node_list.pop(len(node_list)-1)

    if LogFacility == None:
        if config.Parameters.has_key("logfacility"): 
            LogFacility = config.Parameters["logfacility"][0]
        else:
            LogFacility = DefaultFacility

    if LimitNodes > 0:
        if len(node_list) > LimitNodes:
            print("Limiting the number of nodes configured=%d (max=%d)"
                  %(len(node_list), LimitNodes))
            while len(node_list) > LimitNodes:
                node_list.pop(len(node_list)-1)

    if StonithParams[0] == "hostlist":
        StonithParams[1] = string.join(node_list, " ")

#    alt_list = []
#    for node in node_list:
#        alt_list.append(string.lower(node))
#    node_list = alt_list

    Environment = CtsLab(node_list)
    Environment["HAdir"] = HAdir
    Environment["ClobberCIB"] = ClobberCIB
    Environment["CIBfilename"] = CIBfilename
    Environment["CIBResource"] = CIBResource 
    Environment["LogFileName"] = LogFile
    Environment["DoStonith"] = DoStonith
    Environment["SyslogFacility"] = LogFacility
    Environment["DoStandby"] = DoStandby
    Environment["DoFencing"] = DoFencing
    Environment["ResCanStop"] = ResCanStop
    Environment["SuppressMonitoring"] = SuppressMonitoring
    Environment["XmitLoss"] = XmitLoss
    Environment["RecvLoss"] = RecvLoss
    Environment["IPBase"] = IPBase
    Environment["SuppressCib"] = SuppressCib
    Environment["DoBSC"]    = 0
    Environment["use_logd"] = 0
    Environment["logfacility"] = LogFacility
    Environment["oprofile"] = oprofile

    if config.Parameters.has_key("use_logd"):
        Environment["use_logd"] = 1
    
    if Version == 2:
        from CM_LinuxHAv2 import LinuxHAv2
        Environment['CMclass']=LinuxHAv2

    if HaveSeed:
        Environment["RandSeed"] = Seed 

    Environment["reset"] = Stonith(sttype=StonithType, pName=StonithParams[0], pValue=StonithParams[1])

    if DoBSC:
        Environment["DoBSC"]       = 1
        Environment["ClobberCIB"]  = 1
        Environment["CIBResource"] = 0 
        Environment["logger"]  = (FileLog(Environment), StdErrLog(Environment))
        scenario = Scenario([ BasicSanityCheck(Environment) ])
    else:
        scenario = Scenario(
            [        InitClusterManager(Environment), PacketLoss(Environment)])
        
    Environment.SupplyDefaults()

    # Your basic start up the world type of test scenario...

    #scenario = Scenario(
    #[        InitClusterManager(Environment)
    #,        PingFest(Environment)])

    # Create the Cluster Manager object
    cm = Environment['CMclass'](Environment)
    if TruncateLog:
        cm.log("Truncating %s" % LogFile)
        lf = open(LogFile, "w");
        if lf != None:
            lf.truncate(0)
            lf.close()

    cm.log(">>>>>>>>>>>>>>>> BEGINNING " + repr(NumIter) + " TESTS ")
    cm.log("HA configuration directory: " + Environment["HAdir"])
    cm.log("System log files: " + Environment["LogFileName"])
    cm.log("Enable Stonith:   %d" % Environment["DoStonith"])
    cm.log("Enable Fencing:   %d" % Environment["DoFencing"])
    cm.log("Enable Standby:   %d" % Environment["DoStandby"])
    cm.log("Enable Resources: %d" % Environment["CIBResource"])
    if Environment.has_key("SuppressMonitoring") \
    and Environment["SuppressMonitoring"]:
        cm.log("Resource Monitoring is disabled")
    cm.ns.WaitForAllNodesToComeUp(config.Parameters["node"])
    cm.log("Cluster nodes: ")
    for node in config.Parameters["node"]:
        (rc, lines) = cm.rsh.remote_py(node, "os", "system", 
                                     "/usr/sbin/crm_uuid")
        if not lines:
            cm.log("    * %s: __undefined_uuid__" % node)
        else:
            out=lines[0]
            out = out[:-1]
            cm.log("    * %s: %s" % (node, out))

    Audits = AuditList(cm)
    Tests = []

    if Environment["DoBSC"]:
        test = BSC_AddResource(cm)
        Tests.append(test)
    elif TestCase != None:
        for test in TestList(cm):
            if test.name == TestCase:
                Tests.append(test)
        if Tests == []:
            usage("--choose: No applicable/valid tests chosen")        
    else:
        Tests = TestList(cm)
    
    if ListTests == 1 :
        cm.log("Total %d tests"%len(Tests))
        for test in Tests :
            cm.log(str(test.name));
        sys.exit(0)
        
    tests = RandomTests(scenario, cm, Tests, Audits)
    Environment.RandomTests = tests
    try :
        overall, detailed = tests.run(NumIter)
    except :
        cm.Env.log("Exception by %s" % sys.exc_info()[0])
        for logmethod in Environment["logger"]:
          traceback.print_exc(50, logmethod)
        
    tests.summarize()
    if tests.Stats["failure"] > 0:
        sys.exit(tests.Stats["failure"])

    elif tests.Stats["success"] != NumIter:
        cm.Env.log("No failure count but success != requested iterations")
        sys.exit(1)