resource-agents 1:3.9.7-1 / usr / sbin / ocft

#!/bin/bash

# Copyright (c) 2010-2013 Novell Inc, John Shi
#           All Rights Reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of version 2 of the GNU General Public License as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it would be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# Further, this software is distributed without any warranty that it is
# free of the rightful claim of any third person regarding infringement
# or the like.  Any license provided herein, whether implied or
# otherwise, applies only to this software file.  Patent licenses, if
# any, provided herein do not apply to combinations of this program with
# other software, or any other product whatsoever.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write the Free Software Foundation,
# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.

die()
{
  local str
  str="$1"

  echo "ERROR: $str" >&2
  exit 1
}

warn()
{
  local str
  str="$1"

  echo "WARNING: $str" >&2
}

parse_die()
{
  local str
  str="$1"

  agent_parse_finish
  die "${agent}: line ${line_num}: ${str}"
}

# add quotes to string for Here Documents
add_quotes()
{
  local typ str a b
  typ="$1"
  str="$2"

  case "$typ" in
    1) a=\'; b=\";;
    2) a=\"; b=\';;
  esac

  echo "$str" | sed "s/$a/$a$b$a$b$a/g; 1 s/^/$a/; $ s/$/$a/"
}

# split strings
explode()
{
  local str
  str="$1"

  echo "$str" | awk -F'"' '{
    if (NF > 0 && NF%2 == 0)
      exit(1);
    for (i=1; i<=NF; i++) {
      if (i%2 == 0)
        print $i;
      else {
        len = split($i, str, /[ \t]+/);
        for (j=1; j<=len; j++) {
          sb = sub(/#.*/, "", str[j]);
          if (str[j] != "")
            print str[j];
          if (sb)
            exit(0);
        }
      }
    }
  }'
}

# phase 1: parse the string to 'command' and 'argument collection'.
line2trunk()
{
  trunk[0]="${line%%[[:blank:]]*}"
  trunk[1]="${line#*[[:blank:]]}"
}

# phase 2: split the argument collection.
trunk2branch()
{
  local IFS

  # Some of statements need one parameter at least.
  if [ "$line" = "${trunk[0]}" ]; then
    parse_die "missing parameter."
  fi

  IFS=$'\n'
  branch=($(explode "${trunk[1]}"))
  if [ $? -ne 0 ]; then
    parse_die "missing '\"'."
  fi
}

preparse_cfg()
{
  local agent line trunk branch macro num host
  agent="$1"

  if [ ! -r "$opt_cfgsdir/$agent" ]; then
    die "${agent}: configuration file not found."
  fi

  line_num=0
  while read -r line; do
    let line_num++
    num=" $line_num"

    case "$line" in
      ""|\#*) continue;;
    esac

    line2trunk
    case "${trunk[0]}" in
      CASE-BLOCK)
        trunk2branch
        macro="$CASES_DIR/${agent}_macro.${branch[0]}"
        continue
        ;;
      Include|Include@*)
        host=$(echo "${trunk[0]}" | awk -F@ '{print $2}')
        trunk2branch
        if [ ! -r "$CASES_DIR/${agent}_macro.${branch[0]}" ]; then
          parse_die "Macro '${branch[0]}' not found."
        fi
        if [ -n "$host" ]; then
          line="$(sed -e 's/^\([^[:blank:]]*\)@[^[:blank:]]*/\1/' -e "s/^[^[:blank:]]*/&@$host/" "$CASES_DIR/${agent}_macro.${branch[0]}")"
        else
          line="$(<"$CASES_DIR/${agent}_macro.${branch[0]}")"
        fi
        num=
        ;;
      *[!A-Z-]*)
        :
        ;;
      *)
        macro=
        ;;
    esac

    if [ -n "$macro" ]; then
      echo "$line$num" >>"$macro"
    else
      echo "$line$num" >>"$CASES_DIR/${agent}_preparse"
    fi
  done <"$opt_cfgsdir/$agent"
}

case_parse_finish()
{
  local host

  if [ -n "$sh" ]; then
    cat >>$sh <<EOF
if [ -n "\$__OCFT__VERBOSE" ]; then
  echo
fi
# Cleanup and exit
EOF
    for host in $hosts; do
      echo "backbash_stop $host" >>$sh
    done
    echo "quit 0" >>$sh
  fi

  atexit_num=0
  hosts=
  sh=
}

init_cfg_vars()
{
  cfg_agent=
  cfg_agent_root=
  cfg_install_package=()
  cfg_hang_timeout=20
}

agent_parse_finish()
{
  local suf

  for suf in preparse setup cleanup var hosts; do
    rm -f $CASES_DIR/${agent}_$suf
  done
  rm -f $CASES_DIR/${agent}_macro.*
  init_cfg_vars
}

need_make()
{
  local src_time obj_time

  if [ ! -f "$CASES_DIR/0_${agent}.sh" ]; then
    return 0
  fi

  src_time=$(stat -c '%Y' "$opt_cfgsdir/$agent")
  obj_time=$(stat -c '%Y' "$CASES_DIR/0_${agent}.sh")

  test $src_time -ge $obj_time
}

parse_cfg()
{
  local agents i line stat sh trunk branch atexit_num host hosts

  if [ $# -eq 0 ]; then
    agents=($opt_cfgsdir/*)
  else
    agents=("$@")
  fi

  for agent in "${agents[@]}"; do
    agent="$(basename "$agent")"

    if ! need_make; then
      continue
    fi

    agent_obj_clean $agent
    agent_parse_finish

    i=0

    echo "Making '$agent': "
    preparse_cfg "$agent"
    while read -r line; do
      line_num="${line##* }"
      line="${line% *}"
      line2trunk

      # state switch
      case "${trunk[0]}" in
        CONFIG)
          case_parse_finish
          stat=1
          continue
          ;;
        VARIABLE)
          case_parse_finish
          stat=2
          continue
          ;;
        SETUP-AGENT)
          case_parse_finish
          stat=3
          continue
          ;;
        CLEANUP-AGENT)
          case_parse_finish
          stat=4
          continue
          ;;
        CASE)
          case_parse_finish
          trunk2branch
          echo "    - case ${i}: ${branch[0]}"
          sh="$CASES_DIR/${i}_${agent}.sh"
          cat >$sh <<EOF
#!/bin/bash

# Agent:   $cfg_agent
# Summary: ${branch[0]}

. $OCFT_DIR/caselib || {
  echo "ERROR: '$OCFT_DIR/caselib' not found."
  exit 2
}

$(test -r $CASES_DIR/${agent}_var && cat $CASES_DIR/${agent}_var)

__OCFT__MYROOT="$cfg_agent_root"

$(test -n "$cfg_install_package" && echo "agent_install ${cfg_install_package[@]}")

if [ -n "\$__OCFT__VERBOSE" ]; then
  echo -e $(add_quotes 1 "Starting '\\033[33m${agent}\\033[0m' case $i '\\033[33m${branch[0]}\\033[0m':")
else
  echo -n "${agent}: ${branch[0]} - "
fi
EOF
          chmod a+x $sh

          let i++
          stat=5
          continue
          ;;
      esac

      case "$stat" in
        1)
          case "${trunk[0]}" in
            Agent)
              trunk2branch
              cfg_agent="${branch[0]}"
              ;;
            AgentRoot)
              trunk2branch
              cfg_agent_root="${branch[0]}"
              ;;
            InstallPackage)
              trunk2branch
              cfg_install_package=(${cfg_install_package[@]} ${branch[@]})
              ;;
            HangTimeout)
              trunk2branch
              if ! echo "${branch[0]}" | grep -qxE '[0-9]+'; then
                parse_die "numeric argument required."
              fi
              cfg_hang_timeout="${branch[0]}"
              ;;
            *)
              parse_die "unimplemented statement: ${trunk[0]}"
              ;;
          esac
          ;;
        2)
          if echo "$line" | grep -q '^__OCFT__'; then
            parse_die "reserved key word '__OCFT__'."
          fi
          echo "declare $line" >>$CASES_DIR/${agent}_var
          ;;
        3)
          echo "$line" >>$CASES_DIR/${agent}_setup
          ;;
        4)
          echo "$line" >>$CASES_DIR/${agent}_cleanup
          ;;
        5)
          host=$(echo ${trunk[0]} | awk -F@ '{print $2}')
          if [ -n "$host" ]; then
            if ! echo "$hosts" | grep -q "$host"; then
              echo "$host" >>$CASES_DIR/${agent}_hosts
              hosts=$hosts$'\n'$host
              cat >>$sh <<EOF
# Initialize remote shell
backbash_start $host
backbash $host <<CMD
__OCFT__VERBOSE=\$__OCFT__VERBOSE
CMD
backbash $host <$OCFT_DIR/caselib
backbash $host <<'CMD'
$(test -r $CASES_DIR/${agent}_var && cat $CASES_DIR/${agent}_var)

__OCFT__MYROOT="$cfg_agent_root"
__OCFT__showhost="${host}: "

$(test -n "$cfg_install_package" && echo "agent_install ${cfg_install_package[@]}")
CMD
EOF
            fi
          fi

          echo "
# CASE statement: $line" >>$sh

          if [ -n "$host" ]; then
            echo "backbash $host <<'CMD'" >>$sh
          fi

          case "${trunk[0]}" in
            Env|Env@*)
              cat >>$sh <<EOF
if [ -n "\$__OCFT__VERBOSE" ]; then
  echo $(add_quotes 2 "    \${__OCFT__showhost}Setting agent environment:    export ${trunk[1]}")
fi
export ${trunk[1]}
check_success \$? $(add_quotes 1 "export ${trunk[1]}")
EOF
              ;;
            Unenv|Unenv@*)
              cat >>$sh <<EOF
if [ -n "\$__OCFT__VERBOSE" ]; then
  echo $(add_quotes 2 "    \${__OCFT__showhost}Removing agent environment:   unset ${trunk[1]}")
fi
unset ${trunk[1]}
check_success \$? $(add_quotes 1 "unset ${trunk[1]}")
EOF
              ;;
            AgentRun|AgentRun@*)
              trunk2branch
              if [ -z "${branch[1]}" ]; then
                if [ "${branch[0]}" = "start" ]; then
                  cat >>$sh <<EOF
agent_run $(add_quotes 1 "$cfg_agent") monitor $cfg_hang_timeout
__OCFT__rc=\$?

if [ \$__OCFT__rc -eq \$OCF_SUCCESS -o \$__OCFT__rc -eq \$OCF_RUNNING_MASTER ]; then
  : #The status I want, so I can do nothing.
elif [ \$__OCFT__rc -eq \$OCF_NOT_RUNNING ]; then
  if [ -n "\$__OCFT__VERBOSE" ]; then
    echo $(add_quotes 2 "    \${__OCFT__showhost}Running agent:                ./$cfg_agent ${branch[0]}")
  fi
  agent_run $(add_quotes 1 "$cfg_agent") start $cfg_hang_timeout
  check_success \$? $(add_quotes 1 "./$cfg_agent ${branch[0]}")
else
  check_success \$__OCFT__rc $(add_quotes 1 "./$cfg_agent monitor")
fi
EOF
                elif [ "${branch[0]}" = "stop" ]; then
                  cat >>$sh <<EOF
agent_run $(add_quotes 1 "$cfg_agent") monitor $cfg_hang_timeout
__OCFT__rc=\$?

if [ \$__OCFT__rc -eq \$OCF_NOT_RUNNING ]; then
  : #The status I want, so I can do nothing.
elif [ \$__OCFT__rc -eq \$OCF_SUCCESS -o \$__OCFT__rc -eq \$OCF_RUNNING_MASTER ]; then
  if [ -n "\$__OCFT__VERBOSE" ]; then
    echo $(add_quotes 2 "    \${__OCFT__showhost}Running agent:                ./$cfg_agent ${branch[0]}")
  fi
  agent_run $(add_quotes 1 "$cfg_agent") stop $cfg_hang_timeout
  check_success \$? $(add_quotes 1 "./$cfg_agent ${branch[0]}")
else
  check_success \$__OCFT__rc $(add_quotes 1 "./$cfg_agent monitor")
fi
EOF
                elif [ "${branch[0]}" = "monitor" ]; then
                  cat >>$sh <<EOF
if [ -n "\$__OCFT__VERBOSE" ]; then
  echo $(add_quotes 2 "    \${__OCFT__showhost}Running agent:                ./$cfg_agent ${branch[0]}")
fi
agent_run $(add_quotes 1 "$cfg_agent") $(add_quotes 1 "${branch[0]}") $cfg_hang_timeout
EOF
                else
                  cat >>$sh <<EOF
if [ -n "\$__OCFT__VERBOSE" ]; then
  echo $(add_quotes 2 "    \${__OCFT__showhost}Running agent:                ./$cfg_agent ${branch[0]}")
fi
agent_run $(add_quotes 1 "$cfg_agent") $(add_quotes 1 "${branch[0]}") $cfg_hang_timeout
check_success \$? $(add_quotes 1 "./$cfg_agent ${branch[0]}")
EOF
                fi
              else
                cat >>$sh <<EOF
test -n $(add_quotes 2 "\$${branch[1]}")
check_success \$? $(add_quotes 1 "test -n \"\$${branch[1]}\"")
if [ -n "\$__OCFT__VERBOSE" ]; then
  echo $(add_quotes 2 "    \${__OCFT__showhost}Running agent:                ./$cfg_agent ${branch[0]}")
fi
agent_run $(add_quotes 1 "$cfg_agent") $(add_quotes 1 "${branch[0]}") $cfg_hang_timeout
__OCFT__ret=\$?
if [ -n "\$__OCFT__VERBOSE" ]; then
  echo -n "    \${__OCFT__showhost}Checking return value:"
fi
if [ -n "\${__OCFT__retval[__OCFT__ret]}" ]; then
  __OCFT__retstr="\${__OCFT__retval[__OCFT__ret]}"
else
  __OCFT__retstr=\$__OCFT__ret
fi
if [ \$__OCFT__ret -eq \$${branch[1]} ]; then
  if [ -n "\$__OCFT__VERBOSE" ]; then
    echo -e $(add_quotes 2 "        \\033[32mOK\\033[0m. The return value '\\033[34m\$__OCFT__retstr\\033[0m' == '\\033[34m${branch[1]}\\033[0m'")
  else
    echo -e "\\033[32mOK\\033[0m."
  fi
else
  if [ -n "\$__OCFT__VERBOSE" ]; then
    echo -en $(add_quotes 2 "        \\033[31mFAILED\\033[0m. The return value '\\033[34m\$__OCFT__retstr\\033[0m' != '\\033[34m${branch[1]}\\033[0m'. ")
  else
    echo -en "\\033[31mFAILED\\033[0m. Agent returns unexpected value: '\$__OCFT__retstr'. "
  fi
  echo "See details below:"
  cat /tmp/.ocft_runlog
  echo
  quit 1
fi
EOF
              fi
              ;;
            Bash|Bash@*)
              cat >>$sh <<EOF
if [ -n "\$__OCFT__VERBOSE" ]; then
  echo $(add_quotes 2 "    \${__OCFT__showhost}Setting system environment:   ${trunk[1]}")
fi
${trunk[1]}
check_success \$? $(add_quotes 1 "${trunk[1]}")
EOF
              ;;
            BashAtExit|BashAtExit@*)
              let atexit_num++
              cat >>$sh <<EOF
atexit${atexit_num}()
{
  if [ -n "\$__OCFT__VERBOSE" ]; then
    echo $(add_quotes 2 "    \${__OCFT__showhost}Setting system environment:   ${trunk[1]}")
  fi
  ${trunk[1]}
}
let __OCFT__atexit_num++
EOF
              ;;
            *)
              parse_die "unimplemented statement: ${trunk[0]}"
              ;;
          esac
          if [ -n "$host" ]; then
            echo 'CMD' >>$sh
          fi
          ;;
        *)
          parse_die "unimplemented statement: ${trunk[0]}"
          ;;
      esac
    done <$CASES_DIR/${agent}_preparse

    if [ -r "$CASES_DIR/${agent}_setup" ]; then
      cat >$CASES_DIR/setup_${agent}.sh <<EOF
#!/bin/bash

# Agent:   $cfg_agent
# Summary: SETUP before test

echo "Initializing '$cfg_agent' ..."

. $OCFT_DIR/caselib || {
  echo "ERROR: '$OCFT_DIR/caselib' not found."
  exit 2
}

$(test -r "$CASES_DIR/${agent}_var" && cat $CASES_DIR/${agent}_var)
$(test -n "$cfg_install_package" && echo "agent_install ${cfg_install_package[@]}")

EOF
      for host in $(test -r $CASES_DIR/${agent}_hosts && cat $CASES_DIR/${agent}_hosts); do
        cat >>$CASES_DIR/setup_${agent}.sh <<EOF
# Initialize remote shell
backbash_start $host
backbash $host <<CMD
__OCFT__VERBOSE=\$__OCFT__VERBOSE
CMD
backbash $host <$OCFT_DIR/caselib
backbash $host <<'CMD'
$(test -r "$CASES_DIR/${agent}_var" && cat $CASES_DIR/${agent}_var)

__OCFT__MYROOT="$cfg_agent_root"
__OCFT__showhost="${host}: "

$(test -n "$cfg_install_package" && echo "agent_install ${cfg_install_package[@]}")

$(cat $CASES_DIR/${agent}_setup)
check_success \$? "SETUP-AGENT"
CMD
backbash_stop $host 
EOF
      done
      cat >>$CASES_DIR/setup_${agent}.sh <<EOF
$(cat $CASES_DIR/${agent}_setup)
check_success \$? "SETUP-AGENT"

echo "Done."
echo
quit 0
EOF
      chmod a+x $CASES_DIR/setup_${agent}.sh
    fi

    if [ -r "$CASES_DIR/${agent}_cleanup" ]; then
      cat >$CASES_DIR/cleanup_${agent}.sh <<EOF
#!/bin/bash

# Agent:   $cfg_agent
# Summary: CLEANUP after test

echo "Cleaning '$cfg_agent' ..."

. $OCFT_DIR/caselib || {
  echo "ERROR: '$OCFT_DIR/caselib' not found."
  exit 2
}

$(test -r "$CASES_DIR/${agent}_var" && cat $CASES_DIR/${agent}_var)
$(test -n "$cfg_install_package" && echo "agent_install ${cfg_install_package[@]}")

$(cat $CASES_DIR/${agent}_cleanup)
check_success \$? "CLEANUP-AGENT"

EOF
      for host in $(test -r $CASES_DIR/${agent}_hosts && cat $CASES_DIR/${agent}_hosts); do
        cat >>$CASES_DIR/cleanup_${agent}.sh <<EOF
# Initialize remote shell
backbash_start $host
backbash $host <<CMD
__OCFT__VERBOSE=\$__OCFT__VERBOSE
CMD
backbash $host <$OCFT_DIR/caselib
backbash $host <<'CMD'
$(test -r "$CASES_DIR/${agent}_var" && cat $CASES_DIR/${agent}_var)

__OCFT__MYROOT="$cfg_agent_root"
__OCFT__showhost="${host}: "

$(test -n "$cfg_install_package" && echo "agent_install ${cfg_install_package[@]}")

$(cat $CASES_DIR/${agent}_cleanup)
check_success \$? "CLEANUP-AGENT"
CMD
backbash_stop $host 
EOF
      done
      cat >>$CASES_DIR/cleanup_${agent}.sh <<EOF
echo "Done."
echo
quit 0
EOF
      chmod a+x $CASES_DIR/cleanup_${agent}.sh
    fi
    case_parse_finish
    agent_parse_finish

  done
}

start_test()
{
  local sh shs testsh agents line ret
  local rc=0
  local varlib
  local rc_f

  if ! cd $CASES_DIR >/dev/null 2>&1; then
    die "cases directory not found."
  fi

  if [ ! -d logs ]; then
    mkdir logs
  fi

  export __OCFT__VERBOSE=$opt_verbose

  if [ $# -eq 0 ]; then
    agents=($(ls -1 *.sh 2>/dev/null | sed 's/.*_\([^_]*\)\.sh$/\1/' | sort | uniq))
  else
    agents=("$@")
  fi

  for shs in "${agents[@]}"; do
    if [ -z "$opt_incremental" ]; then
      testsh="setup_${shs}.sh
              $(ls -1 [0-9]*_${shs}.sh 2>/dev/null | sort -n)
              cleanup_${shs}.sh"
    else
      testsh="setup_${shs}.sh
              $(ls -1 [0-9]*_${shs}.retest 2>/dev/null | sed 's/retest$/sh/' | sort -n)
              cleanup_${shs}.sh"
    fi

    if [ -n "$opt_trace_ra" ]; then
      varlib=${HA_VARLIB:="/var/lib/heartbeat"}
      export OCF_RESKEY_trace_ra=1
      echo "RA trace on, output in $varlib/trace_ra"
    fi

    rc_f=`mktemp`
    (for sh in $testsh; do
      if [ -r "$sh" ]; then
        if [ -n "$opt_trace_ra" ]; then
          export OCF_RESOURCE_INSTANCE="`echo $sh | sed 's/_.*//'`"
        fi
        ./$sh
        ret=$?

        case "$sh" in
          setup*)
              rc=$((rc|ret))
              if [ $ret -ne 0 ]; then
                warn "SETUP failed, break all tests of '$shs'."
                break
              fi
              ;;
          cleanup*)
              if [ $ret -ne 0 ]; then
                warn "CLEANUP failed."
              fi
              ;;
          [0-9]*)
              case $ret in
                3) die "core function failed, break all tests." ;;
                2) warn "core function failed, break all tests of '$shs'."; break ;;
                1) touch ${sh%.*}.retest ;;
                0) rm -f ${sh%.*}.retest ;;
              esac
              rc=$((rc|ret))
              ;;
        esac
      fi
    done 2>&1; echo $rc > $rc_f) | while read -r line; do
      echo "$line"
      echo "$(date '+%F %T'): $line" | cat -A |
      sed -r 's/\^\[\[[0-9]+m|\^I|.$//g' >>logs/$shs.log
    done
  done
  rc=`cat $rc_f`
  rm -f $rc_f
  return $rc
}

agent_clean()
{
  local typ ra
  typ=$1
  
  shift

  if [ $# -eq 0 ]; then
    rm -f $CASES_DIR/*.$typ
  else
    for ra in "$@"; do
      rm -f $CASES_DIR/*_${ra}.$typ
    done
  fi
}

agent_retest_clean()
{
  agent_clean retest "$@"
}

agent_obj_clean()
{
  agent_clean sh "$@"
}

usage()
{
  cat <<EOF
$0 ACTION [OPTION] [agent1 [agent2] [...]]
ACTIONs include:
     make [-d dir]   Generate the testing shell scripts.
                       -d  The directory that contains 
           configuration of cases.
     test [-v|-i|-X]  Execute the testing shell scripts.
                       -v  Verbose output mode.
                       -i  Incremental mode, skip case 
                           which succeeded. If cleaning 
                           the status of incremental mode
                           is needed, try to '$0 clean RA_NAME'.
                       -X  Trace the RA
     clean           Delete the testing shell scripts.
     help [-v]       Show this help and exit.
                       -v  Show HOWTO and exit.
Version 0.44
See '$OCFT_DIR/README' for detail.
EOF
}

howto()
{
  cat <<EOF
HOW TO USE THIS TOOL

  - Ocft is a testing tool for resource agents. Instead of the policy of HA,
    it mainly concerns whether resource agents run correct locally. It can 
    design types of complicated environments to test the reliability of 
    resource agents. Precisely, it is to display whether resource agents can 
    return to correct or expected value. The advantage of the tool provides 
    us with competence to design conditions which can be recorded or reproduced. 
    Hence it is useful to debuggers.

* Components
    ** Test case generator (/usr/sbin/ocft)
      - Turning configuration files of test case to executable scripts.

    ** Configuration file  ($CONFIGS_DIR/)
      - Every configuration file directs only one resource agent and share the same 
        name with resource agent but contains more test cases.

    ** The testing script  ($CASES_DIR/)
      - After the generator reads configuration files and generates many testing 
        scripts and the script is underway, the test begins.

* How to customize the environment of testing
  - Ocft designs the running conditions through two ways, one is changing the 
    environment variables of resource agents (it is the interface left by OCF itself), 
    the other is modifying the OS environment of resource agents, such as altering 
    the permission of some key file or IP address of the machine.

* How to test
  - Firstly, you need to sketch the all complex and uncommon environments against 
    a certain resource agent and keep in mind what consequences may be caused by 
    these uncommon environments. 
    Secondly, write the designed conditions and foreknown consequences into 
    configuration files, and then run the generator to translate the test case to 
    executable scripts. 
    Finally, you need running these scripts to observe the output and learn 
    the running status of each test case, which will compares the predicated result 
    with the actual one. If they differ, you will be able to find the bugs of the 
    resource agent.
  - All of the output with test will be recorded into the log files, you can find them
    in $CASES_DIR/logs.
EOF
}


export LANG=C

# system variable
OCFT_DIR=/usr/share/resource-agents/ocft
CONFIGS_DIR=/usr/share/resource-agents/ocft/configs
CASES_DIR=/var/lib/resource-agents/ocft/cases

# global variable
agent=
line_num=

# default configuration
init_cfg_vars

# default option
opt_verbose=
opt_incremental=
opt_cfgsdir=$CONFIGS_DIR

command="$1"
shift

case "$command" in
  make)
    if [ "$1" = "-d" ]; then
      if [ ! -d "$2" ]; then
        usage
        exit 1
      fi
      opt_cfgsdir="$2"
      shift 2
    fi
    if [ ! -d "$CASES_DIR" ]; then
      mkdir -p "$CASES_DIR" || die "Can not create directory: ${CASES_DIR}."
    fi
    parse_cfg "$@"
    ;;
  test)
    for v in 1 2 3; do
      case "$1" in
        -v)
          opt_verbose=1
          shift
          ;;
        -X)
          opt_trace_ra=1
          shift
          ;;
        -i)
          opt_incremental=1
          shift
          ;;
        -*)
          die "bad option $1"
          ;;
      esac
    done
    start_test "$@"
    ;;
  clean)
    agent_obj_clean "$@"
    agent_retest_clean "$@"
    ;;
  help)
    if [ "$1" = "-v" ]; then
      howto
    else
      usage
    fi
    exit 0
    ;;
  *)
    usage
    exit 1
    ;;
esac

# vim:ts=2:sw=2:et: