lustre-utils 1.8.5+dfsg-3ubuntu1 / usr / sbin / lustre_config

#!/bin/bash

# vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4:

#
# lustre_config - format and set up multiple lustre servers from a csv file
#
# This script is used to parse each line of a spreadsheet (csv file) and 
# execute remote commands to format (mkfs.lustre) every Lustre target 
# that will be part of the Lustre cluster.
# 
# In addition, it can also verify the network connectivity and hostnames in 
# the cluster, configure Linux MD/LVM devices and produce High-Availability
# software configurations for Heartbeat or CluManager.
#
################################################################################

# Usage
usage() {
    cat <<EOF

Usage: $(basename $0) [options] <-a|-w|-x> <csv file>

    This script is used to format and set up multiple lustre servers from a
    csv file.

    Options:
    -h          help and examples
    -a          select all the nodes from the csv file to operate on
    -w hostname,hostname,...
                select the specified list of nodes (separated by commas) to
                operate on rather than all the nodes in the csv file
    -x hostname,hostname,...
                exclude the specified list of nodes (separated by commas)
    -t HAtype   produce High-Availability software configurations
                The argument following -t is used to indicate the High-
                Availability software type. The HA software types which 
                are currently supported are: hbv1 (Heartbeat version 1)
                and hbv2 (Heartbeat version 2).
    -n          no net - don't verify network connectivity and hostnames
                in the cluster
    -d          configure Linux MD/LVM devices before formatting the
                Lustre targets
    -f          force-format the Lustre targets using --reformat option
    -m          no fstab change - don't modify /etc/fstab to add the new
                Lustre targets
                If using this option, then the value of "mount options"
                item in the csv file will be passed to mkfs.lustre, else
                the value will be added into the /etc/fstab.
    -u          upgrade Lustre targets from 1.4 to 1.6
    -v          verbose mode
    csv file    a spreadsheet that contains configuration parameters
                (separated by commas) for each target in a Lustre cluster

EOF
}

# Samples 
sample() {
    cat <<EOF

This script is used to parse each line of a spreadsheet (csv file) and 
execute remote commands to format (mkfs.lustre) every Lustre target 
that will be part of the Lustre cluster.

It can also optionally: 
 * upgrade Lustre targets from 1.4 to 1.6
   It does not support "rolling upgrade". Before upgrading, the entire Lustre
   filesystem (all servers and clients) need be shut down. After upgrading, old
   configuration logs for the filesystem would be erased and new configuration
   logs would be regenerated as Lustre servers restart.
 * verify the network connectivity and hostnames in the cluster
 * configure Linux MD/LVM devices
 * modify /etc/modprobe.conf to add Lustre networking info
 * add the Lustre server info to /etc/fstab
 * produce configurations for Heartbeat or CluManager.

There are 5 kinds of line formats in the csv file. They represent the following 
targets:
1) Linux MD device
The format is:
hostname,MD,md name,operation mode,options,raid level,component devices

hostname            hostname of the node in the cluster
MD                  marker of MD device line
md name             MD device name, e.g. /dev/md0
operation mode      create or remove, default is create
options             a "catchall" for other mdadm options, e.g. "-c 128"
raid level          raid level: 0,1,4,5,6,10,linear and multipath
component devices   block devices to be combined into the MD device
                    Multiple devices are separated by space or by using
                    shell expansions, e.g. "/dev/sd{a,b,c}"

2) Linux LVM PV (Physical Volume)
The format is:
hostname,PV,pv names,operation mode,options

hostname            hostname of the node in the cluster
PV                  marker of PV line
pv names            devices or loopback files to be initialized for later
                    use by LVM or to be wiped the label, e.g. /dev/sda
                    Multiple devices or files are separated by space or by
                    using shell expansions, e.g. "/dev/sd{a,b,c}"
operation mode      create or remove, default is create
options             a "catchall" for other pvcreate/pvremove options
                    e.g. "-vv"

3) Linux LVM VG (Volume Group)
The format is:
hostname,VG,vg name,operation mode,options,pv paths

hostname            hostname of the node in the cluster
VG                  marker of VG line
vg name             name of the volume group, e.g. ost_vg
operation mode      create or remove, default is create
options             a "catchall" for other vgcreate/vgremove options
                    e.g. "-s 32M"
pv paths            physical volumes to construct this VG, required by
                    create mode
                    Multiple PVs are separated by space or by using
                    shell expansions, e.g. "/dev/sd[k-m]1"

4) Linux LVM LV (Logical Volume)
The format is:
hostname,LV,lv name,operation mode,options,lv size,vg name

hostname            hostname of the node in the cluster
LV                  marker of LV line
lv name             name of the logical volume to be created (optional)
                    or path of the logical volume to be removed (required
                    by remove mode)
operation mode      create or remove, default is create
options             a "catchall" for other lvcreate/lvremove options
                    e.g. "-i 2 -I 128"
lv size             size [kKmMgGtT] to be allocated for the new LV
                    Default unit is megabytes.
vg name             name of the VG in which the new LV will be created

5) Lustre target
The format is:
hostname,module_opts,device name,mount point,device type,fsname,mgs nids,index,
format options,mkfs options,mount options,failover nids

hostname            hostname of the node in the cluster, must match "uname -n"
module_opts         Lustre networking module options
device name         Lustre target (block device or loopback file)
mount point         Lustre target mount point
device type         Lustre target type (mgs, mdt, ost, mgs|mdt, mdt|mgs)
fsname              Lustre filesystem name, should be limited to 8 characters 
                    Default is "lustre".
mgs nids            NID(s) of remote mgs node, required for mdt and ost targets
                    If this item is not given for an mdt, it is assumed that
                    the mdt will also be an mgs, according to mkfs.lustre.
index               Lustre target index
format options      a "catchall" contains options to be passed to mkfs.lustre
                    "--device-size", "--param", etc. all goes into this item.
mkfs options        format options to be wrapped with --mkfsoptions="" and
                    passed to mkfs.lustre
mount options       If this script is invoked with "-m" option, then the value of
                    this item will be wrapped with --mountfsoptions="" and passed
                    to mkfs.lustre, else the value will be added into /etc/fstab.
failover nids       NID(s) of failover partner node

All the NIDs in one node are delimited by commas (','). When multiple nodes are
specified, they are delimited by a colon (':').

Items left blank will be set to defaults.

Example 1 - Simple, with combo MGS/MDT:
-------------------------------------------------------------------------------
# combo mdt/mgs
lustre-mgs,options lnet networks=tcp,/tmp/mgs,/mnt/mgs,mgs|mdt,,,,--device-size=10240

# ost0
lustre-ost,options lnet networks=tcp,/tmp/ost0,/mnt/ost0,ost,,lustre-mgs@tcp0,,--device-size=10240

# ost1
lustre-ost,options lnet networks=tcp,/tmp/ost1,/mnt/ost1,ost,,lustre-mgs@tcp0,,--device-size=10240
-------------------------------------------------------------------------------

Example 2 - Separate MGS/MDT, two networks interfaces:
-------------------------------------------------------------------------------
# mgs
lustre-mgs1,options lnet 'networks="tcp,elan"',/dev/sda,/mnt/mgs,mgs,,,,--quiet --param="sys.timeout=50",,"defaults,noauto","lustre-mgs2,2@elan"

# mdt
lustre-mdt1,options lnet 'networks="tcp,elan"',/dev/sdb,/mnt/mdt,mdt,lustre2,"lustre-mgs1,1@elan:lustre-mgs2,2@elan",,--quiet --param="lov.stripesize=4194304",-J size=16,"defaults,noauto",lustre-mdt2

# ost
lustre-ost1,options lnet 'networks="tcp,elan"',/dev/sdc,/mnt/ost,ost,lustre2,"lustre-mgs1,1@elan:lustre-mgs2,2@elan",,--quiet,-I 512,"defaults,noauto",lustre-ost2
-------------------------------------------------------------------------------

Example 3 - with combo MGS/MDT failover pair and OST failover pair:
-------------------------------------------------------------------------------
# combo mgs/mdt
lustre-mgs1,options lnet networks=tcp,/tmp/mgs,/mnt/mgs,mgs|mdt,,,,--quiet --device-size=10240,,,lustre-mgs2@tcp0

# combo mgs/mdt backup (--noformat)
lustre-mgs2,options lnet networks=tcp,/tmp/mgs,/mnt/mgs,mgs|mdt,,,,--quiet --device-size=10240 --noformat,,,lustre-mgs1@tcp0

# ost
lustre-ost1,options lnet networks=tcp,/tmp/ost1,/mnt/ost1,ost,,"lustre-mgs1@tcp0:lustre-mgs2@tcp0",,--quiet --device-size=10240,,,lustre-ost2@tcp0

# ost backup (--noformat) (note different device name)
lustre-ost2,options lnet networks=tcp,/tmp/ost2,/mnt/ost2,ost,,"lustre-mgs1@tcp0:lustre-mgs2@tcp0",,--quiet --device-size=10240 --noformat,,,lustre-ost1@tcp0
-------------------------------------------------------------------------------

Example 4 - Configure Linux MD/LVM devices before formatting Lustre targets:
-------------------------------------------------------------------------------
# MD device on mgsnode
mgsnode,MD,/dev/md0,,-q,1,/dev/sda1 /dev/sdb1

# MD/LVM devices on ostnode
ostnode,MD,/dev/md0,,-q -c 128,5,"/dev/sd{a,b,c}"
ostnode,MD,/dev/md1,,-q -c 128,5,"/dev/sd{d,e,f}"
ostnode,PV,/dev/md0 /dev/md1
ostnode,VG,ost_vg,,-s 32M,/dev/md0 /dev/md1
ostnode,LV,ost0,,-i 2 -I 128,300G,ost_vg
ostnode,LV,ost1,,-i 2 -I 128,300G,ost_vg

# combo mgs/mdt
mgsnode,options lnet networks=tcp,/dev/md0,/mnt/mgs,mgs|mdt,,,,--quiet

# ost0
ostnode,options lnet networks=tcp,/dev/ost_vg/ost0,/mnt/ost0,ost,,mgsnode,,--quiet

# ost1
ostnode,options lnet networks=tcp,/dev/ost_vg/ost1,/mnt/ost1,ost,,mgsnode,,--quiet
-------------------------------------------------------------------------------

EOF
    exit 0
}

# Get the library of functions
. /usr/share/lustre/lc_common

#***************************** Global variables *****************************#
declare -a NODE_NAMES               # node names in the failover group
declare -a TARGET_OPTS              # target services in one failover group

CONFIG_MD_LVM=false
MODIFY_FSTAB=true
UPGRADE_TARGET=false
# Get and check the positional parameters
while getopts "aw:x:t:ndfmuhv" OPTION; do
    case $OPTION in
    a)
        [ -z "${SPECIFIED_NODELIST}" ] && [ -z "${EXCLUDED_NODELIST}" ] \
        && USE_ALLNODES=true
        NODELIST_OPT="${NODELIST_OPT} -a"
        ;;
    w)
        USE_ALLNODES=false
        SPECIFIED_NODELIST=$OPTARG
        NODELIST_OPT="${NODELIST_OPT} -w ${SPECIFIED_NODELIST}"
        ;;
    x)
        USE_ALLNODES=false
        EXCLUDED_NODELIST=$OPTARG
        NODELIST_OPT="${NODELIST_OPT} -x ${EXCLUDED_NODELIST}"
        ;;
    t)
        HATYPE_OPT=$OPTARG
        if [ "${HATYPE_OPT}" != "${HBVER_HBV1}" ] \
        && [ "${HATYPE_OPT}" != "${HBVER_HBV2}" ] \
        && [ "${HATYPE_OPT}" != "${HATYPE_CLUMGR}" ]; then
            error_output "Invalid HA software type" \
                      "- ${HATYPE_OPT}!"
            usage 1>&2
            exit 1
        fi
        ;;
    n)
        VERIFY_CONNECT=false
        ;;
    d)
        CONFIG_MD_LVM=true
        ;;
    f)
        REFORMAT_OPTION=$"--reformat "
        ;;
    m)
        MODIFY_FSTAB=false
        ;;
    u)
        UPGRADE_TARGET=true 
        ;;
    h)
        usage
        sample
        ;;
    v)
        VERBOSE_OPT=$" -v"
        VERBOSE_OUTPUT=true
        ;;
    ?)
        usage 1>&2
        exit 1
        ;;
    esac
done

# Toss out the parameters we've already processed
shift  `expr $OPTIND - 1`

# Here we expect the csv file
if [ $# -eq 0 ]; then
    error_output "Missing csv file!"
    usage 1>&2
    exit 1
fi

CSV_FILE=$1

# Construct the command line of mkfs.lustre
construct_mkfs_cmdline() {
    # Check argument
    if [ $# -eq 0 ]; then
        error_output "construct_mkfs_cmdline():"\
                  "Missing argument for function construct_mkfs_cmdline()!"
        return 1
    fi

    declare -i i=$1
    local mgsnids mgsnids_str
    local failnids failnids_str

    if $UPGRADE_TARGET; then
        MKFS_CMD="$TUNEFS --writeconf"
    else
        MKFS_CMD="$MKFS $REFORMAT_OPTION"
    fi

    case "${DEVICE_TYPE[i]}" in
    "ost")
        MKFS_CMD="$MKFS_CMD --ost"
        ;;
    "mdt")
        MKFS_CMD="$MKFS_CMD --mdt"
        $UPGRADE_TARGET && [ -n "${MGS_NIDS[i]}" ] && \
        MKFS_CMD="$MKFS_CMD --nomgs"
        ;;
    "mgs")
        MKFS_CMD="$MKFS_CMD --mgs"
        ;;
    "mdt|mgs" | "mgs|mdt")
        MKFS_CMD="$MKFS_CMD --mgs --mdt"
        ;;
    *)
        error_output "construct_mkfs_cmdline():"\
                  "Invalid device type - \"${DEVICE_TYPE[i]}\"!"
        return 1
        ;;
    esac

    if [ -n "${FS_NAME[i]}" ]; then
        MKFS_CMD="$MKFS_CMD --fsname=${FS_NAME[i]}"
    fi

    if [ -n "${MGS_NIDS[i]}" ]; then
        mgsnids_str=${MGS_NIDS[i]}
        for mgsnids in ${mgsnids_str//:/ }; do
            MKFS_CMD="$MKFS_CMD --mgsnode=$mgsnids"
        done
    fi

    if [ -n "${INDEX[i]}" ]; then
        MKFS_CMD="$MKFS_CMD --index=${INDEX[i]}"
    fi

    if [ -n "${FORMAT_OPTIONS[i]}" ]; then
        MKFS_CMD="$MKFS_CMD ${FORMAT_OPTIONS[i]}"
    fi

    if ! $UPGRADE_TARGET && [ -n "${MKFS_OPTIONS[i]}" ]; then
        MKFS_CMD="$MKFS_CMD --mkfsoptions=\"${MKFS_OPTIONS[i]}\""
    fi

    if [ -n "${MOUNT_OPTIONS[i]}" ] && ! $MODIFY_FSTAB; then
        MKFS_CMD="$MKFS_CMD --mountfsoptions=\"${MOUNT_OPTIONS[i]}\""
    fi

    if [ -n "${FAILOVERS[i]}" ]; then
        failnids_str=${FAILOVERS[i]}
        for failnids in ${failnids_str//:/ }; do
            MKFS_CMD="$MKFS_CMD --failnode=$failnids"
        done
    fi

    MKFS_CMD="$MKFS_CMD ${DEVICE_NAME[i]}"
    return 0
} 

# Get all the node names in this failover group
get_nodenames() {
    # Check argument
    if [ $# -eq 0 ]; then
        error_output "get_nodenames(): Missing"\
                  "argument for function get_nodenames()!"
        return 1
    fi

    declare -i i=$1
    declare -i idx
    local nids

    # Initialize the NODE_NAMES array
    unset NODE_NAMES

    NODE_NAMES[0]=${HOST_NAME[i]}

    idx=1
    for nids in ${FAILOVERS_NAMES[i]//:/ }
    do
        NODE_NAMES[idx]=$(nids2hostname ${nids})
        if [ ${PIPESTATUS[0]} -ne 0 ]; then
            error_output "${NODE_NAMES[idx]}"
            return 1
        fi
    
        idx=$idx+1
    done

    return 0
}

# Verify whether the format line has HA items
is_ha_line() {
    declare -i i=$1

    [ -n "${FAILOVERS[i]}" ] && return 0

    return 1
}

# Produce HA software's configuration files
gen_ha_config() {
    declare -i i=$1
    declare -i idx
    local  cmd_line

    # Prepare parameters
    # Hostnames option
    HOSTNAME_OPT=${HOST_NAME[i]}

    if ! get_nodenames $i; then
        error_output "gen_ha_config(): Can not get the"\
        "failover nodenames from failover nids - \"${FAILOVERS[i]}\" in"\
        "the \"${HOST_NAME[i]}\" failover group!"
        return 1
    fi

    for ((idx = 1; idx < ${#NODE_NAMES[@]}; idx++)); do
        HOSTNAME_OPT=${HOSTNAME_OPT}$":"${NODE_NAMES[idx]}
    done

    # Target devices option
    DEVICE_OPT=" -d "${TARGET_OPTS[0]}
    for ((idx = 1; idx < ${#TARGET_OPTS[@]}; idx++)); do
        DEVICE_OPT=${DEVICE_OPT}" -d "${TARGET_OPTS[idx]}
    done

    # Construct the generation script command line
    case "${HATYPE_OPT}" in
    "${HBVER_HBV1}"|"${HBVER_HBV2}")    # Heartbeat 
        cmd_line=${GEN_HB_CONFIG}$" -r ${HATYPE_OPT} -n ${HOSTNAME_OPT}"
        cmd_line=${cmd_line}${DEVICE_OPT}${VERBOSE_OPT}
        ;;
    "${HATYPE_CLUMGR}")                 # CluManager
        cmd_line=${GEN_CLUMGR_CONFIG}$" -n ${HOSTNAME_OPT}"
        cmd_line=${cmd_line}${DEVICE_OPT}${VERBOSE_OPT}
        ;;
    esac
    
    # Execute script to generate HA software's configuration files
    verbose_output "Generating HA software's configurations in"\
               "${HOST_NAME[i]} failover group..."
    verbose_output "${cmd_line}"
    eval $(echo "${cmd_line}")
    if [ ${PIPESTATUS[0]} -ne 0 ]; then
        return 1
    fi
    verbose_output "Generate HA software's configurations in"\
               "${HOST_NAME[i]} failover group OK"
    
    return 0
}

# Configure HA software
config_ha() {
    if $UPGRADE_TARGET || [ -z "${HATYPE_OPT}" ]; then
        return 0
    fi

    declare -i i j k
    declare -i prim_idx         # Index for PRIM_HOSTNAMES array
    declare -i target_idx       # Index for TARGET_OPTS and HOST_INDEX arrays

    declare -a PRIM_HOSTNAMES   # Primary hostnames in all the failover
                                # groups in the lustre cluster
    declare -a HOST_INDEX       # Indices for the same node in all the 
                                # format lines in the csv file
    local prim_host

    # Initialize the PRIM_HOSTNAMES array
    prim_idx=0
    unset PRIM_HOSTNAMES

    # Get failover groups and generate HA configuration files
    for ((i = 0; i < ${#HOST_NAME[@]}; i++)); do
        prim_host=${HOST_NAME[i]}

        for ((j = 0; j < ${#PRIM_HOSTNAMES[@]}; j++)); do
            [ "${prim_host}" = "${PRIM_HOSTNAMES[j]}" ] && continue 2
        done

        target_idx=0
        unset HOST_INDEX
        unset TARGET_OPTS
        for ((k = 0; k < ${#HOST_NAME[@]}; k++)); do
            if [ "${prim_host}" = "${HOST_NAME[k]}" ] && is_ha_line "${k}"
            then
                HOST_INDEX[target_idx]=$k
                TARGET_OPTS[target_idx]=${DEVICE_NAME[k]}:${MOUNT_POINT[k]}
                target_idx=$(( target_idx + 1 ))
            fi
        done

        if [ ${#TARGET_OPTS[@]} -ne 0 ]; then
            PRIM_HOSTNAMES[prim_idx]=${prim_host}
            prim_idx=$(( prim_idx + 1 ))

            if ! gen_ha_config ${HOST_INDEX[0]}; then
                return 1
            fi
        fi
    done

    if [ ${#PRIM_HOSTNAMES[@]} -eq 0 ]; then
        verbose_output "There are no \"failover nids\" items in the"\
        "csv file. No HA configuration files are generated!"
    fi

    rm -rf ${TMP_DIRS}
    return 0
}

# Execute remote command to add lnet options lines to remote nodes'
# modprobe.conf/modules.conf and format(mkfs.lustre) Lustre targets
mass_config() {
    local COMMAND
    declare -a REMOTE_PID 
    declare -a REMOTE_CMD 
    declare -i pid_num=0
    declare -i i=0
    local checked_hosts=""

    if [ ${#HOST_NAME[@]} -eq 0 ]; then
        verbose_output "There are no Lustre targets specified."
        return 0
    fi

    if ! $UPGRADE_TARGET; then
        # Start lnet network in the MGS node
        start_mgs_lnet || return 1    
    fi

    for ((i = 0; i < ${#HOST_NAME[@]}; i++)); do
        # Construct the command line of mkfs.lustre
        if ! construct_mkfs_cmdline $i; then
            return 1    
        fi

        # create the mount point on the node
        COMMAND="mkdir -p ${MOUNT_POINT[i]}"
        verbose_output "Creating the mount point ${MOUNT_POINT[i]} on" \
                       "${HOST_NAME[i]}"
        ${REMOTE} ${HOST_NAME[i]} "${COMMAND}" >&2 
        if [ ${PIPESTATUS[0]} -ne 0 ]; then
            error_output "mass_config():"\
                 "Failed to execute remote command to"\
                 "create the mountpoint on ${HOST_NAME[i]}!"
            return 1
        fi

        if ! $UPGRADE_TARGET && ! is_mgs_node ${HOST_NAME[i]} && \
        ! host_in_hostlist ${HOST_NAME[i]} $checked_hosts; then
            # Execute remote command to add lnet options lines to 
            # modprobe.conf/modules.conf
            add_module_options $i ${HOST_NAME[i]} || return ${PIPESTATUS[0]}

            # Check lnet networks
            check_lnet $i || return ${PIPESTATUS[0]}

            checked_hosts="$checked_hosts,${HOST_NAME[i]}"
        fi

        # Execute remote command to format or upgrade Lustre target
        local OP
        $UPGRADE_TARGET && OP="Upgrading" || OP="Formatting"
        verbose_output "$OP Lustre target ${DEVICE_NAME[i]} on ${HOST_NAME[i]}..."

        COMMAND="export PATH=\$PATH:/sbin:/usr/sbin; $MKFS_CMD"
        REMOTE_CMD[$pid_num]="$REMOTE ${HOST_NAME[i]} \"$COMMAND\""
        verbose_output "$OP command line is: ${REMOTE_CMD[$pid_num]}"

        $REMOTE ${HOST_NAME[i]} "$COMMAND" &
        REMOTE_PID[$pid_num]=$!
        let pid_num=$pid_num+1
        sleep 1
    done

    # Wait for the exit status of the background remote command
    verbose_output "Waiting for the return of the remote command..."
    fail_exit_status=false
    for ((pid_num = 0; pid_num < ${#REMOTE_PID[@]}; pid_num++)); do
        wait ${REMOTE_PID[${pid_num}]}
        if [ ${PIPESTATUS[0]} -ne 0 ]; then
            error_output "mass_config(): Failed"\
            "to execute \"${REMOTE_CMD[${pid_num}]}\"!"
            fail_exit_status=true
        fi
    done

    if ${fail_exit_status}; then
        return 1
    fi    

    verbose_output "Success on all Lustre targets!"
    return 0
}

# get_mntopts hostname device_name failovers
# Construct the mount options of Lustre target @device_name in host @hostname
get_mntopts() {
    local host_name=$1
    local device_name=$2
    local failovers=$3
    local mnt_opts=
    local ret_str

    [ -n "${failovers}" ] && mnt_opts=defaults,noauto || mnt_opts=defaults

    # Execute remote command to check whether the device
    # is a block device or not
    ret_str=$(${REMOTE} ${host_name} \
            "[ -b ${device_name} ] && echo block || echo loop" 2>&1)
    if [ ${PIPESTATUS[0]} -ne 0 -a -n "${ret_str}" ]; then
        echo "`basename $0`: get_mntopts() error:" \
        "remote command to ${host_name} error: ${ret_str}"
        return 1
    fi

    if [ -z "${ret_str}" ]; then
        echo "`basename $0`: get_mntopts() error: remote error:" \
        "No results from remote!" \
        "Check network connectivity between the local host and ${host_name}!"
        return 1
    fi

    [ "${ret_str}" != "${ret_str#*loop}" ] && mnt_opts=${mnt_opts},loop

    echo ${mnt_opts}
    return 0
}

# Execute remote command to modify /etc/fstab to add the new Lustre targets
modify_fstab() {
    declare -i i
    local mntent mntopts device_name
    local COMMAND

    if ! ${MODIFY_FSTAB}; then
        return 0    
    fi

    for ((i = 0; i < ${#HOST_NAME[@]}; i++)); do
        verbose_output "Modify /etc/fstab of host ${HOST_NAME[i]}"\
                   "to add Lustre target ${DEVICE_NAME[i]}"
        mntent=${DEVICE_NAME[i]}"\t\t"${MOUNT_POINT[i]}"\t\t"${FS_TYPE}

        # Get mount options
        if [ -n "${MOUNT_OPTIONS[i]}" ]; then
            # The mount options already specified in the csv file.
            mntopts="${MOUNT_OPTIONS[i]}"
        else
            mntopts=$(get_mntopts ${HOST_NAME[i]} ${DEVICE_NAME[i]}\
                    ${FAILOVERS[i]})
            if [ ${PIPESTATUS[0]} -ne 0 ]; then
                error_output "${mntopts}"
                return 1
            fi
        fi

        mntent=${mntent}"\t"${mntopts}"\t"0" "0
        verbose_output "`echo -e ${mntent}`"

        # Execute remote command to modify /etc/fstab
        device_name=${DEVICE_NAME[i]//\//\\/}
        COMMAND=". /usr/libexec/lustre/lc_common; \
                sed -i \"/^${device_name}\t/d\" \$(fcanon /etc/fstab); \
                echo -e \"${mntent}\" >> \$(fcanon /etc/fstab)"
        ${REMOTE} ${HOST_NAME[i]} "${COMMAND}" >&2
        if [ ${PIPESTATUS[0]} -ne 0 ]; then
            error_output "modify_fstab():"\
            "Failed to modify /etc/fstab of host ${HOST_NAME[i]}"\
            "to add Lustre target ${DEVICE_NAME[i]}!"
            return 1
        fi
    done

    return 0
}

#********************************* Main Flow **********************************#

# Check the csv file
check_file $CSV_FILE || exit ${PIPESTATUS[0]}

# Get the list of nodes to be operated on
NODES_TO_USE=$(get_nodelist) || error_exit ${PIPESTATUS[0]} "$NODES_TO_USE"

# Check the node list
check_nodelist $NODES_TO_USE || exit ${PIPESTATUS[0]}

if ${VERIFY_CONNECT}; then
# Check the network connectivity and hostnames
    echo "`basename $0`: Checking the cluster network connectivity"\
         "and hostnames..."
    if ! ${VERIFY_CLUSTER_NET} ${NODELIST_OPT} ${VERBOSE_OPT} ${CSV_FILE}; then
        exit 1
    fi
    echo "`basename $0`: Check the cluster network connectivity"\
         "and hostnames OK!"
    echo
fi

if $CONFIG_MD_LVM && ! $UPGRADE_TARGET; then
# Configure Linux MD/LVM devices
    echo "`basename $0`: Configuring Linux MD/LVM devices..."
    if ! ${SCRIPT_CONFIG_MD} ${NODELIST_OPT} ${VERBOSE_OPT} ${CSV_FILE}; then
        exit 1
    fi

    if ! ${SCRIPT_CONFIG_LVM} ${NODELIST_OPT} ${VERBOSE_OPT} ${CSV_FILE}; then
        exit 1
    fi
    echo "`basename $0`: Configure Linux MD/LVM devices OK!"
    echo
fi

# Configure the Lustre cluster
echo "`basename $0`: ******** Lustre cluster configuration BEGIN ********"

get_lustre_items $CSV_FILE || exit ${PIPESTATUS[0]}

check_mgs || exit ${PIPESTATUS[0]}

# Format or upgrade Lustre server targets
mass_config || exit ${PIPESTATUS[0]}

# Modify /etc/fstab to add the new Lustre server targets
modify_fstab || exit ${PIPESTATUS[0]}

# Produce HA software's configuration files
if ! config_ha; then
    rm -rf ${TMP_DIRS}
    exit 1
fi

echo "`basename $0`: ******** Lustre cluster configuration END **********"

exit 0