nordugrid-arc-arex 5.3.0~rc1-1 / usr / share / arc / submit-boinc-job

#!/bin/bash
# set -x
#
#  Input: path to grami file (same as Globus).
#  This script creates a temporary job script and runs it

echo "----- starting submit_boinc_job -----" 1>&2
joboption_lrms=boinc

# ARC1 passes first the config file.
if [ "$1" = "--config" ]; then shift; ARC_CONFIG=$1; shift; fi

basedir=`dirname $0`
basedir=`cd $basedir > /dev/null && pwd` || exit $?

pkgdatadir="$basedir"

. ${pkgdatadir}/submit_common.sh || exit $?

##############################################################
# Parse grami file, read arc config
##############################################################
init $1
echo runtimedir=$CONFIG_runtimedir
echo sratch=$CONFIG_scratchdir
joboption_directory_orig=$joboption_directory
joboption_directory='`pwd`'
joboption_stdout='`pwd`/'`basename $joboption_stdout`
joboption_stderr='`pwd`/'`basename $joboption_stderr`
read_arc_conf

# make sure session is world-writable
chmod 777 $joboption_directory

RUNTIME_NODE_SEES_FRONTEND=yes

##############################################################
# Zero stage of runtime environments
##############################################################
RTE_stage0
echo "project_root=$PROJECT_ROOT"

##############################################################
# create job script
##############################################################
mktempscript
LRMS_JOB_BOINC="${LRMS_JOB_SCRIPT}.boinc"
touch $LRMS_JOB_BOINC
chmod u+x ${LRMS_JOB_SCRIPT}


##############################################################
# Start job script
##############################################################

N=0
x=$joboption_directory_orig
while [ "$x" != "/" ]
do
 x=`dirname $x`
N=$((N+1))
done

echo '#!/bin/sh' > $LRMS_JOB_SCRIPT

echo "#job script built by grid-manager and input file for BOINC job" >> $LRMS_JOB_SCRIPT
echo "" >> $LRMS_JOB_SCRIPT
cat >> $LRMS_JOB_SCRIPT <<"FMARK"
set -x 
export RUNTIME_CONFIG_DIR=`pwd`/
#tar zxvf RTE.tar.gz
cp ../shared/ATLAS.root ./
#rename root file
#unzip RTE file
FMARK
echo tar --strip-components=$N -xvf *input.tar.gz >> $LRMS_JOB_SCRIPT

##############################################################
# non-parallel jobs
##############################################################

set_count
sourcewithargs_jobscript

##############################################################
# Override umask
##############################################################
echo "" >> $LRMS_JOB_SCRIPT
echo "# Overide umask of execution node (sometime values are really strange)" >> $LRMS_JOB_SCRIPT
echo "umask 077" >> $LRMS_JOB_SCRIPT

##############################################################
# Add environment variables
##############################################################
add_user_env

##############################################################
# Check for existance of executable,
##############################################################
if [ -z "${joboption_arg_0}" ] ; then
  echo 'Executable is not specified' 1>&2
  exit 1
fi

#######################################################################
# copy information useful for transfering files to/from node directly
#######################################################################
if [ "$joboption_localtransfer" = 'yes' ] ; then
   setup_local_transfer
fi

setup_runtime_env

##############################################################
# Add std... to job arguments
##############################################################
include_std_streams

##############################################################
#  Move files to local working directory (job is done on node only)
#  RUNTIME_JOB_DIR -> RUNTIME_LOCAL_SCRATCH_DIR/job_id
##############################################################
move_files_to_node 

echo "" >> $LRMS_JOB_SCRIPT
echo "RESULT=0" >> $LRMS_JOB_SCRIPT
echo "" >> $LRMS_JOB_SCRIPT


#####################################################
#  Download input files
#####################################################
download_input_files

echo "" >> $LRMS_JOB_SCRIPT

echo "if [ \"\$RESULT\" = '0' ] ; then" >> $LRMS_JOB_SCRIPT

##############################################################
#  Runtime configuration
##############################################################
echo RUNTIME_JOB_DIAG='`pwd`/'`basename $joboption_directory_orig`.diag >>$LRMS_JOB_SCRIPT
RTE_stage1 
echo "echo \"runtimeenvironments=\$runtimeenvironments\" >> \"\$RUNTIME_JOB_DIAG\"" >> $LRMS_JOB_SCRIPT

#####################################################
#  Go to working dir and start job
#####################################################
cd_and_run
echo "fi" >> $LRMS_JOB_SCRIPT

## print nodename
cat >> $LRMS_JOB_SCRIPT <<"FMARK"
sed -i -e '/nodename=/d' $RUNTIME_JOB_DIAG
hostname=` grep domain_name init_data.xml |awk -F '>' '{print $2}'|awk -F "<" '{print $1}'|sed -e "s# #_#g"`
username=` grep user_name init_data.xml |awk -F '>' '{print $2}'|awk -F "<" '{print $1}'|sed -e "s# #_#g"`
nodename=$username@$hostname
echo "nodename=$nodename" >> "$RUNTIME_JOB_DIAG"

FMARK

#####################################################
#  Upload output files
#####################################################
upload_output_files
echo "" >> $LRMS_JOB_SCRIPT

##############################################################
#  Runtime (post)configuration at computing node
##############################################################
configure_runtime


##############################################################
#  Move files back to session directory (job is done on node only)
#  RUNTIME_JOB_DIR -> RUNTIME_LOCAL_SCRATCH_DIR/job_id
##############################################################
#move_files_to_frontend

echo 'echo exitcode=$RESULT >> $RUNTIME_JOB_DIAG'  >> $LRMS_JOB_SCRIPT
#############################################################
# zip the result files into 1 file zip.tar.gz
#############################################################
notnull ()
{
   if [ -z $1 ];then
	echo 0
   else
	echo 1
   fi
}
result_list=
i=0
eval opt=\${joboption_outputfile_$i}
ret=`notnull $opt`
while [ $ret = "1" ]
do
   output_file=$(echo $opt|sed -e "s#^/#./#")
   output_file=$(echo $output_file|sed -e "s#@##")
   result_list=$result_list" "$output_file
   i=$((i+1))
   eval opt=\${joboption_outputfile_$i}
   ret=`notnull $opt`
   echo $ret
done

files=$(echo $result_list|tr " " "\n")

cat >> $LRMS_JOB_SCRIPT <<'EOF'
echo "zip all output files"
flist="*.diag "
EOF
echo for f in $files >>$LRMS_JOB_SCRIPT

cat >> $LRMS_JOB_SCRIPT <<'EOF'
do
 if [  -e $f ];then
 flist=$flist" "$f
fi
done
EOF
#echo $flist
cat <<'EOF' >>$LRMS_JOB_SCRIPT
if [ -f output.list ];then
ol=$(awk '{print $1}' output.list)
for i in $ol
do
 if [ -f $i ];then
 flist=$flist" "$i
 fi
done
fi
EOF

echo 'tar czvf result.tar.gz $flist' >>$LRMS_JOB_SCRIPT
echo 

chmod g+r $LRMS_JOB_SCRIPT


#######################################
#  Submit the job
#######################################
## generate RTE file on the fly
joboption_num=0
rflist=$RUNTIME_CONFIG_DIR/APPS/HEP/ATLAS-SITE
eval "var_is_set=\${joboption_runtime_$joboption_num+yes}"
while [ ! -z "${var_is_set}" ] ; do
   eval "var_value=\${joboption_runtime_$joboption_num}"
   if [ -r "$RUNTIME_CONFIG_DIR/${var_value}" ] ; then
      rflist=$rflist" $RUNTIME_CONFIG_DIR/${var_value}"
   fi
   joboption_num=$(( $joboption_num + 1 ))
   eval "var_is_set=\${joboption_runtime_$joboption_num+yes}"
 done
 
RN=0
x=$RUNTIME_CONFIG_DIR
while [ "$x" != "/" ]
do
 x=`dirname $x`
 RN=$((RN+1))
done
sed -i -e "s/#unzip RTE file/tar --strip-components=$RN -zxvf RTE.tar.gz/g" $LRMS_JOB_SCRIPT

echo "job script ${LRMS_JOB_SCRIPT} built" 1>&2
JobId=`basename $joboption_directory_orig`
JobInput=$joboption_directory_orig/$JobId"_input.tar.gz"
wu=$JobId
RTE_FILE="rte_"$wu".tar.gz"
RTE_LOCATION="$joboption_directory_orig/RTE.tar.gz"

echo "#!/bin/bash" >> $LRMS_JOB_BOINC
tflist=""
Root_basename=""
RootFileOrig=""
for file in `ls $joboption_directory_orig`
  do
    [[ $file == *.root.* ]] && echo skip root file $file && RootFileOrig=$file && Root_basename=$JobId"_"$file && continue
    tflist=$tflist" "$joboption_directory_orig/$file
  done

RootFile=$joboption_directory_orig/$RootFileOrig
sed -i -e "s/#rename root file/mv ATLAS.root $RootFileOrig/g" $LRMS_JOB_SCRIPT
echo tar zhcvf $JobInput $tflist
echo "tar zhcvf $JobInput $tflist" >> $LRMS_JOB_BOINC
echo tar zhcvf $RTE_LOCATION $rflist
echo "tar zhcvf $RTE_LOCATION $rflist" >> $LRMS_JOB_BOINC

echo "cd $PROJECT_ROOT " >>$LRMS_JOB_BOINC

JobInput_basename=`basename $JobInput`
Script_basename=`basename $LRMS_JOB_SCRIPT` 


echo "ln $JobInput "'`bin/dir_hier_path '$(basename $JobInput)'`' >> $LRMS_JOB_BOINC
echo "chmod g+r "'`bin/dir_hier_path '$(basename $JobInput)'`' >> $LRMS_JOB_BOINC
echo "cp $LRMS_JOB_SCRIPT " '`bin/dir_hier_path' $(basename $LRMS_JOB_SCRIPT)'`' >>$LRMS_JOB_BOINC
echo "chmod g+r " '`bin/dir_hier_path' $(basename $LRMS_JOB_SCRIPT)'`' >>$LRMS_JOB_BOINC
echo "ln $RTE_LOCATION " '`bin/dir_hier_path' $RTE_FILE '`' >> $LRMS_JOB_BOINC
echo "chmod g+r " '`bin/dir_hier_path' $RTE_FILE '`' >> $LRMS_JOB_BOINC

[ -L $RootFile ] && RootFile=`ls -l $RootFile|awk '{print $11}'`
echo "ln  $RootFile "'`bin/dir_hier_path' $Root_basename '`' >> $LRMS_JOB_BOINC
echo "chmod g+r "'`bin/dir_hier_path ' $Root_basename '`' >> $LRMS_JOB_BOINC

if [ -z $joboption_memory ];then
memreq=2000000000
else
memreq=$((joboption_memory*1000000))
fi
if [ -z $joboption_cputime ];then
maxcputime=$((2*3600*3000000000))
else
maxcputime=$((joboption_cputime*3000000000))
fi

priority=
if [ -z $joboption_priority ]; then
  $priority="--priority $joboption_priority"
fi

echo "bin/create_work -appname $BOINC_APP -wu_name $wu  -wu_template $WU_TEMPLATE -result_template $RESULT_TEMPLATE  --rsc_memory_bound $memreq  --rsc_fpops_est $maxcputime $priority $(basename $JobInput) $Root_basename $RTE_FILE $(basename $LRMS_JOB_SCRIPT)" >> $LRMS_JOB_BOINC

echo 'ret=$?' >>$LRMS_JOB_BOINC
if [ $ret -eq 0 ];then
echo "successfully submitted workunit $wu"
fi
echo 'exit $ret' >>$LRMS_JOB_BOINC
sh $LRMS_JOB_BOINC
#cat $LRMS_JOB_BOINC
rc=$?

if [ $rc -eq 0 ];then
echo "job $wu submitted successfully!" 1>&2
echo "joboption_jobid=$wu" >> $arg_file
fi
echo "----- removing intermediate files ----"
rm -fr $LRMS_JOB_BOINC $LRMS_JOB_ERR $LRMS_JOB_OUT $LRMS_JOB_SCRIPT
#rm -fr $JobInput
echo "----- exiting submit_boinc_job -----" 1>&2
echo "" 1>&2
exit $rc