#!/bin/sh --login
set -x
usage="\
Usage: $0 [options] executable [args]
where the options are:
-a account account (default: none)
-b binding run smt binding or not (default:NO)
-d dirin initial directory (default: cwd)
-e envars copy comma-separated environment variables
-g group group name
-i append standard input to command file
-j jobname specify jobname (default: executable basename)
-m machine machine on which to run (default: current)
-n write command file to stdout rather than submitting it
-o output specify output file (default: jobname.out)
-p procs[/nodes[/ppreq]
number of MPI tasks and optional nodes or Bblocking and
ppreq option (N or S) (defaults: serial, Bunlimited, S)
-q queue[/qpreq] queue name and optional requirement, e.g. dev/P
(defaults: 1 if serial or dev if parallel and none)
(queue 3 or 4 is dev or prod with twice tasks over ip)
(options: P=parallel, B=bigmem, b=batch)
-r rmem[/rcpu] resources memory and cpus/task (default: '1024 mb', 1)
-t timew wall time limit in [[hh:]mm:]ss format (default: 900)
-u userid userid to run under (default: self)
-v verbose mode
-w when when to run, in yyyymmddhh[mm], +hh[mm], thh[mm], or
Thh[mm] (full, incremental, today or tomorrow) format
(default: now)
Function: This command submits a job to the batch queue."
subcmd="$*"
stdin=NO
nosub=NO
account=""
binding="NO"
dirin=""
envars=""
group=""
jobname=""
machine=""
output=""
procs=0
nodes=""
ppreq=""
queue=""
qpreq=""
rmem="1024"
rcpu="1"
timew="900"
userid=""
verbose=NO
when=""
while getopts a:b:d:e:g:ij:m:no:p:q:r:t:u:vw: opt;do
case $opt in
a) account="$OPTARG";;
b) binding="$OPTARG";;
d) dirin="$OPTARG";;
e) envars="$OPTARG";;
g) group="$OPTARG";;
i) stdin=YES;;
j) jobname=$OPTARG;;
m) machine="$OPTARG";;
n) nosub=YES;;
o) output=$OPTARG;;
p) procs=$(echo $OPTARG/|cut -d/ -f1);nodes=$(echo $OPTARG/|cut -d/ -f2);ppreq=$(echo $OPTARG/|cut -d/ -f3);;
q) queue=$(echo $OPTARG/|cut -d/ -f1);qpreq=$(echo $OPTARG/|cut -d/ -f2);;
r) rmem=$(echo $OPTARG/|cut -d/ -f1);rcpu=$(echo $OPTARG/|cut -d/ -f2);;
t) timew=$OPTARG;;
u) userid=$OPTARG;;
v) verbose=YES;;
w) when=$OPTARG;;
\?) echo $0: invalid option >&2;echo "$usage" >&2;exit 1;;
esac
done
shift $(($OPTIND-1))
if [[ $# -eq 0 ]];then
echo $0: missing executable name >&2;echo "$usage" >&2;exit 1
fi
exec=$1
if [[ ! -s $exec ]]&&which $exec >/dev/null 2>&1;then
exec=$(which $exec)
fi
shift
args="$*"
bn=$(basename $exec)
export jobname=${jobname:-$bn}
output=${output:-$jobname.out}
myuser=$LOGNAME
myhost=$(hostname)
DATA=/data/users/$LOGNAME/tmp
mkdir -p $DATA
#partition=${partition:-c1ms}
queue=${queue:-batch}
timew=${timew:-01:20:00}
task_node=${task_node:-$procs}
size=$((nodes*task_node))
envars=$envars
threads=${rcpu:-1}
#envars=$envars,mpi_tasks=$procs
#Options
###SBATCH -l partition=c1ms,size=0528,walltime=01:20:00
##SBATCH -l partition=$queue,size=$size,walltime=$timew
##SBATCH -S /bin/sh
export TZ=GMT
cfile=$DATA/sub$$
> $cfile
#echo "#SBATCH -S /bin/sh" >> $cfile
echo "#!/bin/sh --login" >> $cfile
echo "" >> $cfile
echo "#SBATCH -o $output" >> $cfile
echo "#SBATCH --job-name=$jobname" >> $cfile
echo "#SBATCH --time=$timew" >> $cfile
echo "#SBATCH --ntasks=$procs" >> $cfile
echo "#SBATCH --cpus-per-task=$nodes" >> $cfile
echo "#SBATCH --mem-per-cpu=6000" >> $cfile
echo "#SBATCH --partition=s4" >> $cfile
echo "#SBATCH --exclusive" >> $cfile
echo "#SBATCH --export=ALL"
echo "#SBATCH --account=star" >> $cfile
echo "#SBATCH --distribution=block:block" >> $cfile
#echo "#SBATCH -d" >> $cfile
#. $exec >> $cfile
#echo "/bin/sh -x $exec" >> $cfile
echo "" >>$cfile
echo "export MPI_BUFS_PER_PROC=2048" >> $cfile
echo "export MPI_BUFS_PER_HOST=2048" >> $cfile
echo "export MPI_GROUP_MAX=256" >> $cfile
echo "export MPI_MEMMAP_OFF=1" >> $cfile
echo "export MP_STDOUTMODE=ORDERED" >> $cfile
#echo "export KMP_STACKSIZE=2048000" >> $cfile
#echo "export KMP_AFFINITY=scatter" >> $cfile
echo "" >>$cfile
echo "export OMP_NUM_THREADS=$threads" >> $cfile
echo "" >>$cfile
echo ". "$(awk '{ print $1, $2, $3, $4, $5, $6, $7, $8, $9 }' $regdir/regression_var.out) >>$cfile
echo "" >>$cfile
cat $exec >> $cfile
if [[ $nosub = YES ]];then
cat $cfile
exit
elif [[ $verbose = YES ]];then
set -x
cat $cfile
fi
#msub -I partition=$partition,size=$procs,walltime=$walltime $cfile
#if [[ -n $when ]];then
# whena=$when
# if [[ $when = +* ]];then
# hr=$(echo $when|cut -c2-3)
# mn=$(echo $when|cut -c4-5)
# [[ -n $mn ]] || mn=00
# now=$(date -u +"%Y%m%d%H%M")
# ((mn+=$(echo $now|cut -c11-12)))
# [[ $mn -ge 60 ]] && ((hr+=1)) && ((mn-=60))
# [[ $mn -lt 10 ]] && mn=0$mn
# whena=$(/nwprod/util/exec/ndate +$hr $(echo $now|cut -c1-10))$mn
# elif [[ $when = t* ]];then
# hr=$(echo $when|cut -c2-3)
# mn=$(echo $when|cut -c4-5)
# [[ -n $mn ]] || mn=00
# now=$(date -u +"%Y%m%d")
# whena=$now$hr$mn
# elif [[ $when = T* ]];then
# hr=$(echo $when|cut -c2-3)
# mn=$(echo $when|cut -c4-5)
# [[ -n $mn ]] || mn=00
# now=$(date -u +"%Y%m%d%H")
# whena=$(/nwprod/util/exec/ndate +24 $now|cut -c1-8)$hr$mn
# fi
# yr=$(echo $whena|cut -c1-4)
# mo=$(echo $whena|cut -c5-6)
# dy=$(echo $whena|cut -c7-8)
# hr=$(echo $whena|cut -c9-10)
# mn=$(echo $whena|cut -c11-12)
# [[ -n $mn ]] || mn=00
# echo "#@ startdate = $mo/$dy/$yr $hr:$mn"
#fi >>$cfile
if [[ $stdin = YES ]];then
cat
fi >>$cfile
if [[ $nosub = YES ]];then
cat $cfile
exit
elif [[ $verbose = YES ]];then
set -x
cat $cfile
fi
sbatch=${sbatch:-sbatch}
ofile=$DATA/subout$$
>$ofile
chmod 777 $ofile
$sbatch $cfile >$ofile
rc=$?
cat $ofile
if [[ -w $SUBLOG ]];then
jobn=$(grep -i submitted $ofile|head -n1|cut -d\" -f2)
date -u +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG
fi
#rm $cfile $ofile
[[ $MKDATA = YES ]] && rmdir $DATA
exit $rc