#!/bin/ksh
set -x
#
# May 28, 2013 - Shrinivas Moorthi :now updated for lsf9.1.1 - should handle coupled case also
#
usage="\
Usage:  $0 [options] executable [args]
      where the options are:
      -a account        account (default: none)
      -b binding        run smt binding or not (default:NO)
      -d dirin          initial directory (default: cwd)
      -e envars         copy comma-separated environment variables
      -g group          group name
      -i                append standard input to command file
      -j jobname        specify jobname (default: executable basename)
#     -m machine        machine on which to run (default: current)
      -m mpiver         mpi version (poe or intelmpi) (default: poe)
      -n                write command file to stdout rather than submitting it
      -o output         specify output file (default: jobname.out)
      -p procs[/nodes[/ppreq]
                        number of MPI tasks and optional nodes or Bblocking and
                        ppreq option (N or S) (defaults: serial, Bunlimited, S)
      -q queue[/qpreq]  queue name and optional requirement, e.g. dev/P
                        (defaults: 1 if serial or dev if parallel and none)
                        (queue 3 or 4 is dev or prod with twice tasks over ip)
                        (options: P=parallel, B=bigmem, b=batch)
      -r rmem[/rcpu[/pe_node]  resources memory and cpus/task and cores per node                       (default: '1024 mb', 1, and 16)
      -t timew          wall time limit in [[hh:]mm:]ss format (default: 900)
      -u userid         userid to run under (default: self)
      -v                verbose mode
      -w when           when to run, in yyyymmddhh[mm], +hh[mm], thh[mm], or
                        Thh[mm] (full, incremental, today or tomorrow) format
                        (default: now)

       You can also export variables \"TASK_GEOMETRY\" (default:-\"NONE\") and
        \"INHERIT_ENV\" (default:-\"YES\") - Set this variable to \"NO\" and
        export it if you do not want the next job to inherit current job
        environment.

        Other environmental variables which can be exported from outside are:

          \"KMP_STACKSIZE\"           (default:-\"1024m\")
          \"MP_EUIDEVELOP\"           (default:-\"NULL\")
          \"F_UFMTENDIAN\"            (default:-\"NULL\")
          \"MPICH_ALLTOALL_THROTTLE\" (default:-\"NULL\")
          \"MP_SINGLE_THREAD\"        (default:-\"NULL\")
          \"MP_EAGER_LIMIT\"          (default:-\"NULL\")
          \"MP_USE_BULK_XFER\"        (default:-\"NULL\")
          \"MP_COLLECTIVE_OFFLOAD\"   (default:-\"NULL\")
          \"MP_SHARED_MEMORY\"        (default:-\"NULL\")
          \"MP_MPILIB\"               (default:-\"NULL\")
          \"MP_LABELIO\"              (default:-\"NULL\")
          \"MP_STDOUTMODE\"           (default:-\"NULL\")
          \"DATA\"                    (default:-\"/stmpp1/$LOGNAME/sub\"
                                       - deleted at the end if created)

Function:  This command submits a job to the batch queue."
subcmd="$*"
stdin=NO
nosub=NO
account=""
binding="NO"
dirin=""
envars=""
group=""
jobname=""
machine=""
mpiver=""
output=""
procs=0
nodes=""
ppreq=""
queue=""
qpreq=""
rmem="1024"
rcpu="1"
pe_node="16"
timew="900"
userid=""
verbose=NO
when=""
while getopts a:b:d:e:g:ij:m:no:p:q:r:t:u:vw: opt;do
  case $opt in
    a) account="$OPTARG";;
    b) binding="$OPTARG";;
    d) dirin="$OPTARG";;
    e) envars="$OPTARG";;
    g) group="$OPTARG";;
    i) stdin=YES;;
    j) jobname=$OPTARG;;
#   m) machine="$OPTARG";;
    m) mpiver="$OPTARG";;
    n) nosub=YES;;
    o) output=$OPTARG;;
    p) procs=$(echo $OPTARG/|cut -d/ -f1);nodes=$(echo $OPTARG/|cut -d/ -f2);ppreq=$(echo $OPTARG/|cut -d/ -f3);;
    q) queue=$(echo $OPTARG/|cut -d/ -f1);qpreq=$(echo $OPTARG/|cut -d/ -f2);;
    r) rmem=$(echo $OPTARG/|cut -d/ -f1);rcpu=$(echo $OPTARG/|cut -d/ -f2);pe_node=$(echo $OPTARG/|cut -d/ -f3);;
    t) timew=$OPTARG;;
    u) userid=$OPTARG;;
    v) verbose=YES;;
    w) when=$OPTARG;;
    \?) echo $0: invalid option >&2;echo "$usage" >&2;exit 1;;
  esac
done
shift $(($OPTIND-1))
if [[ $# -eq 0 ]];then
  echo $0: missing executable name >&2;echo "$usage" >&2;exit 1
fi
exec=$1
if [[ ! -s $exec ]]&&which $exec >/dev/null 2>&1;then
  exec=$(which $exec)
fi
shift
args="$*"
bn=$(basename $exec)
export jobname=${jobname:-$bn}
output=${output:-$jobname.out}
myuser=$LOGNAME
myhost=$(hostname)
mpiver=${mpiver:-poe}
envars=$envars

#DATA=/lustre/fs/scratch/$LOGNAME/stmpp1
DATA=${DATA:-/stmpp1/$LOGNAME/sub}
if [ -s $DATA ] ; then
  MKDATA=NO
else
  mkdir -p $DATA
  MKDATA=YES
fi

queue=${queue:-dev}
timew=${timew:-01:20}
timew=$(echo $timew |cut -d: -f1):$(echo $timew |cut -d: -f2)
threads=${rcpu:-1}


max_core=${max_core:-16}
task_node=${pe_node:-${task_node:-$max_core}}

export INHERIT_ENV=${INHERIT_ENV:-YES}
export TASK_GEOMETRY=${TASK_GEOMETRY:-NONE}
if [ "$TASK_GEOMETRY" = NONE ] ; then
 if [ $nodes -eq 1 ] ; then
  task_node=$procs
 fi
 tot_size=$((procs*nodes))
else
 tot_size=$((task_node*nodes))
fi
if [ $((task_node*threads)) -gt $max_core ]; then
   core=cpu
   echo "Hyper-threading is used - setting core=$core"
fi
export core=${core:-core}

export KMP_STACKSIZE=${KMP_STACKSIZE:-1024m}

export TZ=GMT
cfile=$DATA/sub$$
> $cfile

if [ $INHERIT_ENV = YES ] ; then
 echo "#!/bin/ksh"                                                >> $cfile
else
 echo "#!/bin/ksh --login"                                        >> $cfile
 echo "#BSUB -L /bin/ksh"                                         >> $cfile
fi
echo "#BSUB -a $mpiver"                                           >> $cfile
echo "#BSUB -P $account"                                          >> $cfile
echo "#BSUB -e $output"                                           >> $cfile
echo "#BSUB -o $output"                                           >> $cfile
echo "#BSUB -cwd $dirin"                                          >> $cfile
##echo "#BSUB -o $output.%J"                                      >> $cfile
echo "#BSUB -J $jobname"                                          >> $cfile
echo "#BSUB -network type=sn_all:mode=US"                         >> $cfile
echo "#BSUB -q $queue"                                            >> $cfile
echo "#BSUB -n $tot_size"                                         >> $cfile
if [ $mpiver = openmp ] ; then
 echo "#BSUB -R span[ptile=$procs]"                               >> $cfile
 echo "#BSUB -R affinity[$core]"                                  >> $cfile
 echo "#BSUB -R rusage[mem=$rmem]"                                >> $cfile
else
 echo "#BSUB -R span[ptile=$procs]"                               >> $cfile
fi
if [ $threads -gt 1 ] ; then
  if [ $core = core ] ; then
    echo "#BSUB -R affinity[core($threads)]"   >> $cfile
    if [ $queue = shared -o $ppreq = S ] ; then
      echo "#BSUB -R rusage[mem=$rmem]"                           >> $cfile
    else
      echo "#BSUB -x"                                             >> $cfile
    fi
  else
    if [ $queue = shared -o $ppreq = S ] ; then
      echo "#BSUB -R affinity[cpu($threads):distribute=balance]"  >> $cfile
      echo "#BSUB -R rusage[mem=$rmem]"                           >> $cfile
    else
      echo "#BSUB -R affinity[cpu($threads):distribute=balance]"  >> $cfile
      echo "#BSUB -x"                                             >> $cfile
    fi
  fi
else
  if [ $core = core ] ; then
    echo "#BSUB -R affinity[$core]"            >> $cfile
    if [ $queue = shared -o $ppreq = S ] ; then
      echo "#BSUB -R rusage[mem=$rmem]"                           >> $cfile
    else
      echo "#BSUB -x"                                             >> $cfile
    fi
  else
    if [ $queue = shared -o $ppreq = S ] ; then
      echo "#BSUB -R affinity[cpu:distribute=balance]"            >> $cfile
      echo "#BSUB -R rusage[mem=$rmem]"                           >> $cfile
    else
      echo "#BSUB -R affinity[cpu:distribute=balance]"            >> $cfile
      echo "#BSUB -x"                                             >> $cfile
    fi
  fi
fi
echo "#BSUB -W $timew"                                            >> $cfile

if [[ -n $when ]];then
  whena=$when
  if [[ $when = +* ]];then
    hr=$(echo $when|cut -c2-3)
    mn=$(echo $when|cut -c4-5)
    [[ -n $mn ]] || mn=00
    now=$(date -u +"%Y%m%d%H%M")
    ((mn+=$(echo $now|cut -c11-12)))
    [[ $mn -ge 60 ]] && ((hr+=1)) && ((mn-=60))
    [[ $mn -lt 10 ]] && mn=0$mn
    whena=$(/nwprod/util/exec/ndate +$hr $(echo $now|cut -c1-10))$mn
  elif [[ $when = t* ]];then
    hr=$(echo $when|cut -c2-3)
    mn=$(echo $when|cut -c4-5)
    [[ -n $mn ]] || mn=00
    now=$(date -u +"%Y%m%d")
    whena=$now$hr$mn
  elif [[ $when = T* ]];then
    hr=$(echo $when|cut -c2-3)
    mn=$(echo $when|cut -c4-5)
    [[ -n $mn ]] || mn=00
    now=$(date -u +"%Y%m%d%H")
    whena=$(/nwprod/util/exec/ndate +24 $now|cut -c1-8)$hr$mn
  fi
  yr=$(echo $whena|cut -c1-4)
  mo=$(echo $whena|cut -c5-6)
  dy=$(echo $whena|cut -c7-8)
  hr=$(echo $whena|cut -c9-10)
  mn=$(echo $whena|cut -c11-12)
  [[ -n $mn ]] || mn=00
  echo "#BSUB -b $yr:$mo:$dy:$hr:$mn"                            >> $cfile
fi

#echo "source ~${LOGNAME}/.profile"          >> $cfile
#echo "ulimit -s unlimited"                  >> $cfile
#if [ ${MP_EUIDEVICE:-NULL} = sn_all ] ; then
#echo "#BSUB -network \"type=sn_all:mode=US\" "                 >> $cfile
#fi
#if [ ${MP_EULIB:-NULL} != NULL ] ; then
#echo "export MP_EUILIB=$MP_EUILIB"                             >> $cfile
#fi

if [ ${MP_EUIDEVELOP:-NULL} != NULL ] ; then
 echo "export MP_EUIDEVELOP=$MP_EUIDEVELOP"                     >> $cfile
fi
echo "export KMP_STACKSIZE=$KMP_STACKSIZE"                      >> $cfile
if [ ${F_UFMTENDIAN:-NULL} != NULL ] ; then
 echo "export F_UFMTENDIAN=$F_UFMTENDIAN"                       >> $cfile
fi
if [ ${MPICH_ALLTOALL_THROTTLE:-NULL} != NULL ] ; then
 echo "export MPICH_ALLTOALL_THROTTLE=$MPICH_ALLTOALL_THROTTLE" >> $cfile
fi
if [ ${MP_SINGLE_THREAD:-NULL} != NULL ] ; then
 echo "export MP_SINGLE_THREAD=$MP_SINGLE_THREAD"               >> $cfile
fi
if [ ${MP_EAGER_LIMIT:-NULL} != NULL ] ; then
 echo "export MP_EAGER_LIMIT=$MP_EAGER_LIMIT"                   >> $cfile
fi

if [ ${MP_USE_BULK_XFER:-NULL} != NULL ] ; then
 echo "export MP_USE_BULK_XFER=$MP_USE_BULK_XFER"               >> $cfile
fi
if [ ${MP_COLLECTIVE_OFFLOAD:-NULL} != NULL ] ; then
 echo "export MP_COLLECTIVE_OFFLOAD=$MP_COLLECTIVE_OFFLOAD"     >> $cfile
fi
if [ ${MP_SHARED_MEMORY:-NULL} != NULL ] ; then
 echo "export MP_SHARED_MEMORY=$MP_SHARED_MEMORY"               >> $cfile
fi
if [ ${MP_MPILIB:-NULL} != NULL ] ; then
 echo "export MP_MPILIB=$MP_MPILIB"                             >> $cfile
fi
if [ ${MP_LABELIO:-NULL} != NULL ] ; then
 echo "export MP_LABELIO=$MP_LABELIO"                           >> $cfile
fi
if [ ${MP_STDOUTMODE:-NULL} != NULL ] ; then
 echo "export MP_STDOUTMODE=$MP_STDOUTMODE "                    >> $cfile
fi
if [ "$TASK_GEOMETRY" != NONE ] ; then
 echo "export LSB_PJL_TASK_GEOMETRY=\"$TASK_GEOMETRY\""         >> $cfile
fi
for var in $(eval echo $envars | tr , ' ') ; do
 echo "export $var"                                             >> $cfile
done
echo "export OMP_NUM_THREADS=$threads"                          >> $cfile
echo "" >>$cfile
echo ". "$(awk '{ print $1, $2, $3, $4, $5, $6, $7, $8, $9 }' $regdir/regression_var.out) >> $cfile 
echo "" >>$cfile

echo ". $MODULESHOME/init/ksh"              >> $cfile
echo "module purge"                         >> $cfile
echo "module use -a $gsisrc/modulefiles"    >> $cfile
echo "module load modulefile.ProdGSI.wcoss" >> $cfile
echo "module list"                          >> $cfile

echo "$exec"                                                    >> $cfile

if [[ $stdin = YES ]];then
  cat
fi >>$cfile
if [[ $nosub = YES ]];then
  cat $cfile
  exit
elif [[ $verbose = YES ]];then
  set -x
  cat $cfile
fi
bsub=${bsub:-$LSF_BINDIR/bsub}

ofile=$DATA/subout$$
>$ofile
chmod 777 $ofile
$bsub < $cfile
rc=$?
cat $ofile
if [[ -w $SUBLOG ]];then
  jobn=$(grep -i submitted $ofile|head -n1|cut -d\" -f2)
  date +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG
fi
rm $cfile $ofile
[[ $MKDATA = YES ]] && rmdir $DATA
exit $rc