#!/bin/sh --login
set -x
usage="\
Usage:  $0 [options] executable [args]
      where the options are:
      -a account        account (default: none)
      -b binding        run smt binding or not (default:NO)
      -d dirin          initial directory (default: cwd)
      -e envars         copy comma-separated environment variables
      -g group          group name
      -i                append standard input to command file
      -j jobname        specify jobname (default: executable basename)
      -m machine        machine on which to run (default: current)
      -n                write command file to stdout rather than submitting it
      -o output         specify output file (default: jobname.out)
      -p procs[/nodes[/ppreq]
                        number of MPI tasks and optional nodes or Bblocking and
                        ppreq option (N or S) (defaults: serial, Bunlimited, S)
      -q queue[/qpreq]  queue name and optional requirement, e.g. dev/P
                        (defaults: 1 if serial or dev if parallel and none)
                        (queue 3 or 4 is dev or prod with twice tasks over ip)
                        (options: P=parallel, B=bigmem, b=batch)
      -r rmem[/rcpu]    resources memory and cpus/task (default: '1024 mb', 1)
      -t timew          wall time limit in [[hh:]mm:]ss format (default: 900)
      -u userid         userid to run under (default: self)
      -v                verbose mode
      -w when           when to run, in yyyymmddhh[mm], +hh[mm], thh[mm], or
                        Thh[mm] (full, incremental, today or tomorrow) format
                        (default: now)
Function:  This command submits a job to the batch queue."
subcmd="$*"
stdin=NO
nosub=NO
account=""
binding="NO"
dirin=""
envars=""
group=""
jobname=""
machine=""
output=""
procs=0
nodes=""
ppreq=""
queue=""
qpreq=""
rmem="1024"
rcpu="1"
timew="900"
userid=""
verbose=NO
when=""
while getopts a:b:d:e:g:ij:m:no:p:q:r:t:u:vw: opt;do
  case $opt in
    a) account="$OPTARG";;
    b) binding="$OPTARG";;
    d) dirin="$OPTARG";;
    e) envars="$OPTARG";;
    g) group="$OPTARG";;
    i) stdin=YES;;
    j) jobname=$OPTARG;;
    m) machine="$OPTARG";;
    n) nosub=YES;;
    o) output=$OPTARG;;
    p) procs=$(echo $OPTARG/|cut -d/ -f1);nodes=$(echo $OPTARG/|cut -d/ -f2);ppreq=$(echo $OPTARG/|cut -d/ -f3);;
    q) queue=$(echo $OPTARG/|cut -d/ -f1);qpreq=$(echo $OPTARG/|cut -d/ -f2);;
    r) rmem=$(echo $OPTARG/|cut -d/ -f1);rcpu=$(echo $OPTARG/|cut -d/ -f2);;
    t) timew=$OPTARG;;
    u) userid=$OPTARG;;
    v) verbose=YES;;
    w) when=$OPTARG;;
    \?) echo $0: invalid option >&2;echo "$usage" >&2;exit 1;;
  esac
done
shift $(($OPTIND-1))
if [[ $# -eq 0 ]];then
  echo $0: missing executable name >&2;echo "$usage" >&2;exit 1
fi
exec=$1
if [[ ! -s $exec ]]&&which $exec >/dev/null 2>&1;then
  exec=$(which $exec)
fi
shift
args="$*"
bn=$(basename $exec)
export jobname=${jobname:-$bn}
output=${output:-$jobname.out}
myuser=$LOGNAME
myhost=$(hostname)

DATA=/data/users/$LOGNAME/tmp
mkdir -p $DATA

#partition=${partition:-c1ms}
queue=${queue:-batch}
timew=${timew:-01:20:00}
task_node=${task_node:-$procs}
size=$((nodes*task_node))
envars=$envars
threads=${rcpu:-1}
#envars=$envars,mpi_tasks=$procs
#Options
###SBATCH -l partition=c1ms,size=0528,walltime=01:20:00
##SBATCH -l partition=$queue,size=$size,walltime=$timew
##SBATCH -S /bin/sh

export TZ=GMT
cfile=$DATA/sub$$
> $cfile
#echo "#SBATCH -S /bin/sh"                     >> $cfile
echo "#!/bin/sh --login"                   >> $cfile
echo ""                                    >> $cfile
echo "#SBATCH -o $output"                     >> $cfile
echo "#SBATCH --job-name=$jobname"                    >> $cfile
echo "#SBATCH --time=$timew"             >> $cfile
echo "#SBATCH --ntasks=$procs"             >> $cfile
echo "#SBATCH --cpus-per-task=$nodes"     >> $cfile
echo "#SBATCH --mem-per-cpu=6000"     >> $cfile
echo "#SBATCH --partition=s4"                          >> $cfile
echo "#SBATCH --exclusive"                          >> $cfile
echo "#SBATCH --export=ALL" 
echo "#SBATCH --account=star"                          >> $cfile
echo "#SBATCH --distribution=block:block"                          >> $cfile
#echo "#SBATCH -d"                             >> $cfile
#. $exec                                    >> $cfile
#echo "/bin/sh -x $exec"                    >> $cfile

echo "" >>$cfile
echo "export MPI_BUFS_PER_PROC=2048"  >> $cfile
echo "export MPI_BUFS_PER_HOST=2048" >> $cfile
echo "export MPI_GROUP_MAX=256" >> $cfile
echo "export MPI_MEMMAP_OFF=1" >> $cfile
echo "export MP_STDOUTMODE=ORDERED" >> $cfile
#echo "export KMP_STACKSIZE=2048000" >> $cfile
#echo "export KMP_AFFINITY=scatter" >> $cfile

echo "" >>$cfile
echo "export OMP_NUM_THREADS=$threads"      >> $cfile
echo "" >>$cfile
echo ". "$(awk '{ print $1, $2, $3, $4, $5, $6, $7, $8, $9 }' $regdir/regression_var.out)  >>$cfile
echo "" >>$cfile

cat $exec >> $cfile

if [[ $nosub = YES ]];then
  cat $cfile
  exit
elif [[ $verbose = YES ]];then
  set -x
  cat $cfile
fi
#msub -I partition=$partition,size=$procs,walltime=$walltime $cfile

#if [[ -n $when ]];then
#  whena=$when
#  if [[ $when = +* ]];then
#    hr=$(echo $when|cut -c2-3)
#    mn=$(echo $when|cut -c4-5)
#    [[ -n $mn ]] || mn=00
#    now=$(date -u +"%Y%m%d%H%M")
#    ((mn+=$(echo $now|cut -c11-12)))
#    [[ $mn -ge 60 ]] && ((hr+=1)) && ((mn-=60))
#    [[ $mn -lt 10 ]] && mn=0$mn
#    whena=$(/nwprod/util/exec/ndate +$hr $(echo $now|cut -c1-10))$mn
#  elif [[ $when = t* ]];then
#    hr=$(echo $when|cut -c2-3)
#    mn=$(echo $when|cut -c4-5)
#    [[ -n $mn ]] || mn=00
#    now=$(date -u +"%Y%m%d")
#    whena=$now$hr$mn
#  elif [[ $when = T* ]];then
#    hr=$(echo $when|cut -c2-3)
#    mn=$(echo $when|cut -c4-5)
#    [[ -n $mn ]] || mn=00
#    now=$(date -u +"%Y%m%d%H")
#    whena=$(/nwprod/util/exec/ndate +24 $now|cut -c1-8)$hr$mn
#  fi
#  yr=$(echo $whena|cut -c1-4)
#  mo=$(echo $whena|cut -c5-6)
#  dy=$(echo $whena|cut -c7-8)
#  hr=$(echo $whena|cut -c9-10)
#  mn=$(echo $whena|cut -c11-12)
#  [[ -n $mn ]] || mn=00
#  echo "#@ startdate = $mo/$dy/$yr $hr:$mn"
#fi >>$cfile


if [[ $stdin = YES ]];then
  cat
fi >>$cfile
if [[ $nosub = YES ]];then
  cat $cfile
  exit
elif [[ $verbose = YES ]];then
  set -x
  cat $cfile
fi
sbatch=${sbatch:-sbatch}

ofile=$DATA/subout$$
>$ofile
chmod 777 $ofile
$sbatch $cfile >$ofile
rc=$?
cat $ofile
if [[ -w $SUBLOG ]];then
  jobn=$(grep -i submitted $ofile|head -n1|cut -d\" -f2)
  date -u +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG
fi
#rm $cfile $ofile
[[ $MKDATA = YES ]] && rmdir $DATA
exit $rc