# Usage: mpmd.sh CMDFILE
# Imported variables: CMDNULL POE_OPTS DATA LOUD
# Scratch files: $DATA/mpmd.$$.sh* $DATA/mpmd.$$.cmd $DATA/cmdfile.$$.upd


# Imported variables and defaults
CMDFILE=${1:-$CMDFILE:-?}
CMDNULL=${CMDNULL:-"cat /dev/null"}
POE_OPTS=${POE_OPTS:-""}
DATA=${DATA:-`pwd`}
# Effective with transition to frost/snow, DO NOT execute a time command
#  with poe command!!!
#############################TIMEIT=${TIMEIT:-""}
LOUD=${LOUD:-off}

[ $LOUD != off ] && set -x

echo
echo ...................................................................
echo "Entering mpmd.sh: `date`"
echo

# Determine number of processors, tasks, and tasks per processor
set -A procs $LOADL_PROCESSOR_LIST
nprocs=${#procs[*]}

if [ $nprocs -eq 0 ];then
   echo
   echo "##>FATAL ERROR: Must run POE under LoadLeveler"
   echo
   exit 99
fi

ntasks=$(wc -l <$CMDFILE)
ntperp=$((($ntasks-1)/$nprocs+1))

echo
echo "Number of processors requested by Loadleveler = " $nprocs
echo "Number of tasks passed into poe command file  = " $ntasks
echo "Number of tasks per processor =                 " $ntperp
echo
if [ $nprocs -gt $ntasks ]; then
   echo
   echo "-->WARNING: More processors requested than tasks - fill \
in remaining"
   echo "            processors with DUMMY tasks."
   echo
elif [ $nprocs -lt $ntasks ]; then
   echo
   echo "-->WARNING: Fewer processors requested than tasks - restructure \
command"
   echo "            file to handle extra tasks."
   echo
fi

# Create new scripts and command file using split
rm -f $DATA/mpmd.$$.sh*

#VVVVVVVVVVVVVVVVVVVVVVVVVVVV
count=0
cp $CMDFILE $DATA/cmdfile.$$.upd
while [ $ntperp -gt 1 ];do
   count=`expr $count + 1`
   head -n $ntperp $DATA/cmdfile.$$.upd > $DATA/mpmd.$$.sha${count}
   tail -n `expr $ntasks - $ntperp` $DATA/cmdfile.$$.upd > $DATA/cmdfile.$$.upd_udp
   cp $DATA/cmdfile.$$.upd_udp $DATA/cmdfile.$$.upd
   [ `expr $ntasks - $ntperp` -le 0 ]  &&  break

   nprocs=`expr $nprocs - 1`
   ntasks=`expr $ntasks - $ntperp`
   ntperp=$((($ntasks-1)/$nprocs+1))

   echo
   echo "After interation $count :"
   echo "Remaining number of processors =        " $nprocs
   echo "Remaining number of tasks =             " $ntasks
   echo "Updated number of tasks per processor = " $ntperp
   echo
done
[ `cat $DATA/cmdfile.$$.upd | wc -l` -gt 0 ]  && \
  split -l 1 $DATA/cmdfile.$$.upd $DATA/mpmd.$$.sh

########split -l $ntperp $CMDFILE $DATA/mpmd.$$.sh
#AAAAAAAAAAAAAAAAAAAAAAAAAAAA

chmod +x $DATA/mpmd.$$.sh*
ls $DATA/mpmd.$$.sh* >$DATA/mpmd.$$.cmd

# Pad out new command file
ncomms=$(wc -l <$DATA/mpmd.$$.cmd)
while [[ $((ncomms+=1)) -le $nprocs ]];do echo $CMDNULL >>$DATA/mpmd.$$.cmd
done

# Execute poe and clean up
# Effective with transition to frost/snow, DO NOT execute a time command
#  with poe command!!!
#######$TIMEIT /usr/bin/poe -pgmmodel mpmd -cmdfile $DATA/mpmd.$$.cmd $POE_OPTS
/usr/bin/poe -pgmmodel mpmd -cmdfile $DATA/mpmd.$$.cmd $POE_OPTS
rm -f $DATA/mpmd.$$.sh* $DATA/mpmd.$$.cmd $DATA/cmdfile.$$.upd*

echo
echo "Ending   mpmd.sh: `date`"
echo ...................................................................
echo