#!/bin/sh
 
#-----------------------------------------------------
# Run the sst2mdl program on the wcoss cray nodes.
#
# Invoke this script as follows:
# "cat runit.wcoss.cray.lsf | bsub"
#
# When using FLAKE data, use multiple threads to
# reduce wall clock time.  Otherwise, use one
# thread.
#-----------------------------------------------------

#BSUB -oo sst2mdl.log
#BSUB -eo sst2mdl.log
#BSUB -q dev
#BSUB -J sst2mdl
#BSUB -P NAM-T2O
#BSUB -W 0:03
#BSUB -M 500
#BSUB -cwd .
#BSUB -extsched 'CRAYLINUX[]'

set -x

module list

WORK="/gpfs/hps/stmp/$LOGNAME/sst2mdl"
rm -fr $WORK
mkdir -p $WORK
cd $WORK

rundir=${LS_SUBCWD}
execdir=${rundir}/../../exec

cp ${execdir}/emcsfc_sst2mdl     ./

#------------------------------------------------------------
# input/output files are set in the namelist. 
#------------------------------------------------------------

ln -fs ${rundir}/config.wcoss.cray.nml   ./fort.41

#------------------------------------------------------------
# Run executable (intel compiler)
#------------------------------------------------------------

# Just need to change this line to change number of threads.
# With "-j 1" (no hyperthreading) can use up to 24 threads.
export OMP_NUM_THREADS=1

# This line required when running threads with intel compiler.
export KMP_AFFINITY=disabled

export NODES=1
aprun -j 1 -n 1 -N 1 -d $OMP_NUM_THREADS -cc depth ./emcsfc_sst2mdl 

#------------------------------------------------------------
# Run executable (cray compiler).  NOTE!! I am not sure
# what I am doing wrong here, but i can't get the
# program to run with more than one thread.
#------------------------------------------------------------

#export OMP_NUM_THREADS=1
#aprun -j 1 -n 1 -N 1 -d $OMP_NUM_THREADS ./emcsfc_sst2mdl 

#------------------------------------------------------------
# Clean up
#------------------------------------------------------------

rm -f ./fort.41

exit 0