#!/usr/bin/env bash
#BSUB -J nwm_forcing_short_range_%CYC%_ibm
#BSUB -o /gpfs/hps3/ptmp/Cham.Pham/com/logs/jnwm_forcing_short_range_%CYC%.o%J
#BSUB -cwd /gpfs/hps3/ptmp/Cham.Pham/tmpnwprd
#BSUB -q dev
#BSUB -A "NWM-DEV"
#BSUB -U NWM
#BSUB -W 3:20
#BSUB -n 24
#BSUB -R span[ptile=8]
#BSUB -R affinity[cpu(1):distribute=balance]
#BSUB -a poe

#Unlimit the stack size
ulimit -c 0
ulimit -s unlimited

. /usrx/local/Modules/3.2.9/init/ksh

export MODULEPATH=$MODULEPATH:/ibm/save/Carolyn.Pasti/tools/modulefile
%include "/ecf/ecfnets/include/ecf/ecfnets/include/head.h"
%include "/ecf/ecfnets/include/envir-xc40.h"

#source $MODULESHOME/etc/modules.sh
source /gpfs/hps3/nwc/noscrub/Cham.Pham/nwtest/nwm.v2.1/ecf/model_envir.h
export model=nwm
#source /ecf/ecfnets/include/model_ver.h

module load ibmpe #cfp/2.0.0
module load ncarg
module load NetCDF
module load python
module load ics/16.0.3
module load jobdefault

#export I_MPI_HYDRA_BOOTSTRAP=lsf
#
#export I_MPI_HYDRA_BRANCH_COUNT=`echo $LSB_MCPU_HOSTS | sed 's/ [0-9]*\( \|$\)/\1/g' | wc -w`
#
#export I_MPI_LSF_USE_COLLECTIVE_LAUNCH=1
#export I_MPI_EXTRA_FILESYSTEM="on"
#export I_MPI_EXTRA_FILESYSTEM_LIST="gpfs"

export FORT_BUFFERED=true
export SAVE_ALL_TASKS=yes
#export OMP_NUM_THREADS=1
export IOBUF_PARAMS='*.LAKEOUT_DOMAIN1:size=64M:count=2:prefetch=1,*:size=32M:count=4:vbuffer_count=4096:prefetch=1'



#export MP_CSS_INTERRUPT=yes

#export CFP_NPROCS=24  #The total number of cores to be used by CFP
#export CFP_PTILE=8  #The number of cores per node to be used by CFP
#export CFP_MINMEM=3 #The minimum amount of free memory needed on a node 
                    #before starting next task (GB) (optional)
#export CFP_DELAY=2 #The delay in seconds between starting each task on a node 
#                   #(optional)
#export OMP_NUM_THREADS=2 #This variable must be set to 2.

# add NCL to PATH
#export PATH=$NCARG_ROOT:$NCARG_BIN:$NCARG_LIB:$NCARG_INC:$PATH:
export PATH=/gpfs/tp1/u/James.R.Taft/CFP.2.0.0.new/bin:$NCARG_ROOT:$NCARG_BIN:$NCARG_LIB:$NCARG_INC:$PATH:

# EXPORT list here
export cyc=%CYC%

export HOMEnwm=/gpfs/hps3/nwc/noscrub/Zhengtao.Cui/trunk_v1.1/nwm.v2.1.0
#-----set up CFP2 environment
export OMP_NUM_THREADS=2

export CFP_PTILE=8
export CFP_NPROCS=24
export CFP_VERBOSE=1

#export CFPCOMMAND="mpirun.lsf cfp"
#export CFPCOMMAND="mpirun.lsf /gpfs/tp1/u/James.R.Taft/CFP.2.0.0.new/cfp"
#export CFPCOMMANDEX="mpirun.lsf cfp"

# CALL executable job script here
${HOMEnwm}/jobs/JNWM_FORCING_SHORT_RANGE

%include "/ecf/ecfnets/include/tail.h"
%manual
######################################################################
#
#  PURPOSE:
#
######################################################################

######################################################################
# Job specific troubleshooting instructions:
#  see generic troubleshoot manual page
#
######################################################################

# include manual page below
%end