#!/usr/bin/env bash
#BSUB -J nwm_forcing_long_range_mem%MEM%_%CYC%_ibm
#BSUB -o /gpfs/hps3/ptmp/Cham.Pham/com/logs/jnwm_forcing_long_range_mem%MEM%_%CYC%.o%J
#BSUB -cwd /gpfs/hps3/ptmp/Cham.Pham/tmpnwprd
#BSUB -q dev
#BSUB -A "NWM-DEV"
#BSUB -U NWM
#BSUB -W 6:45
#BSUB -n 96
#BSUB -R span[ptile=12]
#BSUB -R affinity[cpu(2):distribute=balance]
#BSUB -a poe

#Unlimit the stack size
ulimit -c 0
ulimit -s unlimited

. /usrx/local/Modules/3.2.9/init/ksh

export MODULEPATH=$MODULEPATH:/ibm/save/Carolyn.Pasti/tools/modulefile
%include "/ecf/ecfnets/include/ecf/ecfnets/include/head.h"
%include "/ecf/ecfnets/include/envir-xc40.h"

source /gpfs/hps3/nwc/noscrub/Cham.Pham/nwtest/nwm.v2.1/ecf/model_envir.h
export model=nwm

# add NCL to PATH
export PATH=/gpfs/tp1/u/James.R.Taft/CFP.2.0.0.new/bin:$NCARG_ROOT:$NCARG_BIN:$NCARG_LIB:$NCARG_INC:$PATH:

module load ibmpe #cfp/2.0.0
module load ncarg
module load NetCDF
module load python
module load ics/16.0.3
module load jobdefault

#export I_MPI_HYDRA_BOOTSTRAP=lsf
#
#export I_MPI_HYDRA_BRANCH_COUNT=`echo $LSB_MCPU_HOSTS | sed 's/ [0-9]*\( \|$\)/\1/g' | wc -w`
#
#export I_MPI_LSF_USE_COLLECTIVE_LAUNCH=1
#export I_MPI_EXTRA_FILESYSTEM="on"
#export I_MPI_EXTRA_FILESYSTEM_LIST="gpfs"

export FORT_BUFFERED=true
export SAVE_ALL_TASKS=yes
#export OMP_NUM_THREADS=1
export IOBUF_PARAMS='*.LAKEOUT_DOMAIN1:size=64M:count=2:prefetch=1,*:size=32M:count=4:vbuffer_count=4096:prefetch=1'

# EXPORT list here
export cyc=%CYC%
# export TIME_FLAG=YES
#export mem=%mem%
export mem=%MEM%
export member=`echo $mem|cut -c2-2`

export HOMEnwm=/gpfs/hps3/nwc/noscrub/Zhengtao.Cui/trunk_v1.1/nwm.v2.1.0
#-----set up CFP2 environment
export OMP_NUM_THREADS=2

export CFP_PTILE=12
export CFP_NPROCS=96
export CFP_VERBOSE=1

#export CFPCOMMAND="mpirun.lsf cfp"
#export CFPCOMMAND="mpirun.lsf /gpfs/tp1/u/James.R.Taft/CFP.2.0.0.new/cfp"

# CALL executable job script here
${HOMEnwm}/jobs/JNWM_FORCING_LONG_RANGE


%include "/ecf/ecfnets/include/tail.h"
%manual
######################################################################
#
#  PURPOSE:
#
######################################################################

######################################################################
# Job specific troubleshooting instructions:
#  see generic troubleshoot manual page
#
######################################################################

# include manual page below
%end