#!/usr/bin/env bash
#BSUB -J nwm_analysis_assim_ibm_%CYC%
#BSUB -o /gpfs/hps3/ptmp/Cham.Pham/com/logs/jnwm_analysis_assim_%CYC%.o%J
#BSUB -cwd /gpfs/hps3/ptmp/Cham.Pham/tmpnwprd/
#BSUB -q dev
#BSUB -W 03:30
#BSUB -P "NWM-T2O"
#BSUB -n 768
#BSUB -R span[ptile=24]
#BSUB -R affinity[cpu(1):distribute=balance]
#BSUB -a poe

#Unlimit the stack size
ulimit -c 0
ulimit -s unlimited

. /usrx/local/Modules/3.2.9/init/ksh

export MODULEPATH=$MODULEPATH:/ibm/save/Carolyn.Pasti/tools/modulefile
%include "/ecf/ecfnets/include/head.h"
%include "/ecf/ecfnets/include/envir-xc40.h"

source /gpfs/hps3/nwc/noscrub/Cham.Pham/nwtest/nwm.v2.1/ecf/model_envir.h
export model=nwm

module load ncarg
module load NetCDF
module load python
module load ics/16.0.3
module load ibmpe #cfp/2.0.0
module load jobdefault

export PATH=/gpfs/tp1/u/James.R.Taft/CFP.2.0.0.new/bin:$NCARG_ROOT:$NCARG_BIN:$NCARG_LIB:$NCARG_INC:$PATH:

#export I_MPI_HYDRA_BOOTSTRAP=lsf
#
#export I_MPI_HYDRA_BRANCH_COUNT=`echo $LSB_MCPU_HOSTS | sed 's/ [0-9]*\( \|$\)/\1/g' | wc -w`
#
#export I_MPI_LSF_USE_COLLECTIVE_LAUNCH=1
#export I_MPI_EXTRA_FILESYSTEM="on"
#export I_MPI_EXTRA_FILESYSTEM_LIST="gpfs"

export FORT_BUFFERED=true
export FORT_BLOCKSIZE=8388608

export SAVE_ALL_TASKS=yes
#export FORT_BUFFERED=true
export OMP_NUM_THREADS=2
export IOBUF_PARAMS='*.LAKEOUT_DOMAIN1:size=64M:count=2:prefetch=1,*:size=32M:count=4:vbuffer_count=4096:prefetch=1'

#-----set up CFP2 environment

export CFP_PTILE=24
export CFP_NPROCS=768
export CFP_VERBOSE=1

export HOMEnwm=/gpfs/hps3/nwc/noscrub/Zhengtao.Cui/trunk_v1.1/nwm.v2.1

# EXPORT list here
export cyc=%CYC%
# export TIME_FLAG=YES
export CASETYPE=analysis_assim
export job=${model}_analysis_assim_${cyc}

#export CFPCOMMAND="mpirun.lsf cfp"
#export CFPCOMMAND="mpirun.lsf /gpfs/tp1/u/James.R.Taft/CFP.2.0.0.new/cfp"
#export MPICOMMAND="mpiexec.hydra"
export MPICOMMAND="mpirun.lsf"

# CALL executable job script here
${HOMEnwm}/jobs/JNWM_MODEL

%include "/ecf/ecfnets/include/tail.h"
%manual
######################################################################
#
#  PURPOSE:
#
######################################################################

######################################################################
# Job specific troubleshooting instructions:
#  see generic troubleshoot manual page
#
######################################################################

# include manual page below
%end