#!/bin/bash

#PBS -N SREF
#PBS -A SREF-DEV
##PBS -j oe
#PBS -o /lfs/h2/emc/lam/noscrub/Jun.Du/sref.v7_cray/run/sref_fcst_ARW_p03.log
#PBS -e /lfs/h2/emc/lam/noscrub/Jun.Du/sref.v7_cray/run/sref_fcst_ARW_p03.log
#PBS -l select=1:ncpus=112:mem=200GB
##PBS -q debug
#PBS -q dev
#PBS -l walltime=03:00:00
##PBS -l hyper=true
#PBS -l debug=true

##BSUB -R span[ptile=28]
##BSUB -J SREF_FCST_ARW_p03
##BSUB -x

set -eux

trap 'echo "SREF error ARW p03 FCST" >> /lfs/h2/emc/ptmp/$LOGNAME/tmpnwprd/jlogfile_sref' INT TERM ERR
echo "SREF started ARW p03 FCST" >> /lfs/h2/emc/ptmp/$LOGNAME/tmpnwprd/jlogfile_sref

#. /usrx/local/prod/lmod/lmod/init/bash
. /usr/share/lmod/lmod/init/bash

module purge
#module reset
module load PrgEnv-intel/8.1.0
module load envvar/1.0
#module load lsf/10.1
#module load ips/18.0.1.163
module load intel/19.1.3.304
module load libjpeg/9c
module load prod_util/2.0.9
module load prod_envir/2.0.6
module load grib_util/1.2.2
module load netcdf/4.7.4
#module load impi/18.0.1
module load cfp/2.0.4
module load craype/2.7.8
module load cray-pals/1.0.12
module load cray-mpich/8.1.9

#module load gcc/10.3.0
module load gcc/11.2.0
module load intel/19.1.3.304
module load libpng/1.6.37
module list

export OMP_NUM_THREADS=1

export RUN_ENVIR=para
export envir=prod
export job=$$

export CONFIG_FILE=/lfs/h2/emc/lam/noscrub/Jun.Du/sref.v7_cray/parm/sref_para_config_cray

export PDY=20210824
export cyc=09
export FLENGTH=87
export INCR=3
export MODEL=ARW
export MEMBER=p03
export IOFORM=2
export RES=16km
export MACHINE=dell
#export N_TASK=$LSB_DJOB_NUMPROC

/lfs/h2/emc/lam/noscrub/Jun.Du/sref.v7_cray/jobs/JSREF_WRF_FCST

echo "SREF finished ARW p03 FCST" >> /lfs/h2/emc/ptmp/$LOGNAME/tmpnwprd/jlogfile_sref

qsub /lfs/h2/emc/lam/noscrub/Jun.Du/sref.v7_cray/run/SREF_POST_ARW_p03.qsub
qsub /lfs/h2/emc/lam/noscrub/Jun.Du/sref.v7_cray/run/SREF_POST2_ARW_p03.qsub
qsub /lfs/h2/emc/lam/noscrub/Jun.Du/sref.v7_cray/run/SREF_POST3_ARW_p03.qsub
qsub /lfs/h2/emc/lam/noscrub/Jun.Du/sref.v7_cray/run/SREF_POST4_ARW_p03.qsub
echo "SREF waiting ARW p03 POST" >> /lfs/h2/emc/ptmp/$LOGNAME/tmpnwprd/jlogfile_sref
echo "SREF waiting ARW p03 POST2" >> /lfs/h2/emc/ptmp/$LOGNAME/tmpnwprd/jlogfile_sref
echo "SREF waiting ARW p03 POST3" >> /lfs/h2/emc/ptmp/$LOGNAME/tmpnwprd/jlogfile_sref
echo "SREF waiting ARW p03 POST4" >> /lfs/h2/emc/ptmp/$LOGNAME/tmpnwprd/jlogfile_sref