#!/bin/bash
################################################################################
# UNIX Script Documentation Block                                                         #
#                                                                                         #
# Script name:         exhiresw_wrfsi_ens.sh.sms                                          #
# Script description:  Runs WRF standard initialization (SI) for the WRF-NMM and          #
#                      WRF-EM model                                                       #
#                                                                                         #
# Author:        Eric Rogers       Org: NP22         Date: 2004-07-02                     #
#                                                                                         #
# Abstract: The scripts gets all the input files needed for the WRF-NMM ands WRF-EM       #
#           hiresw run and runs the WRF SI which interpolates the operational Eta ic's    #
#           to the nested WRF-NMM domain                                                  #
#                                                                                         #
# Script history log:                                                                     #
# 2003-11-01  Matt Pyle   - Original script for parallel                                  #
# 2004-07-02  Eric Rogers - Preliminary modifications for production.                     #
# 2005-08-02  Jun Du - Modified to fit SREF system
#
# 2005-10-06  Xiaoxue Wang - Modified to operational standard
# 2006-04-05  Jun Du - (1) glb ensemble's directory name and file names have been         #
#                      modified to match the new glb ensemble output format; (2) a new
#                      pair of glb ensemble members are assigned to AWR members for more
#                      diversity
# 2007-08-13  Jun Du - Due to the GFS output grid change to Gaussian, all pgrib files
#                      including GFS 0.5d master files and 1.0d GEFS pgriba and pgribb
#                      files are now generated by the SREF code itself as a solution
# 2008-07-18  Jun Du - upgrade to WRFv2.2
# 2010-04-15  Jun Du - generalized the script to run NEMS ensemble (MODEL=NMB)
# 2014-08-10  Jun Du - expanded for SREF.v7.0.0
# 2021-11-18  Jun Du - transition to WCOSS2 or Cray (v.7.1.0)
################################################################################

set -eux
echo $PATH

MEMBER=$1
GLBMEM=$2

model=`echo $MODEL | tr "[:upper:]" "[:lower:]"`

cd $WORK_DIR

START_DATE=$PDY$CYC
START_DATE_P03=`$NDATE +03 $START_DATE`
START_DATE_M03=`$NDATE -03 $START_DATE`
END_DATE=`$NDATE $FLENGTH $START_DATE`

START_YYYY=`echo $START_DATE | cut -c 1-4`
START_MM=`echo $START_DATE | cut -c 5-6`
START_DD=`echo $START_DATE | cut -c 7-8`
START_HH=`echo $START_DATE | cut -c 9-10`

START_P03_YYYY=`echo $START_DATE_P03 | cut -c 1-4`
START_P03_MM=`echo $START_DATE_P03 | cut -c 5-6`
START_P03_DD=`echo $START_DATE_P03 | cut -c 7-8`
START_P03_HH=`echo $START_DATE_P03 | cut -c 9-10`

START_M03_YYYY=`echo $START_DATE_M03 | cut -c 1-4`
START_M03_MM=`echo $START_DATE_M03 | cut -c 5-6`
START_M03_DD=`echo $START_DATE_M03 | cut -c 7-8`
START_M03_HH=`echo $START_DATE_M03 | cut -c 9-10`

END_YYYY=`echo $END_DATE | cut -c 1-4`
END_MM=`echo $END_DATE | cut -c 5-6`
END_DD=`echo $END_DATE | cut -c 7-8`
END_HH=`echo $END_DATE | cut -c 9-10`

CYC=$START_HH

WPS_START_DATE=${START_YYYY}-${START_MM}-${START_DD}_${START_HH}:00:00
WPS_START_DATE_P03=${START_P03_YYYY}-${START_P03_MM}-${START_P03_DD}_${START_P03_HH}:00:00
WPS_END_DATE=${END_YYYY}-${END_MM}-${END_DD}_${END_HH}:00:00

if [ $CYC -eq 03 -o $CYC -eq 09 -o $CYC -eq 15 -o $CYC -eq 21 ];then
 GLB_DATE=`$NDATE -3 $START_DATE`
else
 GLB_DATE=`$NDATE -0 $START_DATE`
fi
GLB_CYC=`echo ${GLB_DATE} | cut -c 9-10`


# Run setup to initialize working directory and utility scripts
## setup.sh  # no need, since using module load prod_util

#
# ungrib
#

case $MODEL in

ARW)

  if [ $IOFORM == 1 ] ; then
    cp $FIXsref/sref_geo_em.d01.int_$RES geo_em.d01.int
  else
    cp $FIXsref/sref_geo_em.d01.nc_$RES geo_em.d01.nc
  fi

  ###################
  # Initial conditions (RUC + GFS): 00h
  ###################

if [ $MEMBER == ctl -o $MEMBER == n01 -o $MEMBER == p01 -o $MEMBER == n02 -o $MEMBER == p02 ]; then
  if [ $MEMBER == ctl ]; then
  # GFS 00 file
   cp $COMINglb/global_pgrb/gfs.t${GLB_CYC}z.master.grbf03 .
   $USHsref/sref_filter_grib.sh gfs.t${GLB_CYC}z.master.grbf03
   $USHsref/sref_link_grib.csh gfs.t${GLB_CYC}z.master.grbf03
   cp $PARMsref/sref_Vtable.GFS Vtable
  else
  # GEFS 00 file
   cp $COMINglb/global_pgrb/gens_${GLBMEM}.t${GLB_CYC}z.pgrbf03 .
   $USHsref/sref_filter_grib.sh gens_${GLBMEM}.t${GLB_CYC}z.pgrbf03
   $USHsref/sref_link_grib.csh gens_${GLBMEM}.t${GLB_CYC}z.pgrbf03
   cp $PARMsref/sref_Vtable.GEFS Vtable
  fi

  cat $PARMsref/sref_namelist.wps.in.${MODEL}_$RES | sed s/WPS_START_DATE/$WPS_START_DATE/ | \
                                                     sed s/WPS_END_DATE/$WPS_START_DATE/ | \
                                                     sed s/IOFORM/$IOFORM/ | \
                                                     sed s/FILEPREFIX/FILE_GFS/ > namelist.wps
  export pgm=sref_ungrib
  . prep_step
  startmsg
  $EXECsref/sref_ungrib >> $pgmout 2>errfile
  export err=$?; err_chk

  # Rapid Refresh (RUC grid 130) 00 file
  $USHsref/sref_getbestRAPanl.sh $START_DATE
  $USHsref/sref_filter_grib.sh rap.t${CYC}z.awp130f00
  $USHsref/sref_link_grib.csh rap.t${CYC}z.awp130f00
  cp $PARMsref/sref_Vtable.RUC Vtable
  cat $PARMsref/sref_namelist.wps.in.${MODEL}_$RES | sed s/WPS_START_DATE/$WPS_START_DATE/ | \
                                                     sed s/WPS_END_DATE/$WPS_START_DATE/ | \
                                                     sed s/IOFORM/$IOFORM/ | \
                                                     sed s/FILEPREFIX/FILE_RUC/ > namelist.wps
  export pgm=sref_ungrib
  . prep_step
  startmsg
  $EXECsref/sref_ungrib >> $pgmout 2>errfile
  export err=$?; err_chk


  cp $PARMsref/sref_METGRID.TBL.$MODEL METGRID.TBL
  cat $PARMsref/sref_namelist.wps.in.${MODEL}_$RES | sed s/WPS_START_DATE/$WPS_START_DATE/ | \
                                                     sed s/WPS_END_DATE/$WPS_START_DATE/ | \
                                                     sed s/IOFORM/$IOFORM/ | \
                                                     sed s/FGNAME/"'.\/FILE_GFS'","'.\/FILE_RUC'"/ > namelist.wps

  export pgm=sref_metgrid
  . prep_step
  startmsg
  if [ $MACHINE == zeus ]; then
  mpiexec_mpt -prefix "[%g]" -np $PBS_NP $EXECsref/sref_metgrid >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss ]; then
  mpirun.lsf $EXECsref/sref_metgrid >> $pgmout 2>errfile
  elif [ $MACHINE == dell ]; then
  mpirun -l -n $N_TASK $EXECsref/sref_metgrid >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss2 ]; then
  mpiexec -n $NCPUS -ppn $NCPUS $EXECsref/sref_metgrid >> $pgmout 2>errfile
  else
  poe $EXECsref/sref_metgrid >> $pgmout 2>errfile
  fi
mv met_em.d01.${WPS_START_DATE}.nc met_em.d01.${WPS_START_DATE}.nc_IC
  export err=$?; err_chk
fi

if [ $MEMBER == c00 -o $MEMBER == n03 -o $MEMBER == p03 -o $MEMBER == n04 -o $MEMBER == p04 ]; then
  # GEFS 00 file
   cp $COMINglb/global_pgrb/gens_${GLBMEM}.t${GLB_CYC}z.pgrbf03 .
   $USHsref/sref_filter_grib.sh gens_${GLBMEM}.t${GLB_CYC}z.pgrbf03
   $USHsref/sref_link_grib.csh gens_${GLBMEM}.t${GLB_CYC}z.pgrbf03

  cp $PARMsref/sref_Vtable.GEFS Vtable
  cat $PARMsref/sref_namelist.wps.in.${MODEL}_$RES | sed s/WPS_START_DATE/$WPS_START_DATE/ | \
                                                     sed s/WPS_END_DATE/$WPS_START_DATE/ | \
                                                     sed s/IOFORM/$IOFORM/ | \
                                                     sed s/FILEPREFIX/FILE_GFS/ | \
                                                     sed s/FGNAME/"'.\/FILE_GFS'"/ > namelist.wps
  export pgm=sref_ungrib
  . prep_step
  startmsg
  $EXECsref/sref_ungrib >> $pgmout 2>errfile
  export err=$?; err_chk

  cp $PARMsref/sref_METGRID.TBL.$MODEL METGRID.TBL
  export pgm=sref_metgrid
  . prep_step
  startmsg
  if [ $MACHINE == zeus ]; then
  mpiexec_mpt -prefix "[%g]" -np $PBS_NP $EXECsref/sref_metgrid >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss ]; then
  mpirun.lsf $EXECsref/sref_metgrid >> $pgmout 2>errfile
  elif [ $MACHINE == dell ]; then
  mpirun -l -n $N_TASK $EXECsref/sref_metgrid >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss2 ]; then
  mpiexec -n $NCPUS -ppn $NCPUS $EXECsref/sref_metgrid >> $pgmout 2>errfile
  else
  poe $EXECsref/sref_metgrid >> $pgmout 2>errfile
  fi
mv met_em.d01.${WPS_START_DATE}.nc met_em.d01.${WPS_START_DATE}.nc_IC
  export err=$?; err_chk
fi

if [ $MEMBER == n05 -o $MEMBER == p05 -o $MEMBER == n06 -o $MEMBER == p06 ]; then
  # GFS 00 file
# cp $COMIN/global_pgrb/gfs.t${GLB_CYC}z.master.grbf03 .
# $USHsref/sref_filter_grib.sh gfs.t${GLB_CYC}z.master.grbf03
# $USHsref/sref_link_grib.csh gfs.t${GLB_CYC}z.master.grbf03
  cp $COMINglb/global_pgrb/gens_${GLBMEM}.t${GLB_CYC}z.pgrbf03 .
  $USHsref/sref_filter_grib.sh gens_${GLBMEM}.t${GLB_CYC}z.pgrbf03
  $USHsref/sref_link_grib.csh gens_${GLBMEM}.t${GLB_CYC}z.pgrbf03
  cp $PARMsref/sref_Vtable.GEFS Vtable
  cat $PARMsref/sref_namelist.wps.in.${MODEL}_$RES | sed s/WPS_START_DATE/$WPS_START_DATE/ | \
                                                     sed s/WPS_END_DATE/$WPS_START_DATE/ | \
                                                     sed s/IOFORM/$IOFORM/ | \
                                                     sed s/FILEPREFIX/FILE_GFS/ | \
                                                     sed s/FGNAME/"'.\/FILE_GFS'"/ > namelist.wps
  export pgm=sref_ungrib
  . prep_step
  startmsg
  $EXECsref/sref_ungrib >> $pgmout 2>errfile
# if [ $MACHINE == zeus ]; then
# mpiexec_mpt -prefix "[%g]" -np $PBS_NP $EXECsref/sref_ungrib >> $pgmout 2>errfile
# elif [ $MACHINE == wcoss ]; then
# mpirun.lsf $EXECsref/sref_ungrib >> $pgmout 2>errfile
# else
# poe $EXECsref/sref_ungrib >> $pgmout 2>errfile
# fi
  export err=$?;err_chk

# # NDAS (Grid 151) f00 file (later start) or NAM f03 file (earlier start)
  $USHsref/sref_getbestndasanl_${MODEL}.sh $START_DATE

  $USHsref/sref_filter_grib.sh bgd2arw.${START_DATE}
  $USHsref/sref_link_grib.csh  bgd2arw.${START_DATE}


  cp $PARMsref/sref_Vtable.NAM Vtable
# cat $PARMsref/sref_namelist.wps.in.${MODEL}_$RES | sed s/WPS_START_DATE/$WPS_START_DATE/ | \
  cat $PARMsref/sref_namelist.wps.in.${MODEL}_${RES}_nems | sed s/WPS_START_DATE/$WPS_START_DATE/ | \
                                                     sed s/WPS_END_DATE/$WPS_START_DATE/ | \
                                                     sed s/IOFORM/$IOFORM/ | \
                                                     sed s/FILEPREFIX/FILE_NDAS/ | \
                                                     sed s/FGNAME/"'.\/FILE_NDAS'"/ > namelist.nps
#                                                    sed s/FGNAME/"'.\/FILE_NDAS'"/ > namelist.wps
#
  export pgm=sref_ungrib_nems
  . prep_step
  startmsg
  if [ $MACHINE == zeus ]; then
  mpiexec_mpt -prefix "[%g]" -np $PBS_NP $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss ]; then
  mpirun.lsf $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  elif [ $MACHINE == dell ]; then
  mpirun -l -n $N_TASK $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss2 ]; then
# mpiexec -n $NCPUS -ppn $NCPUS $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  else
  poe $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  fi
  export err=$?;err_chk

# export pgm=sref_ungrib
# . prep_step
# startmsg
# $EXECsref/sref_ungrib >> $pgmout 2>errfile
# export err=$?;err_chk

# Control vertical levels
# time=`echo $WPS_START_DATE | cut -c 1-13`
# mv FILE_NDAS:${time} FILE_NDAS:${time}_org
# $EXECsref/sref_${MODEL}_levs FILE_NDAS:${time}_org  FILE_NDAS:${time}

  cp $PARMsref/sref_METGRID.TBL.$MODEL METGRID.TBL
  cat $PARMsref/sref_namelist.wps.in.${MODEL}_$RES | sed s/WPS_START_DATE/$WPS_START_DATE/ | \
                                                     sed s/WPS_END_DATE/$WPS_START_DATE/ | \
                                                     sed s/IOFORM/$IOFORM/ | \
                                                     sed s/FGNAME/"'.\/FILE_GFS'","'.\/FILE_NDAS'"/ > namelist.wps
  cp namelist.wps namelist.wps_${MEMBER}4metgrid
  export pgm=sref_metgrid
  . prep_step
  startmsg
  if [ $MACHINE == zeus ]; then
  mpiexec_mpt -prefix "[%g]" -np $PBS_NP $EXECsref/sref_metgrid >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss ]; then
  mpirun.lsf $EXECsref/sref_metgrid >> $pgmout 2>errfile
  elif [ $MACHINE == dell ]; then
  mpirun -l -n $N_TASK $EXECsref/sref_metgrid >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss2 ]; then
  mpiexec -n $NCPUS -ppn $NCPUS $EXECsref/sref_metgrid >> $pgmout 2>errfile
  else
  poe $EXECsref/sref_metgrid >> $pgmout 2>errfile
  fi
mv met_em.d01.${WPS_START_DATE}.nc met_em.d01.${WPS_START_DATE}.nc_IC
  export err=$?; err_chk
fi


  ###################
  # Lateral Boundary conditions: 03h - WPS_END_DATE
  # (Note: If it starts from 00, it also serves IC)
  ###################

  if [ $MEMBER == ctl ]; then

      cp $COMINglb/global_pgrb/gfs.t${GLB_CYC}z.master.grbf[0123456789]? .
      $USHsref/sref_filter_grib.sh gfs.t${GLB_CYC}z.master.grbf[0123456789]?
      $USHsref/sref_link_grib.csh gfs.t${GLB_CYC}z.master.grbf[0123456789]?
      cp $PARMsref/sref_Vtable.GFS Vtable

  else

      cp $COMINglb/global_pgrb/gens_${GLBMEM}.t${GLB_CYC}z.pgrbf[0123456789]? .
      $USHsref/sref_filter_grib.sh gens_${GLBMEM}.t${GLB_CYC}z.pgrbf[0123456789]?
      $USHsref/sref_link_grib.csh gens_${GLBMEM}.t${GLB_CYC}z.pgrbf[0123456789]?
      cp $PARMsref/sref_Vtable.GEFS Vtable

  fi

# cat $PARMsref/sref_namelist.wps.in.${MODEL}_$RES | sed s/WPS_START_DATE/$WPS_START_DATE_P03/ | \
  cat $PARMsref/sref_namelist.wps.in.${MODEL}_$RES | sed s/WPS_START_DATE/$WPS_START_DATE/ | \
                                                     sed s/WPS_END_DATE/$WPS_END_DATE/ | \
                                                     sed s/IOFORM/$IOFORM/ | \
                                                     sed s/FILEPREFIX/FILE_GFS/ | \
                                                     sed s/FGNAME/"'.\/FILE_GFS'"/ > namelist.wps
  export pgm=sref_ungrib
  . prep_step
  startmsg
  $EXECsref/sref_ungrib >> $pgmout 2>errfile
  export err=$?; err_chk

  cp $PARMsref/sref_METGRID.TBL.$MODEL METGRID.TBL
  export pgm=sref_metgrid
  . prep_step
  startmsg
  if [ $MACHINE == zeus ]; then
  mpiexec_mpt -prefix "[%g]" -np $PBS_NP $EXECsref/sref_metgrid >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss ]; then
  mpirun.lsf $EXECsref/sref_metgrid >> $pgmout 2>errfile
  elif [ $MACHINE == dell ]; then
  mpirun -l -n $N_TASK $EXECsref/sref_metgrid >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss2 ]; then
  mpiexec -n $NCPUS -ppn $NCPUS $EXECsref/sref_metgrid >> $pgmout 2>errfile
  else
  poe $EXECsref/sref_metgrid >> $pgmout 2>errfile
  fi
mv met_em.d01.${WPS_START_DATE}.nc met_em.d01.${WPS_START_DATE}.nc_LBC
cp met_em.d01.${WPS_START_DATE}.nc_IC met_em.d01.${WPS_START_DATE}.nc
  export err=$?; err_chk

  ;;

NMM)

  ###################
  # Initial conditions (GFS and GEFS): 00h
  ###################

  if [ $IOFORM == 1 ] ; then
    cp $FIXsref/sref_geo_nmm.d01.int_$RES geo_nmm.d01.int
  else
    cp $FIXsref/sref_geo_nmm.d01.nc_$RES geo_nmm.d01.nc
  fi

  if [ $MEMBER == ctl ]; then
      $USHsref/sref_link_grib.csh $COMINglb/global_pgrb/gfs.t${GLB_CYC}z.master.grbf[0123456789]?
  else
      $USHsref/sref_link_grib.csh $COMINglb/global_pgrb/gens_${GLBMEM}.t${GLB_CYC}z.pgrbf??
  fi

  cp $PARMsref/sref_Vtable.GFS Vtable
  cat $PARMsref/sref_namelist.wps.in.${MODEL}_$RES | sed s/WPS_START_DATE/$WPS_START_DATE/ | \
                                                     sed s/WPS_END_DATE/$WPS_END_DATE/ | \
                                                     sed s/IOFORM/$IOFORM/ | \
                                                     sed s/FILEPREFIX/FILE_GFS/ | \
                                                     sed s/FGNAME/"'.\/FILE_GFS'"/ > namelist.wps

  export pgm=sref_ungrib
  . prep_step
  startmsg
  $EXECsref/sref_ungrib >> $pgmout 2>errfile
  export err=$?; err_chk

  cp $PARMsref/sref_METGRID.TBL.$MODEL METGRID.TBL
  export pgm=sref_metgrid
  . prep_step
  startmsg
  if [ $MACHINE == zeus ]; then
  mpiexec_mpt -prefix "[%g]" -np $PBS_NP $EXECsref/sref_metgrid >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss ]; then
  mpirun.lsf $EXECsref/sref_metgrid >> $pgmout 2>errfile
  elif [ $MACHINE == dell ]; then
  mpirun -l -n $N_TASK $EXECsref/sref_metgrid >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss2 ]; then
  mpiexec -n $NCPUS -ppn $NCPUS $EXECsref/sref_metgrid >> $pgmout 2>errfile
  else
  poe $EXECsref/sref_metgrid >> $pgmout 2>errfile
  fi
  export err=$?; err_chk

  ;;

NMB)

  cp $FIXsref/sref_geo_nmb.d01.int_$RES geo_nmb.d01.dio

if [ $MEMBER == ctl -o $MEMBER == n01 -o $MEMBER == p01 -o $MEMBER == n02 -o $MEMBER == p02 ]; then
  ###################
  # Initial conditions (NDAS grid151): 00h
  ###################

  if [ $MEMBER == ctl ]; then
  # GFS 00 file
   cp $COMINglb/global_pgrb/gfs.t${GLB_CYC}z.master.grbf03 .
   $USHsref/sref_filter_grib.sh gfs.t${GLB_CYC}z.master.grbf03
   $USHsref/sref_link_grib.csh gfs.t${GLB_CYC}z.master.grbf03
   cp $PARMsref/sref_Vtable.GFS Vtable
  else
  # GEFS 00 file
   cp $COMINglb/global_pgrb/gens_${GLBMEM}.t${GLB_CYC}z.pgrbf03 .
   $USHsref/sref_filter_grib.sh gens_${GLBMEM}.t${GLB_CYC}z.pgrbf03
   $USHsref/sref_link_grib.csh gens_${GLBMEM}.t${GLB_CYC}z.pgrbf03
   cp $PARMsref/sref_Vtable.GEFS Vtable
  fi

  cat $PARMsref/sref_namelist.wps.in.${MODEL}_$RES | sed s/WPS_START_DATE/$WPS_START_DATE/ | \
                                                     sed s/WPS_END_DATE/$WPS_START_DATE/ | \
                                                     sed s/FILEPREFIX/FILE_GFS/ > namelist.nps
  export pgm=sref_ungrib_nems
  . prep_step
  startmsg
  if [ $MACHINE == zeus ]; then
  mpiexec_mpt -prefix "[%g]" -np $PBS_NP $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss ]; then
  mpirun.lsf $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  elif [ $MACHINE == dell ]; then
  mpirun -l -n $N_TASK $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss2 ]; then
# mpiexec -n $NCPUS -ppn $NCPUS $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  else
  poe $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  fi
  export err=$?;err_chk

# # NDAS (Grid 151) f00 file (later start) or NAM f03 file (earlier start)
  $USHsref/sref_getbestndasanl_${MODEL}.sh $START_DATE
  $USHsref/sref_filter_grib.sh bgd2nmb.${START_DATE}
  $USHsref/sref_link_grib.csh  bgd2nmb.${START_DATE}

  cp $PARMsref/sref_Vtable.NAM Vtable
  cat $PARMsref/sref_namelist.wps.in.${MODEL}_$RES | sed s/WPS_START_DATE/$WPS_START_DATE/ | \
                                                     sed s/WPS_END_DATE/$WPS_START_DATE/ | \
#                                                    sed s/FILEPREFIX/FILE_NDAS/ > namelist.nps
                                                     sed s/FILEPREFIX/FILE_NDAS/ | \
                                                     sed s/FGNAME/"'.\/FILE_NDAS'"/ > namelist.nps
  export pgm=sref_ungrib_nems
  . prep_step
  startmsg
  if [ $MACHINE == zeus ]; then
  mpiexec_mpt -prefix "[%g]" -np $PBS_NP $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss ]; then
  mpirun.lsf $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  elif [ $MACHINE == dell ]; then
  mpirun -l -n $N_TASK $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss2 ]; then
# mpiexec -n $NCPUS -ppn $NCPUS $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  else
  poe $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  fi
  export err=$?;err_chk

# Control vertical levels
#  time=`echo $WPS_START_DATE | cut -c 1-13`
#  mv FILE_NDAS:${time} FILE_NDAS:${time}_org
#  ln -sf namelist.nps namelist.wps
#  $EXECsref/sref_${MODEL}_levs FILE_NDAS:${time}_org  FILE_NDAS:${time}

  cp $PARMsref/sref_METGRID.TBL.$MODEL METGRID.TBL
  cat $PARMsref/sref_namelist.wps.in.${MODEL}_$RES | sed s/WPS_START_DATE/$WPS_START_DATE/ | \
                                                     sed s/WPS_END_DATE/$WPS_START_DATE/ | \
                                                     sed s/FGNAME/"'.\/FILE_GFS'","'.\/FILE_NDAS'"/ > namelist.nps
fi

if [ $MEMBER == c00 -o $MEMBER == n03 -o $MEMBER == p03 -o $MEMBER == n04 -o $MEMBER == p04 ]; then
  ###################
  # Initial conditions (NDAS grid151): 00h
  ###################

  # GEFS 00 file
   cp $COMINglb/global_pgrb/gens_${GLBMEM}.t${GLB_CYC}z.pgrbf03 .
   $USHsref/sref_filter_grib.sh gens_${GLBMEM}.t${GLB_CYC}z.pgrbf03
   $USHsref/sref_link_grib.csh gens_${GLBMEM}.t${GLB_CYC}z.pgrbf03

  cp $PARMsref/sref_Vtable.GEFS Vtable
  cat $PARMsref/sref_namelist.wps.in.${MODEL}_$RES | sed s/WPS_START_DATE/$WPS_START_DATE/ | \
                                                     sed s/WPS_END_DATE/$WPS_START_DATE/ | \
                                                     sed s/FILEPREFIX/FILE_GFS/ | \
                                                     sed s/FGNAME/"'.\/FILE_GFS'"/ > namelist.nps
  export pgm=sref_ungrib_nems
  . prep_step
  startmsg
  if [ $MACHINE == zeus ]; then
  mpiexec_mpt -prefix "[%g]" -np $PBS_NP $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss ]; then
  mpirun.lsf $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  elif [ $MACHINE == dell ]; then
  mpirun -l -n $N_TASK $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss2 ]; then
# mpiexec -n $NCPUS -pnn $NCPUS $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  else
  poe $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  fi
  export err=$?;err_chk

  cp $PARMsref/sref_METGRID.TBL.$MODEL METGRID.TBL
fi

if [ $MEMBER == n05 -o $MEMBER == p05 -o $MEMBER == n06 -o $MEMBER == p06 ]; then
  if [ $MEMBER == ctl ]; then
  # GFS 00 file
   cp $COMINglb/global_pgrb/gfs.t${GLB_CYC}z.master.grbf03 .
   $USHsref/sref_filter_grib.sh gfs.t${GLB_CYC}z.master.grbf03
   $USHsref/sref_link_grib.csh gfs.t${GLB_CYC}z.master.grbf03
   cp $PARMsref/sref_Vtable.GFS Vtable
  else
  # GEFS 00 file
   cp $COMINglb/global_pgrb/gens_${GLBMEM}.t${GLB_CYC}z.pgrbf03 .
   $USHsref/sref_filter_grib.sh gens_${GLBMEM}.t${GLB_CYC}z.pgrbf03
   $USHsref/sref_link_grib.csh gens_${GLBMEM}.t${GLB_CYC}z.pgrbf03
   cp $PARMsref/sref_Vtable.GEFS Vtable
  fi

  cat $PARMsref/sref_namelist.wps.in.${MODEL}_$RES | sed s/WPS_START_DATE/$WPS_START_DATE/ | \
                                                     sed s/WPS_END_DATE/$WPS_START_DATE/ | \
                                                     sed s/FILEPREFIX/FILE_GFS/ > namelist.nps
  export pgm=sref_ungrib_nems
  . prep_step
  startmsg
  if [ $MACHINE == zeus ]; then
  mpiexec_mpt -prefix "[%g]" -np $PBS_NP $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss ]; then
  mpirun.lsf $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  elif [ $MACHINE == dell ]; then
  mpirun -l -n $N_TASK $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss2 ]; then
# mpirun -n $NCPUS -pnn $NCPUS $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  else
  poe $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  fi
  export err=$?;err_chk

  # Rapid Refresh (RUC grid 130) 00 file
  $USHsref/sref_getbestRAPanl.sh $START_DATE
  $USHsref/sref_filter_grib.sh rap.t${CYC}z.awp130f00
  $USHsref/sref_link_grib.csh rap.t${CYC}z.awp130f00
  cp $PARMsref/sref_Vtable.RUC Vtable
  cat $PARMsref/sref_namelist.wps.in.${MODEL}_$RES | sed s/WPS_START_DATE/$WPS_START_DATE/ | \
                                                     sed s/WPS_END_DATE/$WPS_START_DATE/ | \
                                                     sed s/FILEPREFIX/FILE_RUC/ > namelist.nps
  export pgm=sref_ungrib_nems
  . prep_step
  startmsg
  if [ $MACHINE == zeus ]; then
  mpiexec_mpt -prefix "[%g]" -np $PBS_NP $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss ]; then
  mpirun.lsf $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  elif [ $MACHINE == dell ]; then
  mpirun -l -n $N_TASK $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss2 ]; then
# mpirun -n $NCPUS -pnn $NCPUS $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  else
  poe $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  fi
  export err=$?;err_chk

  cp $PARMsref/sref_METGRID.TBL.$MODEL METGRID.TBL
  cat $PARMsref/sref_namelist.wps.in.${MODEL}_$RES | sed s/WPS_START_DATE/$WPS_START_DATE/ | \
                                                     sed s/WPS_END_DATE/$WPS_START_DATE/ | \
                                                     sed s/FGNAME/"'.\/FILE_GFS'","'.\/FILE_RUC'"/ > namelist.nps
fi

  export pgm=sref_metgrid_nems
  . prep_step
  startmsg
  if [ $MACHINE == zeus ]; then
  mpiexec_mpt -prefix "[%g]" -np $PBS_NP $EXECsref/sref_metgrid_nems >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss ]; then
  mpirun.lsf $EXECsref/sref_metgrid_nems >> $pgmout 2>errfile
  elif [ $MACHINE == dell ]; then
  mpirun -l -n $N_TASK $EXECsref/sref_metgrid_nems >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss2 ]; then
  mpiexec -n $NCPUS -ppn $NCPUS $EXECsref/sref_metgrid_nems >> $pgmout 2>errfile
  else
  poe $EXECsref/sref_metgrid_nems >> $pgmout 2>errfile
  fi
mv met_nmb.d01.${WPS_START_DATE}.dio met_nmb.d01.${WPS_START_DATE}.dio_IC
  export err=$?;err_chk

  ###################
  # Lateral Boundary conditions: 03h - WPS_END_DATE
  # (Note: If it starts from 00, it also serves IC)
  ###################

  if [ $MEMBER == ctl ]; then

      cp $COMINglb/global_pgrb/gfs.t${GLB_CYC}z.master.grbf[0123456789]? .
      $USHsref/sref_filter_grib.sh gfs.t${GLB_CYC}z.master.grbf[0123456789]?
      $USHsref/sref_link_grib.csh gfs.t${GLB_CYC}z.master.grbf[0123456789]?
      cp $PARMsref/sref_Vtable.GFS Vtable

  else

      cp $COMINglb/global_pgrb/gens_${GLBMEM}.t${GLB_CYC}z.pgrbf[0123456789]? .
      $USHsref/sref_filter_grib.sh gens_${GLBMEM}.t${GLB_CYC}z.pgrbf[0123456789]?
      $USHsref/sref_link_grib.csh gens_${GLBMEM}.t${GLB_CYC}z.pgrbf[0123456789]?
      cp $PARMsref/sref_Vtable.GEFS Vtable

  fi

# cat $PARMsref/sref_namelist.wps.in.${MODEL}_$RES | sed s/WPS_START_DATE/$WPS_START_DATE_P03/ | \
  cat $PARMsref/sref_namelist.wps.in.${MODEL}_$RES | sed s/WPS_START_DATE/$WPS_START_DATE/ | \
                                                     sed s/WPS_END_DATE/$WPS_END_DATE/ | \
                                                     sed s/FILEPREFIX/FILE_GFS/ | \
                                                     sed s/FGNAME/"'.\/FILE_GFS'"/ > namelist.nps
  export pgm=sref_ungrib_nems
  . prep_step
  startmsg
  if [ $MACHINE == zeus ]; then
  mpiexec_mpt -prefix "[%g]" -np $PBS_NP $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss ]; then
  mpirun.lsf $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  elif [ $MACHINE == dell ]; then
  mpirun -l -n $N_TASK $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss2 ]; then
# mpiexec -n $NCPUS -ppn $NCPUS $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  else
  poe $EXECsref/sref_ungrib_nems >> $pgmout 2>errfile
  fi
  export err=$?;err_chk

  cp $PARMsref/sref_METGRID.TBL.$MODEL METGRID.TBL

FHR=00                                                                                                  #!!!!!
#FHR=03                                                                                                  #!!!!!
while [ $FHR -le $FLENGTH ]                                                                             #!!!!!
do                                                                                                      #!!!!!
WPS_DATE=`$NDATE +$FHR $START_DATE`                                                            #!!!!!
YYYY=`echo $WPS_DATE | cut -c 1-4`                                                                      #!!!!!
MM=`echo $WPS_DATE | cut -c 5-6`                                                                        #!!!!!
DD=`echo $WPS_DATE | cut -c 7-8`                                                                        #!!!!!
HH=`echo $WPS_DATE | cut -c 9-10`                                                                       #!!!!!
WPS_DATE=${YYYY}-${MM}-${DD}_${HH}:00:00                                                                #!!!!!

cat $PARMsref/sref_namelist.wps.in.${MODEL}_$RES | sed s/WPS_START_DATE/$WPS_DATE/ | \
                                                   sed s/WPS_END_DATE/$WPS_DATE/ | \
                                                   sed s/FILEPREFIX/FILE_GFS/ | \
                                                   sed s/FGNAME/"'.\/FILE_GFS'"/ > namelist.nps
  export pgm=sref_metgrid_nems
  . prep_step
  startmsg
  if [ $MACHINE == zeus ]; then
  mpiexec_mpt -prefix "[%g]" -np $PBS_NP $EXECsref/sref_metgrid_nems >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss ]; then
  mpirun.lsf $EXECsref/sref_metgrid_nems >> $pgmout 2>errfile
  elif [ $MACHINE == dell ]; then
  mpirun -l -n $N_TASK $EXECsref/sref_metgrid_nems >> $pgmout 2>errfile
  elif [ $MACHINE == wcoss2 ]; then
  mpiexec -n $NCPUS -ppn $NCPUS $EXECsref/sref_metgrid_nems >> $pgmout 2>errfile
  else
  poe $EXECsref/sref_metgrid_nems >> $pgmout 2>errfile
  fi
  export err=$?;err_chk

FHR=`expr $FHR + $INCR`                                                                                 #!!!!!
done                                                                                                    #!!!!!
mv met_nmb.d01.${WPS_START_DATE}.dio met_nmb.d01.${WPS_START_DATE}.dio_LBC
cp met_nmb.d01.${WPS_START_DATE}.dio_IC met_nmb.d01.${WPS_START_DATE}.dio

  ;;

*)
   echo "Unknown model ${MODEL}"
   exit
   ;;

esac

grep "Successful completion of program metgrid.exe" metgrid.log.0000
#grep "Successful completion of program metgrid.exe" metgrid.log
err=$?
if [ $err -ne 0 ]; then
   echo "sref_metgrid failed" >> $WORK_DIR/wps.$MEMBER.failed
   err_chk
fi

echo "done" > $WORK_DIR/wps.$MEMBER.done
echo "done" > $WORK_DIR/../wps.$MEMBER.done