#!/bin/sh
################################################################################
####  UNIX Script Documentation Block
#                      .                                             .
# Script name:         exngac_fcst.sh.sms
# Script description:  Runs a global spectral model forecast
#
# Author:        Mark Iredell       Org: NP23         Date: 1999-05-01
#
# Abstract: This script runs a single or an ensemble of global spectral model
#           forecasts. The initial conditions and run parameters are either
#           passed in the argument list or imported.
#
# Script history log:
# 1999-05-01  Mark Iredell
# 2005-01-03  Cheng-Hsuan Lu :add namelist SOIL_VEG
#                             set FSMCL(2:4) = FSMCL2
#                             add FNVMNC,FNVMXC,FNSLPC,FNABSC
# 2006-02     Shrinivas Moorthi Modified to run ESMF - Stand Alone
#                             version of ATM - Only filestyle "L"
#                             allowed - Added a ESMF config file
#                             The script can run up to 21 ENS members
#                             concurrently.
# 2006-06     Shrinivas Moorthi : Added default PE$n values to 0
# 2009-05     Jun Wang, add write grid component option
# 2009-10     Sarah Lu, ESMF_State_Namelist modified:
#                       tracer added; (q, oz, cld) removed
# 2009-11     Jun Wang, activate reduced grid option and digital filter option
# 2009-12     Jun Wang, link atm_namelist.rc to configure_file
# 2009-2010   Shrinvias Moorthi : Added stochastic ensembles, semi-Lagrangian
#                                 high frequency output, G3D outputs and many
#                                 other upgrades related to model changes
# 2010-01     Weiyu Yang, modified for the ensemble GEFS.
# 2010-01-14  Sarah Lu : Added GOCART_CLIM and GOCART_LUTS
# 2010-02     Jun Wang  Add restart
# 2010-02-06  Sarah Lu : modify phy_namelist.rc (set p_export/dp_export to 1)
# 2010-03-23  Sarah Lu : add passive_tracer to atm_namelist.rc 
# 2010-05-31  Sarah Lu : use wildcard to copy files from GOCART_CLIM to DATA
# 2010-07     Jun Wang : add option to output filtered 3hr output
# 2010-07-14  Shrinivas Moorthi : Upgraded for the new physics and nst model
#             and rewrote some parts and added ensemble generality
# 2010-07-23  Sarah Lu : add AER, modify filename_base, file_io_form, file_io
# 2010-12-12  Henry Juang : add ndslfv, process_split, and mass_dp for NDSL
# 2011-03-15  Henry Juang : add JCAPG for NDSL
# 2011-02-28  Sarah Lu : add thermodyn_id and sfcpress_id to nam_dyn
# 2011-04-14  Jun Wang : add copy MAPL/CHEM config files,set default FILE_IO_FORM
# 2012-01-15  Sarah Lu: add WRITE_DOPOST, GOCART_POSTOUT, POST_GRIBVERSION
# 2012-02-03  Jun Wang: add grib2 option for POST
# 2012-02-12  Sarah Lu: change GOCART_POSTOUT to GOCART_AER2POST
# 2012-02-20  Jun Wang: add POST_NCEPGRB2TBL for post
# 2012-03-15  Sarah Lu: modify how POST_GRIBVERSION is specified
# 2012-05-12  Sarah Lu: modify how LATG is specified
# 2012-09-24  Sarah Lu: add SCHEDULER dependence
# 2012-12-05  Sarah Lu: add lsf SCHEDULER as the place holder
#
# Usage:  global_forecast.sh GRDI SFCI SIGO FLXO FHOUT FHMAX IGEN D3DO NSTI NSTO G3DO FHOUT_HF FHMAX_HF
#
#   Input script positional parameters:
#     1             Input grd file 1
#                   defaults to $GRDI; one or the other is required
#     2             Input surface file
#                   defaults to $SFCI; one or the other is required
#     3             Output sigma file with embedded forecast hour '${FH}'
#                   defaults to $SIGO, then to ${COMOUT}/sigf'${FH}'$SUFOUT
#     4             Output flux file with embedded forecast hour '${FH}'
#                   defaults to $FLXO, then to ${COMOUT}/flxf'${FH}'$SUFOUT
#     5             Output frequency in hours
#                   defaults to $FHOUT, then to 3
#     6             Length of forecast in hours
#                   defaults to $FHMAX; otherwise FHSEG is required to be set
#     7             Output generating code
#                   defaults to $IGEN, defaults to 0
#     8             Output flux file with embedded forecast hour '${FH}'
#                   defaults to $D3DO, then to ${COMOUT}/d3df'${FH}'$SUFOUT
#     9             Input NST file
#     10            Output NST file
#     11            output 3d file for GOCART
#                   defaults to $G3DO, then to ${COMOUT}/g3df'${FH}'$SUFOUT
#     12            High frequency output interval ; default 1 hour
#     13            Maximum hour of high frequecny output
#
#   Imported Shell Variables:
#     GRDI          Input sigma file
#                   overridden by $1; one or the other is required
#     SFCI          Input surface file
#                   overridden by $2; one or the other is required
#     SIGO          Output sigma file with embedded forecast hour '${FH}'
#                   overridden by $3; defaults to ${COMOUT}/sigf'${FH}'$SUFOUT
#     FLXO          Output flux file with embedded forecast hour '${FH}'
#                   overridden by $4; defaults to ${COMOUT}/flxf'${FH}'$SUFOUT
#     D3DO          Output d3d file with embedded forecast hour '${FH}'
#                   overridden by $4; defaults to ${COMOUT}/d3df'${FH}'$SUFOUT
#     NSTO         Output nst file with embedded forecast hour '${FH}'
#                   overridden by $4; defaults to ${COMOUT}/nstf'${FH}'$SUFOUT
#     G3DO          Output g3d file with embedded forecast hour '${FH}'
#                   overridden by $4; defaults to ${COMOUT}/g3df'${FH}'$SUFOUT
#     FHOUT         Output frequency in hours
#                   overridden by $5; defaults to 3
#     FHMAX         Length of forecast in hours
#                   overridden by $6; either FHMAX or FHSEG must be set
#     IGEN          Output generating code
#                   overridden by $7; defaults to 0
#     FIXGLOBAL     Directory for global fixed files
#                   defaults to /nwprod/fix
#     EXECGLOBAL    Directory for global executables
#                   defaults to /nwprod/exec
#     DATA          working directory
#                   (if nonexistent will be made, used and deleted)
#                   defaults to current working directory
#     COMOUT        output directory
#                   (if nonexistent will be made)
#                   defaults to current working directory
#     XC            Suffix to add to executables
#                   defaults to none
#     SUFOUT        Suffix to add to output filenames
#                   defaults to none
#     NCP           Copy command
#                   defaults to cp
#     SIGHDR        Command to read sigma header
#                   (required if JCAP, LEVS, or FHINI are not specified)
#                   defaults to ${EXECGLOBAL}/global_sighdr$XC
#     JCAP          Spectral truncation for model wave
#     JCAPG         Spectral truncation for model Grid
#                   defaults in nems script for always quardratic grid for grid number
#     LEVS          Number of levels
#                   defaults to the value in the input sigma file header
#     LEVR          Number of levels over which radiation is computed
#                   defaults to LEVS
#     FCSTEXEC      Forecast executable
#                   defaults to ${EXECGLOBAL}/global_fcst$XC
#     SIGI2         Second time level sigma restart file
#                   defaults to NULL
#     CO2CON        Input CO2 radiation (vertical resolution dependent)
#                   defaults to ${FIXGLOBAL}/global_co2con.l${LEVS}.f77
#     MTNVAR        Input mountain variance (horizontal resolution dependent)
#                   defaults to ${FIXGLOBAL}/global_mtnvar.t${JCAPG}.f77
#     CLTUNE        Input cloud tuning file
#                   defaults to ${FIXGLOBAL}/global_cldtune.f77
#     DTBTHE        Input equivalent potential temperature file
#                   defaults to ${FIXGLOBAL}/global_tbthe.f77
#     O3FORC        Input ozone forcing (production/loss) climatology
#                   defaults to ${FIXGLOBAL}/global_o3prdlos.f77
#     O3CLIM        Input ozone climatology
#                   defaults to ${FIXGLOBAL}/global_o3clim.txt
#     FNGLAC        Input glacier climatology GRIB file
#                   defaults to ${FIXGLOBAL}/global_glacier.2x2.grb
#     FNMXIC        Input maximum sea ice climatology GRIB file
#                   defaults to ${FIXGLOBAL}/global_maxice.2x2.grb
#     FNTSFC        Input SST climatology GRIB file
#                   defaults to ${FIXGLOBAL}/global_sstclim.2x2.grb
#     FNSNOC        Input snow climatology GRIB file
#                   defaults to ${FIXGLOBAL}/global_snoclim.1.875.grb
#     FNZORC        Input roughness climatology GRIB file
#                   defaults to ${FIXGLOBAL}/global_zorclim.1x1.grb
#     FNALBC        Input albedo climatology GRIB file
#                   defaults to ${FIXGLOBAL}/global_albedo4.1x1.grb
#     FNAISC        Input sea ice climatology GRIB file
#                   defaults to ${FIXGLOBAL}/global_iceclim.2x2.grb
#     FNTG3C        Input deep soil temperature climatology GRIB file
#                   defaults to ${FIXGLOBAL}/global_tg3clim.2.6x1.5.grb
#     FNVEGC        Input vegetation fraction climatology GRIB file
#                   defaults to ${FIXGLOBAL}/global_vegfrac.1x1.grb
#     FNVETC        Input vegetation type climatology GRIB file
#                   defaults to ${FIXGLOBAL}/global_vegtype.1x1.grb
#     FNSOTC        Input soil type climatology GRIB file
#                   defaults to ${FIXGLOBAL}/global_soiltype.1x1.grb
#     FNSMCC        Input soil moisture climatology GRIB file
#                   defaults to ${FIXGLOBAL}/global_soilmcpc.1x1.grb
#     FNVMNC        Input min veg frac climatology GRIB file    
#                   defaults to ${FIXGLOBAL}/global_shdmin.0.144x0.144.grb
#     FNVMXC        Input max veg frac climatology GRIB file    
#                   defaults to ${FIXGLOBAL}/global_shdmax.0.144x0.144.grb
#     FNSLPC        Input slope type climatology GRIB file    
#                   defaults to ${FIXGLOBAL}/global_slope.1x1.grb
#     FNABSC        Input max snow albedo climatology GRIB file    
#                   defaults to ${FIXGLOBAL}/global_snoalb.1x1.grb
#     OROGRAPHY     Input orography GRIB file (horiz resolution dependent)
#                   defaults to ${FIXGLOBAL}/global_orography.t$JCAPG.grb
#     FNMSKH        Input high resolution land mask GRIB file
#                   defaults to ${FIXGLOBAL}/seaice_newland.grb
#     FNTSFA        Input SST analysis GRIB file
#                   defaults to none
#     FNACNA        Input sea ice analysis GRIB file
#                   defaults to none
#     FNSNOA        Input snow analysis GRIB file
#                   defaults to none
#     AERODIR       Input aersol climatology directory
#                   defaults to ${FIXGLOBAL}
##########################################################################
#     FIX_RAD       Directory for global fixed files
#                   Defaults to $${FIXGLOBAL}
#     EMISDIR       Input earth's surface emissivity data directory
#                   defaults to ${FIX_RAD} - export IEMS=1 to activate
#     SOLCDIR       11 year cycle Solar constat data directory
#                   defaults to ${FIX_RAD} - export ISOL=1 to activate
#     VOLCDIR       Volcanic aerosol  data directory
#                   defaults to ${FIX_RAD} - export IAER=11 or 12 to activate
#                   IAER=11 uses opac aero_volc; IAER=12 uses gocart+volc
#     CO2DIR        Historical CO2 data directory
#                   defaults to ${FIX_RAD} - export ICO2=1 or 2 to activate
#                   ICO2=1 gives annual mean and ICO2=2 uses monthly 2D data
##########################################################################
#     GOCART_CLIM   Directory for gocart climo files
#                   Defaults to $${FIXGLOBAL}
#     GOCARTC_LUTS  Directory for gocart luts files
#                   Defaults to $${FIXGLOBAL}
#     SIGR1         Output first time level sigma restart file
#                   defaults to ${DATA}/sigr1 which is deleted
#     SIGR2         Output second time level sigma restart file
#                   defaults to ${DATA}/sigr2 which is deleted
#     SFCR          Output surface restart file
#                   defaults to ${DATA}/sfcr which is deleted
#     NSTR          Output nst restart file
#                   defaults to ${DATA}/nstr which is deleted
#     SFCO          Output surface file with embedded forecast hour '${FH}'
#                   defaults to ${COMOUT}/sfcf'${FH}'$SUFOUT
#     LOGO          Output log file with embedded forecast hour '${FH}'
#                   defaults to ${COMOUT}/logf'${FH}'$SUFOUT
#     INISCRIPT     Preprocessing script
#                   defaults to none
#     LOGSCRIPT     Log posting script
#                   defaults to none
#     ERRSCRIPT     Error processing script
#                   defaults to 'eval [[ $err = 0 ]]'
#     ENDSCRIPT     Postprocessing script
#                   defaults to none
#     FHINI         Starting forecast hour
#                   defaults to the value in the input sigma file header
#     FHSEG         Number of hours to integrate
#                   (only required if FHMAX is not specified)
#                   defaults to 0
#     DELTIM        Timestep in seconds
#                   defaults to 3600/($JCAPG/20)
#     FHRES         Restart frequency in hours
#                   defaults to 24
#     FHZER         Zeroing frequency in hours
#                   defaults to 6
#     FHLWR         Longwave radiation frequency in hours
#                   defaults to 3
#     FHSWR         Shortwave radiation frequency in hours
#                   defaults to 1
#     FHROT         Forecast hour to Read One Time level
#                   defaults to 0
#     FHDFI         Half number of hours of digital filter initialization
#                   defaults to 0
#     FHCYC         Surface cycling frequency in hours
#                   defaults to 0 for no cycling
#     IDVC          Integer ID of the vertical coordinate type
#                   defaults to that in the header for the input upperair
#                   file. IDVC=1 for sigma; IDVC=2 for pressure/sigma hybrid
#     TFILTC        Time filter coefficient
#                   defaults to 0.85
#     DYNVARS       Other namelist inputs to the dynamics executable
#                   defaults to none set
#     PHYVARS       Other namelist inputs to the physics executable
#                   defaults to none set
#     TRACERVARS    Other namelist inputs to the forecast executable
#                   defaults to none set
#     FSMCL2        Scale in days to relax to soil moisture climatology
#                   defaults to 99999 for no relaxation
#     FTSFS         Scale in days to relax to SST anomaly to zero
#                   defaults to 90
#     FAISS         Scale in days to relax to sea ice to climatology
#                   defaults to 99999
#     FSNOL         Scale in days to relax to snow to climatology
#                   defaults to 99999
#     FSICL         Scale in days to relax to sea ice to climatology
#                   defaults to 99999
#     CYCLEVARS     Other namelist inputs to the surface cycling
#                   defaults to none set
#     NTHREADS      Number of threads
#                   defaults to 1
#     SPECTRAL_LOOP Number of spectral loops
#                   defaults to 2          
#     NTHSTACK      Size of stack per thread
#                   defaults to 64000000
#     FILESTYLE     File management style flag
#                   ('L' for symbolic links in $DATA is the only allowed style),
#     PGMOUT        Executable standard output
#                   defaults to $pgmout, then to '&1'
#     PGMERR        Executable standard error
#                   defaults to $pgmerr, then to '&1'
#     pgmout        Executable standard output default
#     pgmerr        Executable standard error default
#     REDOUT        standard output redirect ('1>' or '1>>')
#                   defaults to '1>', or to '1>>' to append if $PGMOUT is a file
#     REDERR        standard error redirect ('2>' or '2>>')
#                   defaults to '2>', or to '2>>' to append if $PGMERR is a file
#     VERBOSE       Verbose flag (YES or NO)
#                   defaults to NO
#
#   Exported Shell Variables:
#     PGM           Current program name
#     pgm
#     ERR           Last return code
#     err
#
#   Modules and files referenced:
#     scripts    : $INISCRIPT
#                  $LOGSCRIPT
#                  $ERRSCRIPT
#                  $ENDSCRIPT
#
#     programs   : $FCSTEXEC
#
#     input data : $1 or $SIGI
#                  $2 or $SFCI
#                  $SIGI2
#                  $FNTSFA
#                  $FNACNA
#                  $FNSNOA
#
#     fixed data : $CO2CON
#                  $MTNVAR
#                  $CLTUNE
#                  $DTBTHE
#                  $O3FORC
#                  $O3CLIM
#                  $FNGLAC
#                  $FNMXIC
#                  $FNTSFC
#                  $FNSNOC
#                  $FNZORC
#                  $FNALBC
#                  $FNAISC
#                  $FNTG3C
#                  $FNVEGC
#                  $FNVETC
#                  $FNSOTC
#                  $FNSMCC
#                  $FNVMNC
#                  $FNVMXC
#                  $FNSLPC
#                  $FNABSC
#                  $FNMSKH
#                  $OROGRAPHY
#
#     output data: $3 or $SIGO
#                  $4 or $FLXO
#                  $SFCO
#                  $LOGO
#                  $SIGR1
#                  $SIGR2
#                  $SFCR
#                  $NSTR
#                  $PGMOUT
#                  $PGMERR
#
#     scratch    : ${DATA}/fort.11
#                  ${DATA}/fort.12
#                  ${DATA}/fort.14
#                  ${DATA}/fort.15
#                  ${DATA}/fort.24
#                  ${DATA}/fort.27
#                  ${DATA}/fort.28
#                  ${DATA}/fort.29
#                  ${DATA}/fort.43
#                  ${DATA}/fort.48
#                  ${DATA}/fort.51
#                  ${DATA}/fort.52
#                  ${DATA}/fort.53
#                  ${DATA}/SIG.F*
#                  ${DATA}/SFC.F*
#                  ${DATA}/FLX.F*
#                  ${DATA}/LOG.F*
#                  ${DATA}/D3D.F*
#                  ${DATA}/G3D.F*
#                  ${DATA}/NST.F*
#                  ${DATA}/sigr1
#                  ${DATA}/sigr2
#                  ${DATA}/sfcr
#                  ${DATA}/nstr
#                  ${DATA}/NULL
#
# Remarks:
#
#   Condition codes
#      0 - no problem encountered
#     >0 - some problem encountered
#
#  Control variable resolution priority
#    1 Command line argument.
#    2 Environment variable.
#    3 Inline default.
#
# Attributes:
#   Language: POSIX shell
#   Machine: IBM SP
#
####
################################################################################
#  Set environment.
if [ $SCHEDULER = 'loadleveler' ]; then
  export MP_BINDPROC=${MP_BINDPROC:-yes}
  export MEMORY_AFFINIY=${MEMORY_AFFINIY:-MCM}
  export MP_SYNC_QP=${MP_SYNC_QP:-yes}
  export MP_SHARED_MEMORY=${MP_SHARED_MEMORY:-"NO"}
  export MP_COREFILE_FORMAT=${MP_COREFILE_FORMAT:-"lite"}
elif [ $SCHEDULER = 'pbs' ]; then
  export MPI_BUFS_PER_PROC=256
  export MPI_BUFS_PER_HOST=256
elif [ $SCHEDULER = 'moab' ]; then
  export MPICH_FAST_MEMCPY=${MPICH_FAST_MEMCPY:-"ENABLE"}
  export MPICH_MAX_SHORT_MSG_SIZE=${MPICH_MAX_SHORT_MSG_SIZE:-4096}
  export MPICH_UNEX_BUFFER_SIZE=${MPICH_UNEX_BUFFER_SIZE:-1024000000}
  export MPICH_PTL_UNEX_EVENTS=${MPICH_PTL_UNEX_EVENTS:-400000}
  export MPICH_PTL_OTHER_EVENTS=${MPICH_PTL_OTHER_EVENTS:-100000}
  export MPMD_PROC=${MPMD_PROC:-NO}
elif [ $SCHEDULER = 'lsf' ]; then
  export MP_EUIDEVICE=sn_all
#  export MP_EUILIB=us
#  export MP_TASK_AFFINITY=cpu
  export MP_EUIDEVELOP=min
  export KMP_STACKSIZE=1024m
  export F_UFMTENDIAN=big
  export NLSPATH=/usr/share/locale/en_US/%N
  export MPICH_ALLTOALL_THROTTLE=0
  export MP_SINGLE_THREAD=yes
  export MP_EAGER_LIMIT=65536 
  export MP_USE_BULK_XFER=yes
  export MP_COLLECTIVE_OFFLOAD=yes
  export MP_SHARED_MEMORY=no
#  export MP_MPILIB=mpich2
fi


export VERBOSE=${VERBOSE:-"NO"}
if [[ "$VERBOSE" = "YES" ]]
then
   echo $(date) EXECUTING $0 $* >&2
   set -x
fi
#  Command line arguments.
export NEMSIO_IN=${NEMSIO_IN:-".true."}
if [ $NEMSIO_IN = .false. ] ; then
export SIGI=${1:-${SIGI:-?}}
else
export GRDI=${1:-${GRDI:-?}}
fi
export ENS_NUM=${ENS_NUM:-1}
export FM=${FM}
export GRDI=${1:-${GRDI:-?}}
export SFCI=${2:-${SFCI:-?}}
export SIGO=${3:-${SIGO}}
export FLXO=${4:-${FLXO}}
export FHOUT=${5:-${FHOUT:-3}}
export FHMAX=${6:-${FHMAX:-0}}
export IGEN=${7:-${IGEN:-0}}
export D3DO=${8:-${D3DO}}
export NSTII=${9:-${NSTI:-?}}
export NSTO=${10:-${NSTO}}
export G3DO=${11:-${G3DO}}
export FHOUT_HF=${12:-${FHOUT_HF:-0}}
export FHMAX_HF=${13:-${FHMAX_HF:-0}}
export AERO=${14:-${AERO}}

# DHOU 02/28/2008 Modified for general case
# DHOU 01/07/2008 Added two input for the GEFS_Cpl module
# FHM_FST is the FHMAX for the integration before the first stop
# FH_INC is the FHMAX_increase for the integration before next stop
export FH_INC=${FH_INC:-6}
export ENS_SPS=${ENS_SPS:-.false.}
export ADVANCECOUNT_SETUP=${ADVANCECOUNT_SETUP:-0}
export HOUTASPS=${HOUTASPS:-10000}
[[ $ENS_NUM -le 2 ]]&&ENS_SPS=.false.
if [ $ENS_SPS = .false. ] ; then export FH_INC=$FHMAX ; fi

#  Directories.

export HOMEDIR=${HOMEDIR:-/nwprod}
export NWPROD=${NWPROD:-$HOMEDIR}
export FIXSUBDA=${FIXSUBDA:-fix}
#  export FIXGLOBAL=${FIXGLOBAL:-$NWPROD/$FIXSUBDA}
export FIXGLOBAL=
if [ $SCHEDULER = 'loadleveler' ]; then
   export FIXGLOBAL=${FIXGLOBAL:-/nwprod/fix}
elif [ $SCHEDULER = 'moab' ]; then
   export FIXGLOBAL=${FIXGLOBAL}
elif [ $SCHEDULER = 'pbs' ]; then
   export FIXGLOBAL=${FIXGLOBAL:-/scratch2/portfolios/NCEPDEV/rstprod/nwprod/fix}
elif [ $SCHEDULER = 'lsf' ]; then
   export FIXGLOBAL=${FIXGLOBAL:-/nwprod/fix}
fi
export FIX_RAD=${FIX_RAD:-$FIXGLOBAL}
export EXECGLOBAL=${EXECGLOBAL:-$NWPROD/exec}
export DATA=${DATA:-$(pwd)}
export COMOUT=${COMOUT:-$(pwd)}

#  Filenames.
MN=${MN:-""}
export XC=${XC}
export SUFOUT=${SUFOUT}
export SIGI=${SIGI:-$GRDI}
##export SIGHDR=${SIGHDR:-${EXECGLOBAL}/nemsio_get$XC}
if [ $NEMSIO_IN = .true. ]; then
  if [ $SCHEDULER = 'loadleveler' ]; then
    export SIGHDR=${SIGHDR:-/climate/save/wx20wa/gfsio/nems/nemsio_get}
  elif [ $SCHEDULER = 'moab' ]; then
    export SIGHDR=${SIGHDR}
  elif [ $SCHEDULER = 'pbs' ]; then
    export SIGHDR=${SIGHDR:-/scratch1/portfolios/NCEPDEV/nems/save/Jun.Wang/nems/util/nemsio_get}
  elif [ $SCHEDULER = 'lsf' ]; then
    export SIGHDR=${SIGHDR:-/nems/save/Jun.Wang/util/nemsio_get}
  fi
else
  export SIGHDR=${SIGHDR:-${EXECGLOBAL}/global_sighdr$XC}
fi
export JCAP=${JCAP:-$($SIGHDR ${GRDI}$FM jcap |grep -i "jcap" |awk -F"= " '{print $2}')}
export JCAPG=${JCAPG:-$($SIGHDR ${GRDI}$FM jcap|grep -i "jcap" |awk -F"= " '{print $2}')}
export LEVS=${LEVS:-$($SIGHDR ${GRDI}$FM levs|grep -i "levs" |awk -F"= " '{print $2}')}
export LEVR=${LEVR:-$LEVS}
export LONF=${LONF:-$($SIGHDR ${GRDI}$FM LONF|grep -i "lonf" |awk -F"= " '{print $2}')}
#export LATG=${LATG:-$($SIGHDR ${GRDI}$FM LATG|grep -i "latg" |awk -F"= " '{print $2}')}
export LONR=$LONF   
#export LATR=$LATG  
export NTRAC=${NTRAC:-$($SIGHDR ${GRDI}$FM NTRAC|grep -i "NTRAC" |awk -F"= " '{print $2}')}
export IDVC=${IDVC:-$($SIGHDR ${GRDI}$FM IDVC |grep -i "IDVC" |awk -F"= " '{print $2}')}
export IDVM=${IDVM:-$($SIGHDR ${GRDI}$FM IDVM |grep -i "IDVM" |awk -F"= " '{print $2}')}
export THERMODYN_ID=${THERMODYN_ID:-$((IDVM/10))}
export SFCPRESS_ID=${SFCPRESS_ID:-$((IDVM-(IDVM/10)*10))}
export NMTVR=${NMTVR:-14}
export LSOIL=${LSOIL:-4}
export NTOZ=${NTOZ:-2}
export NTCW=${NTCW:-3}
export NCLD=${NCLD:-1}
export NGPTC=${NGPTC:-30}
#jw
export ADIAB=${ADIAB:-.false.}
export nsout=${nsout:-0}
export LDFIFLTO=${LDFIFLTO:-.false.}
export QUILTING=${QUILTING:-.true.}
export REDUCED_GRID=${REDUCED_GRID:-.true.}
export SPECTRAL_LOOP=${SPECTRAL_LOOP:-2}                 
##export nemsioget=${nemsioget:-${EXECGLOBAL}/nemsio_get}  
export nemsioget=${nemsioget:-${SIGHDR}}
export PASSIVE_TRACER=${PASSIVE_TRACER:-.false.}
export NST_FCST=${NST_FCST:-0}
export GOCART=${GOCART:-0}
export NDSLFV=${NDSLFV:-.false.}

## for post
export WRITE_DOPOST=${WRITE_DOPOST:-.false.}
export GOCART_AER2POST=${GOCART_AER2POST:-.false.}
export POST_GRIBVERSION=${POST_GRIBVERSION:-grib1}
export POSTCTLFILE=${POSTCTLFILE:-$PARMngac/ngac_postcntrl.parm}
export POST_PARM=${POST_PARM:-$PARMngac/ngac_postcntrl.xml}
export POST_AVBLFLDSXML=${POST_AVBLFLDSXML:-$PARMngac/ngac_post_avblflds.xml}
export POST_NCEPGRB2TBL=${POST_NCEPGRB2TBL:-$HOMEGLOBAL/lib/sorc/g2tmpl/params_grib2_tbl_new}


if [ $NDSLFV = .true. ]
then
 if [[ $waveg -eq  62 ]] ; then export DELTIM=1200 ; fi
 if [[ $waveg -eq 126 ]] ; then export DELTIM=1200 ; fi
 if [[ $waveg -eq 254 ]] ; then export DELTIM=900  ; fi
 if [[ $waveg -eq 382 ]] ; then export DELTIM=600  ; fi
 if [[ $waveg -eq 574 ]] ; then export DELTIM=400  ; fi
 if [[ $waveg -eq 852 ]] ; then export DELTIM=300  ; fi
 export MASS_DP=.true.
 export PROCESS_SPLIT=.false.
else
 export MASS_DP=.false.
 export PROCESS_SPLIT=.false.
fi
#
export EXPLICIT=${EXPLICIT:-.false.}
export MASS_DP=${MASS_DP:-.false.}
export PROCESS_SPLIT=${PROCESS_SPLIT:-.false.}
export NDSLFV=${NDSLFV:-.false.}
#export ZFLXTVD=${ZFLXTVD:-.false.}
export ZFLXTVD=${ZFLXTVD:-.true.}
export SEMI_IMPLICIT_TEMP_PROFILE=${SEMI_IMPLICIT_TEMP_PROFILE:-.false.}
#
export pre_rad=${pre_rad:-.false.}
export random_clds=${random_clds:-.false.}
export FCSTEXEC=${FCSTEXEC:-${EXECGLOBAL}/ngac_fcst}
export GRDI2=${GRDI2:-NULL}
export SIGI1=${SIGI1:-NULL}
export SIGI2=${SIGI2:-NULL}
export CO2CON=${CO2CON:-${FIXGLOBAL}/global_co2con.l${LEVS}.f77}
export MTNVAR=${MTNVAR:-${FIXGLOBAL}/global_mtnvar.t${JCAPG}.f77}
export CLTUNE=${CLTUNE:-${FIXGLOBAL}/global_cldtune.f77}
export DTBTHE=${DTBTHE:-${FIXGLOBAL}/global_tbthe.f77}
export O3FORC=${O3FORC:-${FIXGLOBAL}/global_o3prdlos.f77}
export O3CLIM=${O3CLIM:-${FIXGLOBAL}/global_o3clim.txt}
export FNGLAC=${FNGLAC:-${FIXGLOBAL}/global_glacier.2x2.grb}
export FNMXIC=${FNMXIC:-${FIXGLOBAL}/global_maxice.2x2.grb}
export FNTSFC=${FNTSFC:-${FIXGLOBAL}/cfs_oi2sst1x1monclim19822001.grb}
export FNSNOC=${FNSNOC:-${FIXGLOBAL}/global_snoclim.1.875.grb}
export FNZORC=${FNZORC:-${FIXGLOBAL}/global_zorclim.1x1.grb}
export FNALBC=${FNALBC:-${FIXGLOBAL}/global_albedo4.1x1.grb}
export FNAISC=${FNAISC:-${FIXGLOBAL}/cfs_ice1x1monclim19822001.grb}
export FNTG3C=${FNTG3C:-${FIXGLOBAL}/global_tg3clim.2.6x1.5.grb}
export FNVEGC=${FNVEGC:-${FIXGLOBAL}/global_vegfrac.0.144.decpercent.grb}
export FNVETC=${FNVETC:-${FIXGLOBAL}/global_vegtype.1x1.grb}
export FNSOTC=${FNSOTC:-${FIXGLOBAL}/global_soiltype.1x1.grb}
export FNSMCC=${FNSMCC:-${FIXGLOBAL}/global_soilmcpc.1x1.grb}
export FNVMNC=${FNVMNC:-${FIXGLOBAL}/global_shdmin.0.144x0.144.grb}
export FNVMXC=${FNVMXC:-${FIXGLOBAL}/global_shdmax.0.144x0.144.grb}
export FNSLPC=${FNSLPC:-${FIXGLOBAL}/global_slope.1x1.grb}
export FNABSC=${FNABSC:-${FIXGLOBAL}/global_snoalb.1x1.grb}
export FNMSKH=${FNMSKH:-${FIXGLOBAL}/seaice_newland.grb}
export OROGRAPHY=${OROGRAPHY:-${FIXGLOBAL}/global_orography.t$JCAPG.grb}
export FNTSFA=${FNTSFA}
export FNACNA=${FNACNA}
export FNSNOA=${FNSNOA}
#
export AERODIR=${AERODIR:-${FIX_RAD}}
export EMISDIR=${EMISDIR:-${FIX_RAD}}
export SOLCDIR=${SOLCDIR:-${FIX_RAD}}
export VOLCDIR=${VOLCDIR:-${FIX_RAD}}
export CO2DIR=${CO2DIR:-${FIX_RAD}}
export GOCART_CLIM=${GOCART_CLIM:-${FIX_RAD}}
export GOCART_LUTS=${GOCART_LUTS:-${FIX_RAD}}
#export ALBDIR=${ALBDIR:-${FIX_RAD}}
export IEMS=${IEMS:-0}
export ISOL=${ISOL:-0}
export IAER=${IAER:-0}
export ICO2=${ICO2:-0}
#export IALB=${IALB:-0}
#
LOCD=${LOCD:-""}
## Define Restart Files
export COMENS=$DATA'$LOCD'
export GRDR1=${GRDR1:-${COMENS}/grdr1}
export GRDR2=${GRDR2:-${COMENS}/grdr2}
export SIGR1=${SIGR1:-${COMENS}/sigr1}
export SIGR2=${SIGR2:-${COMENS}/sigr2}
export SFCR=${SFCR:-${COMENS}/sfcr}
export NSTR=${NSTR:-${COMENS}/nstr}
## Define History Files 
export SIGO=${SIGO:-${COMENS}/sigf'${FH}''${MN}'$SUFOUT}
export SFCO=${SFCO:-${COMENS}/sfcf'${FH}''${MN}'$SUFOUT}
export FLXO=${FLXO:-${COMENS}/flxf'${FH}''${MN}'$SUFOUT}
export LOGO=${LOGO:-${COMENS}/logf'${FH}''${MN}'$SUFOUT}
export D3DO=${D3DO:-${COMENS}/d3df'${FH}''${MN}'$SUFOUT}
export NSTO=${NSTO:-${COMENS}/nstf'${FH}''${MN}'$SUFOUT}
export G3DO=${G3DO:-${COMENS}/g3df'${FH}''${MN}'$SUFOUT}
export AERO=${AERO:-${COMOUT}/aerf'${FH}''${MN}'$SUFOUT}
export INISCRIPT=${INISCRIPT}
export ERRSCRIPT=${ERRSCRIPT:-'eval [[ $err = 0 ]]'}
export LOGSCRIPT=${LOGSCRIPT}
export ENDSCRIPT=${ENDSCRIPT}
#  Other variables.
export FHINI=${FHINI:-$($SIGHDR ${GRDI}$FM NFHOUR |grep -i "NFHOUR" |awk -F"= " '{print $2}')}
export FHSEG=${FHSEG:-0}
export FHMAX=${FHMAX:-$((10#$FHINI+10#$FHSEG))}
export DELTIM=${DELTIM:-$((3600/(JCAPG/20)))}
export FHRES=${FHRES:-24}
export FHZER=${FHZER:-6}
export FHLWR=${FHLWR:-3}
export FHSWR=${FHSWR:-1}
export FHROT=${FHROT:-0}
export FHDFI=${FHDFI:-1}
export FHCYC=${FHCYC:-0}
export GB=${GB:-0}
export gfsio_in=${gfsio_in:-.false.}
export gfsio_out=${gfsio_out:-.false.}
if [ $gfsio_in = .true. ] ; then
 export GB=1
fi

#
# Total pe = WRT_GROUP*WRTPE_PER_GROUP + fcst pes
#
export WRT_GROUP=${WRT_GROUP:-1}
export WRTPE_PER_GROUP=${WRTPE_PER_GROUP:-1}
# number of output files, default =3, for adiab num_file=1
if [[ $ADIAB = .true. ]] ; then
  export NUM_FILE=1 ;
  export FILENAME_BASE="'SIG.F'"
  export FILE_IO_FORM="'grib'"
else
  export FILENAME_BASE="'SIG.F' 'SFC.F' 'FLX.F'"
  export FILE_IO_FORM="'grib' 'bin4' 'grib'"
  export NUM_FILE=3

  if [ $NST_FCST -gt 0 ] ; then
    export FILENAME_BASE=${FILENAME_BASE}" 'NST.F'"
    export FILE_IO_FORM=${FILE_IO_FORM}" 'bin4'"
    NUM_FILE=`expr $NUM_FILE + 1`
    export NUM_FILE
    if [ $NST_FCST -eq 1 ]; then
          NST_SPINUP=.true.
      fi
  fi
  if [ $GOCART == 1 ] ; then
      export FILENAME_BASE=${FILENAME_BASE}" 'AER.F'"
      export FILE_IO_FORM=${FILE_IO_FORM}" 'grib'"
      NUM_FILE=`expr $NUM_FILE + 1`
      export NUM_FILE
  fi
  echo "NUM_FILE=$NUM_FILE,GOCART=$GOCART,NST_FCST=$NST_FCST,FILENAME_BASE=$FILENAME_BASE"
 fi
NST_SPINUP=${NST_SPINUP:-.false.}
#

if [ $IDVC = 1 ] ; then
 export HYBRID=.false.
 export GEN_COORD_HYBRID=.false.
elif [ $IDVC = 2 ] ; then
 export HYBRID=.true.
 export GEN_COORD_HYBRID=.false.
elif [ $IDVC = 3 ] ; then
 export HYBRID=.false.
 export GEN_COORD_HYBRID=.true.
fi
export TFILTC=${TFILTC:-0.85}
export DYNVARS=${DYNVARS:-""}
export PHYVARS=${PHYVARS:-""}
export TRACERVARS=${TRACERVARS:-""}
export FSMCL2=${FSMCL2:-99999}
export FTSFS=${FTSFS:-90}
export FAISS=${FAISS:-99999}
export FSNOL=${FSNOL:-99999}
export FSICL=${FSICL:-99999}
export CYCLVARS=${CYCLVARS}
export NTHREADS=${NTHREADS:-1}
export NTHSTACK=${NTHSTACK:-128000000}
export XLSMPOPTS=${XLSMPOPTS:-"parthds=$NTHREADS:stack=$NTHSTACK"}
export FILESTYLE=${FILESTYLE:-'L'}
export PGMOUT=${PGMOUT:-${pgmout:-'&1'}}
export PGMERR=${PGMERR:-${pgmerr:-'&2'}}
export MEMBER_NAMES=${MEMBER_NAMES:-''}

if [ $SCHEDULER = 'loadleveler' ]; then
  export BIND_TASKS=${BIND_TASKS:-no}
  typeset -L1 l=$PGMOUT
  [[ $l = '&' ]]&&a=''||a='>'
  export REDOUT=${REDOUT:-'1>'$a}
  typeset -L1 l=$PGMERR
  [[ $l = '&' ]]&&a=''||a='>'
  export REDERR=${REDERR:-'2>'$a}
else
  export REDOUT=${REDOUT:-'1>'}
  export REDERR=${REDERR:-'2>'}
fi
################################################################################
#  Preprocessing
$INISCRIPT
pwd=$(pwd)
if [[ -d $DATA ]] ; then
   mkdata=NO
else
   mkdir -p $DATA
   mkdata=YES
fi
cd $DATA||exit 99
[[ -d $COMOUT ]]||mkdir -p $COMOUT
################################################################################
#  Make forecast
export XLFRTEOPTS="unit_vars=yes:intrinthds=1"
export PGM=$DATA/$(basename $FCSTEXEC)
export pgm=$PGM
$LOGSCRIPT
${NCP:-cp} $FCSTEXEC $DATA
rm -f NULL
FH=$((10#$FHINI))
[[ $FH -lt 10 ]]&&FH=0$FH
if [[ $FHINI -gt 0 ]] ; then
   if [ $FHOUT_HF -ne $FHOUT -a $FH -lt $FHMAX_HF ] ; then
    FH=$((10#$FHINI+10#$FHOUT_HF))
   else
    FH=$((10#$FHINI+10#$FHOUT))
   fi
   [[ $FH -lt 10 ]]&&FH=0$FH
fi
while [[ $FH -le $FHMAX ]] ; do
   if [[ $FH -le $HOUTA ]] ; then
     FNSUB=$NMSUB
   else
     FNSUB=""
   fi
#  eval rm -f ${LOGO}${FNSUB}  ## Comment this line to prevent the log file from being removed
   if [ $FHOUT_HF -ne $FHOUT -a $FH -lt $FHMAX_HF ] ; then
     ((FH=10#$FH+10#$FHOUT_HF))
   else
     ((FH=10#$FH+10#$FHOUT))
   fi
   [[ $FH -lt 10 ]]&&FH=0$FH
done
if [[ $FILESTYLE = "L" ]] ; then
   ln -fs $CO2CON fort.15
   ln -fs $MTNVAR fort.24
   ln -fs $DTBTHE fort.27
   ln -fs $O3FORC fort.28
   ln -fs $CLTUNE fort.43
   ln -fs $O3CLIM fort.48
else
  echo 'FILESTYLE' $FILESTYLE 'NOT SUPPORTED'
  exit 222
fi
#for m in 01 02 03 04 05 06 07 08 09 10 11 12
#do
# ln -fs $AERODIR/global_aeropac3a.m$m.txt aeropac3a.m$m
#done
AEROSOL_FILE=${AEROSOL_FILE:-global_climaeropac_global.txt}
EMMISSIVITY_FILE=${EMMISSIVITY_FILE:-global_sfc_emissivity_idx.txt}
ln -fs $AERODIR/$AEROSOL_FILE     aerosol.dat
ln -fs $EMISDIR/$EMMISSIVITY_FILE sfc_emissivity_idx.txt
ln -fs $OROGRAPHY                 orography
if [ $IEMS -gt 0 ] ; then
 EMMISSIVITY_FILE=${EMMISSIVITY_FILE:-global_sfc_emissivity_idx.txt}
 ln -fs $EMISDIR/$EMMISSIVITY_FILE sfc_emissivity_idx.txt
fi
if [ $ISOL -gt 0 ] ; then
 ${NCP:-cp} $SOLCDIR/*solarconstantdata.txt solarconstantdata.txt
fi
if [ $IAER -gt 0 ] ; then
 cd $VOLCDIR
 for file in `ls | grep volcanic_aerosols` ; do
  ${NCP:-cp} $file $DATA/$(echo $file |sed -e "s/global_//g")
 done
 cd $DATA
#  ${NCP:-cp} $GOCART_CLIM/* $DATA
#  ${NCP:-cp} $GOCART_LUTS/NCEP_AEROSOL.bin $DATA
fi
if [ $ICO2 -gt 0 ] ; then
 cd $CO2DIR
 for file in `ls | grep co2historicaldata` ; do
  ${NCP:-cp} $file $DATA/$(echo $file |sed -e "s/global_//g")
 done
fi
cd $DATA
export PHYVARS="IEMS=$IEMS,ISOL=$ISOL,IAER=$IAER,ICO2=$ICO2,$PHYVARS"
#
#     For one member case i.e. control
#     --------------------------------
mins=$((DELTIM/60))
secs=$((DELTIM-(DELTIM/60)*60))
[[ $mins -lt 10 ]] &&mins=0$mins
[[ $secs -lt 10 ]] &&secs=0$secs

if [[ $ENS_NUM -le 1 ]] ; then
  FH=$((10#$FHINI))
  [[ $FH -lt 10 ]]&&FH=0$FH
  if [[ $FHINI -gt 0 ]] ; then
    if [ $FHOUT_HF -ne $FHOUT -a $FH -lt $FHMAX_HF ] ; then
     FH=$((10#$FHINI+10#$FHOUT_HF))
    else
     FH=$((10#$FHINI+10#$FHOUT))
    fi
    [[ $FH -lt 10 ]]&&FH=0$FH
  fi
#        For Initial Conditions
#        ----------------------
  if [ $((FHINI+0)) -eq  $((FHROT+0)) ]; then
    ln -fs $GRDI  grid_ini
    ln -fs $SFCI  sfc_ini
    ln -fs $NSTI  nst_ini
    export RESTART=.false.
  else
    ln -fs $GRDI  grid_ini
    ln -fs $GRDI2 grid_ini2
    ln -fs $SIGI  sig_ini
    ln -fs $SIGI2 sig_ini2
    ln -fs $SFCI  sfc_ini
    ln -fs $NSTI  nst_ini
    export RESTART=.true.
  fi
#        For output
#        ----------
  while [[ $FH -le $FHMAX ]] ; do
    if [[ $FH -le $HOUTA ]] ; then
      FNSUB=$NMSUB
    else
      FNSUB=""
    fi
    if [ $FH -eq 00 ] ; then
      SUF2=:${mins}:${secs}
    else
      SUF2=""
    fi
    eval ln -fs ${SIGO}$FNSUB SIG.F${FH}$SUF2
    eval ln -fs ${SFCO}$FNSUB SFC.F${FH}$SUF2
    eval ln -fs ${FLXO}$FNSUB FLX.F${FH}$SUF2
    eval ln -fs ${LOGO}$FNSUB LOG.F${FH}$SUF2
    eval ln -fs ${D3DO}$FNSUB D3D.F${FH}$SUF2
    eval ln -fs ${NSTO}$FNSUB NST.F${FH}$SUF2
    eval ln -fs ${G3DO}$FNSUB G3D.F${FH}$SUF2
    eval ln -fs ${AERO}$FNSUB AER.F${FH}$SUF2

    if [ $FHOUT_HF -ne $FHOUT -a $FH -lt $FHMAX_HF ] ; then
     ((FH=10#$FH+10#$FHOUT_HF))
    else
     ((FH=10#$FH+10#$FHOUT))
    fi
    [[ $FH -lt 10 ]]&&FH=0$FH

  done
  eval ln -fs $GRDR1 GRDR1
  eval ln -fs $GRDR2 GRDR2
  eval ln -fs $SIGR1 SIGR1
  eval ln -fs $SIGR2 SIGR2
  eval ln -fs $SFCR  SFCR
  eval ln -fs $NSTR  NSTR
else
#
#   For Ensemble runs (members > 1)
#   -------------------------------
  for MN in $MEMBER_NAMES ; do
    IMN=`echo $MN|cut -c2-3`
    IMN=$((IMN+0))
    if [ $IMN -eq 0 ] ; then IMN=$ENS_NUM ; fi
    if [ $IMN -lt 10 ] ; then IMN=0$IMN ; fi
    echo 'IMN=' $IMN
    if [ ${USESUBDIR:-NO} = YES ] ; then
     if [ $IMN -eq $ENS_NUM ] ; then LOCD=/c00 ; else LOCD=/p$IMN ; fi
    fi
    mkdir -p `eval echo \$COMENS`

#      This is just faking the ensemble ICs.
#   ${NCP:-cp} $SIGI  ${SIGI}${MN}
#   ${NCP:-cp} $SFCI  ${SFCI}${MN}
#   ${NCP:-cp} $SIGI2 ${SIGI2}${MN}
#        For Initial Conditions
#        ----------------------
    eval ln -fs ${GRDI}${MN}  grid_ini_${IMN}
    eval ln -fs ${GRDI2}${MN} grid_ini2_${IMN}
    eval ln -fs ${SIGI}${MN}  sig_ini_${IMN}
    eval ln -fs ${SIGI2}${MN} sig_ini2_${IMN}
    eval ln -fs ${SFCI}${MN}  sfc_ini_${IMN}
    eval ln -fs ${NSTI}${MN}  nst_ini_${IMN}

    if [ $((FHINI+0)) -eq  $((FHROT+0)) ]; then
       export RESTART=.false.
    else
       export RESTART=.true.
    fi

echo 'wy, RESTART=' $RESTART ############################????????????????????????
echo 'wy, FHINI=' $FHINI ############################????????????????????????
echo 'wy, FHROT=' $FHROT ############################????????????????????????

#        For output
#        ----------
    FH=$((10#$FHINI))
    [[ $FH -lt 10 ]]&&FH=0$FH
    if [[ $FHINI -gt 0 ]] ; then
      if [ $FHOUT_HF -ne $FHOUT -a $FH -lt $FHMAX_HF ] ; then
       FH=$((10#$FHINI+10#$FHOUT_HF))
      else
       FH=$((10#$FHINI+10#$FHOUT))
      fi
      [[ $FH -lt 10 ]]&&FH=0$FH
    fi
    while [[ $FH -le $FHMAX ]] ; do
      if [[ $FH -le $HOUTA ]] ; then
        FNSUB=$NMSUB
      else
        FNSUB=""
      fi
      if [ $FH -eq 00 ] ; then
        SUF2=:${mins}:${secs}
      else
        SUF2=""
      fi
      eval ln -fs ${SIGO}$FNSUB SIG.F${FH}${SUF2}_${IMN}
      eval ln -fs ${SFCO}$FNSUB SFC.F${FH}${SUF2}_${IMN}
      eval ln -fs ${FLXO}$FNSUB FLX.F${FH}${SUF2}_${IMN}
      eval ln -fs ${LOGO}$FNSUB LOG.F${FH}${SUF2}_${IMN}
      eval ln -fs ${D3DO}$FNSUB D3D.F${FH}${SUF2}_${IMN}
      eval ln -fs ${NSTO}$FNSUB NST.F${FH}${SUF2}_${IMN}
      eval ln -fs ${G3DO}$FNSUB G3D.F${FH}${SUF2}_${IMN}
      eval ln -fs ${AERO}$FNSUB AER.F${FH}${SUF2}_${IMN}

      if [ $FHOUT_HF -ne $FHOUT -a $FH -lt $FHMAX_HF ] ; then
       ((FH=10#$FH+10#$FHOUT_HF))
      else
       ((FH=10#$FH+10#$FHOUT))
      fi
      [[ $FH -lt 10 ]]&&FH=0$FH
    done

# 02/29/2008 DHOU,  added new files for the output after SPS
#     FH=$FHMAX
# 09/09/2008 DHOU,  changed the time of output after SPS, fro end to earlier
#             for the digital filtering after resolution change
      FH=$HOUTASPS
    if [[ $FH -lt 10000 ]] ; then
      [[ $FH -lt 10 ]]&&FH=0$FH
      eval ln -fs $SIGS SIG.S${FH}_${IMN}
      eval ln -fs $SFBS SFB.S${FH}_${IMN}
      eval ln -fs $FLXS FLX.S${FH}_${IMN}
    fi

    eval ln -fs ${SIGR1}_${IMN} SIGR1_${IMN}
    eval ln -fs ${SIGR2}_${IMN} SIGR2_${IMN}
    eval ln -fs ${SFCR}_{IMN}   SFCR_${IMN}
    eval ln -fs ${NSTR}_${IMN}  NSTR_${IMN}
# 02/29/2008 DHOU,  added new files for the re-start-files after SP
    if [ $ENS_NUM -gt 2 ] ; then
      eval ln -fs ${SIGS1}_${IMN} SIGS1_${IMN}
      eval ln -fs ${SIGS2}_${IMN} SIGS2_${IMN}
      eval ln -fs ${SFCS}_${IMN}  SFCS_${IMN}
      eval ln -fs ${NSTS}_${IMN}  NSTS_${IMN}
    fi
  done
fi

#
# Create Configure file (i.e. .rc file) here
# PE$n are to be imported from outside.  If PE$n are not set from outside, the
# model would give equal processors for all ensembel members.
#
c=1
while [ $c -le $ENS_NUM ] ; do
 eval export PE$c=\${PE$c:-0}
 c=$((c+1))
done

##export wgrib=${wgrib:-/nwprod/util/exec/wgrib}
if [ $SCHEDULER = 'loadleveler' ]; then
   export wgrib=${wgrib:-/nwprod/util/exec/wgrib}
elif [ $SCHEDULER = 'moab' ]; then
   export wgrib=${wgrib:-/nwprod/util/exec/wgrib}
elif [ $SCHEDULER = 'pbs' ]; then
   export wgrib=${wgrib:-/home/George.Vandenberghe/nwprod/util/exec/wgrib}
elif [ $SCHEDULER = 'lsf' ]; then
   export wgrib=${wgrib:-/nwprod/util/exec/wgrib}
fi

if [ $FHINI -eq 0 ]; then
  if [[ $ENS_NUM -le 1 ]] ; then
     CDATE=$($wgrib -4yr $GRDI | grep -i hgt |awk -F: '{print $3}' |awk -F= '{print $2}')
  else
     MN=c00
     CDATE=$($wgrib -4yr ${GRDI}${MN} | grep -i hgt |awk -F: '{print $3}' |awk -F= '{print $2}')
  fi
  INI_YEAR=$(echo $CDATE | cut -c1-4)
  echo "now"
  echo ${INI_YEAR}
  INI_MONTH=$(echo $CDATE | cut -c5-6)
  INI_DAY=$(echo $CDATE | cut -c7-8)
  INI_HOUR=$(echo $CDATE | cut -c9-10)
else
  if [[ $ENS_NUM -le 1 ]] ; then
     CDATE=$($nemsioget $GRDI idate | grep -i "idate" |awk -F= '{print $2}')
  else
     MN=c00
     CDATE=$($nemsioget ${GRDI}${MN} idate | grep -i "idate" |awk -F= '{print $2}')
  fi
  CDATE=$($nemsioget $GRDI idate | grep -i "idate" |awk -F= '{print $2}')
  INI_YEAR=$(echo $CDATE | awk -F" " '{print $1}')
  echo "now"
  echo ${INI_YEAR}
  INI_MONTH=$(echo $CDATE | awk -F" " '{print $2}')
  INI_DAY=$(echo $CDATE | awk -F" " '{print $3}')
  INI_HOUR=$(echo $CDATE | awk -F" " '{print $4}')
fi

## copy configure files needed for NEMS GFS
${NCP:-cp} $PARMngac/atmos.configure  atmos.configure
${NCP:-cp} ${MAPL:-$PARMngac/MAPL.rc} MAPL.rc
${NCP:-cp} ${CHEM_REGISTRY:-$PARMngac/Chem_Registry.rc}  Chem_Registry.rc

## copy/link post related files
if [[ $WRITE_DOPOST = .true. ]] ; then
 if [[ $POST_GRIBVERSION = grib1 ]] ; then
   ln -sf ${POSTCTLFILE} fort.14
 elif [[ $POST_GRIBVERSION = grib2 ]] ; then
   ${NCP:-cp} ${POST_PARM}        postcntrl.xml
   ${NCP:-cp} ${POST_AVBLFLDSXML} post_avblflds.xml
   ${NCP:-cp} ${POST_NCEPGRB2TBL} params_grib2_tbl_new
 fi
ln -sf griddef.out fort.110
${NCP:-cp} /nwprod/parm/nam_micro_lookup.dat ./eta_micro_lookup.dat
fi

## copy configure files and fixed files needed for GOCART
if [ $GOCART == 1 ] ; then
## copy Chem_Registry.rc with GOCART turned on
 ${NCP:-cp} ${CONFIG_DU:-$PARMngac/DU_GridComp.rc}         DU_GridComp.rc
 ${NCP:-cp} ${CONFIG_SU:-$PARMngac/SU_GridComp.rc}         SU_GridComp.rc
 ${NCP:-cp} ${CONFIG_OC:-$PARMngac/OC_GridComp.rc}         OC_GridComp.rc
 ${NCP:-cp} ${CONFIG_OCx:-$PARMngac/OC_GridComp---full.rc} OC_GridComp---full.rc
 ${NCP:-cp} ${CONFIG_BC:-$PARMngac/BC_GridComp.rc}         BC_GridComp.rc
 ${NCP:-cp} ${CONFIG_SS:-$PARMngac/SS_GridComp.rc}         SS_GridComp.rc
 ${NCP:-cp} ${AOD_REGISTRY:-$PARMngac/Aod-550nm_Registry.rc}   $DATA/Aod-550nm_Registry.rc

# ${NCP:-cp} -r $FIXngac/ngac_fix  $DATA/ngac_fix
 ln -sf $FIXngac/ngac_fix  .

fi


## determine LATG here
if [[ $RESTART = .true. ]] ; then
export LATG=${LATG:-$($SIGHDR ${GRDI}$FM LATF|grep -i "latf" |awk -F"= " '{print $2}')}
else
export LATG=${LATG:-$($SIGHDR ${GRDI}$FM LATG|grep -i "latg" |awk -F"= " '{print $2}')}
fi
export LATR=$LATG  


cat << EOF > atm_namelist.rc

core: gfs

#nam_atm +++++++++++++++++++++++++++
nlunit:                  35
deltim:                  ${DELTIM}.0
namelist:                atm_namelist
total_member:            $ENS_NUM
grib_input:              $GB
PE_MEMBER01:             $PE1
PE_MEMBER02:             $PE2
PE_MEMBER03:             $PE3
PE_MEMBER04:             $PE4
PE_MEMBER05:             $PE5
PE_MEMBER06:             $PE6
PE_MEMBER07:             $PE7
PE_MEMBER08:             $PE8
PE_MEMBER09:             $PE9
PE_MEMBER10:             $PE10
PE_MEMBER11:             $PE11
PE_MEMBER12:             $PE12
PE_MEMBER13:             $PE13
PE_MEMBER14:             $PE14
PE_MEMBER15:             $PE15
PE_MEMBER16:             $PE16
PE_MEMBER17:             $PE17
PE_MEMBER18:             $PE18
PE_MEMBER19:             $PE19
PE_MEMBER20:             $PE20
PE_MEMBER21:             $PE21

# For stachastic purturbed runs -  added by Dhou and Wyang
  --------------------------------------------------------
#  ENS_SPS, logical control for application of stochastic perturbation scheme
#  HH_START, start hour of forecast, and modified ADVANCECOUNT_SETUP
#  HH_INCREASE and HH_FINAL are fcst hour increment and end hour of forecast
#  ADVANCECOUNT_SETUP is an integer indicating the number of time steps between
#  integrtion_start and the time when model state is saved for the _ini of the
#  GEFS_Coupling, currently is 0h.

HH_INCREASE:             $FH_INC
HH_FINAL:                $FHMAX
HH_START:                $FHINI
ADVANCECOUNT_SETUP:      $ADVANCECOUNT_SETUP

ENS_SPS:                  $ENS_SPS
HOUTASPS:                 $HOUTASPS

#ESMF_State_Namelist +++++++++++++++

RUN_CONTINUE:             .false.

#
dt_int:                   $DELTIM
dt_num:                   0
dt_den:                   1
start_year:               $INI_YEAR
start_month:              $INI_MONTH
start_day:                $INI_DAY
start_hour:               $INI_HOUR
start_minute:             0
start_second:             0
nhours_fcst:              $FHMAX
restart:                  $RESTART
nhours_fcst1:             6
im:                       $LONB
jm:                       $LATB
global:                   .true.
nhours_dfini:             $FHDFI
adiabatic:                $ADIAB
lsoil:                    $LSOIL
passive_tracer:           $PASSIVE_TRACER
ldfiflto:                 $LDFIFLTO
nemsio_in:                $NEMSIO_IN


#jwstart added quilt
###############################
#### Specify the I/O tasks ####
###############################


quilting:                 $QUILTING   #For asynchronous quilting/history writes
read_groups:              0
read_tasks_per_group:     0
write_groups:             $WRT_GROUP
write_tasks_per_group:    $WRTPE_PER_GROUP
write_dopost:             $WRITE_DOPOST
post_gribversion:         $POST_GRIBVERSION

gocart_aer2post:          $GOCART_AER2POST
num_file:                 $NUM_FILE                   #
filename_base:            $FILENAME_BASE
file_io_form:             $FILE_IO_FORM                     
file_io:                  'DEFERRED' 'DEFERRED' 'DEFERRED' 'DEFERRED'  #
write_nemsioflag:         .TRUE.       # True--> Write nemsio run history files
nfhout:                   $FHOUT
nsout:                    $nsout

io_recl:                  100
io_position:              ' '
io_action:                'WRITE'
io_delim:                 ' '
io_pad:                   ' '

#jwend

EOF

#
cat atm_namelist.rc > dyn_namelist.rc
cat << EOF >> dyn_namelist.rc

#ESMF_State_Namelist +++++++++++++++
idate1_import:                    1
z_import:                         1
ps_import:                        1
div_import:                       0
vor_import:                       0
u_import:                         1
v_import:                         1
temp_import:                      1
tracer_import:                    1
p_import:                         0
dp_import:                        0
dpdt_import:                      0

idate1_export:                    1
z_export:                         1
ps_export:                        1
div_export:                       0
vor_export:                       0
u_export:                         1
v_export:                         1
temp_export:                      1
tracer_export:                    1
p_export:                         1
dp_export:                        1
dpdt_export:                      1

EOF

cat atm_namelist.rc > phy_namelist.rc
cat << EOF >> phy_namelist.rc

#Upper_Air_State_Namelist +++++++++++++++
idate1_import:                    1
z_import:                         1
ps_import:                        1
div_import:                       0
vor_import:                       0
u_import:                         1
v_import:                         1
temp_import:                      1
tracer_import:                    1
p_import:                         1
dp_import:                        1
dpdt_import:                      1

idate1_export:                    1
z_export:                         1
ps_export:                        1
div_export:                       0
vor_export:                       0
u_export:                         1
v_export:                         1
temp_export:                      1
tracer_export:                    1
p_export:                         1
dp_export:                        1
dpdt_export:                      0

# Surface state.
#---------------
orography_import:                 1
t_skin_import:                    1
soil_mois_import:                 1
snow_depth_import:                1
soil_t_import:                    1
deep_soil_t_import:               1
roughness_import:                 1
conv_cloud_cover_import:          1
conv_cloud_base_import:           1
conv_cloud_top_import:            1
albedo_visible_scattered_import:  1
albedo_visible_beam_import:       1
albedo_nearir_scattered_import:   1
albedo_nearir_beam_import:        1
sea_level_ice_mask_import:        1
vegetation_cover_import:          1
canopy_water_import:              1
m10_wind_fraction_import:         1
vegetation_type_import:           1
soil_type_import:                 1
zeneith_angle_facsf_import:       1
zeneith_angle_facwf_import:       1
uustar_import:                    1
ffmm_import:                      1
ffhh_import:                      1
sea_ice_thickness_import:         1
sea_ice_concentration_import:     1
tprcp_import:                     1
srflag_import:                    1
actual_snow_depth_import:         1
liquid_soil_moisture_import:      1
vegetation_cover_min_import:      1
vegetation_cover_max_import:      1
slope_type_import:                1
snow_albedo_max_import:           1

orography_export:                 1
t_skin_export:                    1
soil_mois_export:                 1
snow_depth_export:                1
soil_t_export:                    1
deep_soil_t_export:               1
roughness_export:                 1
conv_cloud_cover_export:          1
conv_cloud_base_export:           1
conv_cloud_top_export:            1
albedo_visible_scattered_export:  1
albedo_visible_beam_export:       1
albedo_nearir_scattered_export:   1
albedo_nearir_beam_export:        1
sea_level_ice_mask_export:        1
vegetation_cover_export:          1
canopy_water_export:              1
m10_wind_fraction_export:         1
vegetation_type_export:           1
soil_type_export:                 1
zeneith_angle_facsf_export:       1
zeneith_angle_facwf_export:       1
uustar_export:                    1
ffmm_export:                      1
ffhh_export:                      1
sea_ice_thickness_export:         1
sea_ice_concentration_export:     1
tprcp_export:                     1
srflag_export:                    1
actual_snow_depth_export:         1
liquid_soil_moisture_export:      1
vegetation_cover_min_export:      1
vegetation_cover_max_export:      1
slope_type_export:                1
snow_albedo_max_export:           1

EOF

#
#   WARNING WARNING FILESTYLE "C" will not work for Component Ensembles!!!
#
#eval $PGM <<EOF $REDOUT$PGMOUT $REDERR$PGMERR
#totalview poe -a $PGM <<EOF $REDOUT$PGMOUT $REDERR$PGMERR
#
cat  > atm_namelist <<EOF
 &nam_dyn
  FHOUT=$FHOUT, FHMAX=$FHMAX, IGEN=$IGEN, DELTIM=$DELTIM,
  FHRES=$FHRES, FHROT=$FHROT, FHDFI=$FHDFI, nsout=$nsout,
  nxpt=1, nypt=2, jintmx=2, lonf=$LONF, latg=$LATG,
  jcap=$JCAP, jcapg=$JCAPG, levs=$LEVS,  levr=$LEVR, 
  ntrac=$NTRAC, ntoz=$NTOZ, ntcw=$NTCW, ncld=$NCLD,
  ngptc=$NGPTC, hybrid=$HYBRID, tfiltc=$TFILTC,
  gen_coord_hybrid=$GEN_COORD_HYBRID, zflxtvd=$ZFLXTVD,
  spectral_loop=$SPECTRAL_LOOP, explicit=$EXPLICIT,
  ndslfv=$NDSLFV,mass_dp=$MASS_DP,process_split=$PROCESS_SPLIT,
  reduced_grid=$REDUCED_GRID,
  semi_implicit_temp_profile=$SEMI_IMPLICIT_TEMP_PROFILE,
  thermodyn_id=$THERMODYN_ID, sfcpress_id=$SFCPRESS_ID,
  $DYNVARS /
&nam_phy
  FHOUT=$FHOUT, FHMAX=$FHMAX, IGEN=$IGEN, DELTIM=$DELTIM,
  FHRES=$FHRES, FHROT=$FHROT, FHCYC=$FHCYC, FHDFI=$FHDFI,
  FHZER=$FHZER, FHLWR=$FHLWR, FHSWR=$FHSWR,nsout=$nsout,
  nxpt=1, nypt=2, jintmx=2, lonr=$LONR, latr=$LATR,
  jcap=$JCAP, levs=$LEVS, levr=$LEVR, reduced_grid=$REDUCED_GRID,
  ntrac=$NTRAC, ntoz=$NTOZ, ntcw=$NTCW, ncld=$NCLD,
  lsoil=$LSOIL, nmtvr=$NMTVR, ICTM=$ICTM,
  ngptc=$NGPTC, hybrid=$HYBRID, tfiltc=$TFILTC,
  gen_coord_hybrid=$GEN_COORD_HYBRID,
  thermodyn_id=$THERMODYN_ID, sfcpress_id=$SFCPRESS_ID,
  $PHYVARS /
&TRACER_CONSTANT
  $TRACERVARS /
&SOIL_VEG
  LPARAM = .FALSE./
&NAMSFC
  FNGLAC="$FNGLAC",
  FNMXIC="$FNMXIC",
  FNTSFC="$FNTSFC",
  FNSNOC="$FNSNOC",
  FNZORC="$FNZORC",
  FNALBC="$FNALBC",
  FNAISC="$FNAISC",
  FNTG3C="$FNTG3C",
  FNVEGC="$FNVEGC",
  FNVETC="$FNVETC",
  FNSOTC="$FNSOTC",
  FNSMCC="$FNSMCC",
  FNMSKH="$FNMSKH",
  FNTSFA="$FNTSFA",
  FNACNA="$FNACNA",
  FNSNOA="$FNSNOA",
  FNVMNC="$FNVMNC",
  FNVMXC="$FNVMXC",
  FNSLPC="$FNSLPC",
  FNABSC="$FNABSC",
  LDEBUG=.false.,
  FSMCL(2)=$FSMCL2,
  FSMCL(3)=$FSMCL2,
  FSMCL(4)=$FSMCL2,
  FTSFS=$FTSFS,
  FAISS=$FAISS,
  FSNOL=$FSNOL,
  FSICL=$FSICL,
  FTSFL=99999,
  FAISL=99999,
  FVETL=99999,
  FSOTL=99999,
  FvmnL=99999,
  FvmxL=99999,
  FSLPL=99999,
  FABSL=99999,
  FSNOS=99999,
  FSICS=99999,

  $CYCLVARS /
EOF

ln -sf atm_namelist.rc ./model_configure

##eval $PGM $REDOUT$PGMOUT $REDERR$PGMERR
#eval hpmcount -o gfs_test $PGM $REDOUT$PGMOUT $REDERR$PGMERR
if [ $SCHEDULER = 'loadleveler' ]; then
   eval $PGM $REDOUT$PGMOUT $REDERR$PGMERR
elif [ $SCHEDULER = 'moab' ]; then
   aprun -n $TASKSIZE $PGM
elif [ $SCHEDULER = 'pbs' ]; then
   mpiexec_mpt -np $TASKSIZE $PGM
elif [ $SCHEDULER = 'lsf' ]; then
  echo " WCOSS option, work in progress"
fi

export ERR=$?
export err=$ERR
$ERRSCRIPT||exit 2


rm -f NULL
rm -f fort.11 fort.12 fort.14
rm -f fort.15 fort.24 fort.27 fort.28 fort.29 fort.43 fort.48
rm -f orography
rm -f fort.51 fort.52 fort.53
#rm -f SIG.F* SFC.F* FLX.F* LOG.F* D3D.F AER.F*
##rm -f sigr1 sigr2 sfcr
################################################################################
#  Postprocessing
cd $pwd
[[ $mkdata = YES ]]&&rmdir $DATA
$ENDSCRIPT
set +x
if [[ "$VERBOSE" = "YES" ]] ; then
   echo $(date) EXITING $0 with return code $err >&2
fi
exit $err
