#PBS -N hwrf%STORMNUM%_failsafe_%CYC%
#PBS -j oe
#PBS -S /bin/bash
#PBS -q %QUEUE%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l walltime=02:20:00
#PBS -l select=16:mpiprocs=60:ompthreads=2:ncpus=120+1:mpiprocs=12:ompthreads=2:ncpus=24
#PBS -l debug=true

export NODES=17
export TOTAL_TASKS=972
export OMP_NUM_THREADS=2
export ATM_THREADS=2

model=hwrf
%include <head.h>
%include <envir-p1.h>
export FORECAST_CONF_OVERRIDE=" %FORECAST_CONF_OVERRIDE%"

export cyc="%CYC%"
export storm_num="%STORMNUM%"

# versions file for hwrf sets $model_ver and $code_ver

module load envvar/${envvar_ver}
module load PrgEnv-intel/${PrgEnv_intel_ver}
module load craype/${craype_ver}
module load intel/${intel_ver}
module load cray-mpich/${cray_mpich_ver}
module load cray-pals/${cray_pals_ver}
module load libjpeg/${libjpeg_ver}
module load grib_util/${grib_util_ver}
module load wgrib2/${wgrib2_ver}
module load bufr/${bufr_ver}
module load hdf5/${hdf5_ver}
module load netcdf/${netcdf_ver}
# module load pnetcdf/${pnetcdf_ver}
module load udunits/${udunits_ver}
module load nco/${nco_ver}
module load python/${python_ver}
module list

${HOMEhwrf}/jobs/JHWRF_FORECAST

%include <tail.h>

%manual

TASK failsafe

PURPOSE: Runs the failsafe forecast job.  This job disables several
parts of the HWRF system, and enables extra vertical acceleration
damping.  It is likely to result in a degraded forecast, but it is
unlikely to fail.  It will never be run automatically since it starts
in a completed state.  Instead, it must be manually submitted, and
should only be used as a last resort.

ACTIVATING THE FAILSAFE FORECAST

1. Suspend the forecast family
2. Kill all jobs in the forecast family
3. Requeue the forecast/post and forecast/output
4. Set the forecast/wave_post to complete
5. Execute the forecast/model/failsafe
6. Set the other three forecast/model/wrf* jobs to complete
7. Resume the forecast family

The post-processing should start soon after the failsafe starts.  The
rest of the HWRF workflow will run as normal, but without ocean and
wave outputs.

%end