#PBS -N hwrf%STORMNUM%_failsafe_%CYC%
#PBS -j oe
#PBS -S /bin/bash
#PBS -q %QUEUE%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l walltime=02:20:00
#PBS -l select=16:mpiprocs=60:ompthreads=2:ncpus=120+1:mpiprocs=12:ompthreads=2:ncpus=24
#PBS -l debug=true

export NODES=17
export TOTAL_TASKS=972
export OMP_NUM_THREADS=2
export ATM_THREADS=2

model=hwrf
%include <head.h>
%include <envir-p1.h>
export FORECAST_CONF_OVERRIDE=" %FORECAST_CONF_OVERRIDE%"

export cyc="%CYC%"
export storm_num="%STORMNUM%"

# versions file for hwrf sets $model_ver and $code_ver

module load envvar/$::env(envvar_ver)
module load PrgEnv-intel/$::env(PrgEnv_intel_ver)
module load craype/$::env(craype_ver)
module load intel/$::env(intel_ver)
module load cray-mpich/$::env(cray_mpich_ver)
module load cray-pals/$::env(cray_pals_ver)
module load libpng/$::env(libpng_ver)
module load zlib/$::env(zlib_ver)
module load jasper/$::env(jasper_ver)
module load libjpeg/$::env(libjpeg_ver)
module load grib_util/$::env(grib_util_ver)
module load wgrib2/$::env(wgrib2_ver)
module load w3emc/$::env(w3emc_ver)
module load w3nco/$::env(w3nco_ver)
module load bufr/$::env(bufr_ver)
module load hdf5/$::env(hdf5_ver)
module load netcdf/$::env(netcdf_ver)
module load pnetcdf/$::env(pnetcdf_ver)
module load udunits/$::env(udunits_ver)
module load nco/$::env(nco_ver)
module load python/$::env(python_ver)
module list

${HOMEhwrf}/jobs/JHWRF_FORECAST

%include <tail.h>

%manual

TASK failsafe

PURPOSE: Runs the failsafe forecast job.  This job disables several
parts of the HWRF system, and enables extra vertical acceleration
damping.  It is likely to result in a degraded forecast, but it is
unlikely to fail.  It will never be run automatically since it starts
in a completed state.  Instead, it must be manually submitted, and
should only be used as a last resort.

ACTIVATING THE FAILSAFE FORECAST

1. Suspend the forecast family
2. Kill all jobs in the forecast family
3. Requeue the forecast/post and forecast/output
4. Set the forecast/wave_post to complete
5. Execute the forecast/model/failsafe
6. Set the other three forecast/model/wrf* jobs to complete
7. Resume the forecast family

The post-processing should start soon after the failsafe starts.  The
rest of the HWRF workflow will run as normal, but without ocean and
wave outputs.

%end