#!/bin/ksh -x if [ $# -ne 1 ]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" echo "anal fcst post vrfy metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" exit 1 fi step=$1 export npe_node_max=40 export launcher="srun --export=ALL" # Configure MPI environment #export I_MPI_ADJUST_ALLREDUCE=5 #export MPI_BUFS_PER_PROC=2048 #export MPI_BUFS_PER_HOST=2048 #export MPI_GROUP_MAX=256 #export MPI_MEMMAP_OFF=1 #export MP_STDOUTMODE="ORDERED" export OMP_STACKSIZE=2048000 export NTHSTACK=1024000000 ##export LD_BIND_NOW=1 export job=${PBS_JOBNAME:-$step} export jobid=${job}.${PBS_JOBID:-$$} if [ $step = "prep" -o $step = "prepbufr" ]; then nth_max=$(($npe_node_max / $npe_node_prep)) export POE="NO" export BACK="NO" export sys_tp="HERA" export launcher_PREP="srun" elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostbndpntbll" -o $step = "wavepostpnt" ]; then export mpmd="--multi-prog" export CFP_MP="YES" if [ $step = "waveprep" ]; then export MP_PULSE=0 ; fi export wavempexec=${launcher} export wave_mpmd=${mpmd} elif [ $step = "anal" ]; then export MKL_NUM_THREADS=4 export MKL_CBWR=AUTO nth_max=$(($npe_node_max / $npe_node_anal)) export NTHREADS_GSI=${nth_anal:-$nth_max} [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max export APRUN_GSI="$launcher" export NTHREADS_CALCINC=${nth_calcinc:-1} [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max export APRUN_CALCINC="$launcher" export NTHREADS_CYCLE=${nth_cycle:-12} [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max npe_cycle=${ntiles:-6} export APRUN_CYCLE="$launcher -n $npe_cycle" export NTHREADS_GAUSFCANL=1 npe_gausfcanl=${npe_gausfcanl:-1} export APRUN_GAUSFCANL="$launcher -n $npe_gausfcanl" export NTHREADS_CHGRES=${nth_echgres:-1} [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max export APRUN_CHGRES="" export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} export APRUNCFP="$launcher -n \$ncmd --multi-prog" elif [ $step = "gldas" ]; then export USE_CFP="NO" nth_max=$(($npe_node_max / $npe_node_gldas)) export NTHREADS_GLDAS=${nth_gldas:-$nth_max} [[ $NTHREADS_GLDAS -gt $nth_max ]] && export NTHREADS_GLDAS=$nth_max export APRUN_GLDAS="$launcher -n $npe_gldas" export NTHREADS_GAUSSIAN=${nth_gaussian:-1} [[ $NTHREADS_GAUSSIAN -gt $nth_max ]] && export NTHREADS_GAUSSIAN=$nth_max export APRUN_GAUSSIAN="$launcher -n $npe_gaussian" # Must run data processing with exactly the number of tasks as time # periods being processed. npe_gldas_data_proc=$(($gldas_spinup_hours + 12)) export APRUN_GLDAS_DATA_PROC="$launcher -n $npe_gldas_data_proc --multi-prog" elif [ $step = "eobs" ]; then export MKL_NUM_THREADS=4 export MKL_CBWR=AUTO nth_max=$(($npe_node_max / $npe_node_eobs)) export NTHREADS_GSI=${nth_eobs:-$nth_max} [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max export APRUN_GSI="$launcher" export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} export APRUNCFP="$launcher -n \$ncmd --multi-prog" elif [ $step = "eupd" ]; then nth_max=$(($npe_node_max / $npe_node_eupd)) export NTHREADS_ENKF=${nth_eupd:-$nth_max} [[ $NTHREADS_ENKF -gt $nth_max ]] && export NTHREADS_ENKF=$nth_max export APRUN_ENKF="$launcher" export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} export APRUNCFP="$launcher -n \$ncmd --multi-prog" elif [ $step = "fcst" ]; then nth_max=$(($npe_node_max / $npe_node_fcst)) export NTHREADS_FV3=${nth_fv3:-$nth_max} [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max export cores_per_node=$npe_node_max export APRUN_FV3="$launcher" export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max export APRUN_REGRID_NEMSIO="$launcher" export NTHREADS_REMAP=${nth_remap:-2} [[ $NTHREADS_REMAP -gt $nth_max ]] && export NTHREADS_REMAP=$nth_max export APRUN_REMAP="$launcher" elif [ $step = "efcs" ]; then nth_max=$(($npe_node_max / $npe_node_efcs)) export NTHREADS_FV3=${nth_efcs:-$nth_max} [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max export cores_per_node=$npe_node_max export APRUN_FV3="$launcher" export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max export APRUN_REGRID_NEMSIO="$launcher $LEVS" elif [ $step = "post" ]; then nth_max=$(($npe_node_max / $npe_node_post)) export NTHREADS_NP=${nth_np:-1} [[ $NTHREADS_NP -gt $nth_max ]] && export NTHREADS_NP=$nth_max export APRUN_NP="$launcher" export NTHREADS_DWN=${nth_dwn:-1} [[ $NTHREADS_DWN -gt $nth_max ]] && export NTHREADS_DWN=$nth_max export APRUN_DWN="$launcher" elif [ $step = "ecen" ]; then nth_max=$(($npe_node_max / $npe_node_ecen)) export NTHREADS_ECEN=${nth_ecen:-$nth_max} [[ $NTHREADS_ECEN -gt $nth_max ]] && export NTHREADS_ECEN=$nth_max export APRUN_ECEN="$launcher" export NTHREADS_CHGRES=${nth_chgres:-12} [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max export APRUN_CHGRES="time" export NTHREADS_CALCINC=${nth_calcinc:-1} [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max export APRUN_CALCINC="$launcher" elif [ $step = "esfc" ]; then nth_max=$(($npe_node_max / $npe_node_esfc)) export NTHREADS_ESFC=${nth_esfc:-$nth_max} [[ $NTHREADS_ESFC -gt $nth_max ]] && export NTHREADS_ESFC=$nth_max export APRUN_ESFC="$launcher -n $npe_esfc" export NTHREADS_CYCLE=${nth_cycle:-14} [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max export APRUN_CYCLE="$launcher -n $npe_esfc" elif [ $step = "epos" ]; then nth_max=$(($npe_node_max / $npe_node_epos)) export NTHREADS_EPOS=${nth_epos:-$nth_max} [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max export APRUN_EPOS="$launcher" elif [ $step = "fv3ic" ]; then export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max} [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max export APRUN_CHGRES="time" elif [ $step = "postsnd" ]; then nth_max=$(($npe_node_max / $npe_node_postsnd)) export NTHREADS_POSTSND=${nth_postsnd:-1} [[ $NTHREADS_POSTSND -gt $nth_max ]] && export NTHREADS_POSTSND=$nth_max export APRUN_POSTSND="$launcher" export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1} [[ $NTHREADS_POSTSNDCFP -gt $nth_max ]] && export NTHREADS_POSTSNDCFP=$nth_max export APRUN_POSTSNDCFP="$launcher" elif [ $step = "awips" ]; then echo "WARNING: $step is not enabled on $machine!" elif [ $step = "gempak" ]; then echo "WARNING: $step is not enabled on $machine!" fi