###################################################
# This script generates job scripts for MPMC test suite cases
#
# by Guoqing Ge, 2018/8/18, guoqing.ge@noaa.gov
#
###################################################
#
from MPMC_config import ProdGSI_root, build_root, project_acct, queue_name, hostname, rocoto_scheduler
import os
SBATCH_extras=""
MPI_CMD_ORG=' RUN_COMMAND="mpirun -np ${GSIPROC} " ;;\n'
if hostname.startswith("Theia"): ## Theia
crtm_dir="/scratch4/BMC/comgsi/case_data/CRTM_v2.3.0"
data_root="/scratch4/BMC/comgsi/case_data"
myARCH=" ARCH='LINUX_PBS'\n"
few_cpu_res="--ntasks=4"; few_procs="4"
many_cpu_res="--ntasks=24"; many_procs="24" #24 cores/node
MPI_CMD_ORG=' RUN_COMMAND="srun " ;;\n'
elif hostname.startswith("Hera"): ## Hera
crtm_dir="/scratch1/BMC/comgsi/case_data/CRTM_v2.3.0"
data_root="/scratch1/BMC/comgsi/case_data"
myARCH=" ARCH='LINUX_PBS'\n"
few_cpu_res="--ntasks=4"; few_procs="4"
many_cpu_res="--ntasks=40"; many_procs="40" #40 cores/node
MPI_CMD_ORG=' RUN_COMMAND="srun " ;;\n'
elif hostname.startswith("Jet"): ## Jet
crtm_dir="/lfs1/projects/wrfruc/gge/MPMC/case_data/CRTM_v2.3.0"
data_root="/lfs1/projects/wrfruc/gge/MPMC/case_data"
myARCH=" ARCH='LINUX_PBS'\n"
few_cpu_res="--ntasks=4"; few_procs="4"
#SBATCH_extras="#SBATCH --partition=tjet\n" #12 cores/node
#SBATCH_extras="#SBATCH --partition=xjet\n" #24 cores/node
SBATCH_extras="#SBATCH --partition=kjet\n" #40 cores/node
many_cpu_res="--ntasks=40"; many_procs="40"
MPI_CMD_ORG=' RUN_COMMAND="srun " ;;\n'
elif hostname.startswith("Orion"): ## Orion
crtm_dir="/work/noaa/comgsi/MPMC/case_data/CRTM_v2.3.0"
data_root="/work/noaa/comgsi/MPMC/case_data"
myARCH=" ARCH='LINUX_PBS'\n"
few_cpu_res="--ntasks=4"; few_procs="4"
SBATCH_extras="#SBATCH --partition=orion\n" #40 cores/node
many_cpu_res="--ntasks=40"; many_procs="40" #40 cores/node
MPI_CMD_ORG=' RUN_COMMAND="srun " ;;\n'
elif hostname.startswith("Cheyenne"): ## Cheyenne
crtm_dir="/glade/p/ral/jntp/DAtask/case_data/CRTM_v2.3.0"
data_root="/glade/p/ral/jntp/DAtask/case_data"
myARCH=" ARCH='LINUX_PBS'\n"
few_cpu_res="select=1:ncpus=4:mpiprocs=4"; few_procs="4"
many_cpu_res="select=2:ncpus=36:mpiprocs=36"; many_procs="36" #Dual-socket nodes, 18 cores per socket, 20 ensembles, need at least 20cores
elif hostname.startswith("GSI_Docker"): ## GSI_Docker
crtm_dir="/tutorial/case_data/CRTM_v2.3.0"
data_root="/tutorial/case_data"
myARCH=" ARCH='LINUX'\n"
few_cpu_res="docker"; few_procs="1"
many_cpu_res="docker"; many_procs="1"
######################### variables definition #######################################
hybrid_yes=" if_hybrid=Yes # Yes, or, No -- case sensitive !\n"
EnVar4D_yes=" if_4DEnVar=Yes # Yes, or, No -- case sensitive (set if_hybrid=Yes first)!\n"
linkallobs =" ln -s ${srcobsfile[$ii]} ${gsiobsfile[$ii]}\n"
#----------------------
#**** substitutions to be made in run scripts based on template scripts in ProdGSI/ush
jobproc_few=" GSIPROC="+few_procs+"\n"
jobproc_many=" GSIPROC="+many_procs+"\n"
#
com_obs=" OBS_ROOT="+data_root+"/com_2018081212/obs\n"
com_bkg=" BK_ROOT="+data_root+"/com_2018081212/bkg\n"
com_ens=" ENS_ROOT="+data_root+"/com_2018081212/gfsens\n"
#
nmmb_obs=" OBS_ROOT="+data_root+"/nmmb_2012062812/obs\n"
nmmb_bkg=" BK_ROOT="+data_root+"/nmmb_2012062812/bkg\n"
nmmb_ens=" ENS_ROOT="+data_root+"/nmmb_2012062812/gfsens\n"
#
chem_obs=" OBS_ROOT="+data_root+"/chemdata/obs\n"
chem_bkg=" BK_ROOT="+data_root+"/chemdata/bkg\n"
#
gfs_obs=" OBS_ROOT="+data_root+"/T62.gfs/obs\n"
gfs_bkg=" BK_ROOT="+data_root+"/T62.gfs/bkg\n"
#
case2124_obs=" OBS_ROOT="+data_root+"/enkf_arw_2014021300/obs\n"
case2124_bkg=" BK_ROOT="+data_root+"/enkf_arw_2014021300/bkg\n"
#
case2526_obs=" OBS_ROOT="+data_root+"/enkf_glb_T62/obs\n"
case2526_bkg=" BK_ROOT="+data_root+"/enkf_glb_T62/bkg\n"
#
cases={ \
"case01-oneobs-glb":{" if_oneob=No":" if_oneob=Yes\n", " bkcv_option=NAM":" bkcv_option=GLOBAL\n", \
"ln -s ${PREPBUFR} ./prepbufr":"#ln -s ${PREPBUFR} ./prepbufr\n", \
" PREPBUFR=":" PREPBUFR=${OBS_ROOT}/rap.t${HH}z.prepbufr.tm00\n", \
" ANAL_TIME=":" ANAL_TIME=2018081212\n", \
" BK_FILE=${BK_ROOT}":" BK_FILE=${BK_ROOT}/wrfout_d01_2018-08-12_12:00:00\n", \
" OBS_ROOT=":com_obs," BK_ROOT=":com_bkg," ENS_ROOT=":com_ens," GSIPROC=":jobproc_few } \
,"case02-oneobs-nam":{" if_oneob=No":" if_oneob=Yes\n", \
"ln -s ${PREPBUFR} ./prepbufr":"#ln -s ${PREPBUFR} ./prepbufr\n", \
" PREPBUFR=":" PREPBUFR=${OBS_ROOT}/rap.t${HH}z.prepbufr.tm00\n", \
" ANAL_TIME=":" ANAL_TIME=2018081212\n", \
" BK_FILE=${BK_ROOT}":" BK_FILE=${BK_ROOT}/wrfout_d01_2018-08-12_12:00:00\n", \
" OBS_ROOT=":com_obs," BK_ROOT=":com_bkg," ENS_ROOT=":com_ens," GSIPROC=":jobproc_few} \
,"case03-conv":{" PREPBUFR=":" PREPBUFR=${OBS_ROOT}/rap.t${HH}z.prepbufr.tm00\n", \
" BK_FILE=${BK_ROOT}":" BK_FILE=${BK_ROOT}/wrfout_d01_2018-08-12_12:00:00\n", \
" ANAL_TIME=":" ANAL_TIME=2018081212\n", \
" OBS_ROOT=":com_obs," BK_ROOT=":com_bkg," ENS_ROOT=":com_ens," GSIPROC=":jobproc_few} \
,"case04-allobs":{"# ln -s ${srcobsfile[$ii]} ${gsiobsfile[$ii]}":linkallobs, \
" PREPBUFR=":" PREPBUFR=${OBS_ROOT}/rap.t${HH}z.prepbufr.tm00\n", \
" ANAL_TIME=":" ANAL_TIME=2018081212\n", \
" BK_FILE=${BK_ROOT}":" BK_FILE=${BK_ROOT}/wrfout_d01_2018-08-12_12:00:00\n", \
" OBS_ROOT=":com_obs," BK_ROOT=":com_bkg," ENS_ROOT=":com_ens," GSIPROC=":jobproc_few} \
,"case05-3DEnVar":{" if_hybrid=":hybrid_yes," if_nemsio=No":" if_nemsio=Yes\n", \
"# ln -s ${srcobsfile[$ii]} ${gsiobsfile[$ii]}":linkallobs, \
" PREPBUFR=":" PREPBUFR=${OBS_ROOT}/rap.t${HH}z.prepbufr.tm00\n", \
" ANAL_TIME=":" ANAL_TIME=2018081212\n", \
" BK_FILE=${BK_ROOT}":" BK_FILE=${BK_ROOT}/wrfout_d01_2018-08-12_12:00:00\n", \
" OBS_ROOT=":com_obs," BK_ROOT=":com_bkg," ENS_ROOT=":com_ens," GSIPROC=":jobproc_few} \
,"case07-4DEnVar":{" if_hybrid=":hybrid_yes," if_4DEnVar=":EnVar4D_yes, " if_nemsio=No":" if_nemsio=Yes\n", \
" BK_FILE=${BK_ROOT}":" BK_FILE=${BK_ROOT}/wrfout_d01_2018-08-12_12:00:00\n", \
" PREPBUFR=":" PREPBUFR=${OBS_ROOT}/rap.t${HH}z.prepbufr.tm00\n", \
" ANAL_TIME=":" ANAL_TIME=2018081212\n", \
" OBS_ROOT=":com_obs," BK_ROOT=":com_bkg," ENS_ROOT=":com_ens," GSIPROC=":jobproc_few} \
,"case08-nmmb":{" ANAL_TIME=":" ANAL_TIME=2012062812\n"," bk_core=ARW":" bk_core=NMMB\n", \
" OBS_ROOT=":nmmb_obs," BK_ROOT=":nmmb_bkg," ENS_ROOT=":nmmb_ens," GSIPROC=":jobproc_few,
" BK_FILE=${BK_ROOT}":" BK_FILE=${BK_ROOT}/input_domain_01_nemsio\n", " PREPBUFR=":" PREPBUFR=${OBS_ROOT}/gfs.t${HH}z.prepbufr.nr\n"} \
,"case09-wrfchem":{" OBS_ROOT=":chem_obs," BK_ROOT=":chem_bkg," GSIPROC=":jobproc_few } \
,"case10-cmaq":{" OBS_ROOT=":chem_obs," BK_ROOT=":chem_bkg," GSIPROC=":jobproc_few," ANAL_TIME=":" ANAL_TIME=2013062112\n"," bk_core=":" bk_core=CMAQ\n", \
" BK_FILE=":" BK_FILE=${BK_ROOT}/cmaq2gsi_4.7_20130621_120000.bin\n"," PREPBUFR=":" PREPBUFR=${OBS_ROOT}/anow.2013062112.bufr\n"} \
,"case11-gfs":{" OBS_ROOT=":gfs_obs," BK_ROOT=":gfs_bkg," GSIPROC=":jobproc_few } \
,"case21-observer-conv":{" OBS_ROOT=":case2124_obs," BK_ROOT=":case2124_bkg," GSIPROC=":jobproc_few, " ANAL_TIME=":" ANAL_TIME=2014021300\n", \
" PREPBUFR=":" PREPBUFR=${OBS_ROOT}/gdas1.t${HH}z.prepbufr.nr\n", " BK_FILE=${BK_ROOT}":" BK_FILE=${BK_ROOT}/wrfarw.ensmean\n", \
" if_observer":" if_observer=Yes\n"} \
,"case22-enkf-conv":{" OBS_ROOT=":case2124_obs," BK_ROOT=":case2124_bkg," GSIPROC=":jobproc_many, " diag_ROOT":" diag_ROOT=../case21-observer-conv\n" } \
,"case23-observer-allobs":{" OBS_ROOT=":case2124_obs," BK_ROOT=":case2124_bkg," GSIPROC=":jobproc_few, " ANAL_TIME=":" ANAL_TIME=2014021300\n", \
" PREPBUFR=":" PREPBUFR=${OBS_ROOT}/gdas1.t${HH}z.prepbufr.nr\n", " BK_FILE=${BK_ROOT}":" BK_FILE=${BK_ROOT}/wrfarw.ensmean\n", \
" if_observer":" if_observer=Yes\n", "# ln -s ${srcobsfile[$ii]} ${gsiobsfile[$ii]}":linkallobs } \
,"case24-enkf-allobs":{" OBS_ROOT=":case2124_obs," BK_ROOT=":case2124_bkg," GSIPROC=":jobproc_many," diag_ROOT":" diag_ROOT=../case23-observer-allobs\n", \
' list="conv"':' list="conv amsua_n18 mhs_n18 hirs4_n19"\n' } \
,"case25-observer-gfs":{" OBS_ROOT=":case2526_obs," BK_ROOT=":case2526_bkg," GSIPROC=":jobproc_few, " ANAL_TIME=":" ANAL_TIME=2014092918\n", \
" PREPBUFR=":" PREPBUFR=${OBS_ROOT}/gdas1.t18z.prepbufr.nr\n"," if_observer":" if_observer=Yes\n", " GFSCASE=":" GFSCASE=enkf_glb_t62\n" } \
,"case26-enkf-gfs":{" OBS_ROOT=":case2526_obs," BK_ROOT=":case2526_bkg," GSIPROC=":jobproc_many," diag_ROOT":" diag_ROOT=../case25-observer-gfs\n" } \
}
allcases=[\
"case01-oneobs-glb","case02-oneobs-nam",
"case03-conv","case04-allobs","case05-3DEnVar","case07-4DEnVar", \
# "case08-nmmb", \ #remove case08 from test suite due to difficulties to fix bugs and nmmb is phasing away to fv3
"case09-wrfchem","case10-cmaq","case11-gfs", \
# "case12-fv3", "case13-cloudana"
"case21-observer-conv", "case22-enkf-conv", "case23-observer-allobs","case24-enkf-allobs","case25-observer-gfs","case26-enkf-gfs" ]
fullLIST='1,2,3,4,5,7,9,10,11,21,22,23,24,25,26'
caselist=fullLIST.split(',')
mycases=[]
for x in caselist:
k=int(x)
for i in range(len(allcases)):
if allcases[i].find("case{0:02d}".format(k)) >=0:
mycases.append(allcases[i])
##################### generate runing job scripts for a build option #########
def create_run_scripts(build_ID, modules, job_dir):
if modules.find("module load mpt") >=0:
MPI_CMD=' RUN_COMMAND="mpiexec_mpt dplace -s 1 " ;;\n'
else:
MPI_CMD=MPI_CMD_ORG
for x in mycases:
if (x.find("wrfchem")>=0 ) or (x.find("cmaq")>=0):
srcfname=ProdGSI_root+"/ush/comgsi_run_chem.ksh"
elif (x.find("enkf")>=0 and x.find("gfs")>=0):
srcfname=ProdGSI_root+"/ush/comenkf_run_gfs.ksh"
elif (x.find("enkf")>=0): #enkf regional
srcfname=ProdGSI_root+"/ush/comenkf_run_regional.ksh"
elif (x.find("gfs")>=0):
srcfname=ProdGSI_root+"/ush/comgsi_run_gfs.ksh"
else:
srcfname=ProdGSI_root+"/ush/comgsi_run_regional.ksh"
jobfname=generateAjob(x,srcfname, build_ID, modules, job_dir, MPI_CMD)
##################### generate a job script for a case #########
def generateAjob(case_name, srcfname, build_ID, modules, job_dir, MPI_CMD):
global jobknt
if case_name.find('enkf')>=0:
cpu_res=many_cpu_res
else:
cpu_res=few_cpu_res
if rocoto_scheduler.find('slurm')>=0:
PBScmds="#SBATCH --account "+project_acct+"\n" \
+"#SBATCH -t 00:20:00\n" \
+"#SBATCH --job-name "+build_ID+"-"+case_name+"\n" \
+"#SBATCH "+cpu_res+"\n" \
+"#SBATCH --qos "+queue_name+"\n" \
+SBATCH_extras \
+"#SBATCH -o out."+case_name+"\n" \
+modules+"\n\n" \
+"set -x\n"
else:
PBScmds="### in PBS, cannnot put comments after job decription commands\n" \
+"#PBS -A "+project_acct+"\n" \
+"#PBS -l walltime=00:20:00\n" \
+"#PBS -N "+build_ID+"-"+case_name+"\n" \
+"#PBS -l "+cpu_res+"\n" \
+"#PBS -q "+queue_name+"\n" \
+"#PBS -o out."+case_name+"\n" \
+"#PBS -j oe\n\n" \
+modules+"\n\n" \
+"set -x\n"
common={"set -x" :PBScmds \
,' RUN_COMMAND="mpirun -np ${GSIPROC} "' :MPI_CMD \
," ARCH=" :myARCH \
,"cp ${GSI_EXE} gsi.x" :"ln -sf ${GSI_EXE} gsi.x\n" #link to gsi to avoid unneccessary duplicates
,"cp $ENKF_EXE enkf.x" :"ln -sf ${ENKF_EXE} enkf.x\n" #link to enkf to avoid unneccessary duplicates
," JOB_DIR=" :" JOB_DIR="+job_dir+"\n" \
," RUN_NAME=" :" RUN_NAME="+case_name+"\n" \
," GSI_ROOT=" :" GSI_ROOT="+ProdGSI_root+"\n" \
," CRTM_ROOT=" :" CRTM_ROOT="+crtm_dir+"\n" \
}
jobfname=job_dir+'/run.'+case_name
jobfile = open(jobfname, "w")
mydict=cases[case_name]
file1=open(srcfname, "r")
while True:
line=file1.readline()
if not line:
break
found=False
for key in common.keys():
if line.startswith(key):
jobfile.write(common[key])
found=True
break
for key in mydict.keys():
if line.startswith(key):
jobfile.write(mydict[key])
found=True
break
if not found:
jobfile.write(line)
jobfile.close()
file1.close()
os.system("chmod +x "+jobfname)
return jobfname
#-----------------------End of generateAjob(...)---------------|