U $gBZ@sBdZdddgZddlZddlZddlZddlZddlZddlZddl Zddl Zddl Zddl Zddl ZddlZddlZddlZddlZddlZddlZddlmZmZmZddl mZddl mZdd lmZmZmZmZdd l mZmZmZm Z m!Z!dd lm"Z"m#Z#m$Z$m%Z%dd l&m'Z'd ddddddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;dd?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcdddedfgZZ(Gdgddej)j*Z+Gdhdidie+Z,Gdjdkdke+Z-Gdldmdme+Z.dvdndoZ/dwdpdqZ0dxdrdsZ1dydtduZ2dS)zz3!Runs the GSI data assimilation on the HWRF system.GSI_DATA_TYPESGSIBaseZFGATGSIGSIMeanHxGSIHxN) setrlimitrusage getrlimit) COMPLETED)NamedDir) make_symlink deliver_file isnonemptymake_symlinks_in)mpirunmpiopenmpcheckrunbigexe) to_datetimeto_datetime_rel to_fraction to_timedelta) GSIInputErrorZ hirs2_n14Zmsu_n14Zsndr_g08Zsndr_g11Zsndr_g12Zsndr_g13Z sndr_g08_prepZ sndr_g11_prepZ sndr_g12_prepZ sndr_g13_prepZ sndrd1_g11Z sndrd2_g11Z sndrd3_g11Z sndrd4_g11Z sndrd1_g12Z sndrd2_g12Z sndrd3_g12Z sndrd4_g12Z sndrd1_g13Z sndrd2_g13Z sndrd3_g13Z sndrd4_g13Z sndrd1_g14Z sndrd2_g14Z sndrd3_g14Z sndrd4_g14Z sndrd1_g15Z sndrd2_g15Z sndrd3_g15Z sndrd4_g15Z hirs3_n15Z hirs3_n16Z hirs3_n17Z amsua_n15Z amsua_n16Z amsua_n17Z amsub_n15Z amsub_n16Z amsub_n17Zhsb_aquaZ airs_aquaZ amsua_aquaZimgr_g08imgr_g11imgr_g12Zimgr_g14imgr_g15Z pcp_ssmi_dmspZ pcp_tmi_trmmconvZ sbuv2_n16Z sbuv2_n17Z sbuv2_n18Z sbuv2_n19z gome_metop-aZomi_auraZmls_aurassmi_f13ssmi_f14ssmi_f15Z hirs4_n18z hirs4_metop-aZ amsua_n18z amsua_metop-aZmhs_n18z mhs_metop-aZamsre_low_aquaZamsre_mid_aquaZamsre_hig_aqua ssmis_f16 ssmis_f17 ssmis_f18 ssmis_f19 ssmis_f20z iasi_metop-aZ hirs4_n19Z amsua_n19Zmhs_n19Z seviri_m08Z seviri_m09 seviri_m10Zcris_nppz cris-fsr_nppz cris-fsr_n20Zatms_nppz hirs4_metop-bz amsua_metop-bz mhs_metop-bz iasi_metop-bz gome_metop-bcsJeZdZdZdKfdd ZeddZeddZed d Zd d Z d dZ ddZ dLddZ dMddZ dNddZdOddZddZddZddZdPd d!ZdQd#d$Zd%d&Zd'd(Zd)d*Zed+d,ZdRd.d/ZdSd1d2Zd3d4Zd5d6Zd7d8Zd9d:Zd;d<Zd=d>Z d?d@Z!dAdBZ"dCdDZ#dEdFZ$dGdHZ%dIdJZ&Z'S)Trz@!Base class of anything that runs the GSI. Do not use directly.Nc  sttj|||fd|ij}| _|dkr:|j}| dkrLtd|} t|_t| _ |_ |_ |_ t _tjjdd_d_d_d} j| d d _jd d d_d_| dk r| D]}j|q|dk st|jdk st|jd ks$t| l}djkrHdj!j"}n&djkrbdj!j"}n dj!j"}t#j$j%|||tjj&|d}|_'W5QRXfdd}|dd_(|dd_)|dd_*|dj*d_+|dd_,|d d!_-|d"d#_.|d$d_/|d%d&_0|d'd(_1|d)d*_2|d+d,_3d_4d}d-krd-}t5|t6j7j8rv|_4n8t5|t9r|}n&|dkrnt:d.t;|j_4dS)0a!The GSIBase constructor: @param dstore passed to Datum: the Datastore object for this Task @param conf the conf object for this task (passed to HWRFTask) @param section the conf section for this task (passed to HWRFTask) @param domain the WRFDomain for this GSI. Must have been initialized by a WRFSimulation @param wrf_in_prod the Product for the wrfinput_d01 or ghost_d0* file for that domain @param sim the hwrf.wrf.WRFSimulation that will be run as the forecast @param taskname Optional: the taskname for this product in the datastore @param atime the analysis time as a datetime.datetime. Default: conf.cycle @param parent_atime the analysis time of the parent model. This is relevant if the parent model's forecast is used as the background. @param enkf_domains a list of WRF domains that should be copied from the hwrf.ensda.EnsembleDA. @param ensda a subclass of hwrf.ensda.DAEnsemble that provides regional ensemble forecasts to generate the forecast error covariance @param kwargs ignored; passed to HWRFTasktasknameNicomz{stormlabel}.tdrFP nml_sectionHYBENS_REGIONALTconfighybrid_fallbackmeanhxZwrf_out_ensmean_hxZwrf_out_Zgsi_out_)locationcs8|krt|S|d}|dk r4|dkr4|S|S)Nr+)strconfstr)whatdefaultskwargsself6/lfs/h1/ops/prod/packages/hwrf.v13.2.9/ush/hwrf/gsi.pygetc~s   zGSIBase.__init__..getcZ sigma_datasetgdas1Z sigma_itemZgdas1_sfZ bufr_datasetgfsZ bufr_itemZ_bufr prepbufr_itemZ prepbufr_nr enkf_datasetenkf enkf_itemenkf_sfgZbiascr_datasetZ biascr_itemZ gdas1_biascrZbiascr_pc_itemZgdas1_biascr_pcZ abias_itemZ gdas1_abiasZ satang_itemZ gdas1_satang in_catalogzIn hwrf.gsi.GSIBase.__init__, in_catalog must be None, a basestring or a DataCatalog. You provided an object of type %s, with value %s.catalog)?superr__init__r$_ensdacyclerr_atime _parent_atime_domain _wrf_in_prod_simlist _enkf_domainsospathjoinconfgetdiricstr _tdrflagfileused_regional_ensembleglobal_ensemble_sizer0getbool hybrid_dar* _other_prodsappendAssertionErrorr. transactiondomainnameprodutil datastore FileProductoutdir _wrf_out_prod_sigma_dataset _sigma_item _bufr_datasetZ _bufr_item_prepbufr_item _enkf_dataset _enkf_item_biascr_dataset _biascr_item_biascr_pc_item _abias_item _satang_item _in_catalog isinstancehwrfinput DataCatalogr/ TypeErrortype__name__repr parent_atime)r6dstorerQsectionr] wrf_in_prodsimr$atimerx enkf_domainsensdar5nmltprodnameprodr9 incat_nameink __class__r4r8rD=s                          zGSIBase.__init__cCs|jjdd}t|S)z!The WRF model top in pascalsdomainsp_top_requested)rKnlnl_getfloat)r6rr7r7r8 wrf_top_PaszGSIBase.wrf_top_PacCs|jS)z!The analysis time of this GSI.)rGr6r7r7r8r}sz GSIBase.atimecCs|jS)z!Parent model analysis time. The analysis time of the parent data assimilation models (ie.: GDAS, GFS ENKF). This is usually six hours before self.atime.)rHrr7r7r8rxszGSIBase.parent_atimecCs(t|_||_|D]}|j|qdS)a!Sets the hwrf.ensda.DAEnsemble to use, and the enkf domains. Specifies the ensemble to use for forecast error covariances, and the list of hwrf.wrf.WRFDomain domains that should be input to the GSI. @param ensda the hwrf.ensda.DAEnsemble that provides short simulation output, for the forecast error covariances @param enkf_domains an iterable of enkf domains, WRFDomain objects, identifying which domains in ensda should be used.N)rLrMrErZ)r6rr~domr7r7r8 set_ensdas zGSIBase.set_ensdac cs|dd}|dkr|jr|dd}|dd}|j}t| d|}td||}t|D]*}d |}t|j|j|j |||d d Vq^|j } | d d rt|j|j |j | | dVt|j|j |j| | dVnB|jrt|j|j |j| | dV|jrt|j|j |j| | dV|j}t|j|j|j|dV|d} t} | dD]} | } t| dksL| | krtqL|j| d}|j| d}|j| d}|dkr| dd sqL|dkr| dd sڐqL|j| D]0\}}|dkrqt|j||||d dVqqL|jdd}|dkrpd} |j| d}|j| d}|j}t|j|||d dV|j }|D]0}|tj|d }t|j|j |j!||d!Vq~d"S)#a!Iterate over needed inputs. Iterates over all files external to this workflow that are required to run the GSI. This may include the GFS ENKF, prepbufr, bufr, and other files. This is used by the hwrf.input module to obtain those inputs.num_enkfr&renkf_fhr@ enkf_age_hr @T)datasetitemr}enkfmemftimeoptional use_newradbcF)rrrr})rrr}obstypes,rrru satellitesat_radiance_dasatwnd sat_wnd_darrru)rrobstyper}rbufrprep prepbufrprepZtempdrop_obstype)rrr}rhours)rrr}rN)"confintrX conffloatr}rrangedicttaskvarsrhrirxconfboolrjrkrlrmrnrfrgr0setsplitstriplenrQgetloweritemsgetint parent_fhrsdatetime timedeltardre)r6maxmembfhourenkf_agemy_atimer}rzmembimembrxolisttouchedrztrimrrotype localnamerrZtempdrop_datasetZ tempdrop_itemhrr7r7r8 inputiters                   zGSIBase.inputitercCsd|_|dd}|dd}|jdd}|dd}|d d }|rt|j dkr|j d k rt |j s|d ks||ks|dkr|j j|kr||j |_|js|jr|d S) ac!Link or copy ensemble inputs. Links or copies the output of the last ENKF or hwrf.ensda cycle. Calls grab_wrf_enkf() if any enkf_domains were given to the constructor. Otherwise, calls grab_gfs_enkf() to get the global enkf. Will also revert to grab_gfs_enkf() if the ensda should be used, but was unavailable.Fuse_hwrf_ensemble ensda_whentdr_next_cycler) stormlabel ensda_optrZuse_hwrf_ensemble_wmaxgNalways)rUrr0rrQrrrrrMrEr rT storminfowmax grab_wrf_enkfrX grab_gfs_enkf)r6rrrrZ wmaxthresholdr7r7r8grab_enkf_input#s,     zGSIBase.grab_enkf_inputcCsP|d}t}|dD]0}|}t|dkr||kr|j|||dqdS)a!Link bufr files. Links or copies all needed bufr files to the local directory. If sat_da is False, satellite obs will be omitted. @param atime the analysis time to use when specifying the required bufr files @param morevars additional variables to pass for string replacement when expanding bufr filenames in configuration (hwrf.config.HWRFConfig) sections.rrr)r}morevarsN)r0rrrrgrab_obstype_section)r6r}rrrosectionrr7r7r8 grab_bufr:s zGSIBase.grab_bufrcs|}t|tst|}|dkr*|j}n t||j}|j|d}|j|d}|j|d}|d||||f|dkr| ddr|d |fn|d |fdS|d kr| d dr|d |fn|d|fdSt }|j |} t dd| Dfdd} | D]"\} } | dkr8q"| | q"|D]} |d| ||ft| tstdt| jt| ftjdk} |dkrtj|dd}|jj|||dd}tj||}n8|dks|dks|dkrV|jj||||| d}tj|}t|||ddtd d!}t|rLt ||d"|!n|d#krt"|j#r| rtj|dd}|jj|||| d}tj||}nj|d$krt"|j#r| rtj|dd}|jj|||d%d}tj||}n|jj||||| d}|dkrJ|dkrJ|dkrJ|dksF|d&krrd'| f}||| d(d)rt$|nxtj%|rtj|}| || }|dkst|j&rt|||ddn,d*| |f}||| d(d)rJt$|qJdS)+a!Copies or links observations. @param section the obstype section to read @param atime the atime for string expansion when finding bufr files @param morevars more variables for string expansion when finding bufr files Copies or links observations specified in the obstype sections of the configuration file to the current working directory. The section listed in self.section should contain an "obstype" option, whose value is a comma separated list of section names. This method reads every section in that list. For example, @code{.unformatted} [gsi_d02] catalog = {input_catalog} ... obstypes = hdob_obstype,sat_obstypes,tdr_new_obstype ... [sat_obstypes] type=satellite dataset=gfs item=gfs_bufr gsnd1bufr=goesfv amsuabufr=1bamua satwndbufr=satwnd gpsrobufr=gpsro ... [hdob_obstype] ... [tdr_new_obstype] ... @endcode For each section, the option keys are the local directory filenames expected by GSI, while the values are the data type part of the operational filename (ie.: the satwind in gfs.t12z.tm00.satwind.bufr_d). There are a few special keys: dataset - the name of the dataset for hwrf.input purposes item - the name of the item for hwrf.input purposes type - the type of observation: satellite, or anything else. At present, only "satellite" has any special meaning. If the type is "satellite" then the entire section will be skipped if sat_radiance_da=False in this task's config section. If the type is "satwnd" then the entire section will be skipped if sat_wnd_da=False in this task's config section. Once the section is parsed, the files are all linked to this directory. Nrrruz6process obs section %s with dataset=%s item=%s type=%srrTz4%s: satellite radiance DA is enabled. Continuing...z^%s: satellite radiance DA is disabled in hwrf.conf. Not linking bufr files from this section.rrz=%s: assimilation of satellite wind is enabled. Continuing...zg%s: assimilation of satellite wind is disabled in hwrf.conf. Not linking bufr files from this section.cSsg|]\}}||fqSr7r7).0kvr7r7r8 sz0GSIBase.grab_obstype_section..cs|SNr7)frZotdictr7r8z.GSIBase.grab_obstype_section..rz%Find obstype=%s in dataset=%s item=%szIn gsi.GSIBase.link_bufr, the obstypes parameter must be an iterable container of basestrings. One of the elements was a %s (value %s) instead.)tidegyrelunasurgedogwoodcactustdrintercomrtldplr)r}loggerrZtdrsoZg4tdrsoZhiwrapsorforcetldplrsoa)fileZ nexrad_obsZhd_obsZhdobr+z/%s: Could not find a location for this obstype.Zrequire_all_bufrsFz1%s: Observation file is empty or non-existant: %s)'logrpr/r}rrQrrwarningrrLrrrZrtrurvrwr_clusterr^rNrOrPrRroparselocatebasenamer openr printcloseboolrealtimerfileoprT)r6rzr}rrrrrrrnamerrrZ prodmachinedsitthere tdrsofilebnrmsgonr7rr8rKs6                           zGSIBase.grab_obstype_sectioncKs0|dkr|j}n t||j}|}|jj|j|jf||d|}|jddddkrz|jj|j|jf||d|}n>t j | dd}|jj |jf||d|}t j ||}|dks|dkrdt|jt|jf}||t|n(tj|s|d }||t|t|d |d d dS) a0!Links or copies the prepbufr file to the local directory. @param atime the analysis time, or time relative to self.atime. Used for string expansion in the hwrf.config.HWRFConfig. @param kwargs also passed to the hwrf.config.HWRFConfig for string expansionN)r}rrrrrr+z5Could not find the prepbufr file (item=%s dataset=%s)z: is non-existent or emptyZprepbufrTr)r}rrrorrfrgrQrrNrOrPrRrrwr<rrr_rr errorr )r6r}r5rrrrrr7r7r8 grab_prepbufrsH     zGSIBase.grab_prepbufrtcvitlc CsZ|}|dt|ft|d}||jdW5QRXtj |sVt dS)z!Writes the tcvitals (from self.storminfo) to the specified file. @param filename the file to receive the tcvitalszWriting tcvitals to %swt N) rinforwrwriter as_tcvitalsrNrOexistsr[)r6filenamerrr7r7r8 write_vitals s  zGSIBase.write_vitalscsP||dds2dtjjdddrHfdd}|SdSdS) a!Generate the wrfout file converter. Returns the "copier" argument to deliver_file to use to copy the specified file. Will be None, unless the file is HDF5, in which case it will be "ncks -6 source target" to decompress and convert to NetCDF3 style (uncompressed) with 64-bit indexing. If ncks is missing, such a conversion is impossible, so None is returned. @param file the file that is to be copiedncksr+zAncks path not specified in conf [exe] so I will search your $PATHF) raise_missingcs4tjj|dttdd||fdkddS)Nrz-6z-Oz /dev/null)r_r remove_filerr)r3rxrrr7r8copy&sz#GSIBase.wrfout_copier..copyN)rgetexerr_rfind_exe)r6rrr7rr8 wrfout_copiers   zGSIBase.wrfout_copierc Csx|}|jddd}d}t|rft|d0}|D]$}|ddkr4d}|d |q4W5QRXn||d |S) zv!Should ensemble relocation be run? Reads the RUN_ENSDA_RELOCATE from the run_ensda flag file in COM.ensda_relocate_preens_rlct_flag_filez%{com}/{stormlabel}.run_ensda_relocateFrtzRUN_ENSDA_RELOCATE=YESrTz found RUN_ENSDA_RELOCATE=YES in z is empty or non-existent)rrQgetstrr rfindr )r6r flag_filerun_ens_relocaterliner7r7r8should_run_ens_relocate-s  zGSIBase.should_run_ens_relocatec s|dtt}d}||jD]\}}dt|t|ft|}|jD]}dt|t|t|ft| }| r| rq^q|j j |d}n| rq^n|j||jd}|dkrdt|fq^dt||jfd ||<||q^q.|dk s,tfd d } fd d fdddfdd} fdd} |dd} |jsd} tjd(} | tjj|| | | d}W5QRXt|}||krd||fdS|d||fd|jkr| |dS)zs!Links the WRF ENKF files to this directory. @param ensda the hwrf.ensda.DAEnsemble that provides the fileszin grab_wrf_enkfNzens,member = %s,%szens,member,domain = %s,%s,%s)r])r]r}zNo product for domain %s.zdomain %s prod %sz wrf_en%03dcs|Srr7)plnamemer7r8renamerbrz&GSIBase.grab_wrf_enkf..renamercst||d|ddS)NT)rkeepcopier)r )fromfiletofiler(rr7r8rcsz#GSIBase.grab_wrf_enkf..copycstjj||ddSNrr_rr )r)r*rr7r8linkesz#GSIBase.grab_wrf_enkf..linkcs|j}|d||ftj|dkr>|d|fd}n|d|fd}|dkrl||gn|||gdS)Nz Link %s to %sHDF5zW%s: file is HDF5. I will assume your GSI was built with support for compressed NetCDF4z%s: file is NetCDF3.)r.r r_r netcdfveradd_work)r"nr#r)r()rr-workpoolr7r8actorjsz$GSIBase.grab_wrf_enkf..actorc sFtdd"}|D]}||dqW5QRX|dk rB|dS)N wrf_ens_listrr)rr r)plflr"r$r7r8ensemble_filelistys  z0GSIBase.grab_wrf_enkf..ensemble_filelistmaxwaitr)maxtimez.namercWst|j||ddSr+r r.r"r^rrWr7r7r8r3sz%GSIBase.copy_wrf_inout..actorNrr_r`rErJr6r rrr3r7rXr8copy_wrf_inouts  zGSIBase.copy_wrf_inoutcCs |jd|jjdkr|jSdS)z!Obtain output ghost product for the specified domain. If this GSI is being run on the specified domain, returns the output product of GSI, otherwise returns None. @param domain the WRFDomain of interestr^N__dict__r]rcr6r]r7r7r8 get_ghostszGSIBase.get_ghostcCs |jd|jjdkr|jSdS)z!Obtain output ghost product for the specified domain. If this GSI is being run on the specified domain, returns the output product of GSI, otherwise returns None. @param domain the WRFDomain of interestr^Nr^r`r7r7r8rBszGSIBase.get_wrfanlcCs|jr|jSdSdS)z!Obtain output wrfinput data for the outermost WRF domain. If this GSI is being run on the WRF outermost domain (Mother Of All Domains, or MOAD), returns the output product of GSI. Otherwise, returns None.N)r]r?rcrr7r7r8 get_wrfinputs zGSIBase.get_wrfinputcCs|jS)z+!The WRF domain for which GSI is being run.)rIrr7r7r8r]szGSIBase.domain wrf_out_prodccs`|dkr|jdk r<|jVn |D]}||jkr |jVq|ddr>|j|ddddn|jdddddt|dd d kr|jdd |jdd d }nRt|dd dkr|jdd |jdd d }n|jdd |jdd d}|ddsd}nd}d|jkrd|jkr|d}n|d}|j|j }|j }|s4t |j } | sDt |j } | sTt |j| d|dd| d|d|| d|d|d} | dD]f} | } | dkr|jdd| dkr|jdd| d kr|jdd| d!kr|jddq|jd"d#|d$d}|j|d%|jrJ|d&|jdd'|jjd(n|d)|j|j|jd*|d+}|d,|j}|j}tj|j|}t|d->}t|d.(}||j|f||d|d/|W5QRXW5QRXd0S)1za!Creates the GSI namelist in the specified file. @param filename the destination filenameuse_gfs_stratosphereTrFr&)ZUSE_GFS_STRATOSPHEREZ USE_GFS_OZONEZREGIONAL_OZONEZNDATGFSVERPROD)ZUSE_GFS_NEMSIO)Z USE_GFS_NCIOii>rzupd_pred(1)=0,upd_pred(2)=0,upd_pred(3)=0, upd_pred(4)=0,upd_pred(5)=0,upd_pred(6)=0, upd_pred(7)=0,upd_pred(8)=0,upd_pred(9)=0, upd_pred(10)=0,upd_pred(11)=0,upd_pred(12)=0,anewpc4pred=.true., adp_anglebc=.true., angord=4, passive_bc=.false., use_edges=.false., emiss_bc=.true., diag_precon=.true., step_start=1.e-3, upd_pred(1)=0, upd_pred(2)=0,upd_pred(3)=0,upd_pred(4)=0, upd_pred(5)=0,upd_pred(6)=0,upd_pred(7)=0, upd_pred(8)=0,upd_pred(9)=0,upd_pred(10)=0, upd_pred(11)=0,upd_pred(12)=0,r-r,zomiter=0,niter=1,lread_obs_save=.true., lread_obs_skip=.false.,reduce_diag=.true.zomiter=0,niter=1,lread_obs_save=.false., lread_obs_skip=.true.,reduce_diag=.true.r)ZNLATZNLONZNETCDFZLEVSZNLON_ENS_REGIONALSETUPZNLAT_ENS_REGIONALZJCAP_GFSrrtdr_new_obstype tldplrbufr)Ztdrtypetdr_old_obstyper g4tdr_obstypehiwrap_obstypeZgps_bnd)Z gps_dtype singleobstest)Z ONEOBSTESTzUsing regional ensemble.)ZMERGE_TWO_GRID_ENSPERTSZREGIONAL_ENSEMBLE_OPTIONENSEMBLE_SIZE_REGIONALzUsing global ensemble.)r(rwnml_filer'rrrsource raise_allr}N)rrrupdater=r0rr$rKrInxr[nynzrrUr rEnmembersrXrVrzr}rqnamelistNamelistInserterrQrr r)r6r rinvarsriZjcap_gfsrorr}r~rrrrrurxr'r}ninfofr7r7r8make_gsi_namelists                               zGSIBase.make_gsi_namelistcCsdS)z!Called by run() after the gsi executable completes. This is intended to be overridden by subclasses to perform some action after gsi is complete, but before products are delivered. The default implementation does nothing.Nr7rr7r7r8 after_gsijszGSIBase.after_gsicCsdS)a!Called by run() just before running the gsi program. This is intended to be overridden by subclasses to perform some action after all inputs needed for gsi are available, but before gsi starts. The default implementation does nothing.Nr7rr7r7r8 before_gsiqszGSIBase.before_gsicCsdS)z!Called by run() to obtain additional inputs before before_gsi() This is intended to be overridden by subclasses to copy or link more inputs for GSI. The default implementation does nothing.Nr7rr7r7r8grab_more_inputsxszGSIBase.grab_more_inputscspjfdd}ddrL|jjd|jjdn |jjd|jjddS) z:!Copies or links bias correction and satellite angle filesc sXd|||dfjj||d}d||ft||dddS)Nz1%s: search for this at dataset=%s item=%s time=%sz%Y%m%d%H)r}rrz%s: found at %sTrP)r strftimerorr )rrfnrrrxr6r7r8rs z%GSIBase.grab_bias_satang..getrTZ satbias_inZ satbias_pcZ satbias_angleN)rrxrrjrkrlrmrn)r6rr7rr8grab_bias_satangs zGSIBase.grab_bias_satangc Cs|}t|d}tjdd}t|}|dkr@t||d}t|ddj d|d }|dkrjt||d}|d k}|d k}| t || d t |jf|d d}d}d}||krvz>|d7}t|dt|dt||dW5QRXWqvWqtk rr}zX||kr<|jdt |jt |fddn&|j dt |jt |||fddW5d}~XYqXq| dt |jfdS)z !Runs the actual GSI executable.gsiZ GSI_THREADS1r)threadsTallranks128M) OMP_STACKSIZEOMP_NUM_THREADSrhstdoutzStarting GSI for %s domain sleeptime>@rrrzGSI failed for %s domain: %sexc_infoz8GSI failed for %s domain: %s, will retry %d more time(s)NzGSI succeeded for %s domain)rrrrNenvironrr=rr envrrwrHr/r]rrrr ExceptionrG)r6rcmdrrZntriesmaxtrieser7r7r8 run_gsi_exesJ        zGSIBase.run_gsi_execCs^tj|jj}|}tjj||d|jj dd|dtjj dtj |j dd|ddS)z!Delivers output products. This function is called by run() to deliver output files to the intercom or com directory and record in the database that they are delivered.rrTFfrominfor'rZ satbias_outr'rN) rNrOdirnamercr.rr_rmakedirsdeliverr rPrb)r6drr7r7r8deliver_productss zGSIBase.deliver_productsc Cs|dks tt|tsttj|s*t|}|dtj ||P}| |j ||g| |j ||gtD]}| |j|||gqt|W5QRXdS)av!Creates GSI diagnostic files. Makes some diagnostic files and copies them to the specified delivery location. Part of this routine is threaded: specify the number of worker threads in nthreads. Minimum is 1. @param tgtpre the prefix to the output names, including the full path @param nthreads the maximum number of threads to userzmake diag *.gz filesN)r[rpr/rNrOisabsrr r_r2rDr0_make_stdout_anl_make_ensemble_spreadr_make_diag_forrF)r6tgtprenthreadsrZworkersdtyper7r7r8make_diag_filess  zGSIBase.make_diag_filesc Cs|dtdd|}|dtdd}t||W5QRXtdD]>}t|}|d|ft|d}t||W5QRXqLW5QRX|d }tjj d|d |d d S) ak! Concatenates many "stdout" files into one. Do not call this function directly; it is part of the implementation of make_diag_files. It concatenates "stdout" and fort.2* to a single log file. @param tgtpre the prefix to the output names, including the full path @param logger a logging.Logger to use for logging messagesz-stdout.anl: make this from stdout and fort.2*z stdout.anlwbzstdout.anl: stdoutrrbzfort.2*zstdout.anl: %sz .stdout.anlFrN) r rshutil copyfileobjglobiglobr/r_rr )r6rroutfinfinfile stdoutanlr7r7r8rs"     "zGSIBase._make_stdout_anlcCsV|d|d}|d}tdr6tjjd|d|dtdrRtjjd|d|ddS) zDo not call this function directly; it is part of the implementation of make_diag_files. It delivers ensemble spread binary and ctl files to com directory.zdeliver ensemble spread filesz.ens_spread.ctlz.ens_spread.grdzens_spread.grdFrzens_spread.ctlN)r r r_rr )r6rrZctlZgrdr7r7r8rs" zGSIBase._make_ensemble_spreadc Cs*dD]}|}|dkrd}|dkr&d}d||f}tt|}|sV|d|fqd||f}|d |} |d |t||fd } d } zjt|d } |D]T} |d|| ft| d.} | | }|d ks|dkrq| |qW5QRXqW5| d k r| ~ Xt j j || d|dqd S)a!Generates one diagnostic output file. Do not call this function directly. It is part of the internal implementation of make_diag_files. Each worker thread calls this function for one GSI data type in the GSI_DATA_TYPES array, to create two zlib-compressed (gzipped) copies of various diagnostic files for that data type. There are two such files per datatype: one for the first guess ("ges") and one for the analysis ("anl"). @param tgtpre the prefix to the output names, including the full path @param logger a logging.Logger to use for logging messages @param dtype the string name of this datatype)0103rZgesranlz pe*.%s_%s*zNo %sz diag_%s_%s.gz.z%s: gzip compress %d %sNrz%s: gzip compress %srrFr) sortedrrr rrgziprreadr r_rr )r6rrrZloopstringtheglobglobbedtmpfileoutfile blocksizerrrindatar7r7r8rsH         zGSIBase._make_diag_forc Cs6|}|j}|d|ftj|rD|d|ft|t||j d| | | dds| ||n |d||||||||d|jd}|d d }|d ks|d kr|d n ||||W5QRXt|_d S)a!Runs the GSI and delivers the results. Executes the GSI in a temporary scrub directory, deleting it afterwards if self.scrub is False. Follows this overall pattern: 1. Make temporary area and cd there 2. Copy inputs 3. Run the grab_more_inputs(), which subclasses should override 4. Calls make_gsi_namelist() to generate the namelist 5. Calls before_gsi() which subclasses should override 6. Calls run_gsi_exe() to run the actual GSI program 7. Calls after_gsi() which subclasses should override 8. Calls deliver_products() to copy files to COM 9. Generates diagnostic files. 10. Deletes the temporary directory if self.scrub=FalsezRun gsi in directory %szDelete old data in %s)r'ruFzThis is a single obs test.diagprer% diagthreads Nnoz$GSI diagnostic outputs are disabled.)rworkdirr rNrOr rrmtreerscrub grab_fix_parmrrrrrrrr]rrrrr0rQrRrrrrstate)r6rrrrr7r7r8runEs8        z GSIBase.runc" s|d|j}fdd}|j|j}|j}|s>t|j}|sLtd||ddr|ddr.|dkr|d kr|d }n|d krt | d  d dkr|d}n:t | d  d dkr|d}n d|f|d }n6|dkr|dkr|d}n d|f|d }nT|dkrB|d}n@|d krV|d}n,|dkrj|d}n d|f|d}nT|dkr|d}n@|d kr|d}n,|dkr|d}n d|f|d}|d}|d}|d } |d!} |d"} |d#} |d$} |d%}|d&}|d'}|d(}|d)}|d*}|d+}|d,}|d-}|d.}|d/}|d0}|d1}|d2}|d3}||d4||d5||d6|| d7|| d8|| d9|| d:|| d;||d<||d=||d>||d?||d@||dA||dB||dC||dD||dE||dF||dG||dH||dIt dJdKx}|D]l}|}|rt|dLdLdMkrqt|dL} |dN| dO}!t|!stt|dP|!dQ|dR| dOgdSddTqtW5QRXdUS)VzT!Links or copies to the local directory any fix or parm files needed by GSI.zCopying fix and parm filescstjj||dddSNTrr,)rbrr7r8lnsf~sz#GSIBase.grab_fix_parm..lnsfzgrab_fix_parm: nz=%drTri=z{FIXgsi}/anavinfo_hwrf_L75Krkrlrmz{FIXgsi}/anavinfo_hwrf_L106rnz{FIXgsi}/anavinfo_hwrf_L85zQUnsupported number of vertical levels: %d Using anavinfo_hwrf_L75. Continuing...iz{FIXgsi}/anavinfo_hwrf_L72z{FIXgsi}/anavinfo_hwrf_L60z{FIXgsi}/anavinfo_hwrf_L74+z{FIXgsi}/anavinfo_hwrf_L42zQUnsupported number of vertical levels: %d Using anavinfo_hwrf_L60. Continuing...z{FIXgsi}/anavinfo_hwrf_L60_noozz{FIXgsi}/anavinfo_hwrf_L74_noozz{FIXgsi}/anavinfo_hwrf_L42_noozzVUnsupported number of vertical levels: %d Using anavinfo_hwrf_L60_nooz. Continuing...z{FIXgsi}/nam_glb_berror.f77.gcvzC{FIXcrtm}/EmisCoeff/IR_Water/Big_Endian/Nalli.IRwater.EmisCoeff.binzK{FIXcrtm}/EmisCoeff/IR_Ice/SEcategory/Big_Endian/NPOESS.IRice.EmisCoeff.binzM{FIXcrtm}/EmisCoeff/IR_Land/SEcategory/Big_Endian/NPOESS.IRland.EmisCoeff.binzM{FIXcrtm}/EmisCoeff/IR_Snow/SEcategory/Big_Endian/NPOESS.IRsnow.EmisCoeff.binzM{FIXcrtm}/EmisCoeff/VIS_Ice/SEcategory/Big_Endian/NPOESS.VISice.EmisCoeff.binzO{FIXcrtm}/EmisCoeff/VIS_Land/SEcategory/Big_Endian/NPOESS.VISland.EmisCoeff.binzO{FIXcrtm}/EmisCoeff/VIS_Snow/SEcategory/Big_Endian/NPOESS.VISsnow.EmisCoeff.binzQ{FIXcrtm}/EmisCoeff/VIS_Water/SEcategory/Big_Endian/NPOESS.VISwater.EmisCoeff.binzE{FIXcrtm}/EmisCoeff/MW_Water/Big_Endian/FASTEM6.MWwater.EmisCoeff.binz2{FIXcrtm}/AerosolCoeff/Big_Endian/AerosolCoeff.binz.{FIXcrtm}/CloudCoeff/Big_Endian/CloudCoeff.binz{FIXgsi}/hwrf_satinfo.txtz{FIXgsi}/atms_beamwidth.txtz{FIXgsi}/global_scaninfo.txtz"{FIXgsi}/nam_global_satangbias.txtz{FIXgsi}/nam_global_pcpinfo.txtz{FIXgsi}/global_ozinfo.txtz{FIXgsi}/hwrf_nam_errtable.r3dvz{FIXgsi}/hwrf_convinfo.txtz{FIXgsi}/prepobs_prep.bufrtablez{FIXgsi}/bufrtab.012z ./anavinfoz./berror_statsz./Nalli.IRwater.EmisCoeff.binz./NPOESS.IRice.EmisCoeff.binz./NPOESS.IRsnow.EmisCoeff.binz./NPOESS.IRland.EmisCoeff.binz./NPOESS.VISice.EmisCoeff.binz./NPOESS.VISland.EmisCoeff.binz./NPOESS.VISsnow.EmisCoeff.binz./NPOESS.VISwater.EmisCoeff.binz./FASTEM6.MWwater.EmisCoeff.binz./AerosolCoeff.binz./CloudCoeff.binz ./satinfoz ./scaninfoz ./pcpinfoz./ozinfoz ./convinfoz ./errtablez./atms_beamwidth.txtz./prepobs_prep.bufrtablez./bftab_sstphrsatinforr!z{satsen}.SpcCoeff.bin)satsenz{FIXcrtm}/fix-4-hwrf/{spccoeff})spccoeffz*{FIXcrtm}/fix-4-hwrf/{satsen}.TauCoeff.binrrPN)rr rSrKrIrr[rrr=r0rrrrr r )"r6r3rrrptopanavinfoZberrorZemiscoef_IRwaterZemiscoef_IRiceZemiscoef_IRlandZemiscoef_IRsnowZemiscoef_VISiceZemiscoef_VISlandZemiscoef_VISsnowZemiscoef_VISwaterZemiscoef_MWwaterZaercoefZcldcoefrZ atmsfilterscaninfoZsatanglZpcpinfoozinfoZerrtableconvinfoZ bufrtableZ bftab_sstrr Zsplatrrr7rr8rxs                                               zGSIBase.grab_fix_parm)NNNNN)NN)NN)N)r)N)rT)Nrc)rh)(rv __module__ __qualname____doc__rDpropertyrr}rxrrrrrrrrr!rrr]rarBrbr]rgrrrrrrrrrrrrr __classcell__r7r7rr8r1sZ m   V   " R 2       f0 .3csDeZdZdZdfdd ZddZddZd d Zdd dZZ S)FGATGSIz,!Runs the GSI based on the HWRF FGAT scheme.Nc  s|dk s t|dkrd}t||_dkr0|j|D]}t|tjjs8tq8fdd| D|_ |dk sttt t |j ||||||f| | | d| dS)a!The FGATGSI constructor: @param dstore passed to Datum: the Datastore object for this Task @param conf the conf object for this task (passed to HWRFTask) @param section the conf section for this task (passed to HWRFTask) @param domain the WRFDomain for this GSI. Must have been initialized by a WRFSimulation @param wrf_in_prod the Product for the wrfinput_d01 or ghost_d0* file for that domain @param fgat_in_prods a mapping from analysis time to a product for all of the FGAT times @param sim the hwrf.wrf.WRFSimulation that will be run as the forecast @param cycling_interval Optional: time between HWRF forecast cycles in seconds @param taskname Optional: the taskname for this product in the datastore @param atime the analysis time as a datetime.datetime. Default: conf.cycle @param enkf_domains a list of WRF domains that should be copied from the hwrf.ensda.EnsembleDA. @param ensda a subclass of hwrf.ensda.DAEnsemble that provides regional ensemble forecasts to generate the forecast error covariance @param kwargs ignored; passed to HWRFTaskNi`Tcsg|]\}}|t|fqSr7)r)rtimerr}r7r8rsz$FGATGSI.__init__..r$r}r~r)r[rcycling_intervalrF itervaluesrpr_r`Productr _fgat_in_prodrCrrD)r6ryrQrzr]r{Z fgat_in_prodsr|rr$r}r~rr5rrrr8rDs,     zFGATGSI.__init__ccs4|j}|jD]"\}}ttt||dVq dS)z!Iterates over FGAT forecast hours, relative to the parent analysis time. Iterates over all FGAT forecast hours (as ints) relative to the parent analysis time. For example, 3, 6, and 9 for three-hourly FGAT off of GDAS.rN)rxrr=roundr)r6rxr"rr7r7r8rszFGATGSI.parent_fhrsccs8|j}|jD]&\}}ttt||dddVq dS)z!Iterates over FGAT forecast hours relative to this model. Iterates over all FGAT forecast hours (as ints) relative to this model's analysis time. For example, -3, 0, and 3 for three-hourly FGAT off of GDAS.TnegokrN)r}rr=rr)r6r}r"rr7r7r8 fgat_fhrs'szFGATGSI.fgat_fhrscCs|j}|}|D]f}|tj|d}|jj|j|j|||d}t |sbt d|j|j|fd|f}t ||d|dqdS)ze!Links to the current working directory gdas native spectral output files for all FGAT hours.r)r}rrz7%s %s: required input file is empty or non-existent: %sz gfs_sigf%02dTrPN) rxrrrrrorrdrer rr )r6r}rrrrherer7r7r8r1s&   zFGATGSI.grab_more_inputsrT wrf_inou%dc s|}|j}tt|j}|jD]j\}}t|tjj s>t t t t t||dd|d}||} |dt|jt|f| |<q$d|j<D]$\}} |dt|jt| fqfdd} d d } tjd d D|| | d S)a!Copy WRF analysis or input files to this directory. Copies the WRF analysis or input file to the specified filename. Also copies the wrf input files for other FGAT hours to separate files specified by "others". The "others" must be a format that includes at least one integer argument. @param filename ignored @param others a string format with a %d in it, used to generate output filenamesTrrzadd prod=%s time=%srTzprod %s has name %scs|Srr7rVnamesr7r8r]rz%FGATGSI.copy_wrf_inout..namercWst|j||ddSr+rYrZr7r7r8r3^sz%FGATGSI.copy_wrf_inout..actorcSsg|]}|qSr7r7)rrr7r7r8r`sz*FGATGSI.copy_wrf_inout..N)rr}rrrrrpr_r`rr[r=rrdebugrwrCrJrrEkeys) r6r othersrr}Zfcir"rdtr^r1rr3r7rr8r]Bs,      zFGATGSI.copy_wrf_inout)NNNNN)rTr) rvrrrrDrrrr]rr7r7rr8rs(  rcsteZdZdZdfdd ZddZddZd d Zd d ZdddZ ddZ ddZ ddZ ddZ ddZZS) GSIMeanHxz-!Runs the GSI to get Hx of the ensemble mean.Nc  s:tt|j||||||f||| | d| ||_dS)a!The GSIMeanHx constructor: @param dstore passed to Datum: the Datastore object for this Task @param conf the conf object for this task (passed to HWRFTask) @param section the conf section for this task (passed to HWRFTask) @param domain the WRFDomain for this GSI. Must have been initialized by a WRFSimulation @param wrf_in_prod the Product for the wrfinput_d01 or ghost_d0* file for that domain @param fgat_in_prods a mapping from analysis time to a product for all of the FGAT times (keep for furture extending to 4DEnVar) @param sim the hwrf.wrf.WRFSimulation that will be run as the forecast @param cycling_interval Optional: time between HWRF forecast cycles in seconds @param taskname Optional: the taskname for this product in the datastore @param atime the analysis time as a datetime.datetime. Default: conf.cycle @param enkf_domains a list of WRF domains that should be copied from the hwrf.ensda.EnsembleDA. @param ensda a subclass of hwrf.ensda.DAEnsemble that provides regional ensemble forecasts to generate the forecast error covariance @param kwargs ignored; passed to HWRFTaskrN)rCrrD_make_productsrY) r6ryrQrzr]r{r|r$r}rxr~rr5rr7r8rDfszGSIMeanHx.__init__c Csb|jN}t}tD]<}d|}tjj|j||jd}tj |j ||_ |||<qW5QRX|S)/!Generates FileProduct objects for all outputs.diag_%s_ges.ensmeanrcategory) ryr\rrr_r`rar$rNrOrPrbr.r6rprodsrpnamerr7r7r8rs zGSIMeanHx._make_productscCs||jkr|jSdSdSzZ!Obtain ensemble mean of specified domain. @param domain the WRFDomain of interestN)r]rcr`r7r7r8get_wrf_ensmeans zGSIMeanHx.get_wrf_ensmeancCs ||jkrt|jSdSdSr r]iterrYrdr`r7r7r8get_diags zGSIMeanHx.get_diagcCs&d|_||j|_|js"tddS)z!Link or copy ensemble inputs. Links or copies the output of the last ENKF or hwrf.ensda cycle. Calls grab_wrf_enkf() to get hwrf ensemble. the ensembles are used to calculate ensemble mean, not for hybrid DAFz1hwrf ensemble forecast wrfout files are not foundN)rXrrErUrrr7r7r8rszGSIMeanHx.grab_enkf_inputwrf-ensemble.inputc Cs|}t}|jddddddd|d}|d|j}|j}tj|j |}t |d >}t |d (} | |j |f||d|d |W5QRXW5QRXd S) zk!Creates the hwrf_ensemble namelist in the specified file. @param filename the destination filenameTFr4rTzNOT USED)IS_ENSEMBLE_MEANIS_ENSEMBLE_RECENTERENSEMBLE_MEMBER_FILENAME_LISTENSEMBLE_MEAN_FILENAMEENSEMBLE_RECENTER_FILENAME!ENSEMBLE_RECENTER_INTERP_FILENAMEensemble_nml_fileensemble_nml_sectionrrryN) rrr|r0rzr}rqrrrQrr r) r6r rrrxr'r}rrrr7r7r8make_ensemble_namelists*   z GSIMeanHx.make_ensemble_namelistc Cs|}tt|ddddk}|dk}|t||d||dd}z&t |d t ||d W5QRXWn.t k r}z| d W5d }~XYnX|d d S)ze!Run hwrf_ensemble to generate wrf ensemble mean and copy wrf ensemble mean to wrf_inout hwrf_ensembleTrrstdout_hwrf_ensemble_meanzStarting hwrf_ensemblerrrrzhwrf_ensemble failedNzhwrf_ensemble succeeded) rr rrrrwrHrrrrrrG)r6rrrrr7r7r8rs     zGSIMeanHx.before_gsicCsdSz"Override super class, does nothingNr7r6rrr7r7r8rszGSIMeanHx._make_stdout_anlcCsdSrr7rr7r7r8rszGSIMeanHx._make_ensemble_spreadc Cs|}d|}tt|}|s4|d|fnd|}|d|t||fd}d}zht|d}|D]T} |d|| ft| d .} | |} | dks| d krq| | qW5QRXqlW5|dk r|~XdS) a!Generates one diagnostic output file for ensemble mean. Do not call this function directly. It is part of the internal implementation of make_diag_files. Each worker thread calls this function for one GSI data type in the GSI_DATA_TYPES array, to create one copy of various diagnostic files for that data type. There is one such file per datatype for the ensemble mean first guess ("ges"). @param tgtpre the prefix to the output names, including the full path @param logger a logging.Logger to use for logging messages @param dtype the string name of this datatype pe*.%s_01*GSI failed to generate %sr%s: read from %d %srNr%s: read from %srr) rrrrr rrrrr  r6rrrrrrrrrrrr7r7r8rs.     zGSIMeanHx._make_diag_forc Cs|}tj|j|D]}|j}tj |}d|krb| d|j df|j dd|dqtj |r| d|j |f|j |d|dq| d|j |fqd}t|}|D]<}|d||jftj|j|}tjj||d|d qd S) !Delivers products.Zwrf_out%s: deliver product from ./%srTFr)%s: ./%s does not exist. Cannot deliver. obs_input.*zdeliver %s to %srN)rr_rrrbrgr.rNrOrrrCrr rr rPr ) r6rr"locblocrrrrr7r7r8rs2    zGSIMeanHx.deliver_products)NNNNN)r)rvrrrrDrr r rrrrrrrrr7r7rr8rds"  (rcsteZdZdZdfdd ZddZddZdd d Zd d ZddZ ddZ ddZ ddZ ddZ ddZZS)GSIHxz'!Runs the GSI to get Hx of the ensembleNc  sB||_tt|j|||||||fd| i| d|_||_dS)a!The GSIHx constructor: @param dstore passed to Datum: the Datastore object for this Task @param conf the conf object for this task (passed to HWRFTask) @param section the conf section for this task (passed to HWRFTask) @param domain the WRFDomain for this GSI. Must have been initialized by a WRFSimulation @param wrf_in_prod the Product of a memeber of the ensemble forecast @param fgat_in_prods a mapping from analysis time to a product for all of the FGAT times (keep for furture extending to 4DEnVar) @param sim the hwrf.wrf.WRFSimulation that will be run as the forecast @param member the member id of the ensemble forecast @param cycling_interval Optional: time between HWRF forecast cycles in seconds @param taskname Optional: the taskname for this product in the datastore @param atime the analysis time as a datetime.datetime. Default: conf.cycle @param kwargs ignored; passed to HWRFTaskr}N)rKrCr'rDrcrrY) r6ryrQrzr]r{r|rKr$r}r5rr7r8rDszGSIHx.__init__c Csh|jT}t}tD]B}d||jf}tjj|j||jd}t j |j ||_ |||<qW5QRX|S)rdiag_%s_ges.mem%03dr)ryr\rrrKr_r`rar$rNrOrPrbr.rr7r7r8r:s zGSIHx._make_productscCs ||jkrt|jSdSdSr r r`r7r7r8r Fs zGSIHx.get_diagrTcs6|}fdd}dd}tj|jg|||dS)rUcsSrr7rVrXr7r8rSrz#GSIHx.copy_wrf_inout..namercWst|j||dddSr)r r.rZr7r7r8r3Tsz#GSIHx.copy_wrf_inout..actorNr[r\r7rXr8r]Ns  zGSIHx.copy_wrf_inoutcCs d|_dS)zP!Link or copy ensemble inputs. override superclass, does nothing hereFN)rXrr7r7r8rYszGSIHx.grab_enkf_inputcCsR|}tj|dd}ttj|dD]}t|tj|d|dq0dS)z copy output from meanhx r gsi_meanhxr$TrN)rrNrOrPrRrr r)r6rrrr7r7r8copy_meanhx_output^szGSIHx.copy_meanhx_outputcCs |dS)zd!Run hwrf_ensemble to generate wrf ensemble mean and copy wrf ensemble mean to wrf_inoutN)r*rr7r7r8reszGSIHx.before_gsicCsdSrr7rr7r7r8rjszGSIHx._make_stdout_anlcCsdSrr7rr7r7r8rmszGSIHx._make_ensemble_spreadc Csd|}tt|}|s,|d|fnd||jf}|d|t||fd}d}zht|d}|D]T} |d|| ft| d .} | |} | dks| d krq| | qW5QRXqjW5|dk r|~XdS) a!Generates one diagnostic output file for ensemble mean. Do not call this function directly. It is part of the internal implementation of make_diag_files. Each worker thread calls this function for one GSI data type in the GSI_DATA_TYPES array, to create one copy of various diagnostic files for that data type. There is one such file per datatype for the ensemble mean first guess ("ges"). @param tgtpre the prefix to the output names, including the full path @param logger a logging.Logger to use for logging messages @param dtype the string name of this datatyperrr(rrNrrrr) rrrrKr rrrrr r r7r7r8rps,     zGSIHx._make_diag_forcCs|}tj|j|D]\}|j}tj |}tj |rf| d|j |f|j |d|dq| d|j |fqdS)r!r"Trr#N)rr_rrrbrgr.rNrOrr rrCr)r6rr"r%r&r7r7r8rs   zGSIHx.deliver_products)NN)rT)rvrrrrDrr r]rr*rrrrrrr7r7rr8r's  'r'cCsR|dkr|d}dD]6}|d|}tj|d|}tjj|d|dqdS)aS!Delete the gsi status file. Deletes all GSI status files, whose paths are determined from the given config object. If the logger is not specified, the gsistatus subdomain of the conf default logging domain is used. @param conf the hwrf.config.HWRFConfig object @param logger Optional: a logging.Logger for logging.N gsistatusr+ gsistatus2dirr%T)r r) rrrNrOrPrRr_rr)rQronetwogsistat gsistatfiler7r7r8unset_gsistatuss   r2c Cs(|dkr|d}|dd}|dd}|ddd}|dd d}d D]}|d|}tj|d |}|rvd nd } |rd nd } | d kr|rt|s| d|fd } | d kr|rt|s| d|fd } | d| |f| d| |ft |d} | d| | fW5QRXqJdS)aJ!Sets the GSI status files. Set run_gsi_d02=YES (true) or =NO (false) depending on the configuration. If the logger is not specified, the gsistatus subdomain of the conf default logging domain is used. @param conf the hwrf.config.HWRFConfig object @param logger Optional: the logging.Logger for log messages.Nzset gsi statusr.z{com}/{stormlabel}.tdrr)run_gsiconditional_gsid03Fconditional_gsid02r,r%YESNOzHGSI is disabled for d03 because flag file is empty or does not exist: %szHGSI is disabled for d02 because flag file is empty or does not exist: %sz,Setting run_gsi_d02=%s in gsi status file %sz,Setting run_gsi_d03=%s in gsi status file %srzrun_gsi_d02=%s run_gsi_d03=%s ) r strinterprWrrNrOrPrRr r rr ) rQr tdrflagfiler3r4r5r/r0r1Z gsid02flagZ gsid03flagrr7r7r8 set_gsistatussL         r:c Cs|dkr|dkrtd|dkr*|d}|dd}tj|d|}d}|d ||fzt|d }|D]t}| d |d krd }|d|qr| d|d krd}| d|qr| d|d krrd}| d|qrW5QRXWnxt k r0}z|j dt |fd dW5d}~XYn>tk rl} z|j dt | fd dW5d} ~ XYnX|dkr| d||fd}n$|r|d|n| d||S)ax!Checks the gsi status for a specific domain. Checks the first GSI status file, scanning for information about the specified domain. If the file does not exist or cannot be opened or read, then False is returned. Otherwise, the file is scanned for run_gsi_d02=YES/NO or run_gsi_d03=YES/NO (case insensitive). The last of those run_gsi_d02/d03 lines is used: NO=return False, YES=return True. @param conf the hwrf.config.HWRFConfig object with configuration info @param domain either "gsi_d02" or "gsi_d03", the domain of interest @param logger Optional: the logging.Logger for log messagesgsi_d02gsi_d034In get_gsistatus, domain must be gsi_d02 or gsi_d03.Nget gsi statusr)r+r%-%s: scan gsi status file for run_%s=YES or NOrz run_%s=YESrTz gsi status file says: run_%s=YESz run_%s=NOFzgsi status file says: run_%s=NOz run_%s=FAILz!gsi status file says: run_%s=FAILz"Error checking gsi status file: %sr6Unhandled exception while checking gsi status file: %szGCould not scan gsi status file for run_%s=YESor NO. Assuming run_%s=NO.z4run_%s=YES: gsi status file says gsi init succeeded.zDrun_%s=FAIL/NO: gsi status file says gsi init failed or was aborted.) ValueErrorrrrNrOrPrRr rrrEnvironmentErrorrr/r) rQr]rr0r1 gsi_successrr reer7r7r8 get_gsistatussv     rEc Cs|dkr|dkrtd|dkr*|d}|dd}tj|d|}d}|d ||fzht|d T}| }| d |td d (}|D]} | | q| d|W5QRXW5QRXWnxt k r} z|j dt| fddW5d} ~ XYn>tk r@} z|j dt| fddW5d} ~ XYnXtd |d|d|dd} tj|d| } td | d|ddS)a7!Reset the gsi status for a specific domain. Reset run_gsi_d02 or run_gsi_d03, if GSI job fails @param conf the hwrf.config.HWRFConfig object with configuration info @param domain either "gsi_d02" or "gsi_d03", the domain of interest @param logger Optional: the logging.Logger for log messagesr;r<r=Nr>r)r+r%r?rz run_%s=YES Z tmpstatusrz run_%s=FAIL zError reset gsi status file: %sTrr@rr-F)rArrrNrOrPrRr r readlinesremover rBrr/rr )rQr]rr0r1rCrlinesnew_fr rrDZgsistat2Z gsistatfile2r7r7r8reset_gsistatussL     & rJ)N)N)N)N)3r__all__rNrrr subprocessprodutil.fileopr_produtil.datastore produtil.cd produtil.runprodutil.workpool hwrf.hwrftaskrqhwrf.wrf hwrf.numerics hwrf.namelistrprodutil.rusagerrrrrr r r r r rrrrrrrrhwrf.exceptionsrrhwrftaskHWRFTaskrrrr'r2r:rErJr7r7r7r8s (     Jr:  ( ?