U  gX@sdZdddgZddlZddlZddlZddlZddlZddlZddl Zddl Zddl Zddl Z ddlZ ddlZ ddlZ ddlZddlZddlmZmZmZddlmZddl mZdd lmZmZmZmZdd l mZmZmZmZm Z dd lm!Z!m"Z"m#Z#m$Z$dd l%m&Z&d ddddddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;dd?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcddgXZ'Gdedde j(j)Z*Gdfdde*Z+dmdgdhZ,dndidjZ-dodkdlZ.dS)pz3!Runs the GSI data assimilation on the HWRF system.GSI_DATA_TYPESGSIBaseFGATGSIN) setrlimitrusage getrlimit) COMPLETED)NamedDir) make_symlink deliver_file isnonemptymake_symlinks_in)mpirunmpiopenmpcheckrunbigexe) to_datetimeto_datetime_rel to_fraction to_timedelta) GSIInputErrorZ hirs2_n14Zmsu_n14Zsndr_g08Zsndr_g11Zsndr_g12Zsndr_g13Z sndr_g08_prepZ sndr_g11_prepZ sndr_g12_prepZ sndr_g13_prepZ sndrd1_g11Z sndrd2_g11Z sndrd3_g11Z sndrd4_g11Z sndrd1_g12Z sndrd2_g12Z sndrd3_g12Z sndrd4_g12Z sndrd1_g13Z sndrd2_g13Z sndrd3_g13Z sndrd4_g13Z sndrd1_g14Z sndrd2_g14Z sndrd3_g14Z sndrd4_g14Z sndrd1_g15Z sndrd2_g15Z sndrd3_g15Z sndrd4_g15Z hirs3_n15Z hirs3_n16Z hirs3_n17Z amsua_n15Z amsua_n16Z amsua_n17Z amsub_n15Z amsub_n16Z amsub_n17Zhsb_aquaZ airs_aquaZ amsua_aquaZimgr_g08imgr_g11imgr_g12Zimgr_g14imgr_g15Z pcp_ssmi_dmspZ pcp_tmi_trmmconvZ sbuv2_n16Z sbuv2_n17Z sbuv2_n18Z sbuv2_n19z gome_metop-aZomi_auraZmls_aurassmi_f13ssmi_f14ssmi_f15Z hirs4_n18z hirs4_metop-aZ amsua_n18z amsua_metop-aZmhs_n18z mhs_metop-aZamsre_low_aquaZamsre_mid_aquaZamsre_hig_aqua ssmis_f16 ssmis_f17 ssmis_f18 ssmis_f19 ssmis_f20z iasi_metop-aZ hirs4_n19Z amsua_n19Zmhs_n19Z seviri_m08Z seviri_m09 seviri_m10Zcris_nppZatms_nppz hirs4_metop-bz amsua_metop-bz mhs_metop-bz iasi_metop-bz gome_metop-bcsBeZdZdZdIfdd ZeddZeddZed d Zd d Z d dZ ddZ dJddZ dKddZ dLddZdMddZddZddZdNddZdOd!d"Zd#d$Zd%d&Zd'd(Zed)d*ZdPd,d-ZdQd/d0Zd1d2Zd3d4Zd5d6Zd7d8Zd9d:Zd;d<Zd=d>Z d?d@Z!dAdBZ"dCdDZ#dEdFZ$dGdHZ%Z&S)Rrz@!Base class of anything that runs the GSI. Do not use directly.Nc  sttj|||fd|ij}| _|dkr:|j}| dkrLtd|} t|_t| _ |_ |_ |_ t _tjjdd_d_d_d_| dk r| D]}j|q|dk st|jdk st|jd kst|8} d jj}tj j!|||tjj"|d }|_#W5QRXfd d }|dd_$|dd_%|dd_&|dj&d_'|dd_(|dd_)|dd_*|dd_+|dd_,|dd _-|d!d"_.|d#d$_/d_0d}d%krFd%}t1|t2j3j4r|_0n8t1|t5r |}n&|dkr,nt6d&t7|j8t9|fj0dkr||dkrf:d'}t2j34j|j;_0dS)(a!The GSIBase constructor: @param dstore passed to Datum: the Datastore object for this Task @param conf the conf object for this task (passed to HWRFTask) @param section the conf section for this task (passed to HWRFTask) @param domain the WRFDomain for this GSI. Must have been initialized by a WRFSimulation @param wrf_in_prod the Product for the wrfinput_d01 or ghost_d0* file for that domain @param sim the hwrf.wrf.WRFSimulation that will be run as the forecast @param taskname Optional: the taskname for this product in the datastore @param atime the analysis time as a datetime.datetime. Default: conf.cycle @param parent_atime the analysis time of the parent model. This is relevant if the parent model's forecast is used as the background. @param enkf_domains a list of WRF domains that should be copied from the hwrf.ensda.EnsembleDA. @param ensda a subclass of hwrf.ensda.DAEnsemble that provides regional ensemble forecasts to generate the forecast error covariance @param kwargs ignored; passed to HWRFTasktasknameNicomz{stormlabel}.tdrFPTZgsi_out_)locationcs8|krt|S|d}|dk r4|dkr4|S|S)Nr()strconfstr)whatdefaultskwargsself5/lfs/h1/ops/prod/packages/hmon.v3.2.7/ush/hwrf/gsi.pygetcts   zGSIBase.__init__..getcZ sigma_datasetgdas1Z sigma_itemZgdas1_sfZ bufr_datasetgfsZ bufr_itemZ_bufr prepbufr_itemZ prepbufr_nrZ enkf_datasetZenkfZ enkf_itemZenkf_sfgZbiascr_datasetZ biascr_itemZ gdas1_biascrZbiascr_pc_itemZgdas1_biascr_pcZ abias_itemZ gdas1_abiasZ satang_itemZ gdas1_satangZ in_catalogzIn hwrf.gsi.GSIBase.__init__, in_catalog must be None, a basestring or a DataCatalog. You provided an object of type %s, with value %s.catalog)<superr__init__r%_ensdacyclerr_atime _parent_atime_domain _wrf_in_prod_simlist _enkf_domainsospathjoinconfgetdiricstr _tdrflagfileused_regional_ensembleglobal_ensemble_size hybrid_daappendAssertionErrorr) transactiondomainnameprodutil datastore FileProductoutdir _wrf_out_prod_sigma_dataset _sigma_item _bufr_datasetZ _bufr_item_prepbufr_item _enkf_dataset _enkf_item_biascr_dataset _biascr_item_biascr_pc_item _abias_item _satang_item _in_catalog isinstancehwrfinput DataCatalogr* TypeErrortype__name__reprr+ parent_atime)r1dstorerGsectionrQ wrf_in_prodsimr%atimerl enkf_domainsensdar0tprodnameprodr4 incat_nameZink __class__r/r3r:;s                        zGSIBase.__init__cCs|jjdd}t|S)z!The WRF model top in pascalsdomainsp_top_requested)rAnlnl_getfloat)r1r{r2r2r3 wrf_top_PaszGSIBase.wrf_top_PacCs|jS)z!The analysis time of this GSI.)r=r1r2r2r3rqsz GSIBase.atimecCs|jS)z!Parent model analysis time. The analysis time of the parent data assimilation models (ie.: GDAS, GFS ENKF). This is usually six hours before self.atime.)r>rr2r2r3rlszGSIBase.parent_atimecCs(t|_||_|D]}|j|qdS)a!Sets the hwrf.ensda.DAEnsemble to use, and the enkf domains. Specifies the ensemble to use for forecast error covariances, and the list of hwrf.wrf.WRFDomain domains that should be input to the GSI. @param ensda the hwrf.ensda.DAEnsemble that provides short simulation output, for the forecast error covariances @param enkf_domains an iterable of enkf domains, WRFDomain objects, identifying which domains in ensda should be used.N)rBrCr;rN)r1rsrrdomr2r2r3 set_ensdas zGSIBase.set_ensdac csX|dd}|dkr|dd}|dd}|j}t| d|}td||}t|D](}d |}t|j|j|j|||d VqX|j } | d d rt|j|j |j | | d Vt|j|j |j | | d VnB|jrt|j|j |j| | d V|jr t|j|j |j| | d V|j}t|j|j|j|dV|d} t} | dD]} | } t| dksD| | krlqD|j| d}|j| d}|j| d}|dkr| ddsqD|dkr| ddsҐqD|j| D]0\}}|dkrqt|j||||ddVqސqD|j }|D]0}|tj|d}t|j|j|j||dVq"dS)a!Iterate over needed inputs. Iterates over all files external to this workflow that are required to run the GSI. This may include the GFS ENKF, prepbufr, bufr, and other files. This is used by the hwrf.input module to obtain those inputs.num_enkfr'renkf_fhr@ enkf_age_hr @)datasetitemrqenkfmemftime use_newradbcF)rrrrq)rrrqobstypes,rrri satellitesat_radiance_daTsatwnd sat_wnd_darrri)rrobstyperqoptionalhours)rrrqrN) confint conffloatrqrrangedicttaskvarsr\r]rlconfboolr^r_r`rarbrZr[r+setsplitstriplenrGgetloweritems parent_fhrsdatetime timedeltarXrY)r1maxmembfhourenkf_agemy_atimerqrzmembimembrlolisttouchedrntrimrrotype localnamerhrr2r2r3 inputiters                  zGSIBase.inputitercCsd|_|dd}|dd}|jdd}|dks@||krP||j|_n<|rt|j dkr|jd k rt |j s~|dkr||j|_|js| d S) ac!Link or copy ensemble inputs. Links or copies the output of the last ENKF or hwrf.ensda cycle. Calls grab_wrf_enkf() if any enkf_domains were given to the constructor. Otherwise, calls grab_gfs_enkf() to get the global enkf. Will also revert to grab_gfs_enkf() if the ensda should be used, but was unavailable.Fuse_hwrf_ensemble ensda_whentdr_next_cycleconfig stormlabelalwaysrN) rKrr+rrGr grab_wrf_enkfr;rrCr rJ grab_gfs_enkf)r1rrrr2r2r3grab_enkf_input s  zGSIBase.grab_enkf_inputcCsP|d}t}|dD]0}|}t|dkr||kr|j|||dqdS)a!Link bufr files. Links or copies all needed bufr files to the local directory. If sat_da is False, satellite obs will be omitted. @param atime the analysis time to use when specifying the required bufr files @param morevars additional variables to pass for string replacement when expanding bufr filenames in configuration (hwrf.config.HWRFConfig) sections.rrr)rqmorevarsN)r+rrrrgrab_obstype_section)r1rqrrrZosectionrr2r2r3 grab_bufr#s zGSIBase.grab_bufrcs|}t|tst|}|dkr*|j}n t||j}|j|d}|j|d}|j|d}|d||||f|dkr| ddr|d |fn|d |fdS|d kr| d dr|d |fn|d|fdSt }|j |} t dd| Dfdd} | D]"\} } | dkr8q"| | q"|D]\} |d| ||ft| tstdt| jt| f|dkrt|jrtj|dd} |jj|||dd}tj| |}n|jj||||| d}|dks|dkr.d| f}||| ddrt|nxtj|rztj|}| || }|dksht|j rt!|||ddn,d | |f}||| ddrJt|qJdS)!a!Copies or links observations. @param section the obstype section to read @param atime the atime for string expansion when finding bufr files @param morevars more variables for string expansion when finding bufr files Copies or links observations specified in the obstype sections of the configuration file to the current working directory. The section listed in self.section should contain an "obstype" option, whose value is a comma separated list of section names. This method reads every section in that list. For example, @code{.unformatted} [gsi_d02] catalog = {input_catalog} ... obstypes = hdob_obstype,sat_obstypes,tdr_new_obstype ... [sat_obstypes] type=satellite dataset=gfs item=gfs_bufr gsnd1bufr=goesfv amsuabufr=1bamua satwndbufr=satwnd gpsrobufr=gpsro ... [hdob_obstype] ... [tdr_new_obstype] ... @endcode For each section, the option keys are the local directory filenames expected by GSI, while the values are the data type part of the operational filename (ie.: the satwind in gfs.t12z.tm00.satwind.bufr_d). There are a few special keys: dataset - the name of the dataset for hwrf.input purposes item - the name of the item for hwrf.input purposes type - the type of observation: satellite, or anything else. At present, only "satellite" has any special meaning. If the type is "satellite" then the entire section will be skipped if sat_radiance_da=False in this task's config section. If the type is "satwnd" then the entire section will be skipped if sat_wnd_da=False in this task's config section. Once the section is parsed, the files are all linked to this directory. Nrrriz6process obs section %s with dataset=%s item=%s type=%srrTz4%s: satellite radiance DA is enabled. Continuing...z^%s: satellite radiance DA is disabled in hwrf.conf. Not linking bufr files from this section.rrz=%s: assimilation of satellite wind is enabled. Continuing...zg%s: assimilation of satellite wind is disabled in hwrf.conf. Not linking bufr files from this section.cSsg|]\}}||fqSr2r2).0kvr2r2r3 sz0GSIBase.grab_obstype_section..cs|SNr2)frtZotdictr2r3z.GSIBase.grab_obstype_section..rz%Find obstype=%s in dataset=%s item=%szIn gsi.GSIBase.link_bufr, the obstypes parameter must be an iterable container of basestrings. One of the elements was a %s (value %s) instead.tdrintercombufrpreptldplr)rqloggerrr(z/%s: Could not find a location for this obstype.Zrequire_all_bufrsFrforcez1%s: Observation file is empty or non-existent: %s)"logrdr*rqrrGrrwarningrrBrrrNrhrirjrkboolrealtimerDrErFrHrcparselocaterrSfileopr basenamerJr )r1rnrqrrrrrrrnamerrrdsittheremsgbnonr2rr3r4s6                  zGSIBase.grab_obstype_sectioncKs0|dkr|j}n t||j}|}|jj|j|jf||d|}|jddddkrz|jj|j|jf||d|}n>t j | dd}|jj |jf||d|}t j ||}|dks|dkrdt|jt|jf}||t|n(tj|s|d }||t|t|d |d d dS) a0!Links or copies the prepbufr file to the local directory. @param atime the analysis time, or time relative to self.atime. Used for string expansion in the hwrf.config.HWRFConfig. @param kwargs also passed to the hwrf.config.HWRFConfig for string expansionN)rqrrZ prepbufrpreprrr(z5Could not find the prepbufr file (item=%s dataset=%s)z: is non-existent or emptyZprepbufrTr)rqrrrcrrZr[rGgetintrDrErFrHrrkr7rrrSrr errorr )r1rqr0rrrrrr2r2r3 grab_prepbufrsH     zGSIBase.grab_prepbufrtcvitlc CsZ|}|dt|ft|d}||jdW5QRXtj |sVt dS)z!Writes the tcvitals (from self.storminfo) to the specified file. @param filename the file to receive the tcvitalszWriting tcvitals to %swt N) rinforkopenwrite storminfo as_tcvitalsrDrEexistsrO)r1filenamerrr2r2r3 write_vitalss  zGSIBase.write_vitalscsP||dds2dtjjdddrHfdd}|SdSdS) a!Generate the wrfout file converter. Returns the "copier" argument to deliver_file to use to copy the specified file. Will be None, unless the file is HDF5, in which case it will be "ncks -6 source target" to decompress and convert to NetCDF3 style (uncompressed) with 64-bit indexing. If ncks is missing, such a conversion is impossible, so None is returned. @param file the file that is to be copiedncksr(zAncks path not specified in conf [exe] so I will search your $PATHF) raise_missingcs4tjj|dttdd||fdkddS)Nrz-6z-Oz /dev/null)rSr remove_filerr)r.rtxrrr2r3copysz#GSIBase.wrfout_copier..copyN)rgetexerrSrfind_exe)r1filerr2rr3 wrfout_copiers   zGSIBase.wrfout_copierc s|dtt}d}||jD]\}}dt|t|ft|}|jD]}dt|t|t|ft| }|j ddr| rq^q|j j|d}n| rq^n|j||jd}|dkrd t|fq^d t||jfd ||<||q^q.|dk s2tfd d } fddfdddfdd} |dd} |jsd} tjd(} | tjj|| | | d} W5QRXt|}| |krd| |fdS|d| |fdS)zs!Links the WRF ENKF files to this directory. @param ensda the hwrf.ensda.DAEnsemble that provides the fileszin grab_wrf_enkfNzens,member = %s,%szens,member,domain = %s,%s,%srrun_ens_relocation)rQ)rQrqzNo product for domain %s.zdomain %s prod %sz wrf_en%03dcs|Srr2)pl)namemer2r3renamerrz&GSIBase.grab_wrf_enkf..renamercst||d|ddS)NT)rkeepcopier)r )fromfiletofilerrr2r3r sz#GSIBase.grab_wrf_enkf..copycstjj||ddSNrrSrr )rrrr2r3link"sz#GSIBase.grab_wrf_enkf..linkcs|j}|d||ftj|dkr>|d|fd}n|d|fd}|dkrl||gn|||gdS)Nz Link %s to %sHDF5zW%s: file is HDF5. I will assume your GSI was built with support for compressed NetCDF4z%s: file is NetCDF3.)r)rrSr netcdfveradd_work)rnrrr)rrworkpoolr2r3actor'sz$GSIBase.grab_wrf_enkf..actormaxwait)maxtimez.namercWst|j||ddSrr r)rrRrr-r2r2r3r sz%GSIBase.copy_wrf_inout..actorN)rrSrTrr@)r1rrrr r2r.r3copy_wrf_inoutys  zGSIBase.copy_wrf_inoutcCs||jkr|jSdS)z!Obtain output ghost product for the specified domain. If this GSI is being run on the specified domain, returns the output product of GSI, otherwise returns None. @param domain the WRFDomain of interestNrQrWr1rQr2r2r3 get_ghosts zGSIBase.get_ghostcCs||jkr|jSdS)z!Obtain output ghost product for the specified domain. If this GSI is being run on the specified domain, returns the output product of GSI, otherwise returns None. @param domain the WRFDomain of interestNr2r3r2r2r3rs zGSIBase.get_wrfanlcCs|jr|jSdSdS)z!Obtain output wrfinput data for the outermost WRF domain. If this GSI is being run on the WRF outermost domain (Mother Of All Domains, or MOAD), returns the output product of GSI. Otherwise, returns None.N)rQrrWrr2r2r3 get_wrfinputs zGSIBase.get_wrfinputcCs|jS)z+!The WRF domain for which GSI is being run.)r?rr2r2r3rQszGSIBase.domain wrf_out_prodccs6|dkr|jVn |D]}||jkr|jVq2qdS)z!Iterates over all output products of this Task. @param domains the domains of interest @param prodtype ignoredN)rWrQr@)r1rzprodtyperQr2r2r3productss   zGSIBase.products gsiparm.anlc Cs|}t}|dd}|jdkrD|ddrD|j|ddddn|jddddd|d dshd }nd }|j|j}|j}|st|j }|st|j } | st|j|d |d d| d |d ||d d |jdd|jdd|dd} |j| d|j r&| d|jdd|j jdn| d|j|j|jd|d} |d|j} |j} tj|j| }t| d>}t|d(}||j|f|| d| d|W5QRXW5QRXdS)za!Creates the GSI namelist in the specified file. @param filename the destination filenameuse_gfs_stratosphereTg i@rFr')ZUSE_GFS_STRATOSPHEREZ USE_GFS_OZONEZREGIONAL_OZONEZNDATrzupd_pred(1)=0,upd_pred(2)=0,upd_pred(3)=0, upd_pred(4)=0,upd_pred(5)=0,upd_pred(6)=0, upd_pred(7)=0,upd_pred(8)=0,upd_pred(9)=0, upd_pred(10)=0,upd_pred(11)=0,upd_pred(12)=0,anewpc4pred=.true., adp_anglebc=.true., angord=4, passive_bc=.false., use_edges=.false., emiss_bc=.true., diag_precon=.true., step_start=1.e-3, upd_pred(1)=0, upd_pred(2)=0,upd_pred(3)=0,upd_pred(4)=0, upd_pred(5)=0,upd_pred(6)=0,upd_pred(7)=0, upd_pred(8)=0,upd_pred(9)=0,upd_pred(10)=0, upd_pred(11)=0,upd_pred(12)=0,r)ZNLATZNLONZNETCDFZLEVSZNLON_ENS_REGIONALSETUPZNLAT_ENS_REGIONAL tldplrbufr)ZtdrtypeZgps_bnd)Z gps_dtype singleobstest)Z ONEOBSTESTzUsing regional ensemble.)ZMERGE_TWO_GRID_ENSPERTSZREGIONAL_ENSEMBLE_OPTIONENSEMBLE_SIZE_REGIONALzUsing global ensemble.)ZHYBENS_REGIONALr@nml_file nml_sectionrtr)rsource raise_allrqN)rrrrupdaterAr?nxrOnynzrKrr;nmembersrMrLr+rnrqrenamelistNamelistInserterrGrrr)r1rrZinvarsr:r<rrGrHrIr>rArBrqZniZnfZofr2r2r3make_gsi_namelistsn            zGSIBase.make_gsi_namelistcCsdS)z!Called by run() after the gsi executable completes. This is intended to be overridden by subclasses to perform some action after gsi is complete, but before products are delivered. The default implementation does nothing.Nr2rr2r2r3 after_gsiszGSIBase.after_gsicCsdS)a!Called by run() just before running the gsi program. This is intended to be overridden by subclasses to perform some action after all inputs needed for gsi are available, but before gsi starts. The default implementation does nothing.Nr2rr2r2r3 before_gsiszGSIBase.before_gsicCsdS)z!Called by run() to obtain additional inputs before before_gsi() This is intended to be overridden by subclasses to copy or link more inputs for GSI. The default implementation does nothing.Nr2rr2r2r3grab_more_inputsszGSIBase.grab_more_inputscspjfdd}ddrL|jjd|jjdn |jjd|jjddS) z:!Copies or links bias correction and satellite angle filesc sXd|||dfjj||d}d||ft||dddS)Nz1%s: search for this at dataset=%s item=%s time=%sz%Y%m%d%H)rqrrz%s: found at %sTr&)rstrftimercrr )rrfnrrrlr1r2r3rs z%GSIBase.grab_bias_satang..getrFZ satbias_inZ satbias_pcZ satbias_angleN)rrlrr^r_r`rarb)r1rr2rSr3grab_bias_satang s zGSIBase.grab_bias_satangc Cs|}t|d}tjdd}t|}|dkr@t||d}t|ddj dd }|dkrht||d}|d k}|d k}| t || d t |jf|d d}d}d}||krtz>|d7}t|dt|dt||dW5QRXWqtWqtk rp}zX||kr:|jdt |jt |fddn&|j dt |jt |||fddW5d}~XYqXq| dt |jfdS)z !Runs the actual GSI executable.gsiZ GSI_THREADS1r)threadsT)allranks128M) OMP_STACKSIZEr9stdoutzStarting GSI for %s domain sleeptimeg>@rr)r\zGSI failed for %s domain: %sexc_infoz8GSI failed for %s domain: %s, will retry %d more time(s)NzGSI succeeded for %s domain)rrrrDenvironrrrrenvrrkrr*rQrrrr Exceptionr)r1rcmdrWr\Zntriesmaxtrieser2r2r3 run_gsi_exesJ        zGSIBase.run_gsi_execCs^tj|jj}|}tjj||d|jj dd|dtjj dtj |j dd|ddS)z!Delivers output products. This function is called by run() to deliver output files to the intercom or com directory and record in the database that they are delivered.rr+F)frominforrZ satbias_outrrN) rDrEdirnamerWr)rrSrmakedirsdeliverr rFrV)r1drr2r2r3deliver_productsOs zGSIBase.deliver_productsc Cs|dks tt|tsttj|s*t|}|dtj ||P}| |j ||g| |j ||gtD]}| |j|||gqt|W5QRXdS)av!Creates GSI diagnostic files. Makes some diagnostic files and copies them to the specified delivery location. Part of this routine is threaded: specify the number of worker threads in nthreads. Minimum is 1. @param tgtpre the prefix to the output names, including the full path @param nthreads the maximum number of threads to userzmake diag *.gz filesN)rOrdr*rDrEisabsrrrSr rr _make_stdout_anl_make_ensemble_spreadr_make_diag_forr)r1tgtprenthreadsrZworkersdtyper2r2r3make_diag_files^s  zGSIBase.make_diag_filesc Cs|dtdd|}|dtdd}t||W5QRXtdD]>}t|}|d|ft|d}t||W5QRXqLW5QRX|d }tjj d|d |d d S) ak! Concatenates many "stdout" files into one. Do not call this function directly; it is part of the implementation of make_diag_files. It concatenates "stdout" and fort.2* to a single log file. @param tgtpre the prefix to the output names, including the full path @param logger a logging.Logger to use for logging messagesz-stdout.anl: make this from stdout and fort.2*z stdout.anlwbzstdout.anl: stdoutr[rbzfort.2*zstdout.anl: %sz .stdout.anlFrgN) rrshutil copyfileobjglobiglobr*rSrr )r1rqroutfinfinfileZ stdoutanlr2r2r3rn~s"     "zGSIBase._make_stdout_anlcCsN|d|d}|d}tdrJtjjd|d|dtjjd|d|ddS) zDo not call this function directly; it is part of the implementation of make_diag_files. It delivers ensemble spread binary and ctl files to com directory.zdeliver ensemble spread filesz.ens_spread.ctlz.ens_spread.grdzens_spread.ctlFrgzens_spread.grdN)rr rSrr )r1rqrZctlZgrdr2r2r3ros  zGSIBase._make_ensemble_spreadc Cs*dD]}|}|dkrd}|dkr&d}d||f}tt|}|sV|d|fqd||f}|d |} |d |t||fd } d } zjt|d } |D]T} |d|| ft| d.} | | }|d ks|dkrq| |qW5QRXqW5| d k r| ~ Xt j j || d|dqd S)a!Generates one diagnostic output file. Do not call this function directly. It is part of the internal implementation of make_diag_files. Each worker thread calls this function for one GSI data type in the GSI_DATA_TYPES array, to create two zlib-compressed (gzipped) copies of various diagnostic files for that data type. There are two such files per datatype: one for the first guess ("ges") and one for the analysis ("anl"). @param tgtpre the prefix to the output names, including the full path @param logger a logging.Logger to use for logging messages @param dtype the string name of this datatype)0103r~Zgesranlz pe*.%s_%s*zNo %sz diag_%s_%s.gz.z%s: gzip compress %d %siNruz%s: gzip compress %srvr(Frg) sortedryrrrclosegziprreadrrSrr )r1rqrsrZloopstringZtheglobZglobbedZtmpfileoutfile blocksizer{r}r|indatar2r2r3rpsH         zGSIBase._make_diag_forc Cs.|}|j}|d|ftj|rD|d|ft|t||j d| | | dds| ||n |d|||||||||dd}|d d }|d ks|dkr|d n |||W5QRXt|_d S) a!Runs the GSI and delivers the results. Executes the GSI in a temporary scrub directory, deleting it afterwards if self.scrub is False. Follows this overall pattern: 1. Make temporary area and cd there 2. Copy inputs 3. Run the grab_more_inputs(), which subclasses should override 4. Calls make_gsi_namelist() to generate the namelist 5. Calls before_gsi() which subclasses should override 6. Calls run_gsi_exe() to run the actual GSI program 7. Calls after_gsi() which subclasses should override 8. Calls deliver_products() to copy files to COM 9. Generates diagnostic files. 10. Deletes the temporary directory if self.scrub=FalsezRun gsi in directory %szDelete old data in %s)rr>FzThis is a single obs test.diagprer( diagthreads Nz$GSI diagnostic outputs are disabled.)rworkdirrrDrErrwrmtreer scrub grab_fix_parmrrrrrrTrPr1rMrOrerNrlr+rrtrstate)r1rrhrrr2r2r3runs8        z GSIBase.runc s|d|j}fdd}|ddrz|ddrF|d}q|jdd d d sf|jd d d d rp|d }q|d}n2|jdd d d s|jd d d d r|d}n|d}|d}|d}|d}|d}|d}|d} |d} |d} |d} |d} |d}|d}|d}|d}|d}|d }|d!}|d"}|d#}|d$}|d%}|d&}||d'||d(||d)||d*||d+||d,|| d-|| d.|| d/|| d0|| d1||d2||d3||d4||d5||d6||d7||d8||d9||d:||d;||d<td=d>x}|D]l}|}|rH|d?d?d@krpqH|d?}|dA|dB}t|sHt|dC|dD|dE|dBgdFddGqHW5QRXdHS)IzT!Links or copies to the local directory any fix or parm files needed by GSI.zCopying fix and parm filescstjj||dddS)NTrr)abrr2r3lnsf sz#GSIBase.grab_fix_parm..lnsfrTr:z{FIXgsi}/anavinfo_hwrf_L75hwrf_43lev_confF prelaunch)rnZhwrf_other_confz{FIXgsi}/anavinfo_hwrf_L42z{PARMhwrf}/anavinfo_hwrf_L60z{FIXgsi}/anavinfo_hwrf_L42_noozz!{PARMhwrf}/anavinfo_hwrf_L60_noozz{FIXgsi}/nam_glb_berror.f77.gcvzC{FIXcrtm}/EmisCoeff/IR_Water/Big_Endian/Nalli.IRwater.EmisCoeff.binzK{FIXcrtm}/EmisCoeff/IR_Ice/SEcategory/Big_Endian/NPOESS.IRice.EmisCoeff.binzM{FIXcrtm}/EmisCoeff/IR_Land/SEcategory/Big_Endian/NPOESS.IRland.EmisCoeff.binzM{FIXcrtm}/EmisCoeff/IR_Snow/SEcategory/Big_Endian/NPOESS.IRsnow.EmisCoeff.binzM{FIXcrtm}/EmisCoeff/VIS_Ice/SEcategory/Big_Endian/NPOESS.VISice.EmisCoeff.binzO{FIXcrtm}/EmisCoeff/VIS_Land/SEcategory/Big_Endian/NPOESS.VISland.EmisCoeff.binzO{FIXcrtm}/EmisCoeff/VIS_Snow/SEcategory/Big_Endian/NPOESS.VISsnow.EmisCoeff.binzQ{FIXcrtm}/EmisCoeff/VIS_Water/SEcategory/Big_Endian/NPOESS.VISwater.EmisCoeff.binzE{FIXcrtm}/EmisCoeff/MW_Water/Big_Endian/FASTEM6.MWwater.EmisCoeff.binz2{FIXcrtm}/AerosolCoeff/Big_Endian/AerosolCoeff.binz.{FIXcrtm}/CloudCoeff/Big_Endian/CloudCoeff.binz{FIXgsi}/hwrf_satinfo.txtz{FIXgsi}/atms_beamwidth.txtz{FIXgsi}/global_scaninfo.txtz"{FIXgsi}/nam_global_satangbias.txtz{FIXgsi}/nam_global_pcpinfo.txtz{FIXgsi}/global_ozinfo.txtz{FIXgsi}/hwrf_nam_errtable.r3dvz{FIXgsi}/hwrf_convinfo.txtz{FIXgsi}/prepobs_prep.bufrtablez{FIXgsi}/bufrtab.012z ./anavinfoz./berror_statsz./Nalli.IRwater.EmisCoeff.binz./NPOESS.IRice.EmisCoeff.binz./NPOESS.IRsnow.EmisCoeff.binz./NPOESS.IRland.EmisCoeff.binz./NPOESS.VISice.EmisCoeff.binz./NPOESS.VISland.EmisCoeff.binz./NPOESS.VISsnow.EmisCoeff.binz./NPOESS.VISwater.EmisCoeff.binz./FASTEM6.MWwater.EmisCoeff.binz./AerosolCoeff.binz./CloudCoeff.binz ./satinfoz ./scaninfoz ./pcpinfoz./ozinfoz ./convinfoz ./errtablez./atms_beamwidth.txtz./prepobs_prep.bufrtablez./bftab_sstphrsatinforCr!z{satsen}.SpcCoeff.bin)satsenz({FIXcrtm}/SpcCoeff/Big_Endian/{spccoeff})spccoeffz3{FIXcrtm}/TauCoeff/Big_Endian/{satsen}.TauCoeff.binrr&N)rrrIrrrr r )r1r.rZanavinfoZberrorZemiscoef_IRwaterZemiscoef_IRiceZemiscoef_IRlandZemiscoef_IRsnowZemiscoef_VISiceZemiscoef_VISlandZemiscoef_VISsnowZemiscoef_VISwaterZemiscoef_MWwaterZaercoefZcldcoefrZ atmsfilterZscaninfoZsatanglZpcpinfoZozinfoZerrtableZconvinfoZ bufrtableZ bftab_sstrlinesplatrrr2rr3rs                                   zGSIBase.grab_fix_parm)NNNNN)NN)NN)N)r)N)r+)Nr6)r9)'rj __module__ __qualname____doc__r:propertyrrqrlrrrrrrrrrrr1r4rr5rQr8rMrNrOrPrTrerlrtrnrorprr __classcell__r2r2rxr3r/sX e   J   " H 2      D0 .3csDeZdZdZdfdd ZddZddZd d Zdd dZZ S)rz,!Runs the GSI based on the HWRF FGAT scheme.Nc  s|dk s t|dkrd}t||_dkr0|j|D]}t|tjjs8tq8fdd| D|_ |dk sttt t |j ||||||f| | | d| dS)a!The FGATGSI constructor: @param dstore passed to Datum: the Datastore object for this Task @param conf the conf object for this task (passed to HWRFTask) @param section the conf section for this task (passed to HWRFTask) @param domain the WRFDomain for this GSI. Must have been initialized by a WRFSimulation @param wrf_in_prod the Product for the wrfinput_d01 or ghost_d0* file for that domain @param fgat_in_prods a mapping from analysis time to a product for all of the FGAT times @param sim the hwrf.wrf.WRFSimulation that will be run as the forecast @param cycling_interval Optional: time between HWRF forecast cycles in seconds @param taskname Optional: the taskname for this product in the datastore @param atime the analysis time as a datetime.datetime. Default: conf.cycle @param enkf_domains a list of WRF domains that should be copied from the hwrf.ensda.EnsembleDA. @param ensda a subclass of hwrf.ensda.DAEnsemble that provides regional ensemble forecasts to generate the forecast error covariance @param kwargs ignored; passed to HWRFTaskNi`Tcsg|]\}}|t|fqSr2)r)rtimervrqr2r3r}sz$FGATGSI.__init__..)r%rqrrrs)rOrcycling_intervalr<valuesrdrSrTProductr _fgat_in_prodr9rr:)r1rmrGrnrQroZ fgat_in_prodsrprr%rqrrrsr0rvrxrr3r:^s,     zFGATGSI.__init__ccs4|j}|jD]"\}}ttt||dVq dS)z!Iterates over FGAT forecast hours, relative to the parent analysis time. Iterates over all FGAT forecast hours (as ints) relative to the parent analysis time. For example, 3, 6, and 9 for three-hourly FGAT off of GDAS.rN)rlrrroundr)r1rlrrtr2r2r3rszFGATGSI.parent_fhrsccs8|j}|jD]&\}}ttt||dddVq dS)z!Iterates over FGAT forecast hours relative to this model. Iterates over all FGAT forecast hours (as ints) relative to this model's analysis time. For example, -3, 0, and 3 for three-hourly FGAT off of GDAS.TnegokrN)rqrrrr)r1rqrrtr2r2r3 fgat_fhrsszFGATGSI.fgat_fhrscCs|j}|}|D]f}|tj|d}|jj|j|j|||d}t |sbt d|j|j|fd|f}t ||d|dqdS)ze!Links to the current working directory gdas native spectral output files for all FGAT hours.r)rqrrz7%s %s: required input file is empty or non-existent: %sz gfs_sigf%02dTr&N) rlrrrrrcrrXrYr rr )r1rqrrrrherer2r2r3rPs&   zFGATGSI.grab_more_inputsr+ wrf_inou%dc s|}|j}tt|j}|jD]j\}}t|tjj s>t t t t t||dd|d}||} |dt|jt|f| |<q$d|j<D]$\}} |dt|jt| fqfdd} d d } tjd d D|| | d S)a!Copy WRF analysis or input files to this directory. Copies the WRF analysis or input file to the specified filename. Also copies the wrf input files for other FGAT hours to separate files specified by "others". The "others" must be a format that includes at least one integer argument. @param filename ignored @param others a string format with a %d in it, used to generate output filenamesTrrzadd prod=%s time=%sr+zprod %s has name %scs|Srr2r,namesr2r3rrz%FGATGSI.copy_wrf_inout..namercWst|j||ddSrr/r0r2r2r3r sz%FGATGSI.copy_wrf_inout..actorcSsg|]}|qSr2r2)rrr2r2r3rsz*FGATGSI.copy_wrf_inout..N)rrqrrrrrdrSrTrrOrrr~debugrkrr@rrkeys) r1rothersrrqZfcirrtdtrRr rr r2rr3r1s,      zFGATGSI.copy_wrf_inout)NNNNN)r+r) rjrrrr:rrrPr1rr2r2rxr3r\s(  cCsR|dkr|d}dD]6}|d|}tj|d|}tjj|d|dqdS)aS!Delete the gsi status file. Deletes all GSI status files, whose paths are determined from the given config object. If the logger is not specified, the gsistatus subdomain of the conf default logging domain is used. @param conf the hwrf.config.HWRFConfig object @param logger Optional: a logging.Logger for logging.N gsistatusrZ gsistatus2dirr&T)rr) rrrDrErFrHrSrr)rGronetwogsistat gsistatfiler2r2r3unset_gsistatuss   rc Cs(|dkr|d}|dd}|dd}|ddd}|dd d}d D]}|d|}tj|d |}|rvd nd } |rd nd } | d kr|rt|s| d|fd } | d kr|rt|s| d|fd } | d| |f| d| |ft |d} | d| | fW5QRXqJdS)aJ!Sets the GSI status files. Set run_gsi_d02=YES (true) or =NO (false) depending on the configuration. If the logger is not specified, the gsistatus subdomain of the conf default logging domain is used. @param conf the hwrf.config.HWRFConfig object @param logger Optional: the logging.Logger for log messages.Nzset gsi statusrz{com}/{stormlabel}.tdrrrun_gsiconditional_gsid03Fconditional_gsid02rr&YESNOzHGSI is disabled for d03 because flag file is empty or does not exist: %szHGSI is disabled for d02 because flag file is empty or does not exist: %sz,Setting run_gsi_d02=%s in gsi status file %sz,Setting run_gsi_d03=%s in gsi status file %srzrun_gsi_d02=%s run_gsi_d03=%s ) r strinterprrrDrErFrHr rrr) rGr tdrflagfilerrrrrrZ gsid02flagZ gsid03flagrr2r2r3 set_gsistatussL         rc Cs|dkr|dkrtd|dkr*|d}|dd}tj|d|}d}|d ||fznt|d Z}|D]N}| d |d krd }|d|qr| d|d krrd}| d|qrW5QRXWnxt k r }z|j dt |fd dW5d}~XYn>tk rF} z|j dt | fd dW5d} ~ XYnX|dkrj| d||fd}n$|r|d|n| d||S)ax!Checks the gsi status for a specific domain. Checks the first GSI status file, scanning for information about the specified domain. If the file does not exist or cannot be opened or read, then False is returned. Otherwise, the file is scanned for run_gsi_d02=YES/NO or run_gsi_d03=YES/NO (case insensitive). The last of those run_gsi_d02/d03 lines is used: NO=return False, YES=return True. @param conf the hwrf.config.HWRFConfig object with configuration info @param domain either "gsi_d02" or "gsi_d03", the domain of interest @param logger Optional: the logging.Logger for log messagesgsi_d02gsi_d03z4In get_gsistatus, domain must be gsi_d02 or gsi_d03.Nzget gsi statusrrr&z-%s: scan gsi status file for run_%s=YES or NOrCz run_%s=YESrTz gsi status file says: run_%s=YESz run_%s=NOFzgsi status file says: run_%s=NOz"Error checking gsi status file: %sr]z6Unhandled exception while checking gsi status file: %szGCould not scan gsi status file for run_%s=YESor NO. Assuming run_%s=NO.z4run_%s=YES: gsi status file says gsi init succeeded.z?run_%s=NO: gsi status file says gsi init failed or was aborted.) ValueErrorrrrDrErFrHrrfindrEnvironmentErrorrr*ra) rGrQrrrZ gsi_successrrrdeer2r2r3 get_gsistatussl     r)N)N)N)/r__all__rDrwryrprodutil.fileoprSprodutil.datastore produtil.cd produtil.runprodutil.workpool hwrf.hwrftaskrehwrf.wrf hwrf.numerics hwrf.namelistrprodutil.rusagerrrrr r r r r rrrrrrrrrhwrf.exceptionsrrhwrftaskHWRFTaskrrrrrr2r2r2r3s       5t  (