U $gSN@sdZdgZddlZddlZddlZddlZddlZddlZ ddl Z ddl Z ddl Z ddl Z ddlZ ddlZddlmZmZmZddl mZddlmZmZmZmZmZmZmZmZddlmZmZmZmZmZddl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&dd lm'Z'Gd dde j(j)Z*dS) zZ!PrepHybrid runs the prep_hybrid program to transform GFS spectral files to the HWRF grid. PrepHybridN) setrlimitrusage getrlimit) NoGeogData) to_datetimeto_datetime_rel to_timedelta to_fractiontimedelta_epsilon TimeMappingpartial_orderingr) deliver_file make_symlinkfortlinkrealcwdwait_for_files)aliasbigexecheckrunopenmpmpirunmpi) FileProductc seZdZdZd!fdd Zd"ddZdd Zd d Zd d ZddZ ddZ ddZ d#ddZ ddZ ddZd$ddZd%ddZd&dd ZZS)'ra!Runs the prep_hybrid program on GFS spectral files. Tracks a sequence of GFS spectral files for a list of times. Runs prep_hybrid on them, one by one. This can be done by calling run(), which runs prep_hybrid for all of them before returning, or runpart() which runs only one before returning. If self.realtime is True, then this class will wait for data to appear.NFc s|dkr |}ttj||||f| dkrDdd|D_n t| _| dkrxdd}tj j || } t jt |dd_| dkrd} | _| dkrd } |dkr҈d d }|d kr| }| _|_fd d }|dd_|dd_|dkr"| }|dkr0|}| _t|_t||_|_| _tj_tj_d_|_ !tj"#|d_$%d_&dS)a!PrepHybrid constructor. @param dstore the produtil.datastore.Datastore database @param conf the hwrf.config.HWRFConfig for config info @param section the section to use in conf @param wrf the hwrf.wrf.WRFSimultion being run @param geogdata the geogrid data product @param atime,ftime analysis and forecast times @param taskname taskname in the dstore @param times prep_hybrid input/output times @param inputs the hwrf.input.DataCatalog with the input data @param in_item,in_dataset Input item and dataset @param in_anl_item alternate input item for analysis time @param one_time If True, only one time is prepped but we lie and say should be used for all times. This is used to trick the HWRF into running with the GDAS 9hr forecast even though a 12hr boundary condition would be needed @param kwargs More keyword arguments are passed to hwrf.hwrftask.HWRFTask.__init__NcSsg|]}|qSr).0xrr7/lfs/h1/ops/prod/packages/hwrf.v13.2.9/ush/hwrf/prep.py 9sz'PrepHybrid.__init__..cataloghwrfdata )defaultdatasetitemanl_itemcs8|krt|S|d}|dk r4|dkr4|S|S)Nr&)strconfstr)whatr"skwargsselfrrgetcMs   z!PrepHybrid.__init__..getc enkf_datasetenkf enkf_itemenkf_sfgnamelist)'superr__init__bdytimestimessortedr(hwrfinput DataCatalogconfsimstartr r bdystep_epsilon in_datasetin_item in_anl_item _enkf_dataset _enkf_itemone_timerin_atimerin_ftimegeogdatainputsr bdyprodlogprodinitprodwrf make_productsr3 Conf2Namelistnl init_namelistZ_prep)r-dstorer<sectionrMrHatimeftimetasknamer7rIrAr@rErBr,hdr. __class__r+rr5sX             zPrepHybrid.__init__c csX|j}|jD]F}tj||dr(|j}n|j}t|j|j ||||dV|j r qTq dS)z9!Iterates over the list of data needed to run this class.)r#r$rUrToptionalN) rFr7r9numericswithin_dt_epsilonrBrAdicttaskvarsr@rE)r-r[rTtrArrr inputiters  zPrepHybrid.inputiterc Cs|}t|j|j}|jr"|}n t||}|j}tj||dr\|d||f|j }n|d||f|j }|dt |j t || d| dt|jf|jj|j |f||jd|jS)z!Finds the input needed for the specified time. @param when the time of interest @returns the spectral file at that time.rZzWithin dt epsilon: %s %szNOT within dt epsilon: %s %sz4Check for dataset=%s item=%s ftime=%s atime=%s in %sz%Y%m%d%H)rUrT)logrrGrFrEr9r\r]inforBrAr'r@strftimereprrIlocater_)r-whenloggerrUrTrArrrinput_ats2 zPrepHybrid.input_atcCs |jdS)za!Guesses the WRF I/O form that was used to run geogrid @returns the io_form for auxinput2 auxinput2)rM io_form_forr-rrrio_form_geogridszPrepHybrid.io_form_geogridcCsl|jj}|jjj}|jjj}|ddd|dd|dd|ddt|dd|dd t|d d d S) zS!Constructs the prep_hybrid namelist in the Conf2Namelist object in self.nlprmfldZntimesdomainZlevelsdomains eta_levelsptsgmp_top_requestedptopgY@N)rPnl_set_if_unsetrMnl_get trait_getfloat)r-siuZwgZwtgrrrrQs   zPrepHybrid.init_namelistc Csd}|j}|j}|j}t|d|tj|dd|_tt|j D]`}|j |}|dk sZt t|d|f|tj|d|fd|j |<tj|d|f|j |<q@dS)z!Creates products objects for later file delivery. Creates the produtil.datastore.FileProduct objects for delivery of prep_hybrid output.TZ hwrfinit_0)locationNz hwrfbcs_%dz prep_%d.log) outdirrRrospathjoinrLrangelenr7AssertionErrorrJrK)r-firstr|dscatitimetimerrrrNs    zPrepHybrid.make_productscCs|jS)zE!Return the FileProduct for the prep_hybrid initial time output file.)rLrlrrrprep_init_fileszPrepHybrid.prep_init_fileccs|jD] }|Vq dS)zI!Iterates over the FileProduct for the prep_hybrid boundary output files.N)rJvalues)r-prodrrrprep_bdy_filesszPrepHybrid.prep_bdy_filesccs|dks|dkrDttt||j}|dks<||jkrD|jV|dksT|dkr|j}|dkr||D]\}}|Vqjn | ||j}||kr||VdS)a !Iterates over all output products. Iterates over all products meeting the requirements. @param time the time of interest, or None for all times @param name "init" for initial state, "bdy" for boundary information, or None for all.Ninitbdy) r absr rMr=r?rLrJitemsneartime)r-rnamedtZbdysrrgrrrproductss zPrepHybrid.productscCs|D] }d|_qdS)zP!Marks all products as unavailable, but does not delete delivered files.rN)r available)r-prrrunruns zPrepHybrid.unruncCs|tj|jdS)z0!Deletes all temporary files (the self.workdir).N)rmtreer}r~rworkdirrlrrrcleanszPrepHybrid.cleanTcCs||}|d|jf|jD]"}|d|d||fq tt|jD]}|j ||dqR|d|jfdS)z!Preps all times, even if they have already been prepped. @param keep if True, copy output files instead of moving them to the destination.z%s: prep is startingztime %s: prep %s %Y%m%d-%H%M%Skeepz%s: prep is completedN) rbpostmsgrVr7rcrdrirr run_ipiece)r-rrhnowipiecerrrruns  zPrepHybrid.runc Cs|}tt|jD]}|j|}|j|jr@|dkr|jjs|d|d|fz|j ||dWdSt k r}z(|j d|d|t |fddW5d}~XYqXq|d |d|fqdS) z!Preps one time that has not yet been prepped, and returns. @param keep if True, copy output files instead of moving them to the destination.rz.time %s (piece %d): not done; process this onerrNztime %s (piece %d): %sT)exc_infoz$time %s (piece %d): already complete) rbrrr7rJrrLrcrdr Exceptionwarningr')r-rrhrrerrrrunpart s4    zPrepHybrid.runpartc# Cs|}|jdd}|jdd}|j|}||}t|tsFt|jrt ||| dd| dd| dt d | d d d d d d sd}| |t j||d} |jj} |jjstdt|jjt|jjf|j} |jd||jd} tjj| ||d|d| tt |jjd|ddt!dd} | "d||fW5QRXt!dd} | "d|fW5QRXd|f}t#||ddd|dd|d krd!|f}t#d"|i|ddt!d#d$} | "|j$j%t&d$d%d&gd'W5QRX| d(d)}| d*d+}|'d,}| d-d.}| d/d.}|j(d d0d1krd2}d2}t |)d3*d4d5krxd}d2}t |)d3*d4d6kr d2}d}nn|d ks|d.kr|d7krd}d2}n&|d8krd2}d2}n|d0kr d2}d}n|d7krd}n|d8kr d2}|s|r<|j(d d0d1kr4d9}d:}nd;}d<}|j(d d0d1kr|r|d krt!d=d} | "d>+W5QRXt d?d@|ddt |j(dAdB}|dC||,dDdE}|,dFdE}|j-}|d k stt.| dG|}t.dH||}|j/j0|j1|j2||||dI} |dC|tj34| sH|5| dJn"|jdK|||dL}t | dMd|dNt6t7|8dOdP}t9||dQt:;d?t dRdS|dd|dTkrd7}t<|8dUj=|dVdWdX| j$>dYdZ| j$>dYd[| j$>dYd\| j$>dYd]| j$>dYd^||t|t|f d#kd_k}| d`d } t?|daddbt@|dQtA|dQ|d krv|d.krv|dcnl| d.kr|ddtB|}!n|de| ftB|| df}!| dgdh}"|"dhkr|!j=t|"dit9|!|dQW5QRX|d kr tj3Ct:jDE|jFj|jFjG||d2dj|d kr^|d.kr^|jH|jG|jH|j|d.j|ddjn,|jH|jG||d2djtId_|jJ||d2dkW5QRXd S)la!This is the internal helper function that runs the prep_hybrid for self.run and self.runpart. @param ipiece the index of the output time to run @param keep if True, copy output files instead of moving them to the destination.configdomlatdomlonZmax_spectral_waitiZspectral_sleep_timeZmin_spectral_sizegAZmin_spectral_ageNg?)maxwait sleeptimemin_size min_mtime_age min_atime_age min_ctime_age min_fractionz3Some input spectral files do not exist. Giving up.rz;%s product: WPS geogrid data is not yet available. (loc=%s)z{workdir}/{fYMDH})rUr)rrhz%s: prep in directory %sz geogrid.outT)rhforceZdlocwtz%f %f rz%d z../hwrfbcs00_%d)4 ,-rz../hwrfinit_%d5zhwrf_na12_mkbnd.parmZrgridrnrp)section_sorterimaxijmaxid gfsinit_typero gfsbdy_typerZensdaFZGFSVERZPRODiiiiiizcalc_analysis.nmla &setup datapath="." analysis_filename="anl" firstguess_filename="ges" increment_filename="inc" fhr=6 / zfort.11z./inc.06 zimemb=%dZenkf_fhrg@Z enkf_age_hrg @i)rTrhenkfmemrUz: is empty or non-existentz)sfg_{aYMDH}_fhr{fahr:02d}s_mem{imemb:03d})rTrUimembz./ges.06)rrhZ calc_analysis)rhz./anl.06z ./fort.11rZ hwrf_prepZ4G1)ZIO_FORM OMP_STACKSIZEZMKL_NUM_THREADSrqe_wee_sne_vertdxdyzprep.logthreadsg ZA)rhstackignorezNo need to run hwrf_prepzUse automatic thread count.zUse %d threads)r MALLOC_CHECK_i)r)frominforhr)rhr)Krbr<getfloatr7ri isinstancer'rrealtimerconfintinterrorr9 exceptionsNoSpectralDatardrHr{rrdidrerMget_moadconftimestrinterprprodutilcdNamedDirrcrropenwriterrP make_namelistr rmrVr(stripformat conffloatrFrrIrfrCrDfileop isnonemptyrrrgetexerr}unlinkrenvrwrrrrmakedirsr~dirnamerLdeliverrJrrK)#r-rrrhcenlacenlorZprepmemsgstimeZgeolocmoadtherefZbdyfileZinitfilerriofrrZnemsioZnetcdfrfhourZenkf_ageZmy_atimerTrUlocalcmdexrex2rrrrr sv                              $                             zPrepHybrid.run_ipiece) NNNNNNNFN)F)NN)T)T)T)__name__ __module__ __qualname____doc__r5rarirmrQrNrrrrrrrr __classcell__rrrXrrs0w    )+r__all__r}produtil.datastorer produtil.cdprodutil.fileop produtil.prog hwrf.numericsr9 hwrf.hwrftaskhwrf.wrf hwrf.namelisthwrf.exceptions hwrf.inputZprodutil.rusagerrrrrrr r r r r rrrrr produtil.runrrrrrrrhwrftaskHWRFTaskrrrrrs ( (