U ����� gz>����������������������@���s ��d�Z�dgZddlZddlZddlZddlZddlZddlZ ddl Z ddlZ ddlZ ddl Z ddlZ ddlZddlmZmZmZ�ddl mZ�ddlmZmZmZmZmZmZmZmZ�ddlmZmZmZmZmZ�ddl m!Z!m"Z"m#Z#m$Z$�dd lm%Z%�G�d d��de j&j'�Z(dS�)zZ!PrepHybrid runs the prep_hybrid program to transform GFS spectral files to the HWRF grid.� PrepHybrid�����N)� setrlimit�rusage� getrlimit)� NoGeogData)�to_datetime�to_datetime_rel�to_timedelta�to_fraction�timedelta_epsilon�TimeMapping�partial_orderingr���)�deliver_file�make_symlink�fortlink�realcwd�wait_for_files)�alias�bigexe�checkrun�openmp)�FileProductc���������������� �������s����e�Zd�ZdZd!��fdd� Zdd��Zdd ��Zd d��Zdd ��Zdd��Z dd��Z dd��Zd"dd�Zdd��Z dd��Zd#dd�Zd$dd�Zd%dd �Z���ZS�)&r���a���!Runs the prep_hybrid program on GFS spectral files. Tracks a sequence of GFS spectral files for a list of times. Runs prep_hybrid on them, one by one. This can be done by calling run(), which runs prep_hybrid for all of them before returning, or runpart() which runs only one before returning. If self.realtime is True, then this class will wait for data to appear.NFc��������������������s���|dkr|}t�t|��j||||f|��| dkrDdd��|���D��|�_n t| �|�_| dkrx|��dd�}tj� |�j ||����} t|�jt |����d�d�|�_|dkr�|��d�}||�_|dkr�|��d �}|dkr�|��d d�}|dkr�|}||�_||�_|dkr�|���}|dk�r|}| |�_t|�|�_t||�|�_||�_| |�_t|�j�|�_t|�j�|�_d|�_||�_|�����tj �!||��d��|�_"|��#���d|�_$dS�) a���!PrepHybrid constructor. @param dstore the produtil.datastore.Datastore database @param conf the hwrf.config.HWRFConfig for config info @param section the section to use in conf @param wrf the hwrf.wrf.WRFSimultion being run @param geogdata the geogrid data product @param atime,ftime analysis and forecast times @param taskname taskname in the dstore @param times prep_hybrid input/output times @param inputs the hwrf.input.DataCatalog with the input data @param in_item,in_dataset Input item and dataset @param in_anl_item alternate input item for analysis time @param one_time If True, only one time is prepped but we lie and say should be used for all times. This is used to trick the HWRF into running with the GDAS 9hr forecast even though a 12hr boundary condition would be needed @param kwargs More keyword arguments are passed to hwrf.hwrftask.HWRFTask.__init__Nc�����������������S���s���g�|�]}|�qS���r���)�.0�xr���r����6/lfs/h1/ops/prod/packages/hmon.v3.2.7/ush/hwrf/prep.py� <listcomp>:���s�����z'PrepHybrid.__init__.<locals>.<listcomp>�catalog�hwrfdata� ���)�default�dataset�itemZanl_item���namelist)%�superr����__init__�bdytimes�times�sorted�confstr�hwrf�input�DataCatalog�conf�simstartr���r ����bdystep�_epsilon� in_dataset�in_item�in_anl_item�one_timer����in_atimer����in_ftime�geogdata�inputsr����bdyprod�logprod�initprod�wrf� make_productsr$���� Conf2Namelist�nl� init_namelistZ_prep)�self�dstorer.����sectionr=���r8����atime�ftime�tasknamer(���r9���r3���r2���r5���r4����kwargs�hd�� __class__r���r���r&��� ���sR����� ��� � � � zPrepHybrid.__init__c�����������������c���sV���|�j�}|�jD�]D}tj�||d�r(|�j}n|�j}t|�j|�j |||d�V��|�j r�qRqdS�)z9!Iterates over the list of data needed to run this class.����)r!���r"���rF���rE���N)r6���r(���r+����numerics�within_dt_epsilonr4���r3����dict�taskvarsr2���r5���)rB���rE����tr3���r���r���r���� inputiter����s���� ����zPrepHybrid.inputiterc�������������� ���C���s����|�����}t|�j|�j�}|�jr"|}n t||�}|�j}tj�||d�r\|�d||f���|�j }n|�d||f���|�j }|������dt|�j�t|�|� d�|� d�t|�j�f���|�jj|�j|f||�jd�|�j��S�)z�!Finds the input needed for the specified time. @param when the time of interest @returns the spectral file at that time.rL���zWithin dt epsilon: %s %szNOT within dt epsilon: %s %sz4Check for dataset=%s item=%s ftime=%s atime=%s in %sz%Y%m%d%H)rF���rE���)�logr���r7���r6���r5���r+���rM���rN����infor4���r3����strr2����strftime�reprr9����locaterP���)rB����when�loggerrF���rE���r3���r���r���r����input_at����s2���� ������zPrepHybrid.input_atc�����������������C���s���|�j��d�S�)za!Guesses the WRF I/O form that was used to run geogrid @returns the io_form for auxinput2� auxinput2)r=����io_form_for�rB���r���r���r����io_form_geogrid����s����zPrepHybrid.io_form_geogridc�����������������C���sl���|�j�j}|�jj�j}|�jj�j}|ddd��|dd|dd���|ddt|dd����|dd t|d ��d���dS�) zS!Constructs the prep_hybrid namelist in the Conf2Namelist object in self.nl�prmfldZntimes�����domainZlevels�domains� eta_levels�ptsgm�p_top_requested�ptopg������Y@N)r@����nl_set_if_unsetr=����nl_get� trait_get�float)rB����siuZwgZwtgr���r���r���rA�������s���� zPrepHybrid.init_namelistc�������������� ���C���s����d}|�j�}|�j}|�j�}t|d|tj�|d�d�|�_tt|�j ��D�]`}|�j |�}|dk sZt �t|d|f�|tj�|d|f��d�|�j|<�tj�|d|f��|�j|<�q@dS�)z�!Creates products objects for later file delivery. Creates the produtil.datastore.FileProduct objects for delivery of prep_hybrid output.TZ hwrfinit_0)�locationNz hwrfbcs_%dzprep_%d.log) �outdirrC���r����os�path�joinr<����range�lenr(����AssertionErrorr:���r;���)rB����firstrn����ds�cat�itime�timer���r���r���r>�������s����� �zPrepHybrid.make_productsc�����������������C���s���|�j�S�)zE!Return the FileProduct for the prep_hybrid initial time output file.)r<���r^���r���r���r����prep_init_file����s����zPrepHybrid.prep_init_filec�����������������c���s���|�j����D�] }|V��q dS�)zI!Iterates over the FileProduct for the prep_hybrid boundary output files.N)r:����values)rB����prodr���r���r����prep_bdy_files����s����zPrepHybrid.prep_bdy_filesc�����������������c���s����|dks|dkrDt�tt||�j�������}|dks<||�jk�rD|�jV��|dksT|dkr�|�j}|dkr||���D�]\}}|V��qjn |� ||�j�}||kr�||�V��dS�)a ��!Iterates over all output products. Iterates over all products meeting the requirements. @param time the time of interest, or None for all times @param name "init" for initial state, "bdy" for boundary information, or None for all.N�init�bdy) r ����absr ���r=���r/���r1���r<���r:����items�neartime)rB���ry����name�dtZbdysr|���rY���r���r���r����products����s����� �zPrepHybrid.productsc�����������������C���s���|�����D�] }d|_qdS�)zP!Marks all products as unavailable, but does not delete delivered files.r���N)r����� available)rB����pr���r���r����unrun����s����zPrepHybrid.unrunc�����������������C���s���|���tj�|�j���dS�)z0!Deletes all temporary files (the self.workdir).N)�rmtreero���rp���rq����workdirr^���r���r���r����clean����s����zPrepHybrid.cleanTc�����������������C���s|���|�����}|��d|�jf���|�jD�]"}|�d|�d�|��|�f���q tt|�j��D�]}|�j ||d��qR|��d|�jf���dS�)z�!Preps all times, even if they have already been prepped. @param keep if True, copy output files instead of moving them to the destination.z%s: prep is startingztime %s: prep %s� %Y%m%d-%H%M%S��keepz%s: prep is completedN) rS����postmsgrG���r(���rT���rV���r[���rr���rs���� run_ipiece)rB���r����rZ����now�ipiecer���r���r����run����s���� � zPrepHybrid.runc�����������������C���s����|�����}tt|�j��D�]�}|�j|�}|�j|�jr@|dkr�|�jjs�|�d|�d�|f���z|�j ||d��W��dS��t k r��}�z(|jd|�d�|t|�f�dd����W�5�d}~X�Y�q�X�q|������d |�d�|f���qdS�) z�!Preps one time that has not yet been prepped, and returns. @param keep if True, copy output files instead of moving them to the destination.r���z.time %s (piece %d): not done; process this oner����r����Nztime %s (piece %d): %sT)�exc_infoz$time %s (piece %d): already complete) rS���rr���rs���r(���r:���r����r<���rT���rV���r����� Exception�warningrU���)rB���r����rZ���r����r�����er���r���r����runpart����s4���� ������� �zPrepHybrid.runpartc�����������������C���s���|�����}|�j�dd�}|�j�dd�}|�j|�}|��|�}t|t�sFt�|�jr�t |||�� dd�|�� dd�|�� dtd ��|�� d d�ddd d� s�d}|�|��t j�|��|�d�} |�jj} |�jjs�tdt|�jj�t|�jj�f���|�j���}|�jd||�jd�}tjj|||d�����|�d| t����t |�jjd|dd��t!dd��} | �"d||f���W�5�Q�R�X�t!dd��} | �"d|f���W�5�Q�R�X�d|f�}t#||ddd�|dd��|d k�r�d!|f�}t#d"|i|dd��t!d#d��$} | �"|�j$j%t&d$d%d&g�d'���W�5�Q�R�X�|�� d(d)�}|�� d*d+�}|��'��d,�}|d-k�r0d.}t(|��)d/��j*|d0d1d2�|j$�+d3d4�|j$�+d3d5�|j$�+d3d6�|j$�+d3d7�|j$�+d3d8�||f�d#k�d9k}|�� d:d �}t,|d;dd<��t-|d=��t.|d=��r�|d>k��r�|�d?��t/|�}n|�d@|f���t/||dA�}|�� dBdC�}|dCk�r&|j*t|�dD��t0||d=��W�5�Q�R�X�|d k�rptj1�2t3j4�5|�j6j���|�j6j7||dEdF��|�j8|�j7||dEdF��t9d9|�j:|�|dEdG��W�5�Q�R�X�dS�)Ha��!This is the internal helper function that runs the prep_hybrid for self.run and self.runpart. @param ipiece the index of the output time to run @param keep if True, copy output files instead of moving them to the destination.�config�domlat�domlonZmax_spectral_waiti��Zspectral_sleep_time����Zmin_spectral_sizeg������AZmin_spectral_age����Ng�������?)�maxwait� sleeptime�min_size� min_mtime_age� min_atime_age� min_ctime_age�min_fractionz3Some input spectral files do not exist. Giving up.r����z;%s product: WPS geogrid data is not yet available. (loc=%s)z{workdir}/{fYMDH})rF���r����)r����rZ���z%s: prep in directory %szgeogrid.outT)rZ����forceZdloc�wtz%f %f rx���z%d z../hwrfbcs00_%d)�4��������,����-���r���z../hwrfinit_%d�5���zhwrf_na12_mkbnd.parmZrgridr`���rb���)�section_sorter�imaxi����jmaxi����d���r��������Z hwrf_prep�128M�1)ZIO_FORM� OMP_STACKSIZE�MKL_NUM_THREADSrc����e_we�e_sn�e_vert�dx�dyzprep.log�threadsg����Z�A)rZ����stack�ignore)rZ���ra���zUse automatic thread count.zUse %d threads)r����� MALLOC_CHECK_i���)r����F)�frominforZ���r����)rZ���r����);rS���r.����getfloatr(���r[���� isinstancerU���rt����realtimer����confint�int�errorr+���� exceptions�NoSpectralDatarV���r8���rm���r����r����didrW���r=����get_moad�conftimestrinterpr�����produtil�cd�NamedDirrT���r���r����open�writer���r@���� make_namelistr ���r_���r����getexe�envri���r���r���r���r���r����fileop�makedirsro���rp����dirnamer<����deliverr:���r���r;���)rB���r����r����rZ����cenla�cenlor����Zprepme�msg�stimeZgeoloc�moad�there�fZbdyfileZinitfiler����r�����iof�exr����Zex2r����r���r���r���r������s����� � � � �� �� ���� ��� � ����� � � ���� ����zPrepHybrid.run_ipiece) NNNNNNNFN)NN)T)T)T)�__name__� __module__�__qualname__�__doc__r&���rR���r[���r_���rA���r>���rz���r}���r����r����r����r����r����r����� __classcell__r���r���rJ���r���r������s0������������������k ))r�����__all__ro����produtil.datastorer�����produtil.cd�produtil.fileop� produtil.prog� hwrf.numericsr+���� hwrf.hwrftask�hwrf.wrf� hwrf.namelist�hwrf.exceptions� hwrf.inputZprodutil.rusager���r���r���r���r���r���r ���r ���r���r���r ���r���r���r���r���r����produtil.runr���r���r���r���r����hwrftask�HWRFTaskr���r���r���r���r����<module>���s��� ((