o qg:@sHddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z ddZddZddZd d Zd d Zd dZddZddZddZddZddZddZddZddZdd Zd!d"Zd#d$Zd%d&Zd'd(Z d)d*Z!d+d,Z"d-d.Z#d/d0Z$d1d2Z%d3d4Z&d5d6Z'd7d8Z(d9d:Z)d;d<Z*dS)=N)sleepcstddtfdddDrtjddd}nt}|jdkrAtd d|jd t|jt d d Sd S) zu! Run shell command Args: command - list of agrument entries (string) Returns: z Running  c3s|] }|dvVqdS)rN)join).0ZmarkcommandM/lfs/h1/ops/prod/packages/evs.v1.0.19/ush/global_ens/global_ens_atmos_util.py sz$run_shell_command..)"'|*>T)shellr FATAL ERROR: z gave return code N) printrany subprocessrun returncodeargsstrsysexit)rZ run_commandrrr run_shell_commands    rcCstjtjddd}tjtjddd}tjtjddtjdtjdd tjd tjd |}tj|sGtd |d td|d|d|}|S)z! Write out full call to METplus Args: conf_file_name - METplus conf file name (string) Returns: metplus_cmd - full call to METplus (string) METPLUS_PATHZushzrun_metplus.pyZPARMevsZmetplus_configz machine.conf COMPONENTRUN_ VERIF_CASESTEPr DOES NOT EXISTrz -c ospathrenvironexistsrrr)Zconf_file_nameZ run_metplusZ machine_confZ conf_fileZ metplus_cmdrrr metplus_command$s     r)cCsdtjtjdtjd|}tj|s!td|dtdd|}|D]}|d|}q'|S)a! Write out full call to python Args: python_script_name - python script name (string) script_arg_list - list of script agruments (strings) Returns: python_cmd - full call to python (string) USHevsrrr#rzpython rr$)Zpython_script_nameZscript_arg_listZ python_scriptZ python_cmdZ script_argrrr python_command<s   r+cCsTtj|rtj|dkrd}|Std|dd}|Std|dd}|S)aK! Checks to see if file exists and has size greater than 0 Args: file_name - file path (string) Returns: file_good - boolean - True: file exists,file size >0 - False: file doesn't exist OR file size = 0 rT WARNING: z empty, 0 sizedF does not exist)r%r&r(getsizer) file_nameZ file_goodrrr check_file_exists_sizeQs r0cCs0t|rtd|d|t||dSdS)z! This copies a file from one location to another Args: source_file - source file path (string) dest_file - destination file path (string) Returns: zCopying  to N)r0rshutilcopy2) source_file dest_filerrr copy_filehs r6cCDtd|dd|tjd}t|d|d|ddS)aW! Converts GRIB1 data to GRIB2 Args: grib1_file - string of the path to the GRIB1 file to convert (string) grib2_file - string of the path to save the converted GRIB2 file (string) Returns: zConverting GRIB1 file rto GRIB2 file CNVGRIBz -g12  > /dev/null 2>&1Nrr%r'system) grib1_file grib2_filecnvgribrrr convert_grib1_grib2u   r@cCr7)aE! Converts GRIB2 data to GRIB1 Args: grib2_file - string of the path to the GRIB2 file to convert grib1_file - string of the path to save the converted GRIB1 file Returns: Converting GRIB2 file rzto GRIB1 file r9z -g21 r:Nr;)r>r=r?rrr convert_grib2_grib1rArCcCr7)aK! Converts GRIB2 data to GRIB2 Args: grib2_fileA - string of the path to the GRIB2 file to convert grib2_fileB - string of the path to save the converted GRIB2 file Returns: rBrr8r9z -g22 r:Nr;)Z grib2_fileAZ grib2_fileBr?rrr convert_grib2_grib2rArDcCsxdd|D}dd|D}|dkr|}n|dkr|}|d} |d} t|dkr7tttj|td } nd } tj|| d } tj|| d } g}| }|| kr|dkrZ|}n|dkr`|}|D]I}|d krkd}nt|}|dkr||tj|d }n |dkr|tj|d }| d|vr| d|vri}||d<||d<t ||d<| |qb|tjt| d }|| ksS|S)a! Creates a list of dictionaries containing information on the valid dates and times, the initialization dates and times, and forecast hour pairings Args: date_start - verification start date (string, format:YYYYmmdd) date_end - verification end_date (string, format:YYYYmmdd) date_type - how to treat date_start and date_end (string, values:VALID or INIT) init_hr_list - list of initialization hours (string) valid_hr_list - list of valid hours (string) fhr_list - list of forecasts hours (string) Returns: time_info - list of dictionaries with the valid, initalization, and forecast hour pairings cSg|]}|dqSzfillrhrrrr z!get_time_info..cSrErFrHrJrrr rLrMVALIDINITrrdtype%Y%m%d%Hanlhours%HZ valid_timeZ init_time forecast_hour) lennpmindiffarrayintdatetimestrptime timedeltastrftimerappend)Z date_startZdate_end date_type init_hr_list valid_hr_listZfhr_listZvalid_hr_zfill2_listZinit_hr_zfill2_listZdate_type_hr_listZdate_type_hr_startZdate_type_hr_endZdate_type_hr_incZ date_start_dtZ date_end_dtZ time_infoZdate_dt valid_time_dt init_time_dtfhrrYtrrr get_time_infosd        rlcCsd}gd}tt|dkr|t|}|dD]M}|D]@}|d|d} | dkrcd} | | krc|dvr[|dd d d} |d|dd d d} n|d|dd d d} |d kru|| } n|d kr| dkrt|dkr|d} n|} n| dkr| d } n| dkr| d} n|} n|dkr|| } n|dkr|dd d d} t t|t| }| dkrt|dkr|d} n`|} n]| dkr| d } nS| dkr| d} nI|} nF|dkr|dd d d} |t j t| d}|| } n%|dkr1|dd d d} |t j t| d}|| } n||} |dvrM| d|d| d| d | }n| d|d| d | }| d7} | | ks:q"tj||}q|S)a! Creates a filled file path from a format Args: unfilled_file_format - file naming convention (string) valid_time_dt - valid time (datetime) init_time_dt - initialization time (datetime) forecast_hour - forecast hour (string) str_sub_dict - other strings to substitue (dictionary) Returns: filled_file_format - file_format filled in with verifying time information (string) /)lead lead_shiftvalid valid_shiftinit init_shiftr{z?fmt=r)rorqrszshift=rG}?rprnz%1H z%2Hz%3HrrrorsrVrqz?shift=)rZlistkeyssplitcount partition rpartitionrcr_rIrr`rbreplacer%r&r)Zunfilled_file_formatrhrirYZ str_sub_dictZfilled_file_formatZformat_opt_listZfilled_file_format_chunkZ format_optZ nformat_optZformat_opt_countshiftZformat_opt_count_fmtZreplace_format_opt_countZforecast_hour_shiftZinit_shift_time_dtZvalid_shift_time_dtrrr format_fillers                arc Cs`tjd}tjd}tjtd|dd}|d}|dkrq|dkr*d }n|}gd d d d ddddddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;d<d=d>d?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdW|dXdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdidjdkdldmdndodpdqdrdsdtdudvdwdxdydzd{d|d}d~dddddddddddddddddd} t|rptd|g| D]$} t|dd| d|d|d|d|gtd|d|gt|qKn8d|vrt |ddkrdt t t |d} nd|} t|rt|dd| d|d|d|d|gt ||dS)ai! Do prep work for GFS production files Args: source_file_format - source file format (string) dest_file - destination file (string) forecast_hour - forecast hour (string) prep_method - name of prep method to do (string) Returns: WGRIB2EXECevsatmos.rmrG.tmp1fullrrUz CAPE:surfacezCAPE:90-0 mb above groundz5CWAT:entire atmosphere (considered as a single layer)zDPT:2 m above groundz GUST:surfacez HGT:1000 mbz HGT:925 mbz HGT:850 mbz HGT:700 mbz HGT:500 mbz HGT:400 mbz HGT:300 mbz HGT:250 mbz HGT:200 mbz HGT:150 mbz HGT:100 mbz HGT:50 mbz HGT:20 mbz HGT:10 mbzHGT:5 mbzHGT:1 mbzHGT:cloud ceilingzHGT:tropopausez HPBL:surfacez ICEC:surfacez ICETK:surfacez LHTFL:surfacez O3MR:925 mbz O3MR:100 mbz O3MR:70 mbz O3MR:50 mbz O3MR:30 mbz O3MR:20 mbz O3MR:10 mbz O3MR:5 mbz O3MR:1 mbz PRES:surfacezPRES:tropopausezPRMSL:mean sea levelz5PWAT:entire atmosphere (considered as a single layer)z RH:1000 mbz RH:925 mbz RH:850 mbz RH:700 mbz RH:500 mbz RH:400 mbz RH:300 mbz RH:250 mbz RH:200 mbz RH:150 mbz RH:100 mbzRH:50 mbzRH:20 mbzRH:10 mbzRH:5 mbzRH:1 mbzRH:2 m above groundz SHTFL:surfacez SNOD:surfacez SPFH:1000 mbz SPFH:925 mbz SPFH:850 mbz SPFH:700 mbz SPFH:500 mbz SPFH:400 mbz SPFH:300 mbz SPFH:250 mbz SPFH:200 mbz SPFH:150 mbz SPFH:100 mbz SPFH:50 mbz SPFH:20 mbz SPFH:10 mbz SPFH:5 mbz SPFH:1 mbzSPFH:2 m above groundzSOILW:0-0.1 m below groundzTCDC:entire atmosphere:z TMP:1000 mbz TMP:925 mbz TMP:850 mbz TMP:700 mbz TMP:500 mbz TMP:400 mbz TMP:300 mbz TMP:250 mbz TMP:200 mbz TMP:150 mbz TMP:100 mbz TMP:50 mbz TMP:20 mbz TMP:10 mbzTMP:5 mbzTMP:1 mbzTMP:2 m above groundz TMP:surfacezTMP:tropopausez6TOZNE:entire atmosphere (considered as a single layer)zTSOIL:0-0.1 m below groundz UGRD:1000 mbz UGRD:925 mbz UGRD:850 mbz UGRD:700 mbz UGRD:500 mbz UGRD:400 mbz UGRD:300 mbz UGRD:250 mbz UGRD:200 mbz UGRD:150 mbz UGRD:100 mbz UGRD:50 mbz UGRD:20 mbz UGRD:10 mbz UGRD:5 mbz UGRD:1 mbzUGRD:10 m above groundz VGRD:1000 mbz VGRD:925 mbz VGRD:850 mbz VGRD:700 mbz VGRD:500 mbz VGRD:400 mbz VGRD:300 mbz VGRD:250 mbz VGRD:200 mbz VGRD:150 mbz VGRD:100 mbz VGRD:50 mbz VGRD:20 mbz VGRD:10 mbz VGRD:5 mbz VGRD:1 mbzVGRD:10 m above groundz VIS:surfacez WEASD:surfacerz-matchr r z-iz-gribcatz>>preciprSzAPCP:surface:0-N) r%r'r&rgetcwdr~r0rremover_rr6) r4r5rY prep_methodrr prepped_file working_file1 wgrib_fhrZthin_var_level_listZthin_var_levelrrr prep_prod_gfs_fileos^            !!!!""""####$$$$%%%&'(((())))****++++,----....////0000123 8   rcCs@tjtd|dd}t|rt||t||dS)aR! Do prep work for FNMOC production files Args: source_file - source file format (string) dest_file - destination file (string) forecast_hour - forecast hour (string) prep_method - name of prep method to do (string) Returns: rrmrGN)r%r&rrr~r0rDr6)r4r5rYrrrrr prep_prod_fnmoc_files  rcCsftjd}tjd}tj|d}tjtd|dd}|d}|d} |d kr|d kr4d } nt|d kr=d } nd |d} dD]2} |d| } | dkrT|} n| dkrZ| } t| rwt |d| d| d|d| d| gqEt|rt| rt ||| |gn#d|vr|}t|rt |d|d|d|d|d|gt ||dS)ar! Do prep work for JMA production files Args: source_file_format - source file format (string) dest_file - destination file (string) forecast_hour - forecast hour (string) prep_method - name of prep method to do (string) Returns: WGRIBrZ jma_mergerrmrGr.tmp2rrU:anlr:rK)ns {hem?fmt=str}rrr | grep "" | -i -grib -o rz | grep "0-hr" | N) r%r'r&rrr~r_rr0rr6)source_file_formatr5rYrrrZJMAMERGErr working_file2rhemhem_source_fileZ working_filer4rrr prep_prod_jma_files~       rc CsPtjd}tj|d}tj|d}tjd}tjtd|dd}|d} |d kr}|d kr7d } nt|d kr@d } nd |d} t|rct|d|d| d|d|d| gt| r|tdd| gtdd| gt|| |gnd|vrt|rt|d||gtj |rtdd|gtdd|gt ||dS)aR! Do prep work for ECMWF production files Args: source_file - source file format (string) dest_file - destination file (string) forecast_hour - forecast hour (string) prep_method - name of prep method to do (string) Returns: rZecm_gfs_look_alike_new pcpconformrrrmrGrrrUrrrrKrrrrchmodZ750ZchgrpZrstprodrZecmwfN) r%r'r&rrr~r_r0rr(r6) r4r5rYrrZECMGFSLOOKALIKENEW PCPCONFORMrrrrrrr prep_prod_ecmwf_file$sX       rc Cs>tjd}tjd}tjd}tj|d}tjtd|dd}|d} |d } |d krid d d d ddddddddddddddddddd dd!d"d#d"d$d%d&d'd(d)d*d+d,d-d.d/d0d1d0d2 } |t| vr|d kr| d } d } d }n| |} |} |d krd }n|d3}|d4| }t |rt |d5|d6|d7|d5|d8| gt | rt || || gnOd9|vr|}d:}t |rt |d5|d;d<| gt | rt | | t | rt ||}t |d5| d6t |d=|d>|d5| d8|gt||d?S)@ak! Do prep work for UKMET production files Args: source_file_format - source file format (string) dest_file - destination file (string) forecast_hour - forecast hour (string) prep_method - name of prep method to do (string) Returns: rrrZukm_hires_mergerrmrGrrrrUZAAT06ZBBTZ12ZCCTZ18ZDDTZ24ZEETZ30ZFFTZ36ZGGTZ42ZHHTZ48ZIITZ54ZJJT60Z66ZKKTZ72Z78ZQQTZ84ZLLTZ90ZTTTZMMTZUUTZNNTZVVTZOOTZ11TZPPAZ22T) Z96Z102Z108Z114Z120Z126Z132Z138Z144rKz{letter?fmt=str}rrrrr z -if ":TWATP:" -set_var "APCP" z -fi -grib -rN)r%r'r&rrr~ryrzrr0rrCr_rr6)rr5rYrrrrZ UKMHIRESMERGErrrZukmet_fhr_id_dictZfhr_idZfhr_strrr4Zsource_file_accumZsource_file_accum_fhr_startrrr prep_prod_ukmet_fileWs             rcCstjd}tj|d}tjtd|dd}tjt|ddd}d|vrEt|r9t||t|rEt|d||gt ||d S) ai! Do prep work for DWD production files Args: source_file_format - source file format (string) dest_file - destination file (string) forecast_hour - forecast hour (string) prep_method - name of prep method to do (string) Returns: rrrrmrGz.tmprZdwdN) r%r'r&rrr~r0rCrr6)r4r5rYrrrrrrrr prep_prod_dwd_files$    rc Cstjd}tjd}tjtd|dd}d|vrId}t||}t|rIt|d|d t |d |d |d|d |gt ||d S)aL! Do prep work for METRFRA production files Args: source_file - source file(string) dest_file - destination file (string) forecast_hour - forecast hour (string) prep_method - name of prep method to do (string) Returns: rrrrmrGrrSrrrrrN) r%r'r&rrr~r_r0rrr6) r4r5rYrrrrZ file_accumZfhr_accum_startrrr prep_prod_metfra_files>   rc CsJtjd}tjd}tjtd|dd}tjtd|dd}dD]I} |d| } |dd| d } tjtd| dd} t| rhttj|d d d tj|d d| | g| dkro| } q,| dkru| }q,t| rt|rt | }t |}t j |ddd}| D]6}|dkr| || |d| |q|dkr| |dq|dkr| |dq| || |qt|jD] }||t|j|qdD]J}|||j|j|j|j}|j| D]}|||j| |iq|dkr!|j|ddd|dd<q|j|dd|dd<qdD]l}|||j|jd}|j| D](}|d kr_|||j| |d!d"iqE|||j| |iqEtjt|j|d#dd$ddf|j|d#d$dddff|j|j}||dd<q1t||dS)%a0! Do prep work for OSI-SAF production files Args: daily_source_file_format - daily source file format (string) daily_dest_file - daily destination file (string) weekly_source_file_list - list of daily files to make up weekly average file weekly_dest_file - weekly destination file (string) weekly_dates - date span for weekly dates (tuple of datetimes) Returns: FIXevsCDO_ROOTrrmrG)nhshrzmulti..binZcdoz remapbil,Z cdo_gridsz G004.gridrrwNETCDF3_CLASSICformathistoryrZsouthernmost_latitudez-90ZareaGlobal)timeZ time_bndslatlonrNi)Zice_concZ status_flagZtotal_uncertaintyZsmearing_uncertaintyZalgorithm_uncertainty)rrZ long_nameznorthern hemisphereZglober)r%r'r&rrr~rr0rnetcdfDatasetncattrs setncattr getncattrry dimensionsrzcreateDimensionrZcreateVariable variablesdatatype setncattsr[ma masked_equalvstack _FillValuer6)Zdaily_source_file_formatdaily_dest_fileZweekly_source_file_listZweekly_dest_fileZ weekly_datesrrdaily_prepped_fileZweekly_prepped_filerrZ hem_dest_fileZhem_prepped_fileZnh_prepped_fileZsh_prepped_fileZnh_dataZsh_dataZ merged_dataattrdimvarZ merged_varkZmerged_var_valsrrr prep_prod_osi_saf_files                rcCs^t||||i}tj|st||||i}d|vr#t|||ddSd|vr0t|||ddSd|vr=t|||ddSd|vrJt|||ddSd|vrWt|||ddSd|vrdt|||ddSd |vrqt|||ddSd |vr~t|||ddSd |vrt |||ddStj|rt d |d |t ||dSt d|ddSdS)a! This get a model file and saves it in the specificed destination Args: valid_time_dt - valid time (datetime) init_time_dt - initialization time (datetime) forecast_hour - forecast hour (string) source_file_format - source file format (string) dest_file_format - destination file format (string) Returns: z dcom/navgemrz wgrbbul/jma_z wgrbbul/ecmwfzwgrbbul/ukmet_hiresz qpf_verif/jmarz qpf_verif/UWDzqpf_verif/ukmoz qpf_verif/dwdzqpf_verif/METFRALinking r1r,r#N) rr%r&r(rrrrrrrsymlink)rhrirYrdest_file_formatr5r4rrr get_model_file\sP  rcCsxt|||dgi}tj|s:t|||dgi}tj|r0td|d|t||dStd|ddSdS)a?! This get a model file and saves it in the specificed destination Args: valid_time_dt - valid time (datetime) source_file_format - source file format (string) dest_file_format - destination file format (string) Returns: rUrr1r,r#N)rr%r&r(rr)rhrrr5r4rrr get_truth_files   rcCs~ddddddddddddd}|t|vr-||d}||d}||d}n td |d td |||fS) al! This returns the valid hour start, end, and increment information for a given observation Args: obs - observation name (string) Returns: valid_hr_start - starting valid hour (integer) valid_hr_end - ending valid hour (integer) valid_hr_inc - valid hour increment (integer) rrS)valid_hr_start valid_hr_end valid_hr_incr)24hrCCPA 24hrNOHRSCOSI-SAFrrrzFATAL ERROR: Cannot get z valid hour informationr)ryrzrrr)obsZobs_valid_hr_dictrrrrrr get_obs_valid_hrss(     rc Csd}tjt|d}tjjtjt|d}td|d|td|t|d|dkrit d| d d d |d |d |d d|d|d d|d|d d|d} n2|dvrt d| d d|d d|d|d d|d |d|d|d d} d\} } | | |krt | tdt | | d dt t|d t j| d!d"d#} | d$d%krd'S| d&7} | | |ksd'Sd'S)(a! This submits a job to the transfer queue to get data that does not reside on current machine Args: job_file - path to job submission file (string) job_name - job submission name (string) job_output - path to write job output (string) machine - machine name (string) user - user name (string) queue - submission queue name (string) account - submission account name (string) Returns: r)minutesz Submitting r1zOutput sent to iZWCOSS2zqsub -V -l walltime=z%H:%M:%Srz-q z -A z -o z-e z -N z-l select=1:ncpus=1 zqselect -s QR -u z-N z | wc -l)ZHERAZORIONZS4ZJETzsbatch --ntasks=1 --time=z --partition=z --account=z --output=z --job-name=z squeue -u z -n z-t R,PD -h | wc -l)rrwzWalltime checker: zout of z secondsTUTF-8)rencodingrrrN)r`rbr_ total_secondsr\rrr%rr<rcrrr check_output) Zjob_filejob_nameZ job_outputmachineuserqueueZaccountZwalltimeZwalltime_secondsZ job_check_cmdZ sleep_counterZ sleep_checkerZ check_jobrrr get_off_machine_datas     rcCsgd}|dvr:tjtjdtjddtjdddtjd <tjtjd s2ttjd |gd n |d krE|d d gi}|D] }tj|||<qI|d kr^tjd|d<||d<|dvr ||d<|d krt||d<n||d<tj|d|d<tj|d|d<tj|d|d<|dvrtj|dd}|dd|d <|d!d|d"<t |d#krt t t j |td$}nd%}t||d&<n;|d'krtd(\} } } n|d)krtd*\} } } n|d+krtd,\} } } nd-\} } } t| |d <t| |d"<t| |d&<tj|d.d} | dd|d/<| d!d|d0<t | d#kr.rXcSrrFrrJrrr rLr) r`rarbr_r[arangeastyperangerZrcrddelete)loggerreZ start_dateZend_daterrrrrrrYZ start_date_dtZ end_date_dtZdt_incZdate_type_dates valid_datesZ init_datesZvalid_remove_idx_listrgdZinit_remove_idx_listrfrrr get_plot_datesfsx            rc Cs|ddkr|dd}n |ddkr|}tj|dddd|d }tj|rSt|}|D]}||vrB|d d }nq3Wd n1sMwYn|d |ddt d| ddd} | S)a! Get the MET columns for a specific line type and MET verison Args: logger - logger object met_root - path to MET (string) met_version - MET version number (string) met_line_type - MET line type (string) Returns: met_version_line_type_col_list - list of MET versoin line type colums (strings) rrGrrZshareZmetZ table_filesZmet_header_columns_Vz.txtz : rPNrz DOES NOT EXIST, z*cannot determine MET data column structure r) r|r~r%r&rr(openr{errorrrr) rZmet_rootZ met_versionZ met_line_typeZmet_minor_versionZmet_minor_version_col_fileflineZline_type_colsmet_version_line_type_col_listrrr get_met_line_type_colss2     r cCsdD]}||vr|}||d}q|dvr!d|}d|}||fS|dvr1d|}d|}||fS|d vrAd |}d |}||fS|d vrQd |}d|}||fS|dvrad|}d|}||fS|dvrmd|}d|}||fS)z! Format threshold with letter and symbol options Args: thresh - the threshold (string) Return: thresh_symbol - threshold with symbols (string) thresh_letters - treshold with letters (string) ) >=r==!=<=<gegteqneleltr)rrrr)rrrr)rrrr)rrrr)rrrr)rrrr)r)ZthreshoptZ thresh_optZ thresh_valueZ thresh_symbolZ thresh_letterrrr format_threshs:    rc Cstj|||d} tj| dd} t| dkr!|d| dStj|s|d| t| d } | } Wdn1sDwYd}d |d d |d d |d d |d d |d d | d }| D]/}|d |t j d |d ||dt j t j dd}|d|j||d}qq|d|d|t|d}|| |WddS1swYdSdS)a! Condense the individual date model stat file and thin out unneeded data Args: logger - logger object input_dir - path to input directory (string) output_file - path to output file (string) model - model name (string) obs - observation name (string) grid - verification grid (string) vx_mask - verification masking region (string) fcst_var_name - forecast variable name (string) obs_var_name - observation variable name (string) line_type - MET line type (string) Returns: z_*.statT) recursiverzNO STAT FILES IN MATCHING z$Condensing down stat files matching Nrrz "zGetting data from z grep -R "z " r)rstdoutstderrrzRan z Condensed z .stat file at r)r%r&rglobrZwarningr(debugrreadlinerPopenPIPESTDOUTr communicatewrite)r input_dirZ output_filemodelrgridvx_mask fcst_var_name obs_var_name line_typeZmodel_stat_files_wildcardZmodel_stat_filesZmsfZmet_header_colsZall_grep_outputZ grep_optsZmodel_stat_fileZpsr rrr condense_model_stat_filess|     "r.c  Cstt||d|d|d} i} | d} | D]} | | } | | kr&t| | <qtj| | <qt|D]}|d||dd||d}tjj |g|gdd gd }tj tj || d }||}t j |d |dd |d|d|dd d|dd |d|d|dd d|dd d|dd d|dd d|d|dd |d|ddd |d!dd d"|d#d$d%}t j |st j ||d |dd%}t j |st||||d|d|d|d|d|d|d |dd&kr!t|d\}}n|d}|d}|dd&kr9t|d\}}n|d}|d}t j |r|d'|tj|d(d)d*| d+d,d-d.}||d/|dk|d0|dk@|d1|d#d2d3k@|d4|dk@|d5|dk@|d6|dk@|d7|dk@|d8|dk@|d9|dk@|d:|dk@|d;|dk@|d<|k@|d=|k@|d>|dk@}||d?|}tj|d?d@dA|d?<|jd?dB}|d?jd@|d?<|j|| d-d(dCdDt j |r|dE|ddFdG|n|dH|t j |r|dI|dJ|dtj|d(d)d*| d&gd-dK}|| }|D]W}|j|d?|k}t|dkrk|dL|d(dM|qGt|d)kr|dN|dO|dPn |dQ|dO||j|dd-d-|j||f<qGn| |dR|dSkr|}q2t!||g}q2|S)Ta! Build the data frame for all model stats, Read the model parse file, if doesn't exist parse the model file for need information, and write file Args: logger - logger object input_dir - path to input directory (string) output_dir - path to output directory (string) model_info_dict - model infomation dictionary (strings) plot_info_dict - plot information dictionary (strings) date_info_dict - date information dictionary (strings) met_info_dict - MET information dictionary (strings) dates - array of dates (datetime) met_format_valid_dates - list of valid dates formatted like they are in MET stat files Returns: rootversionr-TOTALrmnameZ plot_namer(r)names)indexcolumnsZfcstr r+Zfcst_var_levelZfcst_var_threshrZobs_namer,Z obs_var_levelZobs_var_threshZlinetyper)Zvxmaskr*interpZ interp_methodZ interp_pointsrerz %Y%m%d%H%M%StorPrjrYrxz.statZNAz Parsing file rrTFrN)sepskiprowsskipinitialspacer3Zkeep_default_narRheaderZMODELZDESCZ FCST_LEADrGZ0000ZFCST_VARZFCST_LEVZOBS_VARZOBS_LEVZOBTYPEZVX_MASKZ INTERP_MTHDZ INTERP_PNTSZ FCST_THRESHZ OBS_THRESHZ LINE_TYPEZFCST_VALID_BEGz %Y%m%d_%H%M%Sr)byr)r;r4r8modezParsed z file zat zCould not create zReading z for )r8r9r:r3Z na_valuesr;zNo data matching valid date zin z#Multiple lines matching valid date z in zusing first onezOne line matching valid date r-Zmodel1)"r r4rr[float64ryrzpdZ MultiIndexZ from_productZ DataFramenanr%r&rlowerrcrIr(r.rr Zread_csvisinZ to_datetimeZ sort_valuesdtZto_csvrtolistrZlocrconcat)rr'Z output_dirZmodel_info_dictZplot_info_dictZdate_info_dictZ met_info_dictdatesZmet_format_valid_datesr Z df_dtype_dictZ float_idxcolZcol_idxZ model_numZmodel_num_nameZmodel_num_df_indexZ model_num_dfZ model_dictZparsed_model_stat_fileZcondensed_model_fileZfcst_var_thresh_symbolZfcst_vat_thresh_letterZobs_var_thresh_symbolZobs_vat_thresh_letterZcondensed_model_dfZparsed_model_dfZmodel_stat_file_dfZ valid_dateZ&model_stat_file_df_valid_date_idx_listZ all_model_dfrrr build_df,s                                  rIcCs|dkr3|jddd}|jddd}|jddd}|jddd}|jddd} n|dkrf|jddd } |jddd } |jddd } |jddd } |jddd } nQ|dkr|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd }|jddd!} |jddd"}!|jddd#}"|jddd$}#|jddd%}$|jddd&}%|jddd'}&|jddd(}'|jddd)}(|jddd*})|jddd+}*|jddd,}+|jddd-},|jddd.}-|jddd/}.|jddd0}/|jddd1}0|jddd2}1|jddd3}2|jddd4}3|jddd5}4|jddd6}5|jddd7}6|jddd8}7|jddd9}8|jddd:}9|jddd;}:|jddd<};|jddd=}<|jddd>}=|jddd?}>|jddd@}?|jdddA}@|jdddB}A|jdddC}B|jdddD}C|jdddE}D|jdddF}E|jdddG}F|jdddH}G|jdddI}H|jdddJ}I|jdddK}J|jdddL}K|jdddM}L|jdddN}M|jdddO}N|jdddP}O|jdddQ}P|jdddR}Q|jdddS}R|jdddT}S|jdddU}T|jdddV}U|jdddW}V|jdddX}W|jdddY}X|jdddZ}Y|jddd[}Z|jddd\}[|jddd]}\|jddd^}]|jddd_}^|jddd`}_|jddda}`|jdddb}a|jdddc}b|jdddd}c|jddde}d|jdddf}e|jdddg}f|jdddh}g|jdddi}h|jdddj}i|jdddk}j|jdddl}k|jdddm}l|jdddn}mn|dokr7|jdddp}n|jdddq}o|jdddr}p|jddds}q|jdddt}r|jdddu}s|jdddv}t|jdddw}u|jdddx}vn|dykrY|jdddz}w|jddd{}x|jddd|}yn^|d}vr|jddd~}z|jddd}{|jddd}||jddd}}n3|dvr|jddd}~|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}ِn|dkrp|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jdddz}w|jddd}|jddd}|jddd|}y|jddd}|jddd}nG|dkr|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}n|dkr|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}n|dk r|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd}|jddd }|jddd }|jddd }|jddd }|jddd }|jddd}|dk r|dk r| | | t| | | | | | }n|dk rt}n|dk r|t||}n|dk r"|dk r||}nv|dk r|*}nl|dk r t|t|}nZ|dk r8|dk r6|z|z|{||}nD|dv ru|dk rj|z|{|||}}|z|{|z|||}|z||z|{|||}n|dk rs|}n|dk r|dk r|}n|dk r|dk r|z|{|z||}n|dk r|}n|dk r|dk r|ݐ}n|d~k r|dk r|z}n|dk r|dk r|z|{|||}}|z|{|z||} |||}|{|}} | | |}|z|}|||}n~|dk r|dk r|z|z||}nj|dAk rF|dk r*t||d|}nR|dk r3|@}nI|dk rEt||d|}n6|dtk rT|dok rS|r}n(|dk rk|dk rjd|{|{|z}n|d|dtdd} i} | |jjk rt |j |  | dt | <| d7} | |jjk s|jjdk rː|j | d| d} || fS(a! Calculate the statistic from the data from the read in MET .stat file(s) Args: data_df - dataframe containing the model(s) information from the MET .stat files line_type - MET line type (string) stat - statistic to calculate (string) Returns: stat_df - dataframe of the statistic stat_array - array of the statistic ZSL1L2NFBAROBARFOBARFFBAROOBARZSAL1L2FABAROABARFOABARFFABAROOABARZCNTFBAR_NCLFBAR_NCUFBAR_BCLFBAR_BCUFSTDEV FSTDEV_NCL FSTDEV_NCU FSTDEV_BCL FSTDEV_BCUOBAR_NCLOBAR_NCUOBAR_BCLOBAR_BCUOSTDEV OSTDEV_NCL OSTDEV_NCU OSTDEV_BCL OSTDEV_BCUPR_CORR PR_CORR_NCL PR_CORR_NCU PR_CORR_BCL PR_CORR_BCUSP_CORRKT_CORRRANKS FRANKS_TIES ORANKS_TIESMEME_NCLME_NCUME_BCLME_BCUESTDEV ESTDEV_NCL ESTDEV_NCU ESTDEV_BCL ESTDEV_BCUMBIAS MBIAS_BCL MBIAS_BCUMAEMAE_BCLMAE_BCUMSEMSE_BCLMSE_BCUBCRMSE BCRMSE_BCL BCRMSE_BCURMSERMSE_BCLRMSE_BCUE10E10_BCLE10_BCUE25E25_BCLE25_BCUE50E50_BCLE50_BCUE75E75_BCLE75_BCUE90E90_BCLE90_BCUIQRIQR_BCLIQR_BCUMADMAD_BCLMAD_BCU ANOM_CORR_NCL ANOM_CORR_NCU ANOM_CORR_BCL ANOM_CORR_BCUME2ME2_BCLME2_BCUMSESS MSESS_BCL MSESS_BCURMSFA RMSFA_BCL RMSFA_BCURMSOA RMSOA_BCL RMSOA_BCUANOM_CORR_UNCNTRANOM_CORR_UNCNTR_BCLANOM_CORR_UNCNTR_BCUSISI_BCLSI_BCUZGRADFGBAROGBARMGBAREGBARS1S1_OG FGOG_RATIODXDYZFHOF_RATEH_RATEO_RATE)CTCZNBRCTCFY_OYFY_ONFN_OYFN_ON)CTSZNBRCTSBASER BASER_NCL BASER_NCU BASER_BCL BASER_BCUFMEAN FMEAN_NCL FMEAN_NCU FMEAN_BCL FMEAN_BCUACCACC_NCLACC_NCUACC_BCLACC_BCUFBIAS FBIAS_BCL FBIAS_BCUPODYPODY_NCLPODY_NCUPODY_BCLPODY_BCUPODNPODN_NCLPODN_NCUPODN_BCLPODN_BCUPOFDPOFD_NCLPOFD_NCUPOFD_BCLPOFD_BCUFARFAR_NCLFAR_NCUFAR_BCLFAR_BCUCSICSI_NCLCSI_NCUCSI_BCLCSI_BCUGSSGSS_BCLGSS_BCUHKHK_NCLHK_NCUHK_BCLHK_BCUHSSHSS_BCLHSS_BCUODDSODDS_NCLODDS_NCUODDS_BCLODDS_BCULODDS LODDS_NCL LODDS_NCU LODDS_BCL LODDS_BCUORSSORSS_NCLORSS_NCUORSS_BCLORSS_BCUEDSEDS_NCLEDS_NCUEDS_BCLEDS_BCUSEDSSEDS_NCLSEDS_NCUSEDS_BCLSEDS_BCUEDIEDI_NCLEDI_NCUEDI_BCLEDI_BCUSEDISEDI_NCLSEDI_NCUSEDI_BCLSEDI_BCUBAGSS BAGSS_BCL BAGSS_BCUZNBRCNTFBSFBS_BCLFBS_BCUFSSFSS_BCLFSS_BCUAFSSAFSS_BCLAFSS_BCUUFSSUFSS_BCLUFSS_BCU F_RATE_BCL F_RATE_BCU O_RATE_BCL O_RATE_BCUZVL1L2UFBARVFBARUOBARVOBARUVFOBARUVFFBARUVOOBARZVAL1L2UFABARVFABARUOABARVOABARUVFOABARUVFFABARUVOOABARZVCNTFS_RMSOS_RMSMSVERMSVEFDIRZODIR FBAR_SPEED OBAR_SPEED VDIFF_SPEED VDIFF_DIR SPEED_ERR SPEED_ABSERRDIR_ERR DIR_ABSERRZBIASr)ZETSrrZPODrGZSRATIOrrz IS NOT AN OPTIONrr4Zindex0Zindex1)rEr[sqrtZ ANOM_CORRr rrr4ZnlevelsrZZget_level_valuesuniquervaluesreshape(rZdata_dfr-statrJrKrLrMrNrOrPrQrRrSrTrUrVrWrXrYrZr[r\r]r^r_r`rarbrcrdrerfrgrhrirjrkrlrmrnrorprqrrrsrtrurvrwrxryrzr{r|r}r~rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr r r r r rrrrrrrrrrrrrrrrrrr r!r"r#r$r%r&r'r(r)r*r+r,r-r.r/r0r1r2r3r4r5r6r7r8r9r:r;r<r=r>r?r@rArBrCrDZORDIRrErFrGrHrIrJrKrLZstat_dfr1CZCAZCBidxidx_dictZ stat_arrayrrr calculate_stats                         "                        rU)+r%r`numpyr[rr2rnetCDF4rrZpandasr?rrrr)r+r0r6r@rCrDrlrrrrrrrrrrrrrrrr rr.rIrUrrrr sR  Ee93b"b2 2oI$#6 :