U go@sddlZddlZddlZddlZddlZddlZddlTddlmZddl m Z m Z ddl Z ddl Z ddlZddlmmZddlZddlmZddlmZGdddZdS) N)*) OrderedDict)datetime timedelta) RFC_Forecast) RFC_Sitesc@seZdZdZeddZejddZeddZejddZedd Zejd d Zd d Z d dZ ddZ ddZ ddZ ddZddZddZddZddZdd Zd!d"Zd#d$Zd%S)&PI_XMLz0 Store one RFC forecast data cCs|jSNZ _timeZoneselfr A/lfs/h1/ops/prod/packages/nwm.v3.0.13/ush/rfc_ingestion/PI_XML.pytimeZone szPI_XML.timeZonecCs ||_dSr r )r tr r rr$scCs|jSr Z_seriesr r r rseries(sz PI_XML.seriescCs ||_dSr r)r sr r rr,scCs|jSr Z _rfcsitesr r r rrfcsites0szPI_XML.rfcsitescCs ||_dSr r)r rr r rr4scCs ||_g|_||_||dS)za Initialize the RFC_Forecast object with a given filename N)sourcerr loadPIxml)r pixmlfilenameZsitesr r r__init__8szPI_XML.__init__c Csz@t|}|}|dj|_|dD]}|j|q,Wn>t k r~}z t dt |d|dW5d}~XYnXdS)Nz'{http://www.wldelft.nl/fews/PI}timeZonez%{http://www.wldelft.nl/fews/PI}serieszWARNING: parsing PI XML error: z: z skipping ...) etreeparsegetrootfindtextriterrappend Exception RuntimeErrorstr)r rZfstwmlrootrer r rrCs  zPI_XML.loadPIxmlcCs&g}|jD]}|t||jq |Sr )rr!rr)r rfc_seriesrr r r toRFCForecastSs zPI_XML.toRFCForecastcCsttdd|S)NcSs|jdko|jdk o| S)NRQOTQINESQIN) parameterIdrfcnameisEmptyrr r r^s z*PI_XML.getFlowTimeseries..)listfilterr(r r r rgetFlowTimeseries]s zPI_XML.getFlowTimeseriescCsttdd|S)NcSs|Sr )get5CharStationIDr0r r rr1dz)PI_XML.getAllStationIDs..)setmapr4r r r rgetAllStationIDscs zPI_XML.getAllStationIDscstfdd|Dd}tfdd|Dd}|dk r|dkrt|}|}t|d}|d}||kr|jr|j ||}qt| d|_ |d}||kr|jj dd ||}q| ||fS) Nc3s&|]}|r|kr|VqdSr ) isForecastr5.0ridr r is z5PI_XML.getObservedAndForecastForID..c3s&|]}|r|kr|VqdSr ) isObservedr5r;r=r rr?ls secondsobservedrFlast) nextr4copydeepcopygetT0rgetTimeStepInSeconds getTimePeriodtimeValueQualitypopitemresetTimePeriod qualifierId)r r>fstobvT0Zdtrr r=rgetObservedAndForecastForIDhs.      z"PI_XML.getObservedAndForecastForIDc Cs|}g}g}tdddg}|D]R}|j|kr"|s"|rL||q"|r`||q"|jdkr"||q"|r|D]}|D] }|j|jkr|j | kr|j |j krt j|j |j dt|jdds|j |j |j |j t|jdfin|j | krt}|j D]"}|| kr*|j |||<q*||_ |j | r|j | |j | i|j j| dd | |_ t|j dt|j d f|_n|j t|d } | | kr.|j | t|jdfi|j j| dd | t|d } q|j | |j | t|jdfi|j j| dd | |_ t|j dt|j d f|_qqqg} |D]>}d} |D]}|j|jkrd } qΐq| s| |q| D]} | j | kr2t}| j D]"}|| kr| j |||<q|| _ nb| j t| d } | | kr| j | t| jdfi| j j| dd | t| d } qF| | _ t| j dt| j d f| _qn|D]} | j | krt}| j D]"}|| kr| j |||<q|| _ nb| j t| d } | | kr|| j | t| jdfi| j j| dd | t| d } q.| | _ t| j dt| j d f| _q|S) Nr*r+r,CNRFCrg-C6?)Zabs_tolFrErAT)r(r7r-r/r@r!r:r. stationIDZ startDaterJrMkeysmathisclosefloatmissValupdategetrZhas_key move_to_endr2Z fstPeriodrrK) r rrDZforecastZ parametersetroZtvqkrZforcast_without_observedfoundfr r rgetReserviorForecastWithT0s                               z!PI_XML.getReserviorForecastWithT0c Csd}d}|d}d|d}d}d|}d|d |}d|d |}d|d |} d} t||j} | rtd } nvt||j} | rd } n^t||j} | rd} nFt||j} | rd} n.t||j} | rd} nt| |j} | rd} | S)NzC[0-9]{4}(0[1-9]|1[0-2])(0[1-9]|[1-2][0-9]|3[0-1])(2[0-3]|[01][0-9])z1[0-9]{4}(0[1-9]|1[0-2])(0[1-9]|[1-2][0-9]|3[0-1])z(0[0-9]|[1-5][0-9]){2}z^(.*/)?CBRFC_Reservoirs_z.xmlz!^(.*/)?NERFC_Reservoir_Export.xmlz!^(.*/)?LMRFC_RESERVOIR_FORECAST\.z^(.*/)?z_RES_NWM_pixml_export.z_QINE_NWM_Res_export.z_RQOT_Forecast_.+_xml_CBRFCNERFCABRFCNWRFCLMRFCMBRFC)rematchr) r YYYYMMDDHHZYYYYMMDDYYYYMMDDHHMMSSZ CBRFCPatternZ NERFCPatternZ LMRFCPattern ABRFCPattern NWRFCPatternZ MBRFCPatternZRFCNameresultr r r getRFCNames: zPI_XML.getRFCNamecCs*t}|}|D]}||q|Sr )r7r(addr5)r Zstationsrrr r r allStationsDs zPI_XML.allStationscCs8|}|jD]$}||}|t|s|SqdSr )rtRFCZ getSitesByRFC isdisjointr7)r ZgagesZstaZrfcZrfcgagesr r rgetRFCNameBySitelistKs   zPI_XML.getRFCNameBySitelistc Cst|d}t|d}|tdd|tddt|}|}|}|d|dkr|jD]\}}||jkrp||j|<qpn|dtdd|dkr0||dtdd|dtdd} | |dkrt |j ddf|j| <| t| d} q|jD]\}}||j|<qn.||d|jD]\}}||j|<qH| |j |_ |j|_|} |} | tdd| dkr|| tddtdd| td d| dkr|| td | d |S) NrCrZhoursdaysTrA )rzrB)rHrIlinearInterpolaterinterForwardPersistrLrMitemspersistForwardr[r\rKrOrPZ forecastDaterJpersistBackward) r ZobvfstrRrQZcombined obvperiod fstperiodravrZcombined_periodrSr r r3combineObvFstAndApplyPresistenceLinearInterpolation`sN     z:PI_XML.combineObvFstAndApplyPresistenceLinearInterpolationcCs|tdd|}|tdd|}|d|dkr|dt|d}||dkr|j||j|i|jj|dd|t|d}qVn:|dtdd|dkr|}||dkr|d}|tdd}||kr|j||j|d|j|d|| || |j|d|j|d|j|d|| || |j|dd fi|jj|dd|tdd}q|}||dkr|j||j|i|jj|dd|tdd}qn|d}|dtdd}||tddkr\|j|t t j d d fi|jj|dd|tdd}q ||kr|j||j|i|jj|dd|tdd}q\||dkr|j||j|i|jj|dd|tdd}qn|d}|dtdd}||tddkrR|j|t t j d d fi|jj|dd|tdd}q||kr|j||j|i|jj|dd|tdd}qR||dkr|j||j|i|jj|dd|tdd}qdS) NrCrxrrAFrEr|ryTr{) rrrLr~rKrMr]r_rJ total_secondsr[rr\)r r`rcrrrt0t1r r r1linearInterpolateObservedPresisForecastAndCombines                z8PI_XML.linearInterpolateObservedPresisForecastAndCombinec Csz|}g}g}|D]>}|jdkr|jdk r|s|rH||q||q|D]}d}|}|jdkr||t dd| |t ddt ddd| |t dddnx|D]}|j |j kr|}qq|r|s| ||n|t dd| |t ddt dd| |t dd|qX||D]&}|jdkrN|rN||qN|S) Nr)APRFCrCrxr|ryr{r})r(r-r.r/r:r!rJZaddLeadingAndTrailingZerosr~rrrrWrZ'linearInterpolateAndInterForwardPersistrOclear) r rZobsrQrrcrRrr`r r r*getReserviorObservedForecastCombinedWithT0 s\   #      z1PI_XML.getReserviorObservedForecastCombinedWithT0N)__name__ __module__ __qualname____doc__propertyrsetterrrrrr(r4r9rTrdrrrtrwrrrr r r rrs6        $'0zr)ossystimeZcsvrkloggingstring collectionsrrrZdateutil.parserZdateutilrYZpytzxml.etree.ElementTreer ElementTreerHrrrr r r rs(