o gb@sdZddlmZddlmZddlmZddlZddlZddlZddlZzddl m Z ddl m Z ddl m Z mZd Zd d ZWn#eycdd lm Z ddlm Z ddlm Z mZd Zdd ZYnwddlmZmZmZmZddlmZddlmZmZmZdZdZddZGddde Z!ddZ"ddZ#ddZ$ d*d d!Z%d"d#Z&Gd$d%d%e'Z(Gd&d'd'e(Z)Gd(d)d)e(Z*dS)+acModel description Model description is the representation of the various externals included in the model. It processes in input data structure, and converts it into a standard interface that is used by the rest of the system. To maintain backward compatibility, externals description files should follow semantic versioning rules, http://semver.org/ )absolute_import)unicode_literals)print_functionN)SafeConfigParser)MissingSectionHeaderError)NoSectionError NoOptionErrorTcCs |dS)z%convert strings into unicode zutf-8)decodetextr c/lfs/h1/nco/ptmp/ethan.collins/aqm/aqm.v7.0.11/sorc/manage_externals/manic/externals_description.pyconfig_string_cleaner$s r) ConfigParserFcCs|S)zgPython3 already uses unicode strings, so just return the string without modification. r r r r r r0s)printlog fatal_error str_to_boolexpand_local_url)execute_subprocess) EMPTY_STRPPRINTERVERSION_SEPERATORexternals_descriptionZschema_versioncCstj|}d|}t|td|tj||}tj|s:| dkr0d|}nd||}t |d}|t j krGt ||}nz t}|||}Wn ty\Ynw|durgd}t ||S)zaRead a file containing an externals description and create its internal representation. In directory : {0}z+Processing externals description file : {0}ZnonezKINTERNAL ERROR: Attempt to read externals file from {0} when not configuredzsERROR: Model description file, "{0}", does not exist at path: {1} Did you run from the root of the source tree?NUnknown file format!)ospathabspathformatlogginginforjoinexistslowerrExternalsDescriptionGIT_SUBMODULES_FILENAMEread_gitmodules_file config_parserreadr)root_dir file_namemsg file_pathrconfigr r r read_externals_description_fileBs8         r/c@sBeZdZdZddZddZdddZd d Zd d Zd dZ dS) LstripReaderzHLstripReader formats .gitmodules files to be acceptable for configparsercCsht|d }|}Wdn1swYt|_t||_d|_|D] }|j|q'dS)Nrr) open readlineslist_lineslen _num_lines_indexappendlstrip)selffilenameZinfilelinesliner r r __init__ls   zLstripReader.__init__cCs|jS)z,Return all the lines from this object's file)r5r;r r r r3uszLstripReader.readlinescCsHz|}Wn tyd}Ynw|dkr"t||kr"|d|S|S)z6Format and return the next line or raise StopIterationr)next StopIterationr6)r;sizer>r r r readlineys   zLstripReader.readlinecCs d|_|S)zBegin an iterationr)r8r@r r r __iter__szLstripReader.__iter__cCs,|j|jkrt|jd|_|j|jdS)z+Return the next line or raise StopIterationr)r8r7rDr5r@r r r rCs  zLstripReader.nextcCs|SN)rCr@r r r __next__szLstripReader.__next__N)rA) __name__ __module__ __qualname____doc__r?r3rFrGrCrIr r r r r0js   r0c Cst}t|gd}t|dd}i}|d}|D]*}|rF|d}|ddd}t|d kr8|d } nd} |d|| d ||d<qt||S) zIRun the git submodule status command to obtain submodule hashes. )git submodulestatusT)Zoutput_to_caller rrN )hashrPtag)rgetcwdchdirrsplitr6) Zrepo_dircwdcmdZ git_outputZ submodulessubmodsZsubmodrPitemsrUr r r git_submodule_statuss"      r]cCs~d}d}|D]4}|d}|dkr|d}q|dkr&|d}q|dkr+qd}||d|}t|q||fS)z4Find the path and url for this submodule descriptionNrrrurlbranchz,WARNING: Ignoring unknown {} property, in {})stripr$rr warning) section_itemsr-rr^itemnamer,r r r parse_submodules_desc_sections recCs tj|}d|}t|td|tj||}tj|s.d||}t |d}d}zt }t rB|j t ||dn |jt ||d|}Wn tyWYnw|durdd}t ||St }t|}|D]}|dd d kr|d dd } || ||} t| |\} } | durd | }t || durd | }t ||| tj| || tjd|| tj| || tjd| |vr| } n| } | |vr|| d}|| tj|qod}|d7}t || qo|t|ttd|S)z_Read a .gitmodules file and convert it to be compatible with an externals description. rz,Processing submodules description file : {0}zJERROR: submodules description file, "{0}", does not exist at path: {1}N)r<)sourcerr rOz "zSubmodule {} missing pathzSubmodule {} missing urlrNTruerTz%submodule status has no section, '{}'z- Check section names in externals config filez1.0.0)rrrrr r!rr"r#rr( USE_PYTHON2Zreadfpr0Z read_filerr]sectionsr`Z add_sectionr\resetr%PATHPROTOCOLREPO_URLREQUIREDHASHDESCRIPTION_SECTION VERSION_ITEM)r*r+r,r-Zsubmodules_descriptionrr.r[sectionZsec_namerbrr^Z submod_nameZgit_hashZemsgr r r r's    2       r'cfgc Cs|d}|dkrt|||d}|S|dkr3t|\}}}|dkr(t||||d}|Sd|}t||Sd|}t||S) zECreate the a externals description object from the provided data Ndict componentsexcludertr)rwrx parent_repoz@Externals description file has unsupported schema version "{0}".zUnknown model data format "{0}")ExternalsDescriptionDictget_cfg_schema_versionExternalsDescriptionConfigV1rr) model_dataZ model_formatrwrxryrmajor_r,r r r create_externals_descriptions*  rc Csd}z|tt}Wnttfydtt}t|Ynwtd|}|d}|t }zt |d }t |d }t |d }Wnt y[d|}t|Ynw|||fS)aExtract the major, minor, patch version of the config file schema Params: model_cfg - config parser object containing the externas description data Returns: major = integer major version minor = integer minor version patch = integer patch version rBzOexternals description file must have the required section: "{0}" and item "{1}"z[-+]rrrSzgConfig file schema version must have integer digits for major, minor and patch versions. Received "{0}") getrqrrrrrrrerXrintr` ValueError) Z model_cfgZ semver_strr,Z version_listZ version_strversionr~minorZpatchr r r r{0s2       r{c@seZdZdZdZdZdZdZdZdZ dZ d Z d Z d Z d Zd ZdZdZdZdZeeegZdZdZdZdZdZe de dedede e de dedededediiZd(ddZddZddZd d!Zd"d#Zd$d%Z d&d'Z!dS))r%aBase externals description class that is independent of the user input format. Different input formats can all be converted to this representation to provide a consistent represtentation for the rest of the objects in the system. NOTE(bja, 2018-03): do NOT define _schema_major etc at the class level in the base class. The nested/recursive nature of externals means different schema versions may be present in a single run! All inheriting classes must overwrite: self._schema_major and self._input_major self._schema_minor and self._input_minor self._schema_patch and self._input_patch where _schema_x is the supported schema, _input_x is the user input value. externalsr_Zfrom_submodulerTrdZ local_pathprotocolZreporepo_urlrequiredrUZsparseZexternals_onlyrNZsvnz .gitmodulesZ TREE_PATHZROOTTAGBRANCHZ REQ_SOURCETstringNcCs8t|d|_d|_d|_d|_d|_d|_||_dS)zkConvert the xml into a standardized dict that can be used to construct the source objects N) rur? _schema_major _schema_minor _schema_patch _input_major _input_minor _input_patch _parent_repo)r;ryr r r r?s  zExternalsDescription.__init__cCs~d|j|j|j}d|j|j|j}|j|jkr$d||}t||j|jkr4d||}t||j|jkr= dSdS)zMUse semantic versioning rules to verify we can process this schema. z {0}.{1}.{2}z=DEV_ERROR: version "{0}" parser received version "{1}" input.zIncompatible schema version: User supplied schema version "{0}" is too new." Can only process version "{1}" files and older.N)rrrrrrrr)r;ZknownZreceivedr,r r r _verify_schema_versions*   z+ExternalsDescription._verify_schema_versioncCs|||dS)aRun a series of checks to attempt to validate the user input and detect errors as soon as possible. NOTE(bja, 2018-03) These checks are called *after* the file is read. That means the schema check can not occur here. Note: the order is important. check_optional will create optional with null data. run check_data first to ensure required data was provided correctly by the user. N) _check_data_check_optional _validater@r r r _check_user_inputs  z&ExternalsDescription._check_user_inputcCsz|D]}|||j|j|jvr&d|||j|j|}t||||j|j|jkrF|j|||jvrFd|}t||||j|j|jkrn|j ||vrnd}t||j||||j|j|||j|j|j krd}d}|j |||jvr|d7}d|j |||j|j |}|j |||jvr|d7}d|j |||j|j |}|j|||jvr|d7}d|j|||j|j|}|j ||vr|||j r|d7}d|j |||j |}|dkr#d}|j ||vr |d 7}n|d 7}|d |7}d ||}t|n|dkr1d |}t||j |||jvrU|j ||vsL|||j sUd|}t||j ||vr|||j r|j |||jvrd|j }d||}t||j||vrd|j}d||}t||j |||jvrt|||j|j |}||||j|j <qdS)z:Check user supplied data is valid where possible. z+Unknown repository protocol "{0}" in "{1}".zSIn repo description for "{0}". svn repositories may not include the "hash" keyword.zYself.SUBMODULE is only supported with {0} protocol, "{1}" is defined as an {2} repositoryrrBrz"{0} = {1}", {2}z%Model description is over specified! zAfrom_submodule is not compatible with "tag", "branch", or "hash" z9 Only one of "tag", "branch", or "hash" may be specified zfor repo description of "{0}".z{0} Found: {1}zwModel description is under specified! One of "tag", "branch", or "hash" must be specified for repo description of "{0}"zXModel description is under specified! Must have "repo_url" in repo description for "{0}"zbModel description is over specified! from_submodule keyword is not compatible with {0} keyword forz{0} repo description of "{1}"N)keysREPOrmKNOWN_PRROTOCOLSrr PROTOCOL_SVNrp PROTOCOL_GIT SUBMODULEPROTOCOL_EXTERNALS_ONLYrrrnrlr)r;Zext_namer,Z ref_countZ found_refsr^r r r rs           z ExternalsDescription._check_datacCsd}|D]}|j||vrt|||j<|j|||jvr(t|||j|j<|j|||jvr.print_compare_differencecsd}d}d}t|tr-t|tr-|D]}|o||v}|r'|o&||||}q|o+|}nt|t|}|sctdt|trO||dd||dd|Std|t|td|t||S)zqCompare a data structure against a schema and validate all required fields are present. FTz Unmatched schema and input:schemainputz schema = {0} ({1})z input = {0} ({1}))rrurrr)rdataZis_validZin_refvalidkeyrvalidate_data_structr r rs0   z.validate_data_structz(ERROR: source for "{0}" did not validateN)_source_schemarrrr)r;rrr,r rr rs#  zExternalsDescription._validaterH)"rJrKrLrMrrrrpNAMErlrmrrnrorrrrrr&rZ _V1_TREE_PATHZ_V1_ROOTZ_V1_TAGZ _V1_BRANCHZ_V1_REQ_SOURCErr?rrrrrrr r r r r%UsV  e> r%c@seZdZdZdddZdS)rzzCreate a externals description object from a dictionary using the API representations. Primarily used to simplify creating model description files for unit testing. NcCst|d|_d|_d|_d|_d|_d|_||r-t | D] }||vr,||=q#|r?t | D] }||vr>||=q5| || dS)z@Parse a native dictionary into a externals description. rrN) r%r?rrrrrrrr4rupdater)r;r}rwrxrr r r r?s(   z!ExternalsDescriptionDict.__init__NN)rJrKrLrMr?r r r r rzsrzc@s0eZdZdZd ddZeddZd ddZdS) r|z^Create a externals description object from a config_parser object, schema version 1. NcCsbtj||dd|_d|_d|_t|\|_|_|_| | ||j |||d| dS)zsConvert the config data into a standardized dict that can be used to construct the source objects )ryrrrvN) r%r?rrrr{rrrr_remove_metadata _parse_cfgr)r;r}rwrxryr r r r?s  z%ExternalsDescriptionConfigV1.__init__cCs|tdS)zRemove the metadata section from the model configuration file so that it is simpler to look through the file and construct the externals description. N)Zremove_sectionrq)r}r r r rsz-ExternalsDescriptionConfigV1._remove_metadatac Csddd}|D]{}t|}|r||vs|r ||vr q i||<|||||i|||j<||}|D]B}||j vr]t |j |t r\t ||||||<qA||j |jvrx||||||j|<|||=qAdj ||d} t| qAq dS)zCParse a config_parser object into a externals description. TcSsFi}|D]}t|d}t|d}|r|}|||<q|S)zAConvert a list of key-value pairs into a dictionary. rr)rr`r$)Z input_listZconvert_to_lower_caseZ output_dictrcrvaluer r r list_to_dicts z=ExternalsDescriptionConfigV1._parse_cfg..list_to_dictz7Invalid input: "{sect}" contains unknown item "{item}".)ZsectrcN)T)rjrr$r`rr\rcopyrrrboolrrr) r;Zcfg_datarwrxrrsrdZ loop_keysrcr,r r r rs.     z'ExternalsDescriptionConfigV1._parse_cfg)NNNr)rJrKrLrMr? staticmethodrrr r r r r|s   r|)rtNNN)+rM __future__rrrr ros.pathrrrr(rrrrir ImportErrorZ configparserZutilsrrrrrZglobal_constantsrrrrqrrr/objectr0r]rer'rr{rur%rzr|r r r r sN          (+Y %~