U  g3@sddlZddlZddlZddlZddlZddlZddlZddlm Z m Z e dZ ej jdddZdZdad d Zdd d Zd dZdddZddZddZddZdddZdddZdS)N)MPIMixedCMDFGenlsf_cray_intelZaprunF) raise_missingmpiexecc Cstdk r tSt}tdd}|D]}tdt||}tdt||D]L}z$tdt||t |WqZt k r}zW5d}~XYqZXqZq&W5QRX| tdt||datd ttd kst tS) a0!Value to send to aprun --p-state option for Intel Turbo Mode. This is the largest value printed out by cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_available_frequencies Which is either the highest allowed sustained clock speed, or the magic number for Turbo Mode.NzB/sys/devices/system/cpu/cpu0/cpufreq/scaling_available_frequenciesrtz saf line: zsaf line split: z saf entry: zPossible --p-states options: z Turbo Mode --p-states option: %dgA) p_state_turbolistopen readlines module_loggerinforeprsplitappendint ValueErrorsortAssertionError)ZstatesZsaflinesplatsZverM/lfs/h1/ops/prod/packages/hmon.v3.2.7/ush/produtil/mpi_impl/lsf_cray_intel.pyget_p_state_turbo#s(     rcCs|dkr t}|ddS)z%!Runs the "sync" command as an exe().NzNot running sync.)rrloggerrrrrunsyncBs rc Cs|dkrXz&tjdd}t|}|dkr,|}Wn(tttfk rV}zW5d}~XYnX|dkrtjdd}t|}td|d}|dkstt|}t |dr|j |d }t |d r||_ |S) z!Adds OpenMP support to the provided object @param arg An produtil.prog.Runner or produtil.mpiprog.MPIRanksBase object tree @param threads the number of threads, or threads per rank, an integerNOMP_NUM_THREADSrPRODUTIL_RUN_NODESIZE128rarginsr threads) osenvirongetrKeyError TypeErrorrmaxrhasattrenvr&)argr&ontenodesizerrropenmpHs&   r3c Cs|dkr t}d|||f}||tjjtddddddt|t|g |d}|dt|t|t|ft |}||kr| d t|t|fd Sd |||f}| |tj ||dS) NzqFilesystem failure (will retry with aprun ln -sf): Cannot symlink "%s" -> "%s". Instead, the symlink is to "%s".-N1-nz/bin/lnz-sfrzaprun ln -sf %s %s = %szaprun ln -sf %s %s SUCCESSTzoFILESYSTEM FAILURE: Python and aprun ln -sf both cannot symlink "%s" -> "%s". Instead, the symlink is to "%s".)rerrorprodutilpipeline simple_run aprun_pathstrwarningrr'readlinkrcriticalfileop WrongSymlink)sourcetargetcontentrmsgstatusrrr aprun_ln_sfrsP   rGcCs"tdk }|rtdttj_|S)zj!Determines if Cray aprun should be used to run MPI programs by looking for the aprun program in $PATHNz6lsf_cray_intel detected; replace produtil.fileop.ln_sf)r;rrrGr8r@Zln_sf)Zdetectedrrrdetects  rHcCsdS)z@!Does this module represent an MPI implementation? Returns True.Trrrrr can_run_mpisrIcKsHt|}ttjdd}|d7}tj|gjt|d}tj |S)a!Returns an ImmutableRunner that will run the specified program. @returns an empty list @note This function does NOT search $PATH. That ensures the $PATH will be expanded on the compute node instead. Use produtil.fileop.find_exe() if you want to explicitly search the PATH before execution @param exe The executable to run on compute nodes. @param kwargs Ignored. INSIDE_APRUN0rrJ) r<rr'r(r)r8progRunnerr.ImmutableRunner)exekwargs inside_aprunrrrr make_bigexes rTcKs@|dkr t}t|f||d|}|dt|t|f|S)N)allranksrzmpirunner: %s => %s)r mpirunner2rr)r/rUrrQmrrr mpirunners rXc Ks6|dkrtd}t|tjjs$t|\}}|r@|r@tdt j dd}t |}t j dd}t |}t j d}t |}tj tg} |j} |d t| |d | dk r| d kr|d t| f| d t| f| jt| dn |dt| f| dd } t t j dd} | d 7} | jt| d} |d kr|r|jddD]\} } t||| }| j}|rt||}| dd|f} d}| rdd| D}d|k}| |} |s| jr| dttf} |dkr| js|st| | } | Sn|r0tdn|rD|dD]J\} } z| j dr`tdWn$t!k r}zW5d}~XYnXq@dd|j"ddd D}tj#$d!dkrt%d"d}tj&|}g}d#}|jddD]\} } t||| }| j}|rt||}d$d|g}d}d}|rPd%d|D}|'|d|k}|sr|jrr|'dttg||kr|| 7}nJ|r|s| }|}n4|s| d&} | |} | dt|d!f} d}|}| }q| dk st||kr|r|s| d&} | |} | dt|d!f} t(d'|f|}| )|} | dk s@t| Sd}d }|jddD]\} } t*| |}qX|jddD]\} } t| ||| }| j}|rt||}|s| d&} | dd| f} d}d}| rd(d| D}| |} d|k}|s| jr| dttf} | | +} d}qz| SdS))a!Turns a produtil.mpiprog.MPIRanksBase tree into a produtil.prog.Runner @param arg a tree of produtil.mpiprog.MPIRanksBase objects @param allranks if True, and only one rank is requested by arg, then all MPI ranks will be used @param logger a logging.Logger for log messages @param kwargs passed to produtil.mpi_impl.mpi_impl_base.CMDFGen when mpiserial is in use. @returns a produtil.prog.Runner that will run the selected MPI programNrzACannot mix serial and parallel MPI ranks in the same MPI program.r"r#ZPRODUTIL_RUN_HYPERTHREADSr5 TOTAL_TASKSzthreads=z&Decide what to do with --depth option.rz*Threaded with %s threads so add -cc depth.--depthr%zNo threads (threads=%s).)rZr5rJrKrLF)expandr6z%dcSsg|] }t|qSrr<.0xrrr szmpirunner2..z --p-stateizWhen using allranks=True, you must provide an mpi program specification with only one MPI rank (to be duplicated across all ranks).zWTrying to run aprun within aprun. In mpiserial, you must run batchexe() programs only.cSsg|]}|qSrr)r^arrrr`sT)to_shellr[ mpiserialzfAttempting to run a serial program via aprun mpiserial but the mpiserial program is not in your $PATH.rr4cSsg|] }t|qSrr\r]rrrr` s: serialcmdfcSsg|] }t|qSrr\r]rrrr`Rs),logging getLogger isinstancer8mpiprog MPIRanksBaser check_serialrr'r(r)rrMrNr;r&rr<rr.nranks expand_iterminranks_per_node haslocalopts localoptiter turbomoderargsMPIAllRanksErrorrunnergetenvr* to_arglistr@find_exeMPISerialMissingcollapseextendrprerunr,rb)r/rUrrQserialparallelr2Z hyperthreadsZmaxtasksrur&rRrankcountZpernoderpnZseenPS localoptskelinesfirstZ prior_arglistZ total_countarglistZseenSZcmdfgenrrrrVs                       rV)N)N)FN)FN)r'socketrfsysprodutil.fileopr8 produtil.progprodutil.mpiprog mpi_impl_baserrrgrr@rxr;r rrr3rGrHrIrTrXrVrrrr s    *