3 dt@sdZddlZddlZddlZddlmZddlZddlmZddl m Z ddl m Z ddl m Z ddlmZdd lmZddlZd d lmZmZmZd d lmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#d d l$m%Z%d dl&m'Z'd dl(m)Z)m*Z*m+Z+m,Z,m-Z-d dl.m/Z/m0Z0d dl1m2Z2d dl3m4Z4d dl5m6Z6m7Z7m8Z8m9Z9ddl:m;Z;ejddZ?d|ddZ@d}ddZAd~dd ZBdd!d"ZCdd$d%ZDd&d'ZEd(d)ZFdd+d,ZGdd-d.ZHdd/d0ZIdd1d2ZJd3d4ZKd5d6ZLd7d8ZMdd9d:ZNdd;d<ZOdd=d>ZPd?d@ZQdAdBZRdCdDZSddEdFZTdGdHZUddIdJZVdKdLZWdMdNZXdOdPZYdQdRZZdSdTZ[dUdVZ\dWdXZ]dYdZZ^d[d\Z_d]d^Z`ddadbZaddcddZbdedfZcdgdhZddidjZedkdlZfddmdnZgdodpfdqdrZhdsdtZiddvdwZjddxdyZkddzd{ZldS)z$Utility routines for workflow graphsN) defaultdict)deepcopy)glob)Path)format_exception)sha1)reduce)loggingconfig LooseVersion) indirectoryrelpathfname_presuffix ensure_listget_related_files save_jsonsavepklloadpklwrite_rst_headerwrite_rst_dictwrite_rst_list)str2bool)create_function_from_source)rebase_path_traitsresolve_path_traitsOutputMultiPath isdefined Undefined)BunchInterfaceResult) CommandLine)IdentityInterface) ProvStorepm nipype_nsget_id) signatureznipype.workflowcCs t|dkrt|jjS|S)z Returns the directory name for the given parameterization string as follows: - If the parameterization is longer than 32 characters, then return the SHA-1 hex digest. - Otherwise, return the parameterization unchanged. )lenrencode hexdigest)paramr->/tmp/pip-build-7vycvbft/nipype/nipype/pipeline/engine/utils.py_parameterization_dir6s r/cCs|yt||Wnhttfk rvtjd}|tkrft|d}|jt|WdQRXtj dn tj d|YnXdS)zStore a hashfilerwtNz2Unable to write a particular type to the json filez)Unable to open the file in write mode: %s) rIOError TypeErrorsysexc_infoopen writelinesstrloggerdebugcritical)ZhashfileZ hashed_inputsZerr_typefdr-r-r. save_hashfileBs   r<Fccsx|D]\}}d}d}zpy|j|d}WnZtk r|r>|j}g}|jrdt|jdrd|jjg}|ttj7}dj |}YnXWd|||fVXqWdS)zX A generator that iterates and over a list of ``nodes`` and executes them. N) updatehash traceback ) run Exceptionresultruntimehasattrr>rr3r4join)nodesr=Z stop_firstinodeerrrBr-r-r.nodelist_runnerTs rJc Cst|jddsdS|j}t|dd}|jjdddtdt|d d td |j d |j gtd dd t |j j g}|dkrtjd||jdj|dStjd||tddd t |j j tddd g7}|j}|dkr |dg7}|jdj|dSt|tr*|jt |jn$|rD|jt |j n |dg7}|r|jtddd ttt|j |jd }g}xFt|D]:} t|dd|j| fdd} |jd| | fqW|jt||jdj|dS|jtddd |jj|jj|jj t|jddd} x,d*D]$} t!|j| r*t|j| | | <q*W|jt | t!|jd!r|td"d#d t|jj"g7}t!|jd$r|td%d#d t|jj#g7}t!|jd&r|td'd#d t|jj$g7}t!|jd(r|td)d#d t |jj%g7}|jdj|dS)+zWrite a report file for a node. executionZ create_reportN_reportz report.rstT)exist_okparentszNode: %sr)levelzHierarchy : %sz Exec ID : %szOriginal Inputsz&[Node] Writing pre-exec report to "%s"r?z'[Node] Writing post-exec report to "%s"zExecution InputszExecution OutputsNonezOutputs object was empty.zSubnode reportsZmapflowz_%s%dzsubnode %d : %sz Runtime infoZprevcwdz )hostnamedurationZ working_dirZprev_wdcmdline mem_peak_gb cpu_percentmergedzTerminal outputstdoutzTerminal - standard outputstderrzTerminal - standard errorenviron Environment)rTrUrV)&rr output_dirrparentmkdirrget_print_namerfullname_idrinputs trait_getr8r9Z write_textrEoutputs isinstancerappendZdictcopyr)rgetattr iterfieldrangenamerCrRrScwdrDrWrYrZr[) rHrB is_mapnoderlZ report_filelinesreZnitemsZsubnode_report_filesrGZ subnode_fileZrst_dictpropr-r-r.write_node_reportns         "     rpcCs8|dkrtjd|dSt|||dkr,|jndddS)z+Write a report file for a node - DEPRECATEDpreexecpostexecz [Node] Unknown report type "%s".N)rmrB)rqrr)r8warningrprB)rHZ report_typermr-r-r. write_reports rtc Csz|dkrtjdd}tjj|}tjj|d|}tjd||jdkr`tj dt ||dSy|jj }Wn(t k rtjdt ||dSX|st ||dSi}zt |~xv|D]n}t|j|}t|r|jj|jtr|jj|jj|j|}|||<t|jj|||} t|j|| qWWdQRXt ||Wdx(t|jD]\}} t|j|| qXWXdS)z#Save a result pklz file to ``cwd``.NrKuse_relative_pathszresult_%s.pklzzSaving results file: '%s'z#Storing result file without outputsz5Storing non-traited results, skipping rebase of paths)r getbooleanospathabspathrEr8r9rersrcopyable_trait_namesAttributeErrorr rhrtrait is_trait_typerhandler get_valuersetattrlistitems) rBrlrkZrebaseZ resultsfileZ output_namesZ backup_traitskeyoldvalr-r-r.save_resultfilesB             rTc Cst|}|jst|t|}|rt|ddry|jj}Wn tk r^tj d||SXtj dxnt |j D]^\}}t |rx|jj |jtr|jj |jj|j|}t|jj |||j}t|j||qxW|S)a] Load InterfaceResult file from path. Parameters ---------- results_file : pathlike Path to an existing pickle (``result_.pklz``) created with ``save_resultfile``. Raises ``FileNotFoundError`` if ``results_file`` does not exist. resolve : bool Determines whether relative paths will be resolved to absolute (default is ``True``). Returns ------- result : InterfaceResult A Nipype object containing the runtime, inputs, outputs and other interface information such as a traceback in the case of errors. reNz.Outputs object of loaded result %s is a Bunch.z4Resolving paths in outputs loaded from results file.)rexistsFileNotFoundErrorrrhregetr2r8r9rrrr|r}rr~rrr^r)Z results_fileresolverBreZ trait_namervaluer-r-r.load_resultfile s(   rcCsNg}xD|D]<}t|tr*|jt||q |j|jtjj|d|q W|S)z%Remove temp from a list of file pathsZ _tempinput)rfrrg strip_tempreplacerwrxrE)fileswdoutfr-r-r.r;s    rcCsg}|jjdd}xt|jjD]\}}t|j|}t|r"t|tt fry t |}Wn(t k r|j d|||fYnpXdd|j Dd}|j tj|||kr|djd|d ||d<d |}|j d |j d |||fq"|j d |||fq"W|S)N._z%s.inputs.%s = '%s'cSsg|]}|dkr|qS) __builtins__r-).0rkr-r-r. Ssz!_write_inputs..rrPz %s(z %s_1(z%s_1z,from nipype.utils.functions import getsourcez%s.inputs.%s = getsource(%s)z%s.inputs.%s = %sr)rarrrcrrhrrfr7bytesr RuntimeErrorrg __globals__pickleloads)rHrnnodenamerrrfuncfuncnamer-r-r. _write_inputsFs,   rpythoncCsFddlm}g}|jjdd}|dkrB|j}d|j|jjf}d|j}t|jj } g} xF| j j D]8} t |jd| j d} | dk rf| j| j d | qfWd j| } |jj}t||rd ||| |j|f}nd ||| |f}|||g}|r|d ||g}|jd||jf|jdk r4|jd||jf|jt||S)z'Format a node in a given output syntax.rP)MapNoderrrzfrom %s import %sz # Node: %sN=z, z-%s = MapNode(%s(%s), iterfield=%s, name="%s")z%s = Node(%s(%s), name="%s")z#from collections import OrderedDictz%s.config = %sz%s.iterables = %s)rFrrar interface __module__ __class____name__r'__init__ parametersvaluesrhrkrgrErfrir iterablesextendr)rHformatZinclude_configrrnrkklassZ importlinecommentspecZ filled_argsr,rargs klass_nameZnodedefr-r-r. format_nodedsD         rcCs4|s tj}t|trPi}x2t|jD]"\}}t|r(t|||d||<q(Wnt|tt frg}x(|D] }t|rh|j t|||dqhWt|t rt |}nt|r"t|t t fotj j|r|rtjddrt||d}n|}ntj jtj j||}tj j|s td|n|}ntdj||S)aiConvert paths in data structure to either full paths or relative paths Supports combinations of lists, dicts, tuples, strs Parameters ---------- relative : boolean indicating whether paths should be set relative to the current directory basedir : default os.getcwd() what base directory to use as default )relativebasedirrKru)startzFile %s not foundzObject {} is undefined)rwgetcwdrfdictsortedrr modify_pathsrtuplergr7rrxisfiler rvrryrErr1r2r)objectrrrrrr-r-r.rs4       rcCs|j}t|drj|jjjjd}|jjj}d}t|dkrFd|d}|rV|j|}ndj|j|g|}|r|jd}t|dkrdj|ddd St|dkr|dS|S) zGet the name of the node For example, a node containing an instance of interfaces.fsl.BET would be called nodename.BET.fsl _interfacerrXz.%sz (rPN)) rarDrrrsplitrr)rE)rH simple_formrkpkglistrZ destclasspartsr-r-r.r`s"        r`c Cstjdddl}|j}xd|jD]X}|j|}t|d|d}t|d|d}|rp|j||t|ddq$|j||q$W|S)zSCreate a graph that can be pickled. Ensures that edge info is pickleable. zcreating dot graphrN)rrPconnect)l) r8r9networkxZDiGraphedges get_edge_datar`add_edger7) graphshow_connectinfornxpklgraphedgedatasrcnameZdestnamer-r-r._create_dot_graphs  rc Cs~ddl}ddg}g}x|j|D]}|j}g}x|j|ddD]\}} } x| dD]} t| dttfrv| d} n | dd} | d} d t| }d t| }|jd |jj d d || jj d d |f| |krV|j| qVWqBWdgddt |Ddg}g}xp|j |ddD]^\}} } xP| dD]D} t| dttfrN| d} n | dd} | |kr,|j| q,WqWdgddt |Ddg}d }t |dr|j jjjd }t|dkr|d}d j|jd dd}d|jd d||f}|d|j d d d j||d j|fg7}q"Wxt |D]}|j|q6W|jdt|d}|jdj|WdQRX|S)a Create a dot file with connection info :: digraph structs { node [shape=record]; struct1 [label=" left| middle| right"]; struct2 [label=" one| two"]; struct3 [label="hello\nworld |{ b |{c| d|e}| f}| g | h"]; struct1:f1 -> struct2:f0; struct1:f0 -> struct2:f1; struct1:f2 -> struct3:here; } rNzdigraph structs {znode [shape=record];T)ZnbunchrrrPzin%szout%sz%s:%s:e -> %s:%s:w;rrz{INcSsg|]}dt||fqS)z | %s) _replacefunk)ripr-r-r.rsz'_write_detailed_dot..}z{OUTcSsg|]}dt||fqS)z | %s)r)rZoportr-r-r.r*srrXz{ %s | %s | %s }z%s [label="%s|%s|%s"];r0r?rr)rtopological_sortiternamein_edgesrfr7rrrgrr out_edgesrDrrrrr)rEr5write)r dotfilenamertextrnrZinportsuvdZcdZoutportinportZipstripZopstripZinputstrZoutportsZ outputstrZ srcpackagerZ srchierarchyZ nodenamestrrZfilepr-r-r._write_detailed_dotsl               rcCs$|jddjddjddjddS)Nrrr@-)r)xr-r-r.rLsrcCsBt|ttfst|}|jtjd}tjdd|}|jdd}|S)zgRemove disallowed characters from path Removes: [][ (){}?:<>#!|"';] Replaces: ',' -> '.' z..z[][ (){}?:<>#!|"';]r,r)rfr7rrrwsepresub)Zpathstrr-r-r._get_valid_pathstrQs  rcCs |r t|Sttt|jS)N)synchronize_iterablesrwalkr)r synchronizer-r-r.expand_iterables_srcCs,|rtndd}t|ddt|jDS)zReturn the number of iterable expansion nodes. If synchronize is True, then the count is the maximum number of iterables value lists. Otherwise, the count is the product of the iterables value list sizes. cSs||S)Nr-)ryr-r-r.msz!count_iterables..cSsg|]\}}t|qSr-)r))rrrr-r-r.rnsz#count_iterables..)maxrrr)rropr-r-r.count_iterablesesrc cs|dkr i}|s|jVdS|d|dd}}|\}}xF|D]<}|rV|||<n|||<x t||d||D] } | VqrWqDWdS)aHGenerate all the full paths in a tree, as a dict. Examples -------- >>> from nipype.pipeline.engine.utils import walk >>> iterables = [('a', lambda: [1, 2]), ('b', lambda: [3, 4])] >>> [val['a'] for val in walk(iterables)] [1, 1, 2, 2] >>> [val['b'] for val in walk(iterables)] [3, 4, 3, 4] rNrP)copyr) childrenrOrxZusenameheadtailrkrchildZ child_pathsr-r-r.rqs    rc Csrg}ddt|jD}xRi}x6|D].\}}yt|||<Wq&tk rRYq&Xq&W|rh|j|qPqW|S)aLSynchronize the given iterables in item-wise order. Return: the {field: value} dictionary list Examples -------- >>> from nipype.pipeline.engine.utils import synchronize_iterables >>> iterables = dict(a=lambda: [1, 2], b=lambda: [3, 4]) >>> synced = synchronize_iterables(iterables) >>> synced == [{'a': 1, 'b': 3}, {'a': 2, 'b': 4}] True >>> iterables = dict(a=lambda: [1, 2], b=lambda: [3], c=lambda: [4, 5, 6]) >>> synced = synchronize_iterables(iterables) >>> synced == [{'a': 1, 'b': 3, 'c': 4}, {'a': 2, 'c': 5}, {'c': 6}] True cSsg|]\}}|t|fqSr-)iter)rfieldZfvalsr-r-r.rsz)synchronize_iterables..)rrnext StopIterationrg)rZout_listZiterable_itemsZcur_dictrZ iter_valuesr-r-r.rs  rcCszt|}y||ft|}WnVtk rt}z:|jdjdr`|jdjdr`|jddf|_|WYdd}~XnX|S)Nrz global namezis not definedzUDue to engine constraints all imports have to be done inside each function definition)rr NameErrorr startswithendswith)Zfunction_sourcerZ first_argrZ output_valueer-r-r.evaluate_connect_functions rcCsZddl}i}xH|j|D]:}d||<x,|j|D]}t||||d||<q0WqW|S)NrrP)rrZ predecessorsr)Grlevelsrpredr-r-r. get_levelss"rcCs|j}dd|D}tt|t|kr2tdi} xt|jD]} |j| j| j} xv|jt|| D]`} | d|jkrn| j| jt| j krg| | j| j<| | j| jj | d|j | fqnWqDW|j |t ||} | s|St| }d|tjtj|f}x|t| D]n\}}t|}dd|jD}|j|}t|j|}d}x.zSExecution graph does not have a unique set of node names. Please rerun the workflowrz .%s%%0%ddcSsg|]}|j|jqSr-)rrb)rrr-r-r.rsrz{}_{}_{}zParameterization: paramstr=%sT)rrP)rFr)setrArindexrrbrkeysrgrremove_nodes_fromrnpceillog10 enumeraterrrrr set_inputr8r9rparameterizationadd_nodes_fromadd_edges_fromr)Z supergraphrFsubgraphZnodeidrprefixrZ supernodesidsedgeinforZnidxrZiterable_paramscounttemplaterGparamsZGcZnodeidxrootnodeZparamstrrrrZ path_lengthZ paramlistrHinfor-r-r. _merge_graphss\        rcCsB|j||dd}|s0d|i}|j|||fgn|dj|dS)z"Add a connection between two nodesN)defaultr)rrr)rsrcnodedestnodeZconnection_inforr-r-r._connect_nodes;s r!cCs.x(t|| D]}t|dst||qW|S)zRemove non-join identity nodes from the given graph Iterable nodes are retained if and only if the keep_iterables flag is set to True. joinsource)_identity_nodesrD_remove_identity_node)rkeep_iterablesrHr-r-r._remove_nonjoin_identity_nodesEs r&cs ddl}fdd|j|DS)zReturn the IdentityInterface nodes in the graph The nodes are in topological sort order. The iterable nodes are included if and only if the include_iterables flag is set to True. rNcs.g|]&}t|jtrs&t|ddkr|qS)rN)rfrr"rh)rrH)include_iterablesr-r.r]s z#_identity_nodes..)rr)rr'rr-)r'r.r#Ss r#cCsht||\}}x|j|ddD],\}}}x |dD]\}}||f||<q,WqWxn|j|ddD]\\}} }xP|dD]D\}}t|tr|d} n|} | |krg|| <|| j| ||fqlWqXW||fS)alReturn the given node's input and output ports The return value is the (inputs, outputs) dictionaries. The inputs is a {destination field: (source node, source field)} dictionary. The outputs is a {source field: destination items} dictionary, where each destination item is a (destination node, destination field, source field) tuple. T)rrr)rrrfrrg) rrHr+r,rrrsrcdestrsrcportr-r-r.r(ps   r(cCsNxH|D]@\}}}t|j|}t|tr:t|d|d|}|j||qWdS)ziPropagates the given graph root node output port field connections to the out-edge destination nodes.rPrXN)rhrcrfrrr)rrHrr-r rr.rr-r-r.r*s   r*cCs>x6|D],\}}}||kr||\}} t| trtt|trt| djdd} |djdd} tdj| | |||j||dgid} t|tr| dj| |d|df|fnd| |fgi} |j||dgid} | d| d7<|j||| fgqt|j |}t|tr*t |d|d|}|j ||qWdS) zPropagates the given graph internal node output port field connections to the out-edge source node and in-edge destination nodes.rPz\nrz}Does not support two inline functions in series ('{}' and '{}'), found when connecting {} to {}. Please use a Function node.r)rrXN) rfrr ValueErrorrrrgrrhrcrr)rrHrr-r+r rr.rr0Zsrc_funcZdst_funcrZ old_connectrr-r-r.r)s*  "  r)c*slddly j}Wntk r,j}YnXtjdtddxjD]}|jrNt |qNWt d}t }tjd|x|r&|dtjdfd d jD}i}x~|D]vi}|<g}x2j dD]"\} } } | || j <|j| | fqWx,|D]$\} } j| | tjd | | qWqWjr4j\} t| ttfrd| g} y$tfd d jDWn&tk rtdfYnXtjdi} fdd | D}t|dkr|dnt|tfdd jD}ddtfdd t |jD} n jj} d_tjd| dd |D}dd |D}tdd |D}|sd}n,|d-dkrtd||j|d.d}tjd|fjd|7_tj tdkrj!|}nj!|j}t"||j#j| |j$x|D]|}t%t }x~D]v}xnt |j&D]^}|j }t'|d r||krq`|j(|r`|t|d}t)j*d!|r`||j|q`WqNWx.t |jD]\}}tjd"|t|qWx&t |j+D]}|j,d#d$d%qWt-| j$}fd&d t.|D}xt |jD]\}}xt/|D]\}}||} t0| }!|!d'}"fd(d |"D}#||}$xNt/|"D]B\}%}&|&\}'}(|(|$kr|$|(})|'|)f|"|%<tjd)||(|)qWj1|f|!tjd*|qbWqPWq2Wt }qWx0jD]$}|j2r2d+d t|j2D|_2q2Wtjd,tS)/aZGenerates an expanded graph based on node parameterization Parameterization is controlled using the `iterables` field of the pipeline elements. Thus if there are two nodes with iterables a=[1,2] and b=[3,4] this procedure will generate a graph with sub-graphs parameterized as (a=1,b=3), (a=1,b=4), (a=2,b=3) and (a=2,b=4). rNzPE: expanding iterablesT)r%abcdefghijklmnopqrstuvwxyzzDetected iterable nodes %sz!Expanding the iterable node %s...cs4g|],}t|drj|jkrj|r|qS)r")rDrkr"has_path)rrH)graph_ininoderr-r.rs  z+generate_expanded_graph..z'Excised the %s -> %s join node in-edge.c3s*|]"}|jkrj|r|VqdS)N)rkr3)rrH)r4r5rsrc_namer-r. sz*generate_expanded_graph..zLThe node %s itersource %s was not found among the iterable predecessor nodesz'The node %s has iterable source node %scsg|]}tj|qSr-)rhrc)rr)iter_srcr-r.rsrPcs$g|]\}}|kr||fqSr-r-)rrlookup)rr-r.rscsdfddfS)NrcsdS)NrPr-r-)pairr-r.rszBgenerate_expanded_graph..make_field_func..r-)r:r-)r:r.make_field_funcsz0generate_expanded_graph..make_field_funccsg|] }|qSr-r-)rr:)r;r-r.r sznode: %s iterables: %scSsg|]}|qSr-r-)rsr-r-r.r(scSs g|]}|jrtjd|jqS)z\.(.)I)rbrfindall)rr<r-r-r.r)scSsg|]}|D]}|q qSr-r-)ritemrr-r-r.r*sazz"Too many iterables in the workflowz subnodes:z.%sI2 joinfieldz((\.[a-z](I\.[a-z])?|J)\d+)?z3The join node %s input %s was expanded to %d nodes.cSs|jS)N)rb)rHr-r-r.rhsz)generate_expanded_graph..)rcsg|] }jqSr-)Z_add_join_item_fields)rr)jnoder-r.rnsrcsg|]\}}|jkr|qSr-)rB)rrr)rCr-r.rsz+Qualified the %s -> %s join field %s as %s.zAConnected the join node %s subgraph to the expanded join point %scSsg|] \}}|qSr-r-)rrr,r-r-r.rsz PE: expanding iterables ... donerr)3r dfs_preorderr{Zdfs_preorder_nodesr8r9r&rFr_standardize_iterablesr_iterable_nodesrrrgZ remove_edge itersourcerfr7rrrr1r)rrrrrr rbr __version__rrrrrr rDrr fullmatchrsortrrjrrrr)*r4rDrHZ allprefixesinodesZjnodesZ jedge_dictrZ edges2remover.r/rZ src_fieldsrZ src_valuesZ iter_dictZsubnodesZ prior_prefixZiterable_prefixrZ old_edge_dictZ expansionsZsrc_idrsuffixin_idZin_nodesZiter_cntZ slot_dictsZold_idZin_idxZin_nodeZolddatanewdataZconnectsZ join_fieldsslotsZcon_idxrZ src_fieldZ dest_fieldZ slot_fieldr-)r4r5r8rCrr;rr6r.generate_expanded_graphs                             rPcCsLddl}|j|}dd|D}dd|D}dd|D}|j||S)aReturns the iterable nodes in the given graph and their join dependencies. The nodes are ordered as follows: - nodes without an itersource precede nodes with an itersource - nodes without an itersource are sorted in reverse topological order - nodes with an itersource are sorted in topological order This order implies the following: - every iterable node without an itersource is expanded before any node with an itersource - every iterable node without an itersource is expanded before any of it's predecessor iterable nodes without an itersource - every node with an itersource is expanded before any of it's successor nodes with an itersource Return the iterable nodes list rNcSsg|]}|jdk r|qS)N)r)rrHr-r-r.rsz#_iterable_nodes..cSsg|]}|js|qSr-)rG)rrHr-r-r.rscSsg|]}|jr|qSr-)rG)rrHr-r-r.rs)rrreverse)r4rrFrKZ inodes_no_srcZ inodes_srcr-r-r.rFs rFcs|js dS|j}t|jj|jrXt|dkrX|\}}tfdd|DrXt||}t|t rh|g}t ||t|t r|j sddfdd|D}t |}||_dS) ztConverts the given iterables to a {field: function} dictionary, if necessary, where the function returns a list.NrXc3s$|]}t|ttfo|kVqdS)N)rfr7r)rr>)fieldsr-r.r7sz)_standardize_iterables..csdfddfS)NrcsdS)NrPr-r-)r:r-r.rszA_standardize_iterables..make_field_func..r-)r:r-)r:r.r;sz/_standardize_iterables..make_field_funccsg|] }|qSr-r-)rZ field_value1)r;r-r.rsz*_standardize_iterables..)rrrcrzrr)all_transpose_iterablesrfr_validate_iterablesrrGr)rHrfirstlastZ iter_itemsr-)rRr;r.rEs&     rEcCst|trt|j}n,t|t rDt|t rDtd|j|jfx|D]z}yt|dkrjtd|jWn4t k r}zt d|j|fWYdd}~XnX|\}}||krJtd|j|fqJWdS)z Raise TypeError if an iterables member is not iterable. Raise ValueError if an iterables member is not a (field, values) pair. Raise ValueError if an iterable field is not in the inputs. z7The %s iterables type is not a list or a dictionary: %srXz0The %s iterables is not a [(field, values)] listz)A %s iterables member is not iterable: %sNz*The %s iterables field is unrecognized: %s) rfrrrrr1rkrr)r2)rHrrRr>rrrr-r-r.rUs$   "rUcCst|trtdd|D}xZt|jD]J\}}x@|D]8}x2t|D]&\}}|dk rF||||j|qFWq8Wq*Wt|jStt|ddt|DS)a Converts the given fields and tuple values into a standardized iterables value. If the input values is a synchronize iterables dictionary, then the result is a (field, {key: values}) list. Otherwise, the result is a list of (field: value list) pairs. cSsg|]}|ttfqSr-)rr)rrr-r-r.rsz(_transpose_iterables..NcSsg|]}ddt|DqS)cSsg|]}|dk r|qS)Nr-)rrr-r-r.r*sz3_transpose_iterables...)r)rZ transposer-r-r.r*s)rfrrrrrgzip)rRrZ transposedrZtuplesZkvalsidxrr-r-r.rTs  " rT graph.dotpngcCs6ddl}t|} |r(t| } tjdn tjd|dkrBtj}tj|ddt|dd|d } t | | t | |d \} } | dk r| j j rtj d | j jt| ||} t|d d|d }|jjj| |t ||d \}} | dk r| j j rtj d | j j|r(|j| d d}|j| ||r(|j| ||r2|S| S)a*Displays the graph layout of the pipeline This function requires that pygraphviz and matplotlib are available on the system. Parameters ---------- show : boolean Indicate whether to generate pygraphviz output fromn networkx. default [False] use_execgraph : boolean Indicates whether to use the specification graph or the execution graph. default [False] show_connectioninfo : boolean Indicates whether to show the edge data on the graph. This makes the graph rather cluttered. default [False] rNzusing execgraphzusing input graphT)rMz _detailed.dotF)rLZuse_extnewpath) format_extz dot2png: %sz.dotdot)prog)rrrPr8r9rwrmakedirsrr_run_dotrC returncodersrZrZdrawingZnx_pydotZ write_dotZgraphviz_layoutZdrawZdraw_networkx_edge_labels)r4base_dirshowZ use_execgraphrrrrrrZout_dotZoutfnameresrZ simple_dotZ simplefnameposr-r-r. export_graph1s8      rgcCsVyt||d\}}Wn<tk rP}z dt|kr.r)rfrrrrr walk_outputsrrr7rrwrxislinkrrpisdir)rrrrr-r-r.rrs    rrccs<x6tj|D](\}}}x|D]}tjj||VqWq WdS)N)rwrrxrE)rlrxrrrr-r-r. walk_filess ruc s|sdSt|jj}|r0t|ddr0|}g}|j} x|D]} |jt| | qBWdd|D} t|ddrg} |j} | jt| | dd| D7} x$dD]}| jttjj ||qW|r| jt |dd|D}|r|jt |x&dD]}|jttjj ||qWg}x| D]}|jt |q2W|} t j ddj | t j ddj |g}t|ddrxt|D]D| kr|s|jn"tfdd|Ds|jqWnlt|ddsHg} |j} | jt| dd| D} x0t|D]$| kr | kr |jq Wt j ddj |x|D]tjq`Wx(|jD]}||kr~t||tq~W|S)zDRemoves all files not needed for further analysis from the directoryNrKZremove_unnecessary_outputscSsg|]\}}|dkr|qS)rr-)rrxtyper-r-r.rsz+clean_working_directory..Z keep_inputscSsg|]\}}|dkr|qS)rr-)rrxrvr-r-r.rs _0x*.json provenance.* pyscript*.m pyjobs*.mat command.txt result*.pklz _inputs.pklz _node.pklz.proc-*cSsg|]\}}|dkr|qS)rr-)rrxrvr-r-r.rs_nipyperLzNeeded files: %s;zNeeded dirs: %scsg|]}j|qSr-)r)rZdname)rr-r.rscSsg|]\}}|dkr|qS)rr-)rrxrvr-r-r.rszRemoving files: %s) rwrxryrzr{r|r}r~r)rrL)rrdr rrrrrrwrxrErrr8r9rurganyremoverzrr)rerlrcZneeded_outputsr Z files2keepZ dirs2keepZoutputs_to_keepZ output_filesZ outputdictoutputZ needed_filesZ input_filesZ inputdictextraZ needed_dirstemprqZ files2removerr-)rr.clean_working_directorys|       rcCs|S)Nr-)rrr-r-r.rsrcCsnt|ts|||St|}|dkr(|Sx@t|jD]0\}}||kr^t||||d||<q6|||<q6W|S)a Merges two dictionaries, non-destructively, combining values on duplicate keys as defined by the optional merge function. The default behavior replaces the values in d1 with corresponding values in d2. (There is no other generally applicable merge strategy, but often you'll have homogeneous types in your dicts, so specifying a merge technique can be valuable.) Examples: >>> d1 = {'a': 1, 'c': 3, 'b': 2} >>> d2 = merge_dict(d1, d1) >>> len(d2) 3 >>> [d2[k] for k in ['a', 'b', 'c']] [1, 2, 3] >>> d3 = merge_dict(d1, d1, lambda x,y: x+y) >>> len(d3) 3 >>> [d3[k] for k in ['a', 'b', 'c']] [2, 4, 6] N)merge)rfrrr merge_dict)Zd1Zd2rrBkrr-r-r.rs   rcCs x|jD]}|j|q W|S)N) get_recordsZ _add_record)Zg1Zg2Zrecr-r-r. merge_bundles"srrScCs|stjjtjd}t}g}|j}x|D]}|j}|jjj }|j \} } } } t j dt |t j ddj||jft d| i} |jjtdd| } t|jtr| jt j dt dixt|jD]\} }t|j| |id}|jr | t|jkr |j| |_|jrdxNt|jjD]<\}} t|j|}t|r$| t|kr$|| |j|<q$Wtj|}t j|j td }|jj!||jj"|j#d t j$id }|jj%|| qWnz| jt j dt d i|j&r|j&}n tj|}t j|j td }|jj!||jj"|j#d t j$id }|jj%|| |j'| q0WxNt|j(D]>\} }|jj)|t|j*|d |t|j*|ddqRW|j+||d|jS)zWrite W3C PROV Model JSON fileZworkflow_provenancervlabelrhashvalNr)re) identifierz prov:type)Zother_attributesNoderPr)Zstarter)r),rwrxrErr#rFrBrrrZ hash_existsr$ZPROVr%rkgZactivityr&rfrCrZadd_attributesrr rcr)rerrhrZ add_resultsZ ProvBundlerZ add_bundleentityrZ PROV_BUNDLEZwasGeneratedByZ provenancergrZ wasStartedByr Zwrite_provenance)rrqrZpsZ processesrFrHrB classnamerrattrsprocessrYrCZ subresultrrZsub_docZ sub_bundleZ bundle_entityZprov_docZ result_bundlerr-r-r.write_workflow_prov(s`         rcCs&ddl}tjdd|}|s,tjjtjd}|dkrFttjddd}ggggggggd}|rtjj|rt |d }|j |}WdQRXxlt |j D]Z\}}|j }|jjj} d } |jrd jd d |jD} y |jj} Wn&tk rtjd|| wYnXt| ts| g} xt | D]\} } yt| jd}Wn2tk rrtjd|| dt| w&YnXx$dD]}||| j|7<qzW|d| g|7<|d|g|7<|d| g|7<|d| g|7<q&WqWt |d}|j||ddWdQRX|S)z Generate a JSON file with profiling traces that can be loaded in a pandas DataFrame or processed with JavaScript like D3.js rNZ monitoringZ summary_filezresource_monitor.jsonZsummary_appendtrue)timerkrrss_GiBvms_GiBcpusmapnoderrrrcSsg|]}dj|qS)z{})r)rpr-r-r.rsz,write_workflow_resources..z8Could not access runtime info for node %s (%s interface)rzGCould not retrieve profiling information for node "%s" (mapflow %d/%d).rPrrrrrkrrwF) ensure_ascii)rrrr)Z simplejsonr rrwrxrErrrr5loadrrFrarrrrrBrCrAr8rsrfrr)Z prof_dictr{dump)rrqrgjsonZbig_dictZrsfrrHrrrZrt_listZsubidxrCZnsamplesrr-r-r.write_workflow_resourcesksh         rc Csddl}t|j|}|s"|dfStjdg}g}d}|j}|j|j|j|j |j |}x|D]} |d7}g} x| D]} | j |j | qW|j tj|tj| tj| jx| D]} |j| qW|j |gt| qlW||fS)z9Returns a depth first sorted order if depth_first is TruerNzPerforming depth first searchrP)rrrr8r9ZGraphrrFrrZconnected_componentsrgr rr arrayZargsorttolistrr)) rZ depth_firstrZnodesortrFgroupsgroupr componentsdescindicesrHr-r-r.rs.    $ r)FF)NF)NF)N)T)rF)TN)T)FT)F)F)rNT)F)F)NFFFrZr[T)r[)NN)NrS)NN)F)m__doc__rwr3r collectionsrrrrrZpathlibrr>rhashlibr functoolsrnumpyr rr r r Zutils.filemanipr rrrrrrrrrrZ utils.miscrZutils.functionsrZ interfaces.base.traits_extensionrrrrrZinterfaces.base.supportrr Zinterfaces.baser!Zinterfaces.utilityr"Zutils.provenancer#r$r%r&inspectr' getLoggerr8r/r<rJrprtrrrrrrr`rrrrrrrrrrrr!r&r#r$r(r*r)rPrFrErUrTrgrirarprrrurrrrrrr-r-r-r.s       4         e - .  . .  ]  !$ h   "u!&!  @    F' C S