3
de                 @   sf   d Z ddlZddlZddlmZ ddlmZ ddlmZ ddl	m
Z
mZ ejd	ZG d
d de
ZdS )zT
Created on Aug 2, 2013

@author: chadcumba

Parallel workflow execution with SLURM
    N)sleep   )logging)CommandLine   )SGELikeBatchManagerBaseloggerznipype.interfacec                   s0   e Zd ZdZ fddZdd Zdd Z  ZS )SLURMPlugina  
    Execute using SLURM

    The plugin_args input to run can be used to control the SLURM execution.
    Currently supported options are:

    - template : template to use for batch job submission

    - sbatch_args: arguments to pass prepend to the sbatch call


    c          
      s  d}d| _ d| _|| _d | _d| _d|kr|d rd|d krL|d d | _ d|d krf|d d | _d|d kr|d d | _d|d kr|d d | _tjj| jrt| j}|j	 | _W d Q R X d	|d kr|d d	 | _i | _
tt| j| jf| d S )
Nz#!/bin/bash   zSubmitted batch job ([0-9]*)plugin_argsZretry_timeoutZ	max_triesZjobid_retemplatesbatch_args)_retry_timeout
_max_tries	_template_sbatch_args	_jobid_reospathisfileopenread_pendingsuperr	   __init__)selfkwargsr   f)	__class__ ?/tmp/pip-build-7vycvbft/nipype/nipype/pipeline/plugins/slurm.pyr   !   s,    zSLURMPlugin.__init__c                s   y8t ddjdd| gdddj }|jjjt|dkS  tk
r   z:t fd	d
dD rpt	j
d| dS dt kr dS d   X nX d S )NZsqueue z-jz%sF	allatonce)argsresource_monitorterminal_outputr   c             3   s   | ]}|t  kV  qd S )N)str).0ss)er   r    	<genexpr>G   s   z*SLURMPlugin._is_pending.<locals>.<genexpr>Socket timed outnot available at the momentzOSLURM timeout encountered while checking job status, treating job %d as pendingTzInvalid job id)r+   r,   )r   joinrunruntimestdoutfindr&   RuntimeErroranyr   warning)r   taskidresr   )r)   r    _is_pending<   s$    

zSLURMPlugin._is_pendingc             C   s,  t dttjddd}tjj|}d}| jr2| j}d|jkrnd|jkr\|jd r\|jd }n|d|jd  7 }d	|krd
|tjj|df }d|krd|tjj|df }|j	rdjttjd |j	|j
f}ndjttjd |j
f}|jd}|j  dj|}d|||f |j_tj}tjtjd d}	xy|j }
W nj tk
r } zL|	| jk rt|	d7 }	t| j n&tj| tdjd|j
 t|fW Y dd}~X nX P q4W tjdj|j tj| dd |
jj jdD }t!t"j#| j$|d j% d }|j& | j'|< tjd||j
f  |S )z
        This is more or less the _submit_batchtask from sge.py with flipped
        variable names, different command line switches, and different output
        formatting/processing
        ZsbatchFr"   )environr$   r%    r   	overwriter!   z-oz%s -o %szslurm-%j.outz-ez%s -e %s.LOGNAMEz%s -J %s %sCRITICALr   r   
z(Could not submit sbatch task for node %sNzRan command ({0})c             S   s   g | ]}|r|qS r   r   )r'   liner   r   r    
<listcomp>   s    z1SLURMPlugin._submit_batchtask.<locals>.<listcomp>z%submitted sbatch task: %d for node %sr-   )(r   dictr   r9   r   dirnamer   r   r.   Z
_hierarchy_idsplitreverseinputsr#   ifloggerlevelsetLevelr   getLevelNamer/   	Exceptionr   r   r   r3   r&   r   debugformatZcmdliner0   r1   intrematchr   groups
output_dirr   )r   Z
scriptfilenodecmdr   r   ZjobnameZjobnameitemsZoldleveltriesresultr)   linesr6   r   r   r    _submit_batchtaskU   s`    
 


 
zSLURMPlugin._submit_batchtask)__name__
__module____qualname____doc__r   r8   rY   __classcell__r   r   )r   r    r	      s   r	   )r]   r   rP   timer   r:   r   Zinterfaces.baser   baser   r   	getLoggerrH   r	   r   r   r   r    <module>   s   
