[FIX] Encoding problems in SGE plugin

.dockerignore

@@ -1,5 +1,6 @@
 .git/
 *.pyc
+.eggs
 *.egg-info
 __pycache__
 docker/nipype_*

docker/nipype_test/Dockerfile_sge

@@ -0,0 +1,94 @@
+# Copyright (c) 2016, The developers of nipype
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of crn_base nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+FROM gawbul/docker-sge:latest
+MAINTAINER Nipype developers
+
+# Preparations
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get -y update && \
+    apt-get install -y curl \
+                       git \
+                       xvfb \
+                       bzip2 \
+                       unzip \
+                       apt-utils \
+                       fusefat \
+                       graphviz \
+                       make \
+                       ruby && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+WORKDIR /opt
+RUN wget  --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q https://2a353b13e8d2d9ac21ce543b7064482f771ce658.googledrive.com/host/0BxI12kyv2olZVFhUcGVpYWF3R3c/FSL-5.0.9-centos6_64.tar.gz && \
+    tar xzf FSL-5.0.9-centos6_64.tar.gz -C /opt/
+
+# Set FSL environment variables
+ENV FSLDIR=/opt/fsl \
+    FSLOUTPUTTYPE=NIFTI_GZ \
+    FSLMULTIFILEQUIT=TRUE \
+    POSSUMDIR=/opt/fsl \
+    FSLTCLSH=/usr/bin/tclsh \
+    FSLWISH=/usr/bin/wish \
+    PATH=/opt/fsl/bin:$PATH \
+    LD_LIBRARY_PATH=/opt/fsl/lib:$LD_LIBRARY_PATH
+
+# Install fake-S3
+ENV GEM_HOME /usr/local/bundle
+ENV BUNDLE_PATH="$GEM_HOME" \
+    BUNDLE_BIN="$GEM_HOME/bin" \
+    BUNDLE_SILENCE_ROOT_WARNING=1 \
+    BUNDLE_APP_CONFIG="$GEM_HOME"
+ENV PATH $BUNDLE_BIN:$PATH
+RUN mkdir -p "$GEM_HOME" "$BUNDLE_BIN" && \
+    chmod 777 "$GEM_HOME" "$BUNDLE_BIN"
+
+RUN gem install fakes3
+
+# # Install Matlab MCR: from the good old install_spm_mcr.sh of @chrisfilo
+# RUN echo "destinationFolder=/opt/mcr" > mcr_options.txt && \
+#     echo "agreeToLicense=yes" >> mcr_options.txt && \
+#     echo "outputFile=/tmp/matlabinstall_log" >> mcr_options.txt && \
+#     echo "mode=silent" >> mcr_options.txt && \
+#     mkdir -p matlab_installer && \
+#     curl -sSL http://www.mathworks.com/supportfiles/downloads/R2015a/deployment_files/R2015a/installers/glnxa64/MCR_R2015a_glnxa64_installer.zip \
+#          -o matlab_installer/installer.zip && \
+#     unzip matlab_installer/installer.zip -d matlab_installer/ && \
+#     matlab_installer/install -inputFile mcr_options.txt && \
+#     rm -rf matlab_installer mcr_options.txt
+# 
+# # Install SPM
+# RUN curl -sSL http://www.fil.ion.ucl.ac.uk/spm/download/restricted/utopia/dev/spm12_r6472_Linux_R2015a.zip -o spm12.zip && \
+#     unzip spm12.zip && \
+#     rm -rf spm12.zip
+# 
+# ENV MATLABCMD="/opt/mcr/v85/toolbox/matlab" \
+#     SPMMCRCMD="/opt/spm12/run_spm12.sh /opt/mcr/v85/ script" \
+#     FORCE_SPMMCR=1
+
+CMD ["/bin/bash"]

docker/nipype_test/Dockerfile_sge_py27

@@ -0,0 +1,74 @@
+# Copyright (c) 2016, The developers of nipype
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of crn_base nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+FROM nipype/nipype_test:sge-base-0.0.2
+MAINTAINER The nipype developers https://github.com/nipy/nipype
+
+COPY docker/circleci/run_* /usr/bin/
+RUN chmod +x /usr/bin/run_*
+
+WORKDIR /home/sgeadmin/
+
+RUN mkdir -p /home/sgeadmin/src/nipype && \
+    chown sgeadmin:sgeadmin -R /home/sgeadmin/*
+
+USER sgeadmin
+# Set FSL environment variables
+ENV FSLDIR=/opt/fsl \
+    FSLOUTPUTTYPE=NIFTI_GZ \
+    FSLMULTIFILEQUIT=TRUE \
+    POSSUMDIR=/opt/fsl \
+    FSLTCLSH=/usr/bin/tclsh \
+    FSLWISH=/usr/bin/wish \
+    PATH=/opt/fsl/bin:$PATH \
+    LD_LIBRARY_PATH=/opt/fsl/lib:$LD_LIBRARY_PATH
+
+RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh && \
+    /bin/bash Miniconda2-latest-Linux-x86_64.sh -b -p /home/sgeadmin/.conda && \
+    rm Miniconda2-latest-Linux-x86_64.sh
+ENV PATH=/home/sgeadmin/.conda/bin:$PATH
+RUN conda install -y numpy scipy
+
+RUN echo 'export PATH=/home/sgeadmin/.conda/bin:$PATH' >> /home/sgeadmin/.bashrc && \
+    echo 'export FSLDIR=/opt/fsl' >> /home/sgeadmin/.bashrc  && \
+    echo 'source ${FSLDIR}/etc/fslconf/fsl.sh' >> /home/sgeadmin/.bashrc 
+
+COPY . /home/sgeadmin/src/nipype
+USER root
+RUN chown sgeadmin:sgeadmin -R /home/sgeadmin/src/nipype/*
+
+USER sgeadmin
+WORKDIR /home/sgeadmin/src/nipype
+RUN pip install --no-cache-dir -r requirements.txt && \
+    pip install --no-cache-dir -e .
+
+ENV SHELL /bin/bash
+USER root
+
+
+CMD ["/bin/bash"]

nipype/interfaces/traits_extension.py

@@ -33,7 +33,7 @@
 from traits.api import BaseUnicode
 from traits.api import Unicode
 
-DictStrStr = traits.Dict(str, (bytes, str))
+DictStrStr = traits.Dict((bytes, str), (bytes, str))
 Str = Unicode
 
 class BaseFile(BaseUnicode):

nipype/pipeline/plugins/sge.py

@@ -3,12 +3,11 @@
 """
 from __future__ import print_function, division, unicode_literals, absolute_import
 
-from builtins import object
+from builtins import object, str, bytes
 
 import os
 import pwd
 import re
-import subprocess
 import time
 
 import xml.dom.minidom
@@ -18,13 +17,13 @@
 from ...interfaces.base import CommandLine
 from .base import (SGELikeBatchManagerBase, logger, iflogger, logging)
 
-DEBUGGING_PREFIX = str(int(random.uniform(100, 999)))
+DEBUGGING_PREFIX = '{0:d}'.format(int(random.uniform(100, 999)))
 
 
 def sge_debug_print(message):
     """  Needed for debugging on big jobs.  Once this is fully vetted, it can be removed.
     """
-    logger.debug(DEBUGGING_PREFIX + " " + "=!" * 3 + "  " + message)
+    logger.debug('%s ' + '=!' * 3 + '  %s', DEBUGGING_PREFIX, message)
     # print DEBUGGING_PREFIX + " " + "=!" * 3 + "  " + message
 
 
@@ -40,8 +39,7 @@ def __init__(self, job_num, job_queue_state, job_time, job_queue_name, job_slots
         self._job_num = int(
             job_num)      # The primary unique identifier for this job, must be an integer!
         # self._jobOwn  = None           # Who owns this job
-        self._job_queue_state = str(
-            job_queue_state)     # ["running","zombie",...??]
+        self._job_queue_state = '%s' % job_queue_state     # ["running","zombie",...??]
         # self._jobActionState = str(jobActionState)  # ['r','qw','S',...??]
         self._job_time = job_time               # The job start time
         self._job_info_creation_time = time.time(
@@ -51,12 +49,10 @@ def __init__(self, job_num, job_queue_state, job_time, job_queue_name, job_slots
         self._qsub_command_line = qsub_command_line
 
     def __repr__(self):
-        return str(self._job_num).ljust(8) \
-            + str(self._job_queue_state).ljust(12) \
-            + str(self._job_slots).ljust(3) \
-            + time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(self._job_time)).ljust(20) \
-            + str(self._job_queue_name).ljust(8) \
-            + str(self._qsub_command_line)
+        return '{:<8d}{:12}{:<3d}{:20}{:8}{}'.format(
+            self._job_num, self._queue_state, self._job_slots,
+            time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(self._job_time)),
+            self._job_queue_name, self._qsub_command_line)
 
     def is_initializing(self):
         return self._job_queue_state == "initializing"
@@ -144,18 +140,19 @@ def _qacct_verified_complete(taskid):
                         "CONTACTING qacct for finished jobs, "
                         "{0}: {1}".format(time.time(), "Verifying Completion"))
 
-        this_command = 'qacct'
+        cmd = CommandLine('qacct', environ=dict(os.environ),
+                          terminal_output='allatonce')
+
         qacct_retries = 10
         is_complete = False
         while qacct_retries > 0:
             qacct_retries -= 1
             try:
-                proc = subprocess.Popen(
-                    [this_command, '-o', pwd.getpwuid(os.getuid())[0], '-j', str(taskid)],
-                    stdout=subprocess.PIPE,
-                    stderr=subprocess.PIPE)
-                qacct_result, _ = proc.communicate()
-                if qacct_result.find(str(taskid)):
+                strtaskid = '{0:d}'.format(int(taskid))
+                cmd.inputs.args = '-o {} -j {}'.format(pwd.getpwuid(os.getuid())[0], strtaskid)
+                qacct_result = cmd.run().runtime.stdout
+
+                if qacct_result.find(strtaskid):
                     is_complete = True
                 sge_debug_print(
                     "NOTE: qacct for jobs\n{0}".format(qacct_result))
@@ -164,6 +161,7 @@ def _qacct_verified_complete(taskid):
                 sge_debug_print("NOTE: qacct call failed")
                 time.sleep(5)
                 pass
+
         return is_complete
 
     def _parse_qstat_job_list(self, xml_job_list):
@@ -248,15 +246,15 @@ def _run_qstat(self, reason_for_qstat, force_instant=True):
         else:
             this_command = self._qstat_cached_executable
 
+        cmd = CommandLine(this_command, environ=dict(os.environ),
+                          terminal_output='allatonce')
+
         qstat_retries = 10
         while qstat_retries > 0:
             qstat_retries -= 1
             try:
-                proc = subprocess.Popen(
-                    [this_command, '-u', pwd.getpwuid(os.getuid())[0], '-xml', '-s', 'psrz'],
-                    stdout=subprocess.PIPE,
-                    stderr=subprocess.PIPE)
-                qstat_xml_result, _ = proc.communicate()
+                cmd.inputs.args = '-u {} -xml -s psrz'.format(pwd.getpwuid(os.getuid())[0])
+                qstat_xml_result = cmd.run().runtime.stdout
                 dom = xml.dom.minidom.parseString(qstat_xml_result)
                 jobs = dom.getElementsByTagName('job_info')
                 run = jobs[0]
@@ -276,7 +274,7 @@ def _run_qstat(self, reason_for_qstat, force_instant=True):
     def print_dictionary(self):
         """For debugging"""
         for vv in list(self._task_dictionary.values()):
-            sge_debug_print(str(vv))
+            sge_debug_print('%s' % vv)
 
     def is_job_pending(self, task_id):
         task_id = int(task_id)  # Ensure that it is an integer
@@ -398,7 +396,7 @@ def _submit_batchtask(self, scriptfile, node):
         oldlevel = iflogger.level
         iflogger.setLevel(logging.getLevelName('CRITICAL'))
         tries = 0
-        result = list()
+        result = None
         while True:
             try:
                 result = cmd.run()
@@ -409,9 +407,8 @@ def _submit_batchtask(self, scriptfile, node):
                         self._retry_timeout)  # sleep 2 seconds and try again.
                 else:
                     iflogger.setLevel(oldlevel)
-                    raise RuntimeError('\n'.join((('Could not submit sge task'
-                                                   ' for node %s') % node._id,
-                                                  str(e))))
+                    raise RuntimeError(
+                        'Could not submit sge task for node %s\n%s' % (node._id, e))
             else:
                 break
         iflogger.setLevel(oldlevel)
@@ -421,6 +418,6 @@ def _submit_batchtask(self, scriptfile, node):
                               lines[-1]).groups()[0])
         self._pending[taskid] = node.output_dir()
         self._refQstatSubstitute.add_startup_job(taskid, cmd.cmdline)
-        logger.debug('submitted sge task: %d for node %s with %s' %
-                     (taskid, node._id, cmd.cmdline))
+        logger.debug('submitted sge task: %d for node %s with %s',
+                     taskid, node._id, cmd.cmdline)
         return taskid