[RF] Detachable run_command process

doc/devel/interface_specs.rst

@@ -12,8 +12,8 @@ In case of trouble, we encourage you to post on `NeuroStars <https://neurostars.
 NeuroStars.org is a platform similar to StackOverflow but dedicated to neuroinformatics.
 You can also post on the nipype developers mailing list: http://mail.python.org/mailman/listinfo/neuroimaging.
 As we are sharing a mailing list with the nipy community, please add ``[nipype]`` to the message title.
-Alternatively, you're welcome to chat with us in the Nipype 
-`Gitter <https://gitter.im/nipy/nipype>`_ channel or in the 
+Alternatively, you're welcome to chat with us in the Nipype
+`Gitter <https://gitter.im/nipy/nipype>`_ channel or in the
 BrainHack `Slack <https://brainhack.slack.com/messages/C1FR76RAL>`_ channel.
 (Click `here <https://brainhack-slack-invite.herokuapp.com>`_ to join the Slack workspace.)
 
@@ -163,62 +163,28 @@ Controlling outputs to terminal
 
 It is very likely that the software wrapped within the interface writes
 to the standard output or the standard error of the terminal.
-Interfaces provide a means to access and retrieve these outputs, by
-using the ``terminal_output`` attribute: ::
+Interfaces redirect both streams to logfiles under the interface's working
+directory (by default: ``.nipype.out`` and ``.nipype.err``, respectively) ::
 
   import nipype.interfaces.fsl as fsl
   mybet = fsl.BET(from_file='bet-settings.json')
-  mybet.terminal_output = 'file_split'
 
-In the example, the ``terminal_output = 'file_split'`` will redirect the
-standard output and the standard error to split files (called
-``stdout.nipype`` and ``stderr.nipype`` respectively).
-The possible values for ``terminal_output`` are:
-
-*file*
-    Redirects both standard output and standard error to the same file
-    called ``output.nipype``.
-    Messages from both streams will be overlapped as they arrive to
-    the file.
-
-*file_split*
-    Redirects the output streams separately, to ``stdout.nipype``
-    and ``stderr.nipype`` respectively, as described in the example.
-
-*file_stdout*
-    Only the standard output will be redirected to ``stdout.nipype``
-    and the standard error will be discarded.
-
-*file_stderr*
-    Only the standard error will be redirected to ``stderr.nipype``
-    and the standard output will be discarded.
+By default, the contents of both logs are copied to the standard output
+and error streams.
 
 *stream*
-    Both output streams are redirected to the current logger printing
-    their messages interleaved and immediately to the terminal.
-
-*allatonce*
-    Both output streams will be forwarded to a buffer and stored
-    separately in the `runtime` object that the `run()` method returns.
-    No files are written nor streams printed out to terminal.
+    Both standard output and standard error are copied to the
+    terminal.
 
-*none*
-    Both outputs are discarded
 
-In all cases, except for the ``'none'`` setting of ``terminal_output``,
-the ``run()`` method will return a "runtime" object that will contain
-the streams in the corresponding properties (``runtime.stdout``
-for the standard output, ``runtime.stderr`` for the standard error, and
-``runtime.merged`` for both when streams are mixed, eg. when using the
-*file* option). ::
+The ``runtime`` object will keep the location of both logging files:
 
   import nipype.interfaces.fsl as fsl
   mybet = fsl.BET(from_file='bet-settings.json')
-  mybet.terminal_output = 'file_split'
   ...
   result = mybet.run()
   result.runtime.stdout
-  ' ... captured standard output ...'
+  '/path/to/interface/cwd/.nipype.out'
 
 
 

doc/devel/matlab_example2.py

@@ -44,7 +44,9 @@ def run(self, **inputs):
         # Inject your script
         self.inputs.script = self._my_script()
         results = super(MatlabCommand, self).run(**inputs)
-        stdout = results.runtime.stdout
+
+        with open(results.runtime.stdout, 'rt') as f:
+            stdout = f.read()
         # Attach stdout to outputs to access matlab results
         results.outputs.matlab_output = stdout
         return results

examples/nipype_tutorial.ipynb

@@ -635,7 +635,7 @@
       "convert = MRIConvert(in_file='../ds107/sub001/BOLD/task001_run001/bold.nii.gz',\n",
       "                     out_file='ds107.nii')\n",
       "print(convert.cmdline)\n",
-      "results = convert.run(terminal_output='none')  # allatonce, stream (default), file"
+      "results = convert.run(terminal_output='default')  # default, stream"
      ],
      "language": "python",
      "metadata": {},

examples/rsfmri_vol_surface_preprocessing.py

@@ -49,7 +49,7 @@
 import os
 
 from nipype.interfaces.base import CommandLine
-CommandLine.set_default_terminal_output('allatonce')
+CommandLine.set_default_terminal_output('default')
 
 from dicom import read_file
 

examples/rsfmri_vol_surface_preprocessing_nipy.py

@@ -51,7 +51,7 @@
 import os
 
 from nipype.interfaces.base import CommandLine
-CommandLine.set_default_terminal_output('allatonce')
+CommandLine.set_default_terminal_output('default')
 
 # https://github.com/moloney/dcmstack
 from dcmstack.extract import default_extractor

nipype/info.py

@@ -107,7 +107,7 @@ def get_nipype_gitversion():
 SCIPY_MIN_VERSION = '0.14'
 TRAITS_MIN_VERSION = '4.6'
 DATEUTIL_MIN_VERSION = '2.2'
-PYTEST_MIN_VERSION = '3.0'
+PYTEST_MIN_VERSION = '3.4'
 FUTURE_MIN_VERSION = '0.16.0'
 SIMPLEJSON_MIN_VERSION = '3.8.0'
 PROV_VERSION = '1.5.2'
@@ -157,7 +157,12 @@ def get_nipype_gitversion():
 if sys.version_info <= (3, 4):
     REQUIRES.append('configparser')
 
-TESTS_REQUIRES = ['pytest-cov', 'codecov', 'pytest-env', 'coverage<5']
+TESTS_REQUIRES = [
+    'pytest-cov',
+    'pytest-env',
+    'codecov',
+    'coverage<5',
+]
 
 EXTRA_REQUIRES = {
     'doc': ['Sphinx>=1.4', 'numpydoc', 'matplotlib', 'pydotplus', 'pydot>=1.2.3'],

nipype/interfaces/afni/base.py

@@ -4,15 +4,14 @@
 """Provide interface to AFNI commands."""
 from __future__ import (print_function, division, unicode_literals,
                         absolute_import)
-from builtins import object, str
 from future.utils import raise_from
 
 import os
 from sys import platform
 from distutils import spawn
 
 from ... import logging, LooseVersion
-from ...utils.filemanip import split_filename, fname_presuffix
+from ...utils.filemanip import split_filename, fname_presuffix, which
 
 from ..base import (CommandLine, traits, CommandLineInputSpec, isdefined, File,
                     TraitedSpec, PackageInfo)
@@ -85,16 +84,12 @@ def standard_image(img_name):
         '''Grab an image from the standard location.
 
         Could be made more fancy to allow for more relocatability'''
-        clout = CommandLine(
-            'which afni',
-            ignore_exception=True,
-            resource_monitor=False,
-            terminal_output='allatonce').run()
-        if clout.runtime.returncode is not 0:
+
+        afni_path = which('afni')
+        if not afni_path:
             return None
 
-        out = clout.runtime.stdout
-        basedir = os.path.split(out)[0]
+        basedir = os.path.split(afni_path)[0]
         return os.path.join(basedir, img_name)
 
 
@@ -104,10 +99,11 @@ class AFNICommandBase(CommandLine):
     See http://afni.nimh.nih.gov/afni/community/board/read.php?1,145346,145347#msg-145347
     """
 
-    def _run_interface(self, runtime):
+    def _run_interface(self, runtime, correct_return_codes=(0, )):
         if platform == 'darwin':
             runtime.environ['DYLD_FALLBACK_LIBRARY_PATH'] = '/usr/local/afni/'
-        return super(AFNICommandBase, self)._run_interface(runtime)
+        return super(AFNICommandBase, self)._run_interface(
+            runtime, correct_return_codes=correct_return_codes)
 
 
 class AFNICommandInputSpec(CommandLineInputSpec):

nipype/interfaces/afni/preprocess.py

@@ -1013,7 +1013,9 @@ def aggregate_outputs(self, runtime=None, needed_outputs=None):
                 return self.run().outputs
         else:
             clip_val = []
-            for line in runtime.stdout.split('\n'):
+            with open(runtime.stdout) as f:
+                stdout = f.read()
+            for line in stdout.splitlines():
                 if line:
                     values = line.split()
                     if len(values) > 1:
@@ -1688,27 +1690,18 @@ class OutlierCount(CommandLine):
     _cmd = '3dToutcount'
     input_spec = OutlierCountInputSpec
     output_spec = OutlierCountOutputSpec
-    _terminal_output = 'file_split'
 
     def _parse_inputs(self, skip=None):
         if skip is None:
             skip = []
 
-        # This is not strictly an input, but needs be
-        # set before run() is called.
-        if self.terminal_output == 'none':
-            self.terminal_output = 'file_split'
-
         if not self.inputs.save_outliers:
             skip += ['outliers_file']
         return super(OutlierCount, self)._parse_inputs(skip)
 
     def _run_interface(self, runtime):
+        runtime.stdout = op.abspath(self.inputs.out_file)
         runtime = super(OutlierCount, self)._run_interface(runtime)
-
-        # Read from runtime.stdout or runtime.merged
-        with open(op.abspath(self.inputs.out_file), 'w') as outfh:
-            outfh.write(runtime.stdout or runtime.merged)
         return runtime
 
     def _list_outputs(self):
@@ -1922,7 +1915,6 @@ class ROIStats(AFNICommandBase):
 
     """
     _cmd = '3dROIstats'
-    _terminal_output = 'allatonce'
     input_spec = ROIStatsInputSpec
     output_spec = ROIStatsOutputSpec
 
@@ -3055,7 +3047,9 @@ def _run_interface(self, runtime):
         if self.inputs.save_warp:
             import numpy as np
             warp_file = self._list_outputs()['warp_file']
-            np.savetxt(warp_file, [runtime.stdout], fmt=str('%s'))
+            with open(runtime.stdout) as f:
+                stdout = f.read().strip()
+            np.savetxt(warp_file, [stdout], fmt=str('%s'))
         return runtime
 
     def _list_outputs(self):

nipype/interfaces/afni/utils.py

@@ -211,10 +211,12 @@ class Autobox(AFNICommand):
     def aggregate_outputs(self, runtime=None, needed_outputs=None):
         outputs = super(Autobox, self).aggregate_outputs(
             runtime, needed_outputs)
-        pattern = 'x=(?P<x_min>-?\d+)\.\.(?P<x_max>-?\d+)  '\
-                  'y=(?P<y_min>-?\d+)\.\.(?P<y_max>-?\d+)  '\
-                  'z=(?P<z_min>-?\d+)\.\.(?P<z_max>-?\d+)'
-        for line in runtime.stderr.split('\n'):
+        pattern = r'x=(?P<x_min>-?\d+)\.\.(?P<x_max>-?\d+)  '\
+                  r'y=(?P<y_min>-?\d+)\.\.(?P<y_max>-?\d+)  '\
+                  r'z=(?P<z_min>-?\d+)\.\.(?P<z_max>-?\d+)'
+        with open(runtime.stderr) as f:
+            stderr = f.read()
+        for line in stderr.splitlines():
             m = re.search(pattern, line)
             if m:
                 d = m.groupdict()
@@ -297,7 +299,9 @@ def aggregate_outputs(self, runtime=None, needed_outputs=None):
                 return self.run().outputs
         else:
             min_val = []
-            for line in runtime.stdout.split('\n'):
+            with open(runtime.stdout, 'rt') as f:
+                stdout = f.read()
+            for line in stdout.splitlines():
                 if line:
                     values = line.split()
                     if len(values) > 1:
@@ -3047,8 +3051,10 @@ class GCOR(CommandLine):
     def _run_interface(self, runtime):
         runtime = super(GCOR, self)._run_interface(runtime)
 
+        with open(runtime.stdout, 'rt') as f:
+            stdout = f.read()
         gcor_line = [
-            line.strip() for line in runtime.stdout.split('\n')
+            line.strip() for line in stdout.splitlines()
             if line.strip().startswith('GCOR = ')
         ][-1]
         setattr(self, '_gcor', float(gcor_line[len('GCOR = '):]))

nipype/interfaces/ants/registration.py

@@ -940,9 +940,11 @@ def _run_interface(self, runtime, correct_return_codes=(0, )):
         runtime = super(Registration, self)._run_interface(runtime)
 
         # Parse some profiling info
-        output = runtime.stdout or runtime.merged
+        with open(runtime.stdout) as f:
+            output = f.read().strip()
+
         if output:
-            lines = output.split('\n')
+            lines = output.splitlines()
             for l in lines[::-1]:
                 # This should be the last line
                 if l.strip().startswith('Total elapsed time:'):
@@ -1489,8 +1491,10 @@ def _format_arg(self, opt, spec, val):
 
     def aggregate_outputs(self, runtime=None, needed_outputs=None):
         outputs = self._outputs()
-        stdout = runtime.stdout.split('\n')
-        outputs.similarity = float(stdout[0])
+        if runtime is not None and runtime.stdout:
+            with open(runtime.stdout, 'rf') as f:
+                stdout = f.read().splitlines()[0]
+            outputs.similarity = float(stdout[0])
         return outputs
 
 

nipype/interfaces/ants/resampling.py

@@ -136,12 +136,16 @@ def _list_outputs(self):
             (name, self.inputs.out_postfix, ext)))
         return outputs
 
-    def _run_interface(self, runtime, correct_return_codes=[0]):
-        runtime = super(WarpTimeSeriesImageMultiTransform,
-                        self)._run_interface(
-                            runtime, correct_return_codes=[0, 1])
-        if "100 % complete" not in runtime.stdout:
-            self.raise_exception(runtime)
+    def _run_interface(self, runtime, correct_return_codes=(0, 1)):
+        runtime = super(
+            WarpTimeSeriesImageMultiTransform, self)._run_interface(
+                runtime, correct_return_codes=correct_return_codes)
+
+        with open(runtime.stdout) as stdoutfh:
+            output_str = stdoutfh.read()
+
+            if "100 % complete" not in output_str:
+                self.raise_exception(runtime)
         return runtime
 
 

nipype/interfaces/ants/segmentation.py

@@ -818,8 +818,11 @@ def _run_interface(self, runtime, correct_return_codes=(0, )):
         runtime = super(BrainExtraction, self)._run_interface(runtime)
 
         # Still, double-check if it didn't found N4
-        if 'we cant find' in runtime.stdout:
-            for line in runtime.stdout.split('\n'):
+        with open(runtime.stdout) as stdoutfh:
+            stdout = stdoutfh.read()
+
+        if 'we cant find' in stdout:
+            for line in stdout.split('\n'):
                 if line.strip().startswith('we cant find'):
                     tool = line.strip().replace('we cant find the',
                                                 '').split(' ')[0]
@@ -828,10 +831,11 @@ def _run_interface(self, runtime, correct_return_codes=(0, )):
             errmsg = (
                 'antsBrainExtraction.sh requires "%s" to be found in $ANTSPATH '
                 '($ANTSPATH="%s").') % (tool, ants_path)
-            if runtime.stderr is None:
-                runtime.stderr = errmsg
-            else:
-                runtime.stderr += '\n' + errmsg
+
+            # Append errmsg to stderr file
+            with open(runtime.stderr, 'w+') as stderrfh:
+                stderrfh.write(errmsg)
+
             runtime.returncode = 1
             self.raise_exception(runtime)
 

nipype/interfaces/ants/tests/test_extra_Registration.py

@@ -1,9 +1,10 @@
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
 # vi: set ft=python sts=4 ts=4 sw=4 et:
 from __future__ import unicode_literals
-from nipype.interfaces.ants import registration
 import os
 import pytest
+from nipype.interfaces.ants import registration
+from nipype.utils.errors import MandatoryInputError
 
 
 def test_ants_mand(tmpdir):
@@ -17,6 +18,6 @@ def test_ants_mand(tmpdir):
     ants.inputs.fixed_image = [os.path.join(datadir, 'T1.nii')]
     ants.inputs.metric = ['MI']
 
-    with pytest.raises(ValueError) as er:
+    with pytest.raises(MandatoryInputError) as er:
         ants.run()
-    assert "ANTS requires a value for input 'radius'" in str(er.value)
+    assert 'Interface "ANTS" requires a value for input radius.' in str(er.value)