diff --git a/examples/fmri_freesurfer_smooth.py b/examples/fmri_freesurfer_smooth.py index 69545b59c0..3a829fca19 100755 --- a/examples/fmri_freesurfer_smooth.py +++ b/examples/fmri_freesurfer_smooth.py @@ -420,7 +420,7 @@ def subjectinfo(subject_id): - from nipype.interfaces.base import Bunch + from nipype.utils.bunch import Bunch from copy import deepcopy print("Subject ID: %s\n" % str(subject_id)) output = [] diff --git a/examples/fmri_fsl.py b/examples/fmri_fsl.py index 9d4ab71423..dadae01aba 100755 --- a/examples/fmri_fsl.py +++ b/examples/fmri_fsl.py @@ -551,7 +551,7 @@ def num_copes(files): def subjectinfo(subject_id): - from nipype.interfaces.base import Bunch + from nipype.utils.bunch import Bunch from copy import deepcopy print("Subject ID: %s\n" % str(subject_id)) output = [] diff --git a/examples/fmri_fsl_feeds.py b/examples/fmri_fsl_feeds.py index f7b0aaf91d..3bdd0da165 100755 --- a/examples/fmri_fsl_feeds.py +++ b/examples/fmri_fsl_feeds.py @@ -24,7 +24,7 @@ from nipype.algorithms import modelgen as model # model generation from nipype.workflows.fmri.fsl import ( create_featreg_preproc, create_modelfit_workflow, create_reg_workflow) -from nipype.interfaces.base import Bunch +from nipype.utils.bunch import Bunch """ Preliminaries ------------- diff --git a/examples/fmri_fsl_reuse.py b/examples/fmri_fsl_reuse.py index 7b24dc24b8..7fe46050ec 100755 --- a/examples/fmri_fsl_reuse.py +++ b/examples/fmri_fsl_reuse.py @@ -190,7 +190,7 @@ def num_copes(files): def subjectinfo(subject_id): - from nipype.interfaces.base import Bunch + from nipype.utils.bunch import Bunch from copy import deepcopy print("Subject ID: %s\n" % str(subject_id)) output = [] diff --git a/examples/fmri_nipy_glm.py b/examples/fmri_nipy_glm.py index 0439050c7a..180aa39f21 100755 --- a/examples/fmri_nipy_glm.py +++ b/examples/fmri_nipy_glm.py @@ -147,7 +147,7 @@ def subjectinfo(subject_id): - from nipype.interfaces.base import Bunch + from nipype.utils.bunch import Bunch from copy import deepcopy print("Subject ID: %s\n" % str(subject_id)) output = [] diff --git a/examples/fmri_spm.py b/examples/fmri_spm.py index 23a16a36fe..2e7af79e3a 100755 --- a/examples/fmri_spm.py +++ b/examples/fmri_spm.py @@ -171,7 +171,7 @@ def subjectinfo(subject_id): - from nipype.interfaces.base import Bunch + from nipype.utils.bunch import Bunch from copy import deepcopy print("Subject ID: %s\n" % str(subject_id)) output = [] diff --git a/examples/fmri_spm_auditory.py b/examples/fmri_spm_auditory.py index e4c690421a..2a767efbf2 100755 --- a/examples/fmri_spm_auditory.py +++ b/examples/fmri_spm_auditory.py @@ -267,7 +267,7 @@ def makelist(item): necessary to generate an SPM design matrix. """ -from nipype.interfaces.base import Bunch +from nipype.utils.bunch import Bunch subjectinfo = [ Bunch( conditions=['Task'], onsets=[list(range(6, 84, 12))], durations=[[6]]) diff --git a/examples/fmri_spm_dartel.py b/examples/fmri_spm_dartel.py index 587ff9b291..b2b32644b4 100755 --- a/examples/fmri_spm_dartel.py +++ b/examples/fmri_spm_dartel.py @@ -314,7 +314,7 @@ def pickFieldFlow(dartel_flow_fields, subject_id): def subjectinfo(subject_id): - from nipype.interfaces.base import Bunch + from nipype.utils.bunch import Bunch from copy import deepcopy print("Subject ID: %s\n" % str(subject_id)) output = [] diff --git a/examples/fmri_spm_face.py b/examples/fmri_spm_face.py index 5644398d54..385deb9b15 100755 --- a/examples/fmri_spm_face.py +++ b/examples/fmri_spm_face.py @@ -259,7 +259,7 @@ def makelist(item): necessary to generate an SPM design matrix. """ -from nipype.interfaces.base import Bunch +from nipype.utils.bunch import Bunch """We're importing the onset times from a mat file (found on http://www.fil.ion.ucl.ac.uk/spm/data/face_rep/) """ diff --git a/examples/fmri_spm_nested.py b/examples/fmri_spm_nested.py index 534b8c960d..f1d46fc7c7 100755 --- a/examples/fmri_spm_nested.py +++ b/examples/fmri_spm_nested.py @@ -285,7 +285,7 @@ def _template_path(in_data): def subjectinfo(subject_id): - from nipype.interfaces.base import Bunch + from nipype.utils.bunch import Bunch from copy import deepcopy print("Subject ID: %s\n" % str(subject_id)) output = [] diff --git a/examples/frontiers_paper/smoothing_comparison.py b/examples/frontiers_paper/smoothing_comparison.py index c4a31dad39..6d2789d775 100644 --- a/examples/frontiers_paper/smoothing_comparison.py +++ b/examples/frontiers_paper/smoothing_comparison.py @@ -17,7 +17,7 @@ import nipype.pipeline.engine as pe # pypeline engine import nipype.algorithms.modelgen as model # model specification import nipype.workflows.fmri.fsl as fsl_wf -from nipype.interfaces.base import Bunch +from nipype.utils.bunch import Bunch import os # system functions preprocessing = pe.Workflow(name="preprocessing") diff --git a/examples/frontiers_paper/workflow_from_scratch.py b/examples/frontiers_paper/workflow_from_scratch.py index 9b1c939d29..fa76f1417a 100644 --- a/examples/frontiers_paper/workflow_from_scratch.py +++ b/examples/frontiers_paper/workflow_from_scratch.py @@ -14,7 +14,7 @@ import nipype.interfaces.spm as spm # spm import nipype.pipeline.engine as pe # pypeline engine import nipype.algorithms.modelgen as model # model specification -from nipype.interfaces.base import Bunch +from nipype.utils.bunch import Bunch import os # system functions """In the following section, to showcase NiPyPe, we will describe how to create and extend a typical fMRI processing pipeline. We will begin with a basic diff --git a/nipype/algorithms/modelgen.py b/nipype/algorithms/modelgen.py index 763e19fbc3..36af3e3b8d 100644 --- a/nipype/algorithms/modelgen.py +++ b/nipype/algorithms/modelgen.py @@ -23,11 +23,11 @@ from scipy.special import gammaln from ..utils import NUMPY_MMAP -from ..interfaces.base import (BaseInterface, TraitedSpec, InputMultiPath, - traits, File, Bunch, BaseInterfaceInputSpec, - isdefined) -from ..utils.filemanip import ensure_list +from ..utils.bunch import Bunch from ..utils.misc import normalize_mc_params +from ..utils.filemanip import ensure_list +from ..interfaces.base import (BaseInterface, TraitedSpec, InputMultiPath, + traits, File, BaseInterfaceInputSpec, isdefined) from .. import config, logging iflogger = logging.getLogger('nipype.interface') @@ -280,7 +280,7 @@ class SpecifyModel(BaseInterface): -------- >>> from nipype.algorithms import modelgen - >>> from nipype.interfaces.base import Bunch + >>> from nipype.utils.bunch import Bunch >>> s = modelgen.SpecifyModel() >>> s.inputs.input_units = 'secs' >>> s.inputs.functional_runs = ['functional2.nii', 'functional3.nii'] @@ -481,7 +481,7 @@ class SpecifySPMModel(SpecifyModel): -------- >>> from nipype.algorithms import modelgen - >>> from nipype.interfaces.base import Bunch + >>> from nipype.utils.bunch import Bunch >>> s = modelgen.SpecifySPMModel() >>> s.inputs.input_units = 'secs' >>> s.inputs.output_units = 'scans' @@ -667,7 +667,7 @@ class SpecifySparseModel(SpecifyModel): -------- >>> from nipype.algorithms import modelgen - >>> from nipype.interfaces.base import Bunch + >>> from nipype.utils.bunch import Bunch >>> s = modelgen.SpecifySparseModel() >>> s.inputs.input_units = 'secs' >>> s.inputs.functional_runs = ['functional2.nii', 'functional3.nii'] diff --git a/nipype/algorithms/tests/test_modelgen.py b/nipype/algorithms/tests/test_modelgen.py index 824a634354..d985820a4c 100644 --- a/nipype/algorithms/tests/test_modelgen.py +++ b/nipype/algorithms/tests/test_modelgen.py @@ -11,7 +11,8 @@ import pytest import numpy.testing as npt -from nipype.interfaces.base import Bunch, TraitError +from nipype.utils.bunch import Bunch +from nipype.interfaces.base import TraitError from nipype.algorithms.modelgen import (SpecifyModel, SpecifySparseModel, SpecifySPMModel) diff --git a/nipype/algorithms/tests/test_rapidart.py b/nipype/algorithms/tests/test_rapidart.py index 9c29648626..81d831d165 100644 --- a/nipype/algorithms/tests/test_rapidart.py +++ b/nipype/algorithms/tests/test_rapidart.py @@ -7,7 +7,7 @@ import numpy.testing as npt from .. import rapidart as ra -from ...interfaces.base import Bunch +from ...utils.bunch import Bunch def test_ad_init(): diff --git a/nipype/interfaces/base/__init__.py b/nipype/interfaces/base/__init__.py index f617064b2f..973dd30081 100644 --- a/nipype/interfaces/base/__init__.py +++ b/nipype/interfaces/base/__init__.py @@ -8,6 +8,7 @@ This module defines the API of all nipype interfaces. """ +from ...utils.bunch import Bunch from .core import (Interface, BaseInterface, SimpleInterface, CommandLine, StdOutCommandLine, MpiCommandLine, SEMLikeCommandLine, LibraryBaseInterface, PackageInfo) @@ -22,5 +23,42 @@ OutputMultiObject, InputMultiObject, OutputMultiPath, InputMultiPath) -from .support import (Bunch, InterfaceResult, load_template, - NipypeInterfaceError) +from .support import (InterfaceRuntime, InterfaceResult, NipypeInterfaceError) + +__all__ = [ + 'Bunch', + 'Interface', + 'BaseInterface', + 'SimpleInterface', + 'CommandLine', + 'StdOutCommandLine', + 'MpiCommandLine', + 'SEMLikeCommandLine', + 'LibraryBaseInterface', + 'PackageInfo', + 'BaseTraitedSpec', + 'TraitedSpec', + 'DynamicTraitedSpec', + 'BaseInterfaceInputSpec', + 'CommandLineInputSpec', + 'StdOutCommandLineInputSpec', + 'traits', + 'Undefined', + 'TraitDictObject', + 'TraitListObject', + 'TraitError', + 'isdefined', + 'File', + 'Directory', + 'Str', + 'DictStrStr', + 'has_metadata', + 'ImageFile', + 'OutputMultiObject', + 'InputMultiObject', + 'OutputMultiPath', + 'InputMultiPath', + 'InterfaceRuntime', + 'InterfaceResult', + 'NipypeInterfaceError', +] diff --git a/nipype/interfaces/base/core.py b/nipype/interfaces/base/core.py index 6177b449f9..2c06d607ce 100644 --- a/nipype/interfaces/base/core.py +++ b/nipype/interfaces/base/core.py @@ -42,7 +42,7 @@ from .specs import (BaseInterfaceInputSpec, CommandLineInputSpec, StdOutCommandLineInputSpec, MpiCommandLineInputSpec, get_filecopy_info) -from .support import (Bunch, InterfaceResult, NipypeInterfaceError) +from .support import (InterfaceRuntime, InterfaceResult, NipypeInterfaceError) from future import standard_library standard_library.install_aliases() @@ -478,7 +478,7 @@ def run(self, cwd=None, ignore_exception=None, **inputs): if self._redirect_x: env['DISPLAY'] = config.get_display() - runtime = Bunch( + runtime = InterfaceRuntime( cwd=cwd, prevcwd=syscwd, returncode=None, @@ -872,9 +872,7 @@ def _run_interface(self, runtime, correct_return_codes=(0, )): """ out_environ = self._get_environ() - # Initialize runtime Bunch - runtime.stdout = None - runtime.stderr = None + # Initialize runtime InterfaceRuntime runtime.cmdline = self.cmdline runtime.environ.update(out_environ) diff --git a/nipype/interfaces/base/support.py b/nipype/interfaces/base/support.py index 543d4b6c40..7ca46244ac 100644 --- a/nipype/interfaces/base/support.py +++ b/nipype/interfaces/base/support.py @@ -9,185 +9,168 @@ """ from __future__ import (print_function, division, unicode_literals, absolute_import) -from builtins import object, str +from builtins import object +from ... import logging, __version__ +from ...utils.bunch import bunch_repr -import os -from copy import deepcopy - -from ... import logging -from ...utils.misc import is_container -from ...utils.filemanip import md5, to_str, hash_infile iflogger = logging.getLogger('nipype.interface') class NipypeInterfaceError(Exception): """Custom error for interfaces""" + pass - def __init__(self, value): - self.value = value - def __str__(self): - return '{}'.format(self.value) +class InterfaceRuntime(object): + """A class to store runtime details of interfaces + Elements can be set at initialization: -class Bunch(object): - """Dictionary-like class that provides attribute-style access to it's items. + >>> rt = InterfaceRuntime(cmdline='/bin/echo', returncode=0) + >>> rt.returncode == 0 + True - A `Bunch` is a simple container that stores it's items as class - attributes. Internally all items are stored in a dictionary and - the class exposes several of the dictionary methods. + Unset elements are initialized to ``None``: - Examples - -------- - >>> from nipype.interfaces.base import Bunch - >>> inputs = Bunch(infile='subj.nii', fwhm=6.0, register_to_mean=True) - >>> inputs - Bunch(fwhm=6.0, infile='subj.nii', register_to_mean=True) - >>> inputs.register_to_mean = False - >>> inputs - Bunch(fwhm=6.0, infile='subj.nii', register_to_mean=False) + >>> rt.cwd is None + True - Notes - ----- - The Bunch pattern came from the Python Cookbook: + And any element can be set by attribute: - .. [1] A. Martelli, D. Hudgeon, "Collecting a Bunch of Named - Items", Python Cookbook, 2nd Ed, Chapter 4.18, 2005. + >>> rt.cwd = '/scratch/workflow' + >>> rt.cwd + '/scratch/workflow' - """ + The structure raises ``AttributeError`` if trying to accessing + an invalid attribute: - def __init__(self, *args, **kwargs): - self.__dict__.update(*args, **kwargs) + >>> rt.nonruntime + Traceback (most recent call last): + ... + AttributeError: 'InterfaceRuntime' object has no attribute 'nonruntime' - def update(self, *args, **kwargs): - """update existing attribute, or create new attribute - Note: update is very much like HasTraits.set""" - self.__dict__.update(*args, **kwargs) + For Nipype to work, it is fundamental that runtime objects + can be pickled/unpickled: - def items(self): - """iterates over bunch attributes as key, value pairs""" - return list(self.__dict__.items()) + >>> import pickle + >>> pickleds = pickle.dumps(rt) + >>> newrt = pickle.loads(pickleds) + >>> newrt + InterfaceRuntime(cmdline='/bin/echo', cwd='/scratch/workflow', returncode=0) - def iteritems(self): - """iterates over bunch attributes as key, value pairs""" - iflogger.warning('iteritems is deprecated, use items instead') - return list(self.items()) + Runtime objects can be compared - def get(self, *args): - """Support dictionary get() functionality - """ - return self.__dict__.get(*args) + >>> newrt == rt + True - def set(self, **kwargs): - """Support dictionary get() functionality - """ - return self.__dict__.update(**kwargs) + >>> newrt != rt + False + + + >>> newrt != 'other' + True + + >>> newrt == 'other' + False + + + """ + + __slots__ = sorted([ + 'cwd', + 'prevcwd', + 'returncode', + 'duration', + 'environ', + 'startTime', + 'endTime', + 'platform', + 'hostname', + 'version', + 'traceback', + 'traceback_args', + 'mem_peak_gb', + 'cpu_percent', + 'prof_dict', + 'cmdline', + 'stdout', + 'stderr', + 'merged', + 'command_path', + 'dependencies', + ]) + + def __init__(self, **inputs): + self.__setstate__(inputs) + + def __setstate__(self, state): + """Necessary for un-pickling""" + for key in self.__class__.__slots__: + setattr(self, key, state.get(key, None)) + + def __getstate__(self): + """Necessary for pickling""" + outdict = {} + for key in self.__class__.__slots__: + value = getattr(self, key, None) + if value is not None: + outdict[key] = value + return outdict def dictcopy(self): - """returns a deep copy of existing Bunch as a dictionary""" - return deepcopy(self.__dict__) + """ + Returns a dictionary of set attributes (backward compatibility) - def __repr__(self): - """representation of the sorted Bunch as a string + >>> rt = InterfaceRuntime() + >>> rt.dictcopy() + {} + + >>> rt = InterfaceRuntime(cmdline='/bin/echo', returncode=0) + >>> rt.dictcopy() + {'cmdline': '/bin/echo', 'returncode': 0} - Currently, this string representation of the `inputs` Bunch of - interfaces is hashed to determine if the process' dirty-bit - needs setting or not. Till that mechanism changes, only alter - this after careful consideration. """ - outstr = ['Bunch('] - first = True - for k, v in sorted(self.items()): - if not first: - outstr.append(', ') - if isinstance(v, dict): - pairs = [] - for key, value in sorted(v.items()): - pairs.append("'%s': %s" % (key, value)) - v = '{' + ', '.join(pairs) + '}' - outstr.append('%s=%s' % (k, v)) - else: - outstr.append('%s=%r' % (k, v)) - first = False - outstr.append(')') - return ''.join(outstr) - - def _get_bunch_hash(self): - """Return a dictionary of our items with hashes for each file. - - Searches through dictionary items and if an item is a file, it - calculates the md5 hash of the file contents and stores the - file name and hash value as the new key value. - - However, the overall bunch hash is calculated only on the hash - value of a file. The path and name of the file are not used in - the overall hash calculation. - - Returns - ------- - dict_withhash : dict - Copy of our dictionary with the new file hashes included - with each file. - hashvalue : str - The md5 hash value of the `dict_withhash` + return self.__getstate__() + + def items(self): + """Provide an interface for items + + >>> rt = InterfaceRuntime() + >>> list(rt.items()) + [] + + >>> rt = InterfaceRuntime(cmdline='/bin/echo', returncode=0) + >>> list(rt.items()) + [('cmdline', '/bin/echo'), ('returncode', 0)] + """ + for key in self.__class__.__slots__: + value = getattr(self, key, None) + if value is not None: + yield (key, value) + + def __repr__(self): + """representation of the runtime object""" + return bunch_repr(self, self.__class__.__name__) + + # Enable when Python 2 support is dropped + # def __str__(self): + # return '%s' % self.__getstate__() + + def __eq__(self, other): + """Overrides the default implementation""" + if isinstance(other, InterfaceRuntime): + return self.__getstate__() == other.__getstate__() + return NotImplemented - infile_list = [] - for key, val in list(self.items()): - if is_container(val): - # XXX - SG this probably doesn't catch numpy arrays - # containing embedded file names either. - if isinstance(val, dict): - # XXX - SG should traverse dicts, but ignoring for now - item = None - else: - if len(val) == 0: - raise AttributeError('%s attribute is empty' % key) - item = val[0] - else: - item = val - try: - if isinstance(item, str) and os.path.isfile(item): - infile_list.append(key) - except TypeError: - # `item` is not a file or string. - continue - dict_withhash = self.dictcopy() - dict_nofilename = self.dictcopy() - for item in infile_list: - dict_withhash[item] = _hash_bunch_dict(dict_withhash, item) - dict_nofilename[item] = [val[1] for val in dict_withhash[item]] - # Sort the items of the dictionary, before hashing the string - # representation so we get a predictable order of the - # dictionary. - sorted_dict = to_str(sorted(dict_nofilename.items())) - return dict_withhash, md5(sorted_dict.encode()).hexdigest() - - def _repr_pretty_(self, p, cycle): - """Support for the pretty module from ipython.externals""" - if cycle: - p.text('Bunch(...)') - else: - p.begin_group(6, 'Bunch(') - first = True - for k, v in sorted(self.items()): - if not first: - p.text(',') - p.breakable() - p.text(k + '=') - p.pretty(v) - first = False - p.end_group(6, ')') - - -def _hash_bunch_dict(adict, key): - """Inject file hashes into adict[key]""" - stuff = adict[key] - if not is_container(stuff): - stuff = [stuff] - return [(afile, hash_infile(afile)) for afile in stuff] + def __ne__(self, other): + """Overrides the default implementation (Python 2)""" + x = self.__eq__(other) + if x is not NotImplemented: + return not x + return NotImplemented class InterfaceResult(object): @@ -215,7 +198,20 @@ class InterfaceResult(object): * stderr : Any error messages output from running ``cmdline``. * returncode : The code returned from running the ``cmdline``. + + >>> rt = InterfaceRuntime(cmdline='/bin/echo', returncode=0) + >>> result = InterfaceResult(interface='CommandLine', + ... runtime=rt) + >>> import pickle + >>> pickleds = pickle.dumps(result) + >>> newresult = pickle.loads(pickleds) + >>> newresult # doctest: +ELLIPSIS + InterfaceResult(interface='CommandLine', runtime=\ +InterfaceRuntime(cmdline='/bin/echo', returncode=0), version=...) + """ + __slots__ = ['interface', 'runtime', 'inputs', 'outputs', 'provenance'] + version = __version__ def __init__(self, interface, @@ -223,26 +219,31 @@ def __init__(self, inputs=None, outputs=None, provenance=None): - self._version = 2.0 self.interface = interface self.runtime = runtime self.inputs = inputs self.outputs = outputs self.provenance = provenance - @property - def version(self): - return self._version + def __setstate__(self, state): + """Necessary for un-pickling""" + if state.get('version', None) != self.version: + iflogger.warning( + 'Restoring an ``InterfaceResult`` from a different ' + 'nipype version (current version %s, object\'s %s', + self.version, state.get('version', '')) + for key in self.__class__.__slots__: + setattr(self, key, state.get(key, None)) + + def __getstate__(self): + """Necessary for pickling""" + outdict = {'version': self.version} + for key in self.__class__.__slots__: + value = getattr(self, key, None) + if value is not None: + outdict[key] = value + return outdict - -def load_template(name): - """ - Deprecated stub for backwards compatibility, - please use nipype.interfaces.fsl.model.load_template - - """ - from ..fsl.model import load_template - iflogger.warning( - 'Deprecated in 1.0.0, and will be removed in 1.1.0, ' - 'please use nipype.interfaces.fsl.model.load_template instead.') - return load_template(name) + def __repr__(self): + return bunch_repr( + self.__getstate__(), self.__class__.__name__) diff --git a/nipype/interfaces/base/tests/test_core.py b/nipype/interfaces/base/tests/test_core.py index bcbd43db28..be51f58635 100644 --- a/nipype/interfaces/base/tests/test_core.py +++ b/nipype/interfaces/base/tests/test_core.py @@ -528,7 +528,7 @@ def _run_interface(self, runtime): class BrokenRuntime(TestInterface): def _run_interface(self, runtime): - del runtime.__dict__['cwd'] + del runtime.cwd return runtime with pytest.raises(RuntimeError): diff --git a/nipype/interfaces/base/tests/test_support.py b/nipype/interfaces/base/tests/test_support.py index e6db69a458..c0da58d619 100644 --- a/nipype/interfaces/base/tests/test_support.py +++ b/nipype/interfaces/base/tests/test_support.py @@ -2,60 +2,3 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: from __future__ import print_function, unicode_literals -import os -import pytest -from builtins import open -from future import standard_library -from pkg_resources import resource_filename as pkgrf - -from ....utils.filemanip import md5 -from ... import base as nib - -standard_library.install_aliases() - - -@pytest.mark.parametrize("args", [{}, {'a': 1, 'b': [2, 3]}]) -def test_bunch(args): - b = nib.Bunch(**args) - assert b.__dict__ == args - - -def test_bunch_attribute(): - b = nib.Bunch(a=1, b=[2, 3], c=None) - assert b.a == 1 - assert b.b == [2, 3] - assert b.c is None - - -def test_bunch_repr(): - b = nib.Bunch(b=2, c=3, a=dict(n=1, m=2)) - assert repr(b) == "Bunch(a={'m': 2, 'n': 1}, b=2, c=3)" - - -def test_bunch_methods(): - b = nib.Bunch(a=2) - b.update(a=3) - newb = b.dictcopy() - assert b.a == 3 - assert b.get('a') == 3 - assert b.get('badkey', 'otherthing') == 'otherthing' - assert b != newb - assert type(dict()) == type(newb) - assert newb['a'] == 3 - - -def test_bunch_hash(): - # NOTE: Since the path to the json file is included in the Bunch, - # the hash will be unique to each machine. - json_pth = pkgrf('nipype', - os.path.join('testing', 'data', 'realign_json.json')) - - b = nib.Bunch(infile=json_pth, otherthing='blue', yat=True) - newbdict, bhash = b._get_bunch_hash() - assert bhash == 'd1f46750044c3de102efc847720fc35f' - # Make sure the hash stored in the json file for `infile` is correct. - jshash = md5() - with open(json_pth, 'r') as fp: - jshash.update(fp.read().encode('utf-8')) - assert newbdict['infile'][0][1] == jshash.hexdigest() - assert newbdict['yat'] is True diff --git a/nipype/interfaces/spm/model.py b/nipype/interfaces/spm/model.py index 3e26ab6e2a..20f210c5dc 100644 --- a/nipype/interfaces/spm/model.py +++ b/nipype/interfaces/spm/model.py @@ -18,9 +18,10 @@ # Local imports from ... import logging +from ...utils.bunch import Bunch from ...utils.filemanip import (ensure_list, simplify_list, split_filename) -from ..base import (Bunch, traits, TraitedSpec, File, Directory, +from ..base import (traits, TraitedSpec, File, Directory, OutputMultiPath, InputMultiPath, isdefined) from .base import (SPMCommand, SPMCommandInputSpec, scans_for_fnames, ImageFileSPM) diff --git a/nipype/pipeline/engine/nodes.py b/nipype/pipeline/engine/nodes.py index b338fd862d..272124a3b2 100644 --- a/nipype/pipeline/engine/nodes.py +++ b/nipype/pipeline/engine/nodes.py @@ -23,6 +23,7 @@ from future import standard_library from ... import config, logging +from ...utils.bunch import Bunch from ...utils.misc import flatten, unflatten, str2bool, dict_diff from ...utils.filemanip import (md5, FileNotFoundError, ensure_list, simplify_list, copyfiles, fnames_presuffix, @@ -30,7 +31,7 @@ emptydirs, savepkl, to_str, indirectory) from ...interfaces.base import (traits, InputMultiPath, CommandLine, Undefined, - DynamicTraitedSpec, Bunch, InterfaceResult, + DynamicTraitedSpec, InterfaceResult, InterfaceRuntime, Interface, isdefined) from ...interfaces.base.specs import get_filecopy_info @@ -569,7 +570,7 @@ def _load_results(self): self._copyfiles_to_wd(linksonly=True) aggouts = self._interface.aggregate_outputs( needed_outputs=self.needed_outputs) - runtime = Bunch( + runtime = InterfaceRuntime( cwd=cwd, returncode=0, environ=dict(os.environ), @@ -604,7 +605,7 @@ def _run_command(self, execute, copyfiles=True): # Run command: either execute is true or load_results failed. result = InterfaceResult( interface=self._interface.__class__, - runtime=Bunch( + runtime=InterfaceRuntime( cwd=outdir, returncode=1, environ=dict(os.environ), diff --git a/nipype/pipeline/engine/tests/test_utils.py b/nipype/pipeline/engine/tests/test_utils.py index 42f8b2434e..f78944c0d8 100644 --- a/nipype/pipeline/engine/tests/test_utils.py +++ b/nipype/pipeline/engine/tests/test_utils.py @@ -220,7 +220,7 @@ def test_mapnode_crash3(tmpdir): wf = pe.Workflow('testmapnodecrash') wf.add_nodes([node]) wf.base_dir = tmpdir.strpath - # changing crashdump dir to cwl (to avoid problems with read-only systems) + # changing crashdump dir to cwd (to avoid problems with read-only systems) wf.config["execution"]["crashdump_dir"] = os.getcwd() - with pytest.raises(RuntimeError): + with pytest.raises(Exception): wf.run(plugin='Linear') diff --git a/nipype/pipeline/engine/utils.py b/nipype/pipeline/engine/utils.py index 7df4fa15ca..b0018af545 100644 --- a/nipype/pipeline/engine/utils.py +++ b/nipype/pipeline/engine/utils.py @@ -39,9 +39,10 @@ write_rst_dict, write_rst_list, ) +from ...utils.bunch import Bunch from ...utils.misc import str2bool from ...utils.functions import create_function_from_source -from ...interfaces.base import (Bunch, CommandLine, isdefined, Undefined, +from ...interfaces.base import (CommandLine, isdefined, Undefined, InterfaceResult, traits) from ...interfaces.utility import IdentityInterface from ...utils.provenance import ProvStore, pm, nipype_ns, get_id @@ -110,7 +111,7 @@ def nodelist_runner(nodes, updatehash=False, stop_first=False): result = node.result err = [] - if result.runtime and hasattr(result.runtime, 'traceback'): + if result.runtime and result.runtime.traceback is not None: err = [result.runtime.traceback] err += format_exception(*sys.exc_info()) @@ -192,40 +193,40 @@ def write_report(node, report_type=None, is_mapnode=False): 'prev_wd': getattr(result.runtime, 'prevcwd', ''), } - if hasattr(result.runtime, 'cmdline'): + if result.runtime.cmdline is not None: rst_dict['command'] = result.runtime.cmdline # Try and insert memory/threads usage if available - if hasattr(result.runtime, 'mem_peak_gb'): + if result.runtime.mem_peak_gb is not None: rst_dict['mem_peak_gb'] = result.runtime.mem_peak_gb - if hasattr(result.runtime, 'cpu_percent'): + if result.runtime.cpu_percent is not None: rst_dict['cpu_percent'] = result.runtime.cpu_percent lines.append(write_rst_dict(rst_dict)) # Collect terminal output - if hasattr(result.runtime, 'merged'): + if result.runtime.merged: lines += [ write_rst_header('Terminal output', level=2), - write_rst_list(result.runtime.merged), + write_rst_list(result.runtime.merged or []), ] - if hasattr(result.runtime, 'stdout'): + if result.runtime.stdout: lines += [ write_rst_header('Terminal - standard output', level=2), - write_rst_list(result.runtime.stdout), + write_rst_list(result.runtime.stdout or []), ] - if hasattr(result.runtime, 'stderr'): + if result.runtime.stderr: lines += [ write_rst_header('Terminal - standard error', level=2), - write_rst_list(result.runtime.stderr), + write_rst_list(result.runtime.stderr or []), ] # Store environment - if hasattr(result.runtime, 'environ'): + if result.runtime.environ: lines += [ write_rst_header('Environment', level=2), - write_rst_dict(result.runtime.environ), + write_rst_dict(result.runtime.environ or {}), ] with open(report_file, 'at') as fp: diff --git a/nipype/utils/bunch.py b/nipype/utils/bunch.py new file mode 100644 index 0000000000..1edc72d468 --- /dev/null +++ b/nipype/utils/bunch.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +"""Nipype Bunch container""" +from __future__ import (print_function, unicode_literals, division, + absolute_import) +from copy import deepcopy +from builtins import object + + +class Bunch(object): + """Dictionary-like class that provides attribute-style access to it's items. + + A `Bunch` is a simple container that stores it's items as class + attributes. Internally all items are stored in a dictionary and + the class exposes several of the dictionary methods. + + Examples + -------- + >>> from nipype.utils.bunch import Bunch + >>> inputs = Bunch(infile='subj.nii', fwhm=6.0, register_to_mean=True) + >>> inputs + Bunch(fwhm=6.0, infile='subj.nii', register_to_mean=True) + >>> inputs.register_to_mean = False + >>> inputs + Bunch(fwhm=6.0, infile='subj.nii', register_to_mean=False) + + Notes + ----- + The Bunch pattern came from the Python Cookbook: + + .. [1] A. Martelli, D. Hudgeon, "Collecting a Bunch of Named + Items", Python Cookbook, 2nd Ed, Chapter 4.18, 2005. + + """ + + def __init__(self, *args, **kwargs): + self.__dict__.update(*args, **kwargs) + + def update(self, *args, **kwargs): + """update existing attribute, or create new attribute + + Note: update is very much like HasTraits.set""" + self.__dict__.update(*args, **kwargs) + + def items(self): + """iterates over bunch attributes as key, value pairs""" + return list(self.__dict__.items()) + + def get(self, *args): + """Support dictionary get() functionality + """ + return self.__dict__.get(*args) + + def set(self, **kwargs): + """Support dictionary get() functionality + """ + return self.__dict__.update(**kwargs) + + def dictcopy(self): + """returns a deep copy of existing Bunch as a dictionary""" + return deepcopy(self.__dict__) + + def __repr__(self): + """representation of the sorted Bunch as a string + + Currently, this string representation of the `inputs` Bunch of + interfaces is hashed to determine if the process' dirty-bit + needs setting or not. Till that mechanism changes, only alter + this after careful consideration. + """ + return bunch_repr(self) + + def _repr_pretty_(self, p, cycle): + """Support for the pretty module from ipython.externals""" + if cycle: + p.text('Bunch(...)') + else: + p.begin_group(6, 'Bunch(') + first = True + for k, v in sorted(self.items()): + if not first: + p.text(',') + p.breakable() + p.text(k + '=') + p.pretty(v) + first = False + p.end_group(6, ')') + + +def bunch_repr(instance, classname=None): + """Represent dict-like objects as a Bunch + + >>> bunch_repr({'b': 2, 'c': 3, 'a': {'n': 1, 'm': 2}}) + "Bunch(a={'m': 2, 'n': 1}, b=2, c=3)" + """ + classname = classname or 'Bunch' + outstr = ['%s(' % classname] + first = True + for k, v in sorted(instance.items()): + if not first: + outstr.append(', ') + if isinstance(v, dict): + pairs = [] + for key, value in sorted(v.items()): + pairs.append("'%s': %s" % (key, value)) + v = '{' + ', '.join(pairs) + '}' + outstr.append('%s=%s' % (k, v)) + else: + outstr.append('%s=%r' % (k, v)) + first = False + outstr.append(')') + return ''.join(outstr) diff --git a/nipype/utils/draw_gantt_chart.py b/nipype/utils/draw_gantt_chart.py index 7a52205090..d63919904b 100644 --- a/nipype/utils/draw_gantt_chart.py +++ b/nipype/utils/draw_gantt_chart.py @@ -322,13 +322,8 @@ def draw_resource_bar(start_time, finish_time, time_series, space_between_minutes = space_between_minutes / scale # Iterate through time series - if PY3: - ts_items = time_series.items() - else: - ts_items = time_series.iteritems() - ts_len = len(time_series) - for idx, (ts_start, amount) in enumerate(ts_items): + for idx, (ts_start, amount) in enumerate(time_series.items()): if idx < ts_len - 1: ts_end = time_series.index[idx + 1] else: diff --git a/nipype/utils/tests/test_bunch.py b/nipype/utils/tests/test_bunch.py new file mode 100644 index 0000000000..bf5eaa1fc9 --- /dev/null +++ b/nipype/utils/tests/test_bunch.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +from __future__ import print_function, unicode_literals + +from ..bunch import Bunch +import pytest + + +@pytest.mark.parametrize("args", [{}, {'a': 1, 'b': [2, 3]}, {'a': 1, 'b': {'c': [2, 3]}}]) +def test_bunch(args): + b = Bunch(**args) + assert b.__dict__ == args + + # test Bunch.get access + for k, v in args.items(): + assert b.get(k) == v + + # test getattr access + for k, v in args.items(): + assert getattr(b, k) == v + + +def test_bunch_attribute(): + b = Bunch(a=1, b=[2, 3], c=None) + assert b.a == 1 + assert b.b == [2, 3] + assert b.c is None + + +def test_bunch_repr(): + b = Bunch(b=2, c=3, a=dict(n=1, m=2)) + assert repr(b) == "Bunch(a={'m': 2, 'n': 1}, b=2, c=3)" + + +def test_bunch_methods(): + b = Bunch(a=2) + b.update(a=3) + newb = b.dictcopy() + assert b.a == 3 + assert b.get('a') == 3 + assert b.get('badkey', 'otherthing') == 'otherthing' + assert b != newb + assert type(dict()) == type(newb) + assert newb['a'] == 3 + +# disabled: _get_bunch_hash wasn't ever used in nipype when removed +# def test_bunch_hash(): +# # NOTE: Since the path to the json file is included in the Bunch, +# # the hash will be unique to each machine. +# json_pth = pkgrf('nipype', +# os.path.join('testing', 'data', 'realign_json.json')) + +# b = Bunch(infile=json_pth, otherthing='blue', yat=True) +# newbdict, bhash = b._get_bunch_hash() +# assert bhash == 'd1f46750044c3de102efc847720fc35f' +# # Make sure the hash stored in the json file for `infile` is correct. +# jshash = md5() +# with open(json_pth, 'r') as fp: +# jshash.update(fp.read().encode('utf-8')) +# assert newbdict['infile'][0][1] == jshash.hexdigest() +# assert newbdict['yat'] is True diff --git a/nipype/utils/tests/test_misc.py b/nipype/utils/tests/test_misc.py index 8896039763..88b6a73e57 100644 --- a/nipype/utils/tests/test_misc.py +++ b/nipype/utils/tests/test_misc.py @@ -1,17 +1,17 @@ # -*- coding: utf-8 -*- # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -from future import standard_library -standard_library.install_aliases() +from __future__ import print_function, unicode_literals import os from shutil import rmtree -from builtins import next - import pytest +from future import standard_library + +from nipype.utils.misc import (container_to_string, str2bool, + flatten, unflatten) -from nipype.utils.misc import (container_to_string, str2bool, flatten, - unflatten) +standard_library.install_aliases() def test_cont_to_str(): diff --git a/nipype/workflows/smri/freesurfer/autorecon3.py b/nipype/workflows/smri/freesurfer/autorecon3.py index 477198d2da..3c7e487842 100644 --- a/nipype/workflows/smri/freesurfer/autorecon3.py +++ b/nipype/workflows/smri/freesurfer/autorecon3.py @@ -847,7 +847,7 @@ def out_aseg(in_aparcaseg, in_aseg, out_file): [('out_file', meas_config['preproc']['out_name'])]), ]) - for value, val_config in meas_config['smooth'].iteritems(): + for value, val_config in meas_config['smooth'].items(): surf2surf = pe.Node( SurfaceSmooth(), name=val_config['name']) surf2surf.inputs.fwhm = value