Merge branch 'dev' into stats_enhancements

Conflicts:
	cmd/connectomestats.cpp
	docs/reference/commands/connectomestats.rst
	testing/data

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "testing/data"]
-	path = testing/data
+	path = testing/binaries/data
 	url = https://github.com/MRtrix3/test_data
+[submodule "testing/script_data"]
+	path = testing/scripts/data
+	url = https://github.com/MRtrix3/script_test_data.git

.travis.yml

@@ -133,9 +133,10 @@ after_failure:
         cat configure.log
         cat build.log
         ;;
-      *)
+      "run")
         cat configure.log
         cat build.log
-        cat testing.log
+        cat testing_units.log
+        cat testing_binaries.log
     esac
   - sleep 10

appveyor.yml

@@ -15,7 +15,8 @@ build_script:
   - cmd: C:\msys64\msys2_shell.cmd -mingw64 -here -defterm -no-start -c "./build -nowarnings -nopaginate" || (appveyor PushArtifact %APPVEYOR_BUILD_FOLDER%/build.log & EXIT 1)
 
 test_script:
-  - cmd: C:\msys64\msys2_shell.cmd -mingw64 -here -defterm -no-start -c "./run_tests" || (appveyor PushArtifact %APPVEYOR_BUILD_FOLDER%/testing.log & EXIT 1)
+  - cmd: C:\msys64\msys2_shell.cmd -mingw64 -here -defterm -no-start -c "./run_tests units" || (appveyor PushArtifact %APPVEYOR_BUILD_FOLDER%/testing_units.log & EXIT 1)
+  - cmd: C:\msys64\msys2_shell.cmd -mingw64 -here -defterm -no-start -c "./run_tests binaries" || (appveyor PushArtifact %APPVEYOR_BUILD_FOLDER%/testing_binaries.log & EXIT 1)
 
 artifacts:
   - path: pacman.log

bin/5ttgen

@@ -1,5 +1,20 @@
 #!/usr/bin/env python
 
+# Copyright (c) 2008-2019 the MRtrix3 contributors.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Covered Software is provided under this License on an "as is"
+# basis, without warranty of any kind, either expressed, implied, or
+# statutory, including, without limitation, warranties that the
+# Covered Software is free of defects, merchantable, fit for a
+# particular purpose or non-infringing.
+# See the Mozilla Public License v. 2.0 for more details.
+#
+# For more details, see http://www.mrtrix.org/.
+
 # Script that generates a five-tissue-type (5TT) segmented image: the format appropriate for ACT
 #
 # In this script, major stages of processing can be performed in one of two ways:
@@ -11,7 +26,7 @@
 
 
 def usage(cmdline): #pylint: disable=unused-variable
-  from mrtrix3 import algorithm #pylint: disable=no-name-in-module
+  from mrtrix3 import algorithm #pylint: disable=no-name-in-module, import-outside-toplevel
 
   cmdline.set_author('Robert E. Smith ([email protected])')
   cmdline.set_synopsis('Generate a 5TT image suitable for ACT')
@@ -29,7 +44,7 @@ def usage(cmdline): #pylint: disable=unused-variable
 
 
 def execute(): #pylint: disable=unused-variable
-  from mrtrix3 import algorithm, app, run #pylint: disable=no-name-in-module
+  from mrtrix3 import algorithm, app, run #pylint: disable=no-name-in-module, import-outside-toplevel
 
   # Find out which algorithm the user has requested
   alg = algorithm.get_module(app.ARGS.algorithm)

bin/blend

@@ -1,4 +1,20 @@
 #!/usr/bin/env python2
+
+# Copyright (c) 2008-2019 the MRtrix3 contributors.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Covered Software is provided under this License on an "as is"
+# basis, without warranty of any kind, either expressed, implied, or
+# statutory, including, without limitation, warranties that the
+# Covered Software is free of defects, merchantable, fit for a
+# particular purpose or non-infringing.
+# See the Mozilla Public License v. 2.0 for more details.
+#
+# For more details, see http://www.mrtrix.org/.
+
 import os
 import sys
 

bin/convert_bruker

@@ -1,5 +1,20 @@
 #!/usr/bin/env python
 
+# Copyright (c) 2008-2019 the MRtrix3 contributors.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Covered Software is provided under this License on an "as is"
+# basis, without warranty of any kind, either expressed, implied, or
+# statutory, including, without limitation, warranties that the
+# Covered Software is free of defects, merchantable, fit for a
+# particular purpose or non-infringing.
+# See the Mozilla Public License v. 2.0 for more details.
+#
+# For more details, see http://www.mrtrix.org/.
+
 import sys, os.path
 
 if len (sys.argv) != 3:

bin/dwi2response

@@ -1,12 +1,27 @@
 #!/usr/bin/env python
 
+# Copyright (c) 2008-2019 the MRtrix3 contributors.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Covered Software is provided under this License on an "as is"
+# basis, without warranty of any kind, either expressed, implied, or
+# statutory, including, without limitation, warranties that the
+# Covered Software is free of defects, merchantable, fit for a
+# particular purpose or non-infringing.
+# See the Mozilla Public License v. 2.0 for more details.
+#
+# For more details, see http://www.mrtrix.org/.
+
 # Script for estimating response functions for spherical deconvolution
 # A number of different approaches are available within this script for performing response function estimation.
 
 
 
 def usage(cmdline): #pylint: disable=unused-variable
-  from mrtrix3 import algorithm, app #pylint: disable=no-name-in-module
+  from mrtrix3 import algorithm, app #pylint: disable=no-name-in-module, import-outside-toplevel
 
   cmdline.set_author('Robert E. Smith ([email protected]) and Thijs Dhollander ([email protected])')
   cmdline.set_synopsis('Estimate response function(s) for spherical deconvolution')
@@ -19,7 +34,7 @@ def usage(cmdline): #pylint: disable=unused-variable
   common_options.add_argument('-voxels', help='Output an image showing the final voxel selection(s)')
   common_options.add_argument('-shells', help='The b-value(s) to use in response function estimation (comma-separated list in case of multiple b-values, b=0 must be included explicitly)')
   common_options.add_argument('-lmax', help='The maximum harmonic degree(s) for response function estimation (comma-separated list in case of multiple b-values)')
-  app.add_dwgrad_import_options()
+  app.add_dwgrad_import_options(cmdline)
 
   # Import the command-line settings for all algorithms found in the relevant directory
   algorithm.usage(cmdline)
@@ -30,8 +45,8 @@ def usage(cmdline): #pylint: disable=unused-variable
 
 
 def execute(): #pylint: disable=unused-variable
-  from mrtrix3 import MRtrixError #pylint: disable=no-name-in-module
-  from mrtrix3 import algorithm, app, image, path, run #pylint: disable=no-name-in-module
+  from mrtrix3 import MRtrixError #pylint: disable=no-name-in-module, import-outside-toplevel
+  from mrtrix3 import algorithm, app, image, path, run #pylint: disable=no-name-in-module, import-outside-toplevel
 
   # Find out which algorithm the user has requested
   alg = algorithm.get_module(app.ARGS.algorithm)
@@ -87,8 +102,11 @@ def execute(): #pylint: disable=unused-variable
 
   if app.ARGS.mask:
     # Check that the brain mask is appropriate
-    if image.Header('mask.mif').size()[:3] != image.Header('dwi.mif').size()[:3]:
+    mask_header = image.Header('mask.mif')
+    if mask_header.size()[:3] != image.Header('dwi.mif').size()[:3]:
       raise MRtrixError('Dimensions of provided mask image do not match DWI')
+    if not (len(mask_header.size()) == 3 or (len(mask_header.size()) == 4 and mask_header.size()[3] == 1)):
+      raise MRtrixError('Provided mask image needs to be a 3D image')
   else:
     app.console('Computing brain mask (dwi2mask)...')
     run.command('dwi2mask dwi.mif mask.mif', show=False)

bin/dwibiascorrect

@@ -1,26 +1,41 @@
 #!/usr/bin/env python
 
+# Copyright (c) 2008-2019 the MRtrix3 contributors.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Covered Software is provided under this License on an "as is"
+# basis, without warranty of any kind, either expressed, implied, or
+# statutory, including, without limitation, warranties that the
+# Covered Software is free of defects, merchantable, fit for a
+# particular purpose or non-infringing.
+# See the Mozilla Public License v. 2.0 for more details.
+#
+# For more details, see http://www.mrtrix.org/.
+
 # Script that performs B1 field inhomogeneity correction for a DWI volume series
 # Bias field is typically estimated using the mean b=0 image, and subsequently used to correct all volumes
 
 
 def usage(cmdline): #pylint: disable=unused-variable
-  from mrtrix3 import algorithm, app #pylint: disable=no-name-in-module
+  from mrtrix3 import algorithm, app #pylint: disable=no-name-in-module, import-outside-toplevel
   cmdline.set_author('Robert E. Smith ([email protected])')
   cmdline.set_synopsis('Perform B1 field inhomogeneity correction for a DWI volume series')
   common_options = cmdline.add_argument_group('Options common to all dwibiascorrect algorithms')
   common_options.add_argument('-mask', metavar='image', help='Manually provide a mask image for bias field estimation')
   common_options.add_argument('-bias', metavar='image', help='Output the estimated bias field')
-  app.add_dwgrad_import_options()
+  app.add_dwgrad_import_options(cmdline)
 
   # Import the command-line settings for all algorithms found in the relevant directory
   algorithm.usage(cmdline)
 
 
 
 def execute(): #pylint: disable=unused-variable
-  from mrtrix3 import MRtrixError #pylint: disable=no-name-in-module
-  from mrtrix3 import algorithm, app, image, path, run #pylint: disable=no-name-in-module
+  from mrtrix3 import MRtrixError #pylint: disable=no-name-in-module, import-outside-toplevel
+  from mrtrix3 import algorithm, app, image, path, run #pylint: disable=no-name-in-module, import-outside-toplevel
 
   # Find out which algorithm the user has requested
   alg = algorithm.get_module(app.ARGS.algorithm)

bin/dwicat

@@ -1,5 +1,20 @@
 #!/usr/bin/env python
 
+# Copyright (c) 2008-2019 the MRtrix3 contributors.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Covered Software is provided under this License on an "as is"
+# basis, without warranty of any kind, either expressed, implied, or
+# statutory, including, without limitation, warranties that the
+# Covered Software is free of defects, merchantable, fit for a
+# particular purpose or non-infringing.
+# See the Mozilla Public License v. 2.0 for more details.
+#
+# For more details, see http://www.mrtrix.org/.
+
 
 
 import json
@@ -21,8 +36,8 @@ def usage(cmdline): #pylint: disable=unused-variable
 
 
 def execute(): #pylint: disable=unused-variable
-  from mrtrix3 import MRtrixError #pylint: disable=no-name-in-module
-  from mrtrix3 import app, image, path, run #pylint: disable=no-name-in-module
+  from mrtrix3 import MRtrixError #pylint: disable=no-name-in-module, import-outside-toplevel
+  from mrtrix3 import app, image, path, run #pylint: disable=no-name-in-module, import-outside-toplevel
 
   num_inputs = len(app.ARGS.inputs)
   if num_inputs < 2:

bin/dwifslpreproc

@@ -1,5 +1,20 @@
 #!/usr/bin/env python
 
+# Copyright (c) 2008-2019 the MRtrix3 contributors.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Covered Software is provided under this License on an "as is"
+# basis, without warranty of any kind, either expressed, implied, or
+# statutory, including, without limitation, warranties that the
+# Covered Software is free of defects, merchantable, fit for a
+# particular purpose or non-infringing.
+# See the Mozilla Public License v. 2.0 for more details.
+#
+# For more details, see http://www.mrtrix.org/.
+
 # Script for performing DWI pre-processing using FSL 5.0 (onwards) tools eddy / topup / applytopup
 
 
@@ -10,7 +25,7 @@ from distutils.spawn import find_executable
 
 
 def usage(cmdline): #pylint: disable=unused-variable
-  from mrtrix3 import app #pylint: disable=no-name-in-module
+  from mrtrix3 import app #pylint: disable=no-name-in-module, import-outside-toplevel
   cmdline.set_author('Robert E. Smith ([email protected])')
   cmdline.set_synopsis('Perform diffusion image pre-processing using FSL\'s eddy tool; including inhomogeneity distortion correction using FSL\'s topup tool if possible')
   cmdline.add_description('This script is intended to provide convenience of use of the FSL software tools topup and eddy for performing DWI pre-processing, by encapsulating some of the surrounding image data and metadata processing steps. It is intended to simply these processing steps for most commonly-used DWI acquisition strategies, whilst also providing support for some more exotic acquisitions. The "example usage" section demonstrates the ways in which the script can be used based on the (compulsory) -rpe_* command-line options.')
@@ -34,6 +49,7 @@ def usage(cmdline): #pylint: disable=unused-variable
   cmdline.add_citation('Andersson, J. L.; Skare, S. & Ashburner, J. How to correct susceptibility distortions in spin-echo echo-planar images: application to diffusion tensor imaging. NeuroImage, 2003, 20, 870-888', condition='If performing EPI susceptibility distortion correction', is_external=True)
   cmdline.add_citation('Andersson, J. L. R.; Graham, M. S.; Zsoldos, E. & Sotiropoulos, S. N. Incorporating outlier detection and replacement into a non-parametric framework for movement and distortion correction of diffusion MR images. NeuroImage, 2016, 141, 556-572', condition='If including "--repol" in -eddy_options input', is_external=True)
   cmdline.add_citation('Andersson, J. L. R.; Graham, M. S.; Drobnjak, I.; Zhang, H.; Filippini, N. & Bastiani, M. Towards a comprehensive framework for movement and distortion correction of diffusion MR images: Within volume movement. NeuroImage, 2017, 152, 450-466', condition='If including "--mporder" in -eddy_options input', is_external=True)
+  cmdline.add_citation('Bastiani, M.; Cottaar, M.; Fitzgibbon, S.P.; Suri, S.; Alfaro-Almagro, F.; Sotiropoulos, S.N.; Jbabdi, S.; Andersson, J.L.R. Automated quality control for within and between studies diffusion MRI data using a non-parametric framework for movement and distortion correction. NeuroImage, 2019, 184, 801-812', condition='If using -eddyqc_test or -eddyqc_all option and eddy_quad is installed', is_external=True)
   cmdline.add_argument('input',  help='The input DWI series to be corrected')
   cmdline.add_argument('output', help='The output corrected image series')
   cmdline.add_argument('-pe_dir', metavar=('PE'), help='Manually specify the phase encoding direction of the input series; can be a signed axis number (e.g. -0, 1, +2), an axis designator (e.g. RL, PA, IS), or NIfTI axis codes (e.g. i-, j, k)')
@@ -45,8 +61,8 @@ def usage(cmdline): #pylint: disable=unused-variable
   cmdline.add_argument('-eddy_options', metavar=('" EddyOptions"'), help='Manually provide additional command-line options to the eddy command (provide a string within quotation marks that contains at least one space, even if only passing a single command-line option to eddy)')
   cmdline.add_argument('-eddyqc_text', metavar=('directory'), help='Copy the various text-based statistical outputs generated by eddy, and the output of eddy_qc (if installed), into an output directory')
   cmdline.add_argument('-eddyqc_all', metavar=('directory'), help='Copy ALL outputs generated by eddy (including images), and the output of eddy_qc (if installed), into an output directory')
-  app.add_dwgrad_export_options()
-  app.add_dwgrad_import_options()
+  app.add_dwgrad_export_options(cmdline)
+  app.add_dwgrad_import_options(cmdline)
   rpe_options = cmdline.add_argument_group('Options for specifying the acquisition phase-encoding design; note that one of the -rpe_* options MUST be provided')
   rpe_options.add_argument('-rpe_none', action='store_true', help='Specify that no reversed phase-encoding image data is being provided; eddy will perform eddy current and motion correction only')
   rpe_options.add_argument('-rpe_pair', action='store_true', help='Specify that a set of images (typically b=0 volumes) will be provided for use in inhomogeneity field estimation only (using the -se_epi option)')
@@ -64,8 +80,8 @@ def usage(cmdline): #pylint: disable=unused-variable
 
 
 def execute(): #pylint: disable=unused-variable
-  from mrtrix3 import CONFIG, MRtrixError #pylint: disable=no-name-in-module
-  from mrtrix3 import app, fsl, image, matrix, path, phaseencoding, run, utils #pylint: disable=no-name-in-module
+  from mrtrix3 import CONFIG, MRtrixError #pylint: disable=no-name-in-module, import-outside-toplevel
+  from mrtrix3 import app, fsl, image, matrix, path, phaseencoding, run, utils #pylint: disable=no-name-in-module, import-outside-toplevel
 
   if utils.is_windows():
     raise MRtrixError('Script cannot run on Windows due to FSL dependency')
@@ -806,7 +822,7 @@ def execute(): #pylint: disable=unused-variable
       app.debug('New: ' + str(new_slice_groups))
       slice_groups = new_slice_groups
 
-    matrix.save_numeric('slspec.txt', slice_groups, header='')
+    matrix.save_numeric('slspec.txt', slice_groups, header='', fmt='%d')
     eddy_manual_options.append('--slspec=slspec.txt')
 
 

bin/dwigradcheck

@@ -1,29 +1,42 @@
 #!/usr/bin/env python
 
-
+# Copyright (c) 2008-2019 the MRtrix3 contributors.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Covered Software is provided under this License on an "as is"
+# basis, without warranty of any kind, either expressed, implied, or
+# statutory, including, without limitation, warranties that the
+# Covered Software is free of defects, merchantable, fit for a
+# particular purpose or non-infringing.
+# See the Mozilla Public License v. 2.0 for more details.
+#
+# For more details, see http://www.mrtrix.org/.
 
 import copy, numbers, os, shutil, sys
 
 
 
 def usage(cmdline): #pylint: disable=unused-variable
-  from mrtrix3 import app #pylint: disable=no-name-in-module
+  from mrtrix3 import app #pylint: disable=no-name-in-module, import-outside-toplevel
   cmdline.set_author('Robert E. Smith ([email protected])')
   cmdline.set_synopsis('Check the orientation of the diffusion gradient table')
   cmdline.add_citation('Jeurissen, B.; Leemans, A.; Sijbers, J. Automated correction of improperly rotated diffusion gradient orientations in diffusion weighted MRI. Medical Image Analysis, 2014, 18(7), 953-962')
   cmdline.add_argument('input', help='The input DWI series to be checked')
   cmdline.add_argument('-mask', metavar='image', help='Provide a brain mask image')
   cmdline.add_argument('-number', type=int, default=10000, help='Set the number of tracks to generate for each test')
 
-  app.add_dwgrad_export_options()
-  app.add_dwgrad_import_options()
+  app.add_dwgrad_export_options(cmdline)
+  app.add_dwgrad_import_options(cmdline)
 
 
 
 
 def execute(): #pylint: disable=unused-variable
-  from mrtrix3 import MRtrixError #pylint: disable=no-name-in-module
-  from mrtrix3 import app, image, matrix, path, run #pylint: disable=no-name-in-module
+  from mrtrix3 import MRtrixError #pylint: disable=no-name-in-module, import-outside-toplevel
+  from mrtrix3 import app, image, matrix, path, run #pylint: disable=no-name-in-module, import-outside-toplevel
 
   image_dimensions = image.Header(path.from_user(app.ARGS.input, False)).size()
   if len(image_dimensions) != 4: