memmap_min_size

mne/__init__.py

@@ -5,7 +5,8 @@
 
 # have to import verbose first since it's needed by many things
 from .utils import set_log_level, set_log_file, verbose, set_config, \
-                   get_config, get_config_path, set_cache_dir
+                   get_config, get_config_path, set_cache_dir,\
+                   set_memmap_min_size
 
 from .cov import read_cov, write_cov, Covariance, \
                  compute_covariance, compute_raw_data_covariance, \

mne/parallel.py

@@ -14,7 +14,7 @@
 
 
 @verbose
-def parallel_func(func, n_jobs, verbose=None, max_nbytes=None):
+def parallel_func(func, n_jobs, verbose=None, max_nbytes='auto'):
     """Return parallel instance with delayed function
 
     Util function to use joblib only if available
@@ -29,10 +29,11 @@ def parallel_func(func, n_jobs, verbose=None, max_nbytes=None):
         If not None, override default verbose level (see mne.verbose).
         INFO or DEBUG will print parallel status, others will not.
     max_nbytes int, str, or None
-        Threshold on the size of arrays passed to the workers that
+        Threshold on the minimum size of arrays passed to the workers that
         triggers automated memmory mapping. Can be an int in Bytes,
         or a human-readable string, e.g., '1M' for 1 megabyte.
-        Use None to disable memmaping of large arrays.
+        Use None to disable memmaping of large arrays. Use 'auto' to
+        use the value set using mne.set_memmap_min_size.
 
     Returns
     -------
@@ -62,10 +63,14 @@ def parallel_func(func, n_jobs, verbose=None, max_nbytes=None):
             parallel = list
             return parallel, my_func, n_jobs
 
+    # check if joblib is recent enough to support memmaping
     aspec = inspect.getargspec(Parallel.__init__)
     joblib_mmap = ('temp_folder' in aspec.args and 'max_nbytes' in aspec.args)
 
     cache_dir = get_config('MNE_CACHE_DIR', None)
+    if isinstance(max_nbytes, basestring) and max_nbytes == 'auto':
+        max_nbytes = get_config('MNE_MEMMAP_MIN_SIZE', None)
+
     if max_nbytes is not None:
         if not joblib_mmap and cache_dir is not None:
             logger.warn('"MNE_CACHE_DIR" is set but a newer version of joblib '

mne/stats/cluster_level.py

@@ -748,8 +748,7 @@ def _permutation_cluster_test(X, threshold, n_permutations, tail, stat_fun,
         slices = [slice(splits_idx[k], splits_idx[k + 1])
                                                     for k in range(len(X))]
 
-    parallel, my_do_perm_func, _ = parallel_func(do_perm_func, n_jobs,
-                                                 max_nbytes=1e6)
+    parallel, my_do_perm_func, _ = parallel_func(do_perm_func, n_jobs)
 
     # Step 2: If we have some clusters, repeat process on permuted data
     # -------------------------------------------------------------------

mne/utils.py

@@ -549,11 +549,31 @@ def set_cache_dir(cache_dir):
         temporary file storage.
     """
     if cache_dir is not None and not op.exists(cache_dir):
-        raise ValueError('Directory %s does not exist' % cache_dir)
+        raise IOError('Directory %s does not exist' % cache_dir)
 
     set_config('MNE_CACHE_DIR', cache_dir)
 
 
+def set_memmap_min_size(memmap_min_size):
+    """Set the minimum size for memmaping of arrays for parallel processing
+
+    Parameters
+    ----------
+    memmap_min_size: str or None
+        Threshold on the minimum size of arrays that triggers automated memmory
+        mapping for parallel processing, e.g., '1M' for 1 megabyte.
+        Use None to disable memmaping of large arrays.
+    """
+    if memmap_min_size is not None:
+        if not isinstance(memmap_min_size, basestring):
+            raise ValueError('\'memmap_min_size\' has to be a string.')
+        if memmap_min_size[-1] not in ['K', 'M', 'G']:
+            raise ValueError('The size has to be given in kilo-, mega-, or '
+                             'gigabytes, e.g., 100K, 500M, 1G.')
+
+    set_config('MNE_MEMMAP_MIN_SIZE', memmap_min_size)
+
+
 # List the known configuration values
 known_config_types = [
     'MNE_BROWSE_RAW_SIZE',
@@ -564,6 +584,7 @@ def set_cache_dir(cache_dir):
     'MNE_USE_CUDA',
     'SUBJECTS_DIR',
     'MNE_CACHE_DIR',
+    'MNE_MEMMAP_MIN_SIZE'
     ]
 
 # These allow for partial matches, e.g. 'MNE_STIM_CHANNEL_1' is okay key