organize IPython.parallel into subpackages

IPython/config/default/ipcluster_config.py

@@ -23,16 +23,16 @@
 # - PBSControllerLauncher
 # - SGEControllerLauncher
 # - WindowsHPCControllerLauncher
-# c.Global.controller_launcher = 'IPython.parallel.launcher.LocalControllerLauncher'
-# c.Global.controller_launcher = 'IPython.parallel.launcher.PBSControllerLauncher'
+# c.Global.controller_launcher = 'IPython.parallel.apps.launcher.LocalControllerLauncher'
+# c.Global.controller_launcher = 'IPython.parallel.apps.launcher.PBSControllerLauncher'
 
 # Options are:
 # - LocalEngineSetLauncher
 # - MPIExecEngineSetLauncher
 # - PBSEngineSetLauncher
 # - SGEEngineSetLauncher
 # - WindowsHPCEngineSetLauncher
-# c.Global.engine_launcher = 'IPython.parallel.launcher.LocalEngineSetLauncher'
+# c.Global.engine_launcher = 'IPython.parallel.apps.launcher.LocalEngineSetLauncher'
 
 #-----------------------------------------------------------------------------
 # Global configuration

IPython/config/default/ipcontroller_config.py

@@ -89,7 +89,7 @@
 # Which class to use for the db backend.  Currently supported are DictDB (the
 # default), and MongoDB. Uncomment this line to enable MongoDB, which will
 # slow-down the Hub's responsiveness, but also reduce its memory footprint.
-# c.HubFactory.db_class = 'IPython.parallel.mongodb.MongoDB'
+# c.HubFactory.db_class = 'IPython.parallel.controller.mongodb.MongoDB'
 
 # The heartbeat ping frequency.  This is the frequency (in ms) at which the
 # Hub pings engines for heartbeats.  This determines how quickly the Hub
@@ -144,11 +144,11 @@
 
 # ----- in-memory configuration --------
 # this line restores the default behavior: in-memory storage of all results.
-# c.HubFactory.db_class = 'IPython.parallel.dictdb.DictDB'
+# c.HubFactory.db_class = 'IPython.parallel.controller.dictdb.DictDB'
 
 # ----- sqlite configuration --------
 # use this line to activate sqlite:
-# c.HubFactory.db_class = 'IPython.parallel.sqlitedb.SQLiteDB'
+# c.HubFactory.db_class = 'IPython.parallel.controller.sqlitedb.SQLiteDB'
 
 # You can specify the name of the db-file.  By default, this will be located
 # in the active cluster_dir, e.g. ~/.ipython/clusterz_default/tasks.db
@@ -165,7 +165,7 @@
 
 # ----- mongodb configuration --------
 # use this line to activate mongodb:
-# c.HubFactory.db_class = 'IPython.parallel.mongodb.MongoDB'
+# c.HubFactory.db_class = 'IPython.parallel.controller.mongodb.MongoDB'
 
 # You can specify the args and kwargs pymongo will use when creating the Connection.
 # For more information on what these options might be, see pymongo documentation.

IPython/external/ssh/tunnel.py

@@ -34,7 +34,7 @@
 except ImportError:
     pexpect = None
 
-from IPython.parallel.entry_point import select_random_ports
+from IPython.parallel.util import select_random_ports
 
 #-----------------------------------------------------------------------------
 # Code

IPython/parallel/__init__.py

@@ -12,14 +12,15 @@
 
 import zmq
 
-if zmq.__version__ < '2.1.3':
-    raise ImportError("IPython.parallel requires pyzmq/0MQ >= 2.1.3, you appear to have %s"%zmq.__version__)
+if zmq.__version__ < '2.1.4':
+    raise ImportError("IPython.parallel requires pyzmq/0MQ >= 2.1.4, you appear to have %s"%zmq.__version__)
 
-from .asyncresult import *
-from .client import Client
-from .dependency import *
-from .remotefunction import *
-from .view import *
 from IPython.utils.pickleutil import Reference
 
+from .client.asyncresult import *
+from .client.client import Client
+from .client.remotefunction import *
+from .client.view import *
+from .controller.dependency import *
+
 

IPython/parallel/ipclusterapp.py → IPython/parallel/apps/ipclusterapp.py

@@ -26,7 +26,7 @@
 
 from IPython.external.argparse import ArgumentParser, SUPPRESS
 from IPython.utils.importstring import import_item
-from IPython.parallel.clusterdir import (
+from .clusterdir import (
     ApplicationWithClusterDir, ClusterDirConfigLoader,
     ClusterDirError, PIDFileError
 )

IPython/parallel/ipcontrollerapp.py → IPython/parallel/apps/ipcontrollerapp.py

@@ -30,16 +30,17 @@
 from zmq.utils import jsonapi as json
 
 from IPython.config.loader import Config
+
 from IPython.parallel import factory
-from IPython.parallel.controller import ControllerFactory
-from IPython.parallel.clusterdir import (
+from .clusterdir import (
     ApplicationWithClusterDir,
     ClusterDirConfigLoader
 )
 from IPython.parallel.util import disambiguate_ip_address, split_url
 # from IPython.kernel.fcutil import FCServiceFactory, FURLError
 from IPython.utils.traitlets import Instance, Unicode
 
+from IPython.parallel.controller.controller import ControllerFactory
 
 
 #-----------------------------------------------------------------------------
@@ -117,11 +118,11 @@ def _add_arguments(self):
         ## Hub Config:
         paa('--mongodb', 
             dest='HubFactory.db_class', action='store_const',
-            const='IPython.parallel.mongodb.MongoDB', 
+            const='IPython.parallel.controller.mongodb.MongoDB', 
             help='Use MongoDB for task storage [default: in-memory]')
         paa('--sqlite', 
             dest='HubFactory.db_class', action='store_const',
-            const='IPython.parallel.sqlitedb.SQLiteDB', 
+            const='IPython.parallel.controller.sqlitedb.SQLiteDB',
             help='Use SQLite3 for DB task storage [default: in-memory]')
         paa('--hb',
             type=int, dest='HubFactory.hb', nargs=2,

IPython/parallel/ipengineapp.py → IPython/parallel/apps/ipengineapp.py

@@ -22,16 +22,17 @@
 import zmq
 from zmq.eventloop import ioloop
 
-from IPython.parallel.clusterdir import (
+from .clusterdir import (
     ApplicationWithClusterDir,
     ClusterDirConfigLoader
 )
 from IPython.zmq.log import EnginePUBHandler
 
 from IPython.parallel import factory
-from IPython.parallel.engine import EngineFactory
-from IPython.parallel.streamkernel import Kernel
+from IPython.parallel.engine.engine import EngineFactory
+from IPython.parallel.engine.streamkernel import Kernel
 from IPython.parallel.util import disambiguate_url
+
 from IPython.utils.importstring import import_item
 
 

IPython/parallel/iploggerapp.py → IPython/parallel/apps/iploggerapp.py

@@ -20,7 +20,7 @@
 
 import zmq
 
-from IPython.parallel.clusterdir import (
+from .clusterdir import (
     ApplicationWithClusterDir,
     ClusterDirConfigLoader
 )

IPython/parallel/launcher.py → IPython/parallel/apps/launcher.py

@@ -46,7 +46,7 @@ def check_output(*args, **kwargs):
 from IPython.utils.path import get_ipython_module_path
 from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError
 
-from .factory import LoggingFactory
+from IPython.parallel.factory import LoggingFactory
 
 # load winhpcjob only on Windows
 try:
@@ -64,15 +64,15 @@ def check_output(*args, **kwargs):
 
 
 ipcluster_cmd_argv = pycmd2argv(get_ipython_module_path(
-    'IPython.parallel.ipclusterapp'
+    'IPython.parallel.apps.ipclusterapp'
 ))
 
 ipengine_cmd_argv = pycmd2argv(get_ipython_module_path(
-    'IPython.parallel.ipengineapp'
+    'IPython.parallel.apps.ipengineapp'
 ))
 
 ipcontroller_cmd_argv = pycmd2argv(get_ipython_module_path(
-    'IPython.parallel.ipcontrollerapp'
+    'IPython.parallel.apps.ipcontrollerapp'
 ))
 
 #-----------------------------------------------------------------------------

IPython/parallel/logwatcher.py → IPython/parallel/apps/logwatcher.py

@@ -21,7 +21,7 @@
 
 from IPython.utils.traitlets import Int, Str, Instance, List
 
-from .factory import LoggingFactory
+from IPython.parallel.factory import LoggingFactory
 
 #-----------------------------------------------------------------------------
 # Classes

IPython/parallel/asyncresult.py → IPython/parallel/client/asyncresult.py

@@ -15,7 +15,7 @@
 from zmq import MessageTracker
 
 from IPython.external.decorator import decorator
-from . import error
+from IPython.parallel import error
 
 #-----------------------------------------------------------------------------
 # Classes

IPython/parallel/client.py → IPython/parallel/client/client.py

@@ -29,11 +29,12 @@
 from IPython.external.decorator import decorator
 from IPython.external.ssh import tunnel
 
-from . import error
-from . import util
-from . import streamsession as ss
+from IPython.parallel import error
+from IPython.parallel import streamsession as ss
+from IPython.parallel import util
+
 from .asyncresult import AsyncResult, AsyncHubResult
-from .clusterdir import ClusterDir, ClusterDirError
+from IPython.parallel.apps.clusterdir import ClusterDir, ClusterDirError
 from .view import DirectView, LoadBalancedView
 
 #--------------------------------------------------------------------------

IPython/parallel/view.py → IPython/parallel/client/view.py

@@ -23,10 +23,11 @@
 
 from IPython.external.decorator import decorator
 
+from IPython.parallel import util
+from IPython.parallel.controller.dependency import Dependency, dependent
+
 from . import map as Map
-from . import util
 from .asyncresult import AsyncResult, AsyncMapResult
-from .dependency import Dependency, dependent
 from .remotefunction import ParallelFunction, parallel, remote
 
 #-----------------------------------------------------------------------------
@@ -68,6 +69,7 @@ def spin_after(f, self, *args, **kwargs):
 # Classes
 #-----------------------------------------------------------------------------
 
+@testdec.skip_doctest
 class View(HasTraits):
     """Base View class for more convenint apply(f,*args,**kwargs) syntax via attributes.
     
@@ -105,7 +107,7 @@ class View(HasTraits):
     history=List()
     outstanding = Set()
     results = Dict()
-    client = Instance('IPython.parallel.client.Client')
+    client = Instance('IPython.parallel.Client')
 
     _socket = Instance('zmq.Socket')
     _flag_names = List(['targets', 'block', 'track'])
@@ -386,11 +388,6 @@ def importer(self):
         """sync_imports(local=True) as a property.
         
         See sync_imports for details.
-
-        In [10]: with v.importer:
-           ....:     import numpy
-           ....:     
-        importing numpy on engine(s)
         
         """
         return self.sync_imports(True)

IPython/parallel/controller.py → IPython/parallel/controller/controller.py

@@ -14,7 +14,6 @@
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
-import logging
 from multiprocessing import Process
 
 import zmq
@@ -23,7 +22,7 @@
 from IPython.utils.importstring import import_item
 from IPython.utils.traitlets import Int, CStr, Instance, List, Bool
 
-from .entry_point import signal_children
+from IPython.parallel.util import signal_children
 from .hub import Hub, HubFactory
 from .scheduler import launch_scheduler
 

IPython/parallel/dependency.py → IPython/parallel/controller/dependency.py

@@ -8,9 +8,9 @@
 
 from types import ModuleType
 
-from .asyncresult import AsyncResult
-from .error import UnmetDependency
-from .util import interactive
+from IPython.parallel.client.asyncresult import AsyncResult
+from IPython.parallel.error import UnmetDependency
+from IPython.parallel.util import interactive
 
 class depend(object):
     """Dependency decorator, for use with tasks.

IPython/parallel/heartmonitor.py → IPython/parallel/controller/heartmonitor.py

@@ -12,15 +12,14 @@
 
 from __future__ import print_function
 import time
-import logging
 import uuid
 
 import zmq
-from zmq.devices import ProcessDevice,ThreadDevice
+from zmq.devices import ProcessDevice, ThreadDevice
 from zmq.eventloop import ioloop, zmqstream
 
 from IPython.utils.traitlets import Set, Instance, CFloat, Bool
-from .factory import LoggingFactory
+from IPython.parallel.factory import LoggingFactory
 
 class Heart(object):
     """A basic heart object for responding to a HeartMonitor.

IPython/parallel/hub.py → IPython/parallel/controller/hub.py

@@ -27,12 +27,11 @@
 from IPython.utils.importstring import import_item
 from IPython.utils.traitlets import HasTraits, Instance, Int, CStr, Str, Dict, Set, List, Bool
 
-from .entry_point import select_random_ports
-from .factory import RegistrationFactory, LoggingFactory
+from IPython.parallel import error
+from IPython.parallel.factory import RegistrationFactory, LoggingFactory
+from IPython.parallel.util import select_random_ports, validate_url_container, ISO8601
 
-from . import error
 from .heartmonitor import HeartMonitor
-from .util import validate_url_container, ISO8601
 
 #-----------------------------------------------------------------------------
 # Code
@@ -160,11 +159,11 @@ def _notifier_port_default(self):
 
     monitor_url = CStr('')
 
-    db_class = CStr('IPython.parallel.dictdb.DictDB', config=True)
+    db_class = CStr('IPython.parallel.controller.dictdb.DictDB', config=True)
 
     # not configurable
-    db = Instance('IPython.parallel.dictdb.BaseDB')
-    heartmonitor = Instance('IPython.parallel.heartmonitor.HeartMonitor')
+    db = Instance('IPython.parallel.controller.dictdb.BaseDB')
+    heartmonitor = Instance('IPython.parallel.controller.heartmonitor.HeartMonitor')
     subconstructors = List()
     _constructed = Bool(False)
 

IPython/parallel/scheduler.py → IPython/parallel/controller/scheduler.py

@@ -36,11 +36,11 @@
 from IPython.external.decorator import decorator
 from IPython.utils.traitlets import Instance, Dict, List, Set
 
-from . import error
-from .dependency import Dependency
-from .entry_point import connect_logger, local_logger
-from .factory import SessionFactory
+from IPython.parallel import error
+from IPython.parallel.factory import SessionFactory
+from IPython.parallel.util import connect_logger, local_logger
 
+from .dependency import Dependency
 
 @decorator
 def logged(f,self,*args,**kwargs):

IPython/parallel/sqlitedb.py → IPython/parallel/controller/sqlitedb.py

@@ -17,7 +17,7 @@
 
 from IPython.utils.traitlets import CUnicode, CStr, Instance, List
 from .dictdb import BaseDB
-from .util import ISO8601
+from IPython.parallel.util import ISO8601
 
 #-----------------------------------------------------------------------------
 # SQLite operators, adapters, and converters

IPython/parallel/engine.py → IPython/parallel/engine/engine.py

@@ -22,11 +22,12 @@
 from IPython.utils.traitlets import Instance, Str, Dict, Int, Type, CFloat
 # from IPython.utils.localinterfaces import LOCALHOST 
 
-from . import heartmonitor
-from .factory import RegistrationFactory
+from IPython.parallel.controller.heartmonitor import Heart
+from IPython.parallel.factory import RegistrationFactory
+from IPython.parallel.streamsession import Message
+from IPython.parallel.util import disambiguate_url
+
 from .streamkernel import Kernel
-from .streamsession import Message
-from .util import disambiguate_url
 
 class EngineFactory(RegistrationFactory):
     """IPython engine"""
@@ -129,7 +130,7 @@ def complete_registration(self, msg):
                     loop=loop, user_ns = self.user_ns, logname=self.log.name)
             self.kernel.start()
             hb_addrs = [ disambiguate_url(addr, self.location) for addr in hb_addrs ]
-            heart = heartmonitor.Heart(*map(str, hb_addrs), heart_id=identity)
+            heart = Heart(*map(str, hb_addrs), heart_id=identity)
             # ioloop.DelayedCallback(heart.start, 1000, self.loop).start()
             heart.start()