diff --git a/luigi/s3.py b/luigi/s3.py index 17cedc5682..b0dbfa3787 100644 --- a/luigi/s3.py +++ b/luigi/s3.py @@ -17,11 +17,9 @@ import os.path import random import tempfile +import warnings import urlparse -import boto -from boto.s3.key import Key - import configuration from ConfigParser import NoSectionError, NoOptionError from luigi.parameter import Parameter @@ -31,12 +29,20 @@ from luigi.task import ExternalTask from luigi.format import FileWrapper +logger = logging.getLogger('luigi-interface') + +try: + import boto + from boto.s3.key import Key +except ImportError: + logger.warning("Loading s3 module without boto installed. Will crash at runtime if s3 functionality is used.") + + # two different ways of marking a directory # with a suffix in S3 S3_DIRECTORY_MARKER_SUFFIX_0 = '_$folder$' S3_DIRECTORY_MARKER_SUFFIX_1 = '/' -logger = logging.getLogger('luigi-interface') class InvalidDeleteException(FileSystemException): pass diff --git a/luigi/webhdfs.py b/luigi/webhdfs.py index 36b04d8b05..96c8d15b73 100644 --- a/luigi/webhdfs.py +++ b/luigi/webhdfs.py @@ -5,10 +5,18 @@ """ import datetime import configuration +import logging import os import posixpath import urlparse -import whoops + +logger = logging.getLogger("luigi-interface") + +try: + import whoops +except ImportError: + logger.warning("Loading webhdfs module without whoops installed. Will crash at runtime if webhdfs functionality is used.") + def get_whoops_defaults(config=None): @@ -60,10 +68,6 @@ class WebHdfsClient(object): use the client, you must specify `namenode_host` and `namenode_port` in the `hdfs` section of your luigi configuration.""" - def __init__(self): - "Configures _homedir so that we can handle relative paths" - self._homedir = get_whoops("/").home() - def _make_absolute(self, inpath): """Makes the given path absolute if it's not already. It is assumed that the path is relative to self._homedir""" @@ -72,7 +76,7 @@ def _make_absolute(self, inpath): if scheme or posixpath.isabs(path): # if a scheme is specified, assume it's absolute. return path - return posixpath.join(self._homedir, path) + return posixpath.join(get_whoops("/").home(), path) def exists(self, path): """Returns true if the path exists and false otherwise"""