Skip to content

Commit

Permalink
Fix luigi.s3 and luigi.webhdfs default imports
Browse files Browse the repository at this point in the history
Don't fail when importing luigi.s3 or luigi.webhdfs if deps
aren't satisified. This improves the ability to generate api
docs for these modules when required packages aren't installed,
and it is in line with the pattern used in luigi.{hdfs,hive,postgres}
and other places.
  • Loading branch information
jcrobak committed Sep 7, 2014
1 parent d74afeb commit e1ec004
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 10 deletions.
14 changes: 10 additions & 4 deletions luigi/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,9 @@
import os.path
import random
import tempfile
import warnings
import urlparse

import boto
from boto.s3.key import Key

import configuration
from ConfigParser import NoSectionError, NoOptionError
from luigi.parameter import Parameter
Expand All @@ -31,12 +29,20 @@
from luigi.task import ExternalTask
from luigi.format import FileWrapper

logger = logging.getLogger('luigi-interface')

try:
import boto
from boto.s3.key import Key
except ImportError:
logger.warning("Loading s3 module without boto installed. Will crash at runtime if s3 functionality is used.")


# two different ways of marking a directory
# with a suffix in S3
S3_DIRECTORY_MARKER_SUFFIX_0 = '_$folder$'
S3_DIRECTORY_MARKER_SUFFIX_1 = '/'

logger = logging.getLogger('luigi-interface')

class InvalidDeleteException(FileSystemException):
pass
Expand Down
16 changes: 10 additions & 6 deletions luigi/webhdfs.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,18 @@
"""
import datetime
import configuration
import logging
import os
import posixpath
import urlparse
import whoops

logger = logging.getLogger("luigi-interface")

try:
import whoops
except ImportError:
logger.warning("Loading webhdfs module without whoops installed. Will crash at runtime if webhdfs functionality is used.")



def get_whoops_defaults(config=None):
Expand Down Expand Up @@ -60,10 +68,6 @@ class WebHdfsClient(object):
use the client, you must specify `namenode_host` and `namenode_port` in the `hdfs` section of
your luigi configuration."""

def __init__(self):
"Configures _homedir so that we can handle relative paths"
self._homedir = get_whoops("/").home()

def _make_absolute(self, inpath):
"""Makes the given path absolute if it's not already. It is assumed that the path is
relative to self._homedir"""
Expand All @@ -72,7 +76,7 @@ def _make_absolute(self, inpath):
if scheme or posixpath.isabs(path):
# if a scheme is specified, assume it's absolute.
return path
return posixpath.join(self._homedir, path)
return posixpath.join(get_whoops("/").home(), path)

def exists(self, path):
"""Returns true if the path exists and false otherwise"""
Expand Down

0 comments on commit e1ec004

Please sign in to comment.