Using requests library for HTTP requests.

CHANGELOG

@@ -1,3 +1,5 @@
+Version 0.18 (TBD)
+
 Version 0.17 (10/May/2015)
 . [NEW] Hurst exponent technical indicator (pyalgotrade.technical.hurst.HurstExponent).
 . [NEW] Added support for slippage models (pyalgotrade.broker.slippage) including a VolumeShareSlippage model like the one in Zipline (https://github.com/quantopian/zipline).

README.md

@@ -40,6 +40,8 @@ PyAlgoTrade is developed using Python 2.7 and depends on:
 
  * [NumPy and SciPy](http://numpy.scipy.org/).
  * [pytz](http://pytz.sourceforge.net/).
+ * [dateutil](https://dateutil.readthedocs.org/en/latest/).
+ * [requests](http://docs.python-requests.org/en/latest/).
  * [matplotlib](http://matplotlib.sourceforge.net/) for plotting support.
  * [ws4py](https://github.com/Lawouach/WebSocket-for-Python) for Bitstamp support.
  * [tornado](http://www.tornadoweb.org/en/stable/) for Bitstamp support.

pyalgotrade/tools/googlefinance.py

@@ -20,33 +20,25 @@
 .. moduleauthor:: Maciej Żok <[email protected]>
 """
 
-import urllib2
 import os
 import datetime
 
 import pyalgotrade.logger
 from pyalgotrade import bar
 from pyalgotrade.barfeed import googlefeed
+from pyalgotrade.utils import csvutils
 
 
 def download_csv(instrument, begin, end):
     url = "http://www.google.com/finance/historical"
-    url += "?q={quote}".format(quote=instrument)
-    url += "&startdate={date}".format(date=begin.strftime("%b+%d,+%Y"))
-    url += "&enddate={date}".format(date=end.strftime("%b+%d,+%Y"))
-    url += "&output=csv"
-
-    f = urllib2.urlopen(url)
-    if f.headers['Content-Type'] != 'application/vnd.ms-excel':
-        raise Exception("Failed to download data: %s" % f.getcode())
-    buff = f.read()
-    f.close()
-
-    # Remove the BOM
-    while not buff[0].isalnum():
-        buff = buff[1:]
-
-    return buff
+    params = {
+        "q": instrument,
+        "startdate": begin.strftime("%Y-%m-%d"),
+        "enddate": end.strftime("%Y-%m-%d"),
+        "output": "csv",
+    }
+
+    return csvutils.download_csv(url, url_params=params, content_type="application/vnd.ms-excel")
 
 
 def download_daily_bars(instrument, year, csvFile):

pyalgotrade/tools/quandl.py

@@ -18,20 +18,20 @@
 .. moduleauthor:: Gabriel Martin Becedillas Ruiz <[email protected]>
 """
 
-import urllib
-import urllib2
 import datetime
 import os
 from pyalgotrade import bar
 from pyalgotrade.barfeed import quandlfeed
 
 from pyalgotrade.utils import dt
+from pyalgotrade.utils import csvutils
 import pyalgotrade.logger
 
 
 # http://www.quandl.com/help/api
 
 def download_csv(sourceCode, tableCode, begin, end, frequency, authToken):
+    url = "http://www.quandl.com/api/v1/datasets/%s/%s.csv" % (sourceCode, tableCode)
     params = {
         "trim_start": begin.strftime("%Y-%m-%d"),
         "trim_end": end.strftime("%Y-%m-%d"),
@@ -40,20 +40,7 @@ def download_csv(sourceCode, tableCode, begin, end, frequency, authToken):
     if authToken is not None:
         params["auth_token"] = authToken
 
-    url = "http://www.quandl.com/api/v1/datasets/%s/%s.csv" % (sourceCode, tableCode)
-    url = "%s?%s" % (url, urllib.urlencode(params))
-
-    f = urllib2.urlopen(url)
-    if f.headers['Content-Type'] != 'text/csv':
-        raise Exception("Failed to download data: %s" % f.getcode())
-    buff = f.read()
-    f.close()
-
-    # Remove the BOM
-    while not buff[0].isalnum():
-        buff = buff[1:]
-
-    return buff
+    return csvutils.download_csv(url, params)
 
 
 def download_daily_bars(sourceCode, tableCode, year, csvFile, authToken=None):

pyalgotrade/tools/yahoofinance.py

@@ -18,14 +18,14 @@
 .. moduleauthor:: Gabriel Martin Becedillas Ruiz <[email protected]>
 """
 
-import urllib2
 import os
 import datetime
 
 import pyalgotrade.logger
 from pyalgotrade import bar
 from pyalgotrade.barfeed import yahoofeed
 from pyalgotrade.utils import dt
+from pyalgotrade.utils import csvutils
 
 
 def __adjust_month(month):
@@ -37,18 +37,7 @@ def __adjust_month(month):
 
 def download_csv(instrument, begin, end, frequency):
     url = "http://ichart.finance.yahoo.com/table.csv?s=%s&a=%d&b=%d&c=%d&d=%d&e=%d&f=%d&g=%s&ignore=.csv" % (instrument, __adjust_month(begin.month), begin.day, begin.year, __adjust_month(end.month), end.day, end.year, frequency)
-
-    f = urllib2.urlopen(url)
-    if f.headers['Content-Type'] != 'text/csv':
-        raise Exception("Failed to download data: %s" % f.getcode())
-    buff = f.read()
-    f.close()
-
-    # Remove the BOM
-    while not buff[0].isalnum():
-        buff = buff[1:]
-
-    return buff
+    return csvutils.download_csv(url)
 
 
 def download_daily_bars(instrument, year, csvFile):

pyalgotrade/utils/csvutils.py

@@ -19,6 +19,7 @@
 """
 
 import csv
+import requests
 
 
 # A faster (but limited) version of csv.DictReader
@@ -47,3 +48,20 @@ def next(self):
             self.__dict[self.__fieldNames[i]] = row[i]
 
         return self.__dict
+
+
+def download_csv(url, url_params=None, content_type="text/csv"):
+    response = requests.get(url, params=url_params)
+
+    response.raise_for_status()
+    response_content_type = response.headers['content-type']
+    if response_content_type != content_type:
+        raise Exception("Invalid content-type: %s" % response_content_type)
+
+    ret = response.text
+
+    # Remove the BOM
+    while not ret[0].isalnum():
+        ret = ret[1:]
+
+    return ret

setup.py

@@ -54,6 +54,7 @@
         "numpy",
         "pytz",
         "python-dateutil",
+        "requests",
     ],
     extras_require={
         'Scipy':  ["scipy"],

testcases/google_test.py

@@ -19,6 +19,7 @@
 """
 
 import os
+import datetime
 
 import common
 
@@ -45,14 +46,15 @@ def testBuildDailyFeed(self):
             instrument = "orcl"
             bf = googlefinance.build_feed([instrument], 2010, 2010, storage=tmpPath)
             bf.loadAll()
+            self.assertEqual(bf[instrument][-1].getDateTime(), datetime.datetime(2010, 12, 31))
             self.assertEqual(bf[instrument][-1].getOpen(), 31.22)
             self.assertEqual(bf[instrument][-1].getClose(), 31.30)
 
     def testInvalidInstrument(self):
         instrument = "inexistent"
 
         # Don't skip errors.
-        with self.assertRaisesRegexp(Exception, "HTTP Error 400: Bad Request"):
+        with self.assertRaisesRegexp(Exception, "400 Client Error: Bad Request"):
             with common.TmpDir() as tmpPath:
                 bf = googlefinance.build_feed([instrument], 2100, 2101, storage=tmpPath, frequency=bar.Frequency.DAY)
 

testcases/quandl_test.py

@@ -158,7 +158,7 @@ def testInvalidInstrument(self):
         instrument = "inexistent"
 
         # Don't skip errors.
-        with self.assertRaisesRegexp(Exception, "HTTP Error 404: Not Found"):
+        with self.assertRaisesRegexp(Exception, "404 Client Error: Not Found"):
             with common.TmpDir() as tmpPath:
                 quandl.build_feed(
                     instrument, [instrument], 2010, 2010, tmpPath, bar.Frequency.WEEK, authToken=auth_token

testcases/yahoo_test.py

@@ -78,7 +78,7 @@ def testInvalidDates(self):
         instrument = "orcl"
 
         # Don't skip errors.
-        with self.assertRaisesRegexp(Exception, "HTTP Error 404: Not Found"):
+        with self.assertRaisesRegexp(Exception, "404 Client Error: Not Found"):
             with common.TmpDir() as tmpPath:
                 bf = yahoofinance.build_feed([instrument], 2100, 2101, storage=tmpPath, frequency=bar.Frequency.DAY)