diff --git a/.travis.yml b/.travis.yml index 01364010..da6b5dd2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,12 +3,9 @@ python: - "2.6" - "2.7" before_script: - - export QINIU_ACCESS_KEY="X0XpjFmLMTJpHB_ESHjeolCtipk-1U3Ok7LVTdoN" - - export QINIU_SECRET_KEY="wenlwkU1AYwNBf7Q9cCoG4VT_GYyrHE9AS_R2u81" - - export QINIU_PIC_KEY="hello_jpg" - - export QINIU_NOEXIST_PIC_KEY="no_exist" - - export QINIU_BUCKET_NAME="pysdk" - - export QINIU_DOMAIN="pysdk.qiniudn.com" -script: + - source env.sh + - export PYTHONPATH="$PYTHONPATH:." +script: - python setup.py nosetests - - export PYTHONPATH="$PYTHONPATH:." && python docs/demo.py && python docs/gist/conf.py + - python docs/demo.py + - python docs/gist/conf.py diff --git a/env.sh b/env.sh new file mode 100644 index 00000000..027451ce --- /dev/null +++ b/env.sh @@ -0,0 +1,6 @@ +export QINIU_ACCESS_KEY="X0XpjFmLMTJpHB_ESHjeolCtipk-1U3Ok7LVTdoN" +export QINIU_SECRET_KEY="wenlwkU1AYwNBf7Q9cCoG4VT_GYyrHE9AS_R2u81" +export QINIU_PIC_KEY="hello_jpg" +export QINIU_NOEXIST_PIC_KEY="no_exist" +export QINIU_BUCKET_NAME="pysdk" +export QINIU_DOMAIN="pysdk.qiniudn.com" diff --git a/qiniu/httplib_chunk.py b/qiniu/httplib_chunk.py new file mode 100644 index 00000000..48afcde2 --- /dev/null +++ b/qiniu/httplib_chunk.py @@ -0,0 +1,122 @@ +""" +CHANGELOG with standdard httplib + +1. HTTPConnection can send trunked data. +2. Remove httplib's automatic Content-Length insertion when data is a file-like object. +""" + +# -*- coding: utf-8 -*- + +import httplib +from httplib import _CS_REQ_STARTED, _CS_REQ_SENT +import string +import os +from array import array + +class HTTPConnection(httplib.HTTPConnection): + + def send(self, data, is_chunked=False): + """Send `data' to the server.""" + if self.sock is None: + if self.auto_open: + self.connect() + else: + raise NotConnected() + + if self.debuglevel > 0: + print "send:", repr(data) + blocksize = 8192 + if hasattr(data,'read') and not isinstance(data, array): + if self.debuglevel > 0: print "sendIng a read()able" + datablock = data.read(blocksize) + while datablock: + print 'chunked:', is_chunked + if is_chunked: + if self.debuglevel > 0: print 'send: with trunked data' + lenstr = string.upper(hex(len(datablock))[2:]) + self.sock.sendall('%s\r\n%s\r\n' % (lenstr, datablock)) + else: + self.sock.sendall(datablock) + datablock = data.read(blocksize) + if is_chunked: + self.sock.sendall('0\r\n\r\n') + else: + self.sock.sendall(data) + + + def _set_content_length(self, body): + # Set the content-length based on the body. + thelen = None + try: + thelen = str(len(body)) + except (TypeError, AttributeError), te: + # Don't send a length if this failed + if self.debuglevel > 0: print "Cannot stat!!" + + if thelen is not None: + self.putheader('Content-Length', thelen) + return True + return False + + + def _send_request(self, method, url, body, headers): + # Honor explicitly requested Host: and Accept-Encoding: headers. + header_names = dict.fromkeys([k.lower() for k in headers]) + skips = {} + if 'host' in header_names: + skips['skip_host'] = 1 + if 'accept-encoding' in header_names: + skips['skip_accept_encoding'] = 1 + + self.putrequest(method, url, **skips) + + is_chunked = False + if body and header_names.get('Transfer-Encoding') == 'chunked': + is_chunked = True + elif body and ('content-length' not in header_names): + is_chunked = not self._set_content_length(body) + if is_chunked: + self.putheader('Transfer-Encoding', 'chunked') + for hdr, value in headers.iteritems(): + self.putheader(hdr, value) + + self.endheaders(body, is_chunked=is_chunked) + + + def endheaders(self, message_body=None, is_chunked=False): + """Indicate that the last header line has been sent to the server. + + This method sends the request to the server. The optional + message_body argument can be used to pass a message body + associated with the request. The message body will be sent in + the same packet as the message headers if it is string, otherwise it is + sent as a separate packet. + """ + if self.__state == _CS_REQ_STARTED: + self.__state = _CS_REQ_SENT + else: + raise CannotSendHeader() + self._send_output(message_body, is_chunked=is_chunked) + + + def _send_output(self, message_body=None, is_chunked=False): + """Send the currently buffered request and clear the buffer. + + Appends an extra \\r\\n to the buffer. + A message_body may be specified, to be appended to the request. + """ + self._buffer.extend(("", "")) + msg = "\r\n".join(self._buffer) + del self._buffer[:] + # If msg and message_body are sent in a single send() call, + # it will avoid performance problems caused by the interaction + # between delayed ack and the Nagle algorithm. + if isinstance(message_body, str): + msg += message_body + message_body = None + self.send(msg) + if message_body is not None: + #message_body was not a string (i.e. it is a file) and + #we must run the risk of Nagle + self.send(message_body, is_chunked=is_chunked) + diff --git a/qiniu/rpc.py b/qiniu/rpc.py index 1afc9041..b35f9da0 100644 --- a/qiniu/rpc.py +++ b/qiniu/rpc.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -import httplib +import httplib_chunk as httplib import json import cStringIO import conf @@ -55,8 +55,8 @@ def call_with_multipart(self, path, fields=None, files=None): * fields => {key} * files => [{filename, data, content_type}] """ - content_type, body = self.encode_multipart_formdata(fields, files) - return self.call_with(path, body, content_type, len(body)) + content_type, mr = self.encode_multipart_formdata(fields, files) + return self.call_with(path, mr, content_type, mr.length()) def call_with_form(self, path, ops): """ @@ -138,9 +138,15 @@ class MultiReader(object): def __init__(self, readers): self.readers = [] self.content_length = 0 + self.valid_content_length = True for r in readers: if hasattr(r, 'read'): - self.content_length += self._get_content_length(r) + if self.valid_content_length: + length = self._get_content_length(r) + if length is not None: + self.content_length += length + else: + self.valid_content_length = False else: buf = r if not isinstance(buf, basestring): @@ -150,17 +156,22 @@ def __init__(self, readers): self.content_length += len(buf) self.readers.append(r) - def __len__(self): - return self.content_length + + # don't name it __len__, because the length of MultiReader is not alway valid. + def length(self): + return self.content_length if self.valid_content_length else None + def _get_content_length(self, reader): - data_len = 0 - try: - reader.seek(0, 2) - data_len= reader.tell() - reader.seek(0, 0) - except (AttributeError, OSError): - print 'can not get content_length' + data_len = None + if hasattr(reader, 'seek') and hasattr(reader, 'tell'): + try: + reader.seek(0, 2) + data_len= reader.tell() + reader.seek(0, 0) + except OSError: + # Don't send a length if this failed + data_len = None return data_len def read(self, n=-1): diff --git a/qiniu/test/io_test.py b/qiniu/test/io_test.py index 0654e0dc..367d9bcb 100644 --- a/qiniu/test/io_test.py +++ b/qiniu/test/io_test.py @@ -89,11 +89,12 @@ def test_put_StringIO(): def test_put_urlopen(): key = "test_%s" % r(9) - data = urllib.urlopen('http://http://cheneya.qiniudn.com/hello_jpg') + data = urllib.urlopen('http://cheneya.qiniudn.com/hello_jpg') ret, err = io.put(policy.token(), key, data) + print 'error', err assert err is None - def test_put_no_length(); + def test_put_no_length(): class test_reader(object): def __init__(self): self.data = 'abc' @@ -108,7 +109,10 @@ def read(self, n=None): return r key = "test_%s" % r(9) data = test_reader() - ret, err = io.put(policy.token(), key, data) + + extra.check_crc = 2 + extra.crc32 = binascii.crc32('abc') & 0xFFFFFFFF + ret, err = io.put(policy.token(), key, data, extra) assert err is None test_put() @@ -120,6 +124,7 @@ def read(self, n=None): test_put_unicode4() test_put_StringIO() test_put_urlopen() + test_put_no_length() def test_put_file(self): localfile = "%s" % __file__ diff --git a/qiniu/test/rpc_test.py b/qiniu/test/rpc_test.py index b3014c71..db35b11a 100644 --- a/qiniu/test/rpc_test.py +++ b/qiniu/test/rpc_test.py @@ -45,7 +45,7 @@ def tripper(client, method, path, body): tpl = "--%s\r\n%s\r\n\r\n%s\r\n--%s--\r\n" % (boundary, dispostion, "auth_string", boundary) self.assertEqual(len(tpl), client._header["Content-Length"]) - self.assertEqual(len(tpl), len(body)) + self.assertEqual(len(tpl), body.length()) round_tripper = tripper client.call_with_multipart("/hello", fields={"auth": "auth_string"}) @@ -123,13 +123,13 @@ def test_encode(self): 'mime_type': 'application/octet-stream', } ] - content_type, body = rpc.Client('localhost').encode_multipart_formdata(fields, files) + content_type, mr = rpc.Client('localhost').encode_multipart_formdata(fields, files) t, b = encode_multipart_formdata2( [('a', '1'), ('b', '2')], [('file', 'key1', 'data1'), ('file', 'key2', 'data2')] ) assert t == content_type - assert len(b) == len(body) + assert len(b) == mr.length() def test_unicode(self): def test1(): diff --git a/qiniu/test/rsf_test.py b/qiniu/test/rsf_test.py index df729300..1aa24d1a 100644 --- a/qiniu/test/rsf_test.py +++ b/qiniu/test/rsf_test.py @@ -11,9 +11,9 @@ class TestRsf(unittest.TestCase): def test_list_prefix(self): c = rsf.Client() - ret, err = c.list_prefix(bucket_name) - assert err is rsf.EOF - self.assertEqual(len(ret.get('items'))>0, True) + ret, err = c.list_prefix(bucket_name, limit = 1) + self.assertEqual(err is rsf.EOF or err is None, True) + assert len(ret.get('items')) == 1 if __name__ == "__main__":