rpc.py/httplib can post chunked data

.travis.yml

@@ -3,12 +3,9 @@ python:
   - "2.6"
   - "2.7"
 before_script:
-  - export QINIU_ACCESS_KEY="X0XpjFmLMTJpHB_ESHjeolCtipk-1U3Ok7LVTdoN"
-  - export QINIU_SECRET_KEY="wenlwkU1AYwNBf7Q9cCoG4VT_GYyrHE9AS_R2u81"
-  - export QINIU_PIC_KEY="hello_jpg"
-  - export QINIU_NOEXIST_PIC_KEY="no_exist"
-  - export QINIU_BUCKET_NAME="pysdk"
-  - export QINIU_DOMAIN="pysdk.qiniudn.com"
-script: 
+  - source env.sh
+  - export PYTHONPATH="$PYTHONPATH:."
+script:
   - python setup.py nosetests
-  - export PYTHONPATH="$PYTHONPATH:." && python docs/demo.py && python docs/gist/conf.py
+  - python docs/demo.py
+  - python docs/gist/conf.py

env.sh

@@ -0,0 +1,6 @@
+export QINIU_ACCESS_KEY="X0XpjFmLMTJpHB_ESHjeolCtipk-1U3Ok7LVTdoN"
+export QINIU_SECRET_KEY="wenlwkU1AYwNBf7Q9cCoG4VT_GYyrHE9AS_R2u81"
+export QINIU_PIC_KEY="hello_jpg"
+export QINIU_NOEXIST_PIC_KEY="no_exist"
+export QINIU_BUCKET_NAME="pysdk"
+export QINIU_DOMAIN="pysdk.qiniudn.com"

qiniu/httplib_chunk.py

@@ -0,0 +1,122 @@
+"""
+CHANGELOG with standdard httplib
+
+1. HTTPConnection can send trunked data.
+2. Remove httplib's automatic Content-Length insertion when data is a file-like object.
+"""
+
+# -*- coding: utf-8 -*-
+
+import httplib
+from httplib import _CS_REQ_STARTED, _CS_REQ_SENT
+import string
+import os
+from array import array
+
+class HTTPConnection(httplib.HTTPConnection):
+
+	def send(self, data, is_chunked=False):
+		"""Send `data' to the server."""
+		if self.sock is None:
+			if self.auto_open:
+				self.connect()
+			else:
+				raise NotConnected()
+
+		if self.debuglevel > 0:
+			print "send:", repr(data)
+		blocksize = 8192
+		if hasattr(data,'read') and not isinstance(data, array):
+			if self.debuglevel > 0: print "sendIng a read()able"
+			datablock = data.read(blocksize)
+			while datablock:
+				print 'chunked:', is_chunked
+				if is_chunked:
+					if self.debuglevel > 0: print 'send: with trunked data'
+					lenstr = string.upper(hex(len(datablock))[2:])
+					self.sock.sendall('%s\r\n%s\r\n' % (lenstr, datablock))
+				else:
+					self.sock.sendall(datablock)
+				datablock = data.read(blocksize)
+			if is_chunked:
+				self.sock.sendall('0\r\n\r\n')
+		else:
+			self.sock.sendall(data)
+
+
+	def _set_content_length(self, body):
+		# Set the content-length based on the body.
+		thelen = None
+		try:
+			thelen = str(len(body))
+		except (TypeError, AttributeError), te:
+			# Don't send a length if this failed
+			if self.debuglevel > 0: print "Cannot stat!!"
+
+		if thelen is not None:
+			self.putheader('Content-Length', thelen)
+			return True
+		return False
+
+
+	def _send_request(self, method, url, body, headers):
+		# Honor explicitly requested Host: and Accept-Encoding: headers.
+		header_names = dict.fromkeys([k.lower() for k in headers])
+		skips = {}
+		if 'host' in header_names:
+			skips['skip_host'] = 1
+		if 'accept-encoding' in header_names:
+			skips['skip_accept_encoding'] = 1
+
+		self.putrequest(method, url, **skips)
+
+		is_chunked = False
+		if body and header_names.get('Transfer-Encoding') == 'chunked':
+			is_chunked = True
+		elif body and ('content-length' not in header_names):
+			is_chunked = not self._set_content_length(body)
+			if is_chunked:
+				self.putheader('Transfer-Encoding', 'chunked')
+		for hdr, value in headers.iteritems():
+			self.putheader(hdr, value)
+
+		self.endheaders(body, is_chunked=is_chunked)
+
+
+	def endheaders(self, message_body=None, is_chunked=False):
+		"""Indicate that the last header line has been sent to the server.
+
+		This method sends the request to the server.  The optional
+		message_body argument can be used to pass a message body
+		associated with the request.  The message body will be sent in
+		the same packet as the message headers if it is string, otherwise it is
+		sent as a separate packet.
+		"""
+		if self.__state == _CS_REQ_STARTED:
+			self.__state = _CS_REQ_SENT
+		else:
+			raise CannotSendHeader()
+		self._send_output(message_body, is_chunked=is_chunked)
+
+
+	def _send_output(self, message_body=None, is_chunked=False):
+		"""Send the currently buffered request and clear the buffer.
+
+		Appends an extra \\r\\n to the buffer.
+		A message_body may be specified, to be appended to the request.
+		"""
+		self._buffer.extend(("", ""))
+		msg = "\r\n".join(self._buffer)
+		del self._buffer[:]
+		# If msg and message_body are sent in a single send() call,
+		# it will avoid performance problems caused by the interaction
+		# between delayed ack and the Nagle algorithm.
+		if isinstance(message_body, str):
+			msg += message_body
+			message_body = None
+		self.send(msg)
+		if message_body is not None:
+			#message_body was not a string (i.e. it is a file) and
+			#we must run the risk of Nagle
+			self.send(message_body, is_chunked=is_chunked)
+

qiniu/rpc.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-import httplib
+import httplib_chunk as httplib
 import json
 import cStringIO
 import conf
@@ -55,8 +55,8 @@ def call_with_multipart(self, path, fields=None, files=None):
 		 *  fields => {key}
 		 *  files => [{filename, data, content_type}]
 		"""
-		content_type, body = self.encode_multipart_formdata(fields, files)
-		return self.call_with(path, body, content_type, len(body))
+		content_type, mr = self.encode_multipart_formdata(fields, files)
+		return self.call_with(path, mr, content_type, mr.length())
 
 	def call_with_form(self, path, ops):
 		"""
@@ -138,9 +138,15 @@ class MultiReader(object):
 	def __init__(self, readers):
 		self.readers = []
 		self.content_length = 0
+		self.valid_content_length = True
 		for r in readers:
 			if hasattr(r, 'read'):
-				self.content_length += self._get_content_length(r)
+				if self.valid_content_length:
+					length = self._get_content_length(r)
+					if length is not None:
+						self.content_length += length
+					else:
+						self.valid_content_length = False
 			else:
 				buf = r
 				if not isinstance(buf, basestring):
@@ -150,17 +156,22 @@ def __init__(self, readers):
 				self.content_length += len(buf)
 			self.readers.append(r)
 
-	def __len__(self):
-		return self.content_length
+
+	# don't name it __len__, because the length of MultiReader is not alway valid.
+	def length(self):
+		return self.content_length if self.valid_content_length else None
+
 
 	def _get_content_length(self, reader):
-		data_len = 0
-		try:
-			reader.seek(0, 2)
-			data_len= reader.tell()
-			reader.seek(0, 0)
-		except (AttributeError, OSError):
-			print 'can not get content_length'
+		data_len = None
+		if hasattr(reader, 'seek') and hasattr(reader, 'tell'):
+			try:
+				reader.seek(0, 2)
+				data_len= reader.tell()
+				reader.seek(0, 0)
+			except OSError:
+				# Don't send a length if this failed
+				data_len = None
 		return data_len
 
 	def read(self, n=-1):

qiniu/test/io_test.py

@@ -89,11 +89,12 @@ def test_put_StringIO():
 
 		def test_put_urlopen():
 			key = "test_%s" % r(9)
-			data = urllib.urlopen('http://http://cheneya.qiniudn.com/hello_jpg')
+			data = urllib.urlopen('http://cheneya.qiniudn.com/hello_jpg')
 			ret, err = io.put(policy.token(), key, data)
+			print 'error', err
 			assert err is None
 
-		def test_put_no_length();
+		def test_put_no_length():
 			class test_reader(object):
 				def __init__(self):
 					self.data = 'abc'
@@ -108,7 +109,10 @@ def read(self, n=None):
 					return r
 			key = "test_%s" % r(9)
 			data = test_reader()
-			ret, err = io.put(policy.token(), key, data)
+
+			extra.check_crc = 2
+			extra.crc32 = binascii.crc32('abc') & 0xFFFFFFFF
+			ret, err = io.put(policy.token(), key, data, extra)
 			assert err is None
 
 		test_put()
@@ -120,6 +124,7 @@ def read(self, n=None):
 		test_put_unicode4()
 		test_put_StringIO()
 		test_put_urlopen()
+		test_put_no_length()
 
 	def test_put_file(self):
 		localfile = "%s" % __file__

qiniu/test/rpc_test.py

@@ -45,7 +45,7 @@ def tripper(client, method, path, body):
 			tpl = "--%s\r\n%s\r\n\r\n%s\r\n--%s--\r\n" % (boundary, dispostion,
 					"auth_string", boundary)
 			self.assertEqual(len(tpl), client._header["Content-Length"])
-			self.assertEqual(len(tpl), len(body))
+			self.assertEqual(len(tpl), body.length())
 
 		round_tripper = tripper
 		client.call_with_multipart("/hello", fields={"auth": "auth_string"})
@@ -123,13 +123,13 @@ def test_encode(self):
 				'mime_type': 'application/octet-stream',
 			}
 		]
-		content_type, body = rpc.Client('localhost').encode_multipart_formdata(fields, files)
+		content_type, mr = rpc.Client('localhost').encode_multipart_formdata(fields, files)
 		t, b = encode_multipart_formdata2(
 			[('a', '1'), ('b', '2')],
 			[('file', 'key1', 'data1'), ('file', 'key2', 'data2')]
 		)
 		assert t == content_type
-		assert len(b) == len(body)
+		assert len(b) == mr.length()
 
 	def test_unicode(self):
 		def test1():

qiniu/test/rsf_test.py

@@ -11,9 +11,9 @@
 class TestRsf(unittest.TestCase):
 	def test_list_prefix(self):
 		c = rsf.Client()
-		ret, err = c.list_prefix(bucket_name)
-		assert err is rsf.EOF
-		self.assertEqual(len(ret.get('items'))>0, True)
+		ret, err = c.list_prefix(bucket_name, limit = 1)
+		self.assertEqual(err is rsf.EOF or err is None, True)
+		assert len(ret.get('items')) == 1
 
 
 if __name__ == "__main__":