Add response body and headers to diagnostic information on HTTP 400 a…

…nd above

Also should fix #13 because stops AuthAgent from using the same response multiple times

libgrive/src/drive/Syncer1.cc

@@ -225,7 +225,7 @@ bool Syncer1::Upload( Resource *res,
 
 		if ( http_code == 410 || http_code == 412 )
 		{
-			Log( "request failed with %1%, retrying whole upload in 5s", http_code, log::warning ) ;
+			Log( "request failed with %1%, body: %2%, retrying whole upload in 5s", http_code, m_http->LastError(), log::warning ) ;
 			retrying = true;
 			continue;
 		}

libgrive/src/drive2/Syncer2.cc

@@ -140,7 +140,7 @@ bool Syncer2::Upload( Resource *res )
 			long http_code = m_http->Put( upload_base + "/" + valr["id"].Str() + "?uploadType=media", &file, &vrsp, hdr ) ;
 			if ( http_code == 410 || http_code == 412 )
 			{
-				Log( "request failed with %1%, retrying whole upload in 5s", http_code, log::warning ) ;
+				Log( "request failed with %1%, body: %2%. retrying whole upload in 5s", http_code, m_http->LastError(), log::warning ) ;
 				os::Sleep( 5 );
 			}
 			else

libgrive/src/http/Agent.hh

@@ -64,6 +64,9 @@ public :
 		DataStream			*dest,
 		const Header&		hdr ) = 0 ;
 
+	virtual std::string LastError() const = 0 ;
+	virtual std::string LastErrorHeaders() const = 0 ;
+
 	virtual std::string RedirLocation() const = 0 ;
 
 	virtual std::string Escape( const std::string& str ) = 0 ;

libgrive/src/http/CurlAgent.cc

@@ -85,6 +85,10 @@ struct CurlAgent::Impl
 {
 	CURL			*curl ;
 	std::string		location ;
+	bool			error ;
+	std::string		error_headers ;
+	std::string		error_data ;
+	DataStream		*dest ;
 } ;
 
 CurlAgent::CurlAgent() :
@@ -96,11 +100,15 @@ CurlAgent::CurlAgent() :
 void CurlAgent::Init()
 {
 	::curl_easy_reset( m_pimpl->curl ) ;
-	::curl_easy_setopt( m_pimpl->curl, CURLOPT_SSL_VERIFYPEER,	0L ) ; 
+	::curl_easy_setopt( m_pimpl->curl, CURLOPT_SSL_VERIFYPEER,	0L ) ;
 	::curl_easy_setopt( m_pimpl->curl, CURLOPT_SSL_VERIFYHOST,	0L ) ;
 	::curl_easy_setopt( m_pimpl->curl, CURLOPT_HEADERFUNCTION,	&CurlAgent::HeaderCallback ) ;
-	::curl_easy_setopt( m_pimpl->curl, CURLOPT_WRITEHEADER ,	this ) ;
-	::curl_easy_setopt( m_pimpl->curl, CURLOPT_HEADER, 			0L ) ;
+	::curl_easy_setopt( m_pimpl->curl, CURLOPT_HEADERDATA,		this ) ;
+	::curl_easy_setopt( m_pimpl->curl, CURLOPT_HEADER,			0L ) ;
+	m_pimpl->error = false;
+	m_pimpl->error_headers = "";
+	m_pimpl->error_data = "";
+	m_pimpl->dest = NULL;
 }
 
 CurlAgent::~CurlAgent()
@@ -110,9 +118,16 @@ CurlAgent::~CurlAgent()
 
 std::size_t CurlAgent::HeaderCallback( void *ptr, size_t size, size_t nmemb, CurlAgent *pthis )
 {
-	char *str = reinterpret_cast<char*>(ptr) ;
+	char *str = static_cast<char*>(ptr) ;
 	std::string line( str, str + size*nmemb ) ;
 
+	// Check for error (HTTP 400 and above)
+	if ( line.substr( 0, 5 ) == "HTTP/" && line[9] >= '4' )
+		pthis->m_pimpl->error = true;
+
+	if ( pthis->m_pimpl->error )
+		pthis->m_pimpl->error_headers += line;
+
 	static const std::string loc = "Location: " ;
 	std::size_t pos = line.find( loc ) ;
 	if ( pos != line.npos )
@@ -124,10 +139,16 @@ std::size_t CurlAgent::HeaderCallback( void *ptr, size_t size, size_t nmemb, Cur
 	return size*nmemb ;
 }
 
-std::size_t CurlAgent::Receive( void* ptr, size_t size, size_t nmemb, DataStream *recv )
+std::size_t CurlAgent::Receive( void* ptr, size_t size, size_t nmemb, CurlAgent *pthis )
 {
-	assert( recv != 0 ) ;
-	return recv->Write( static_cast<char*>(ptr), size * nmemb ) ;
+	assert( pthis != 0 ) ;
+	if ( pthis->m_pimpl->error && pthis->m_pimpl->error_data.size() < 65536 )
+	{
+		// Do not feed error responses to destination stream
+		pthis->m_pimpl->error_data.append( static_cast<char*>(ptr), size * nmemb ) ;
+		return size * nmemb ;
+	}
+	return pthis->m_pimpl->dest->Write( static_cast<char*>(ptr), size * nmemb ) ;
 }
 
 long CurlAgent::ExecCurl(
@@ -142,21 +163,23 @@ long CurlAgent::ExecCurl(
 	::curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, 	error ) ;
 	::curl_easy_setopt(curl, CURLOPT_URL, 			url.c_str());
 	::curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,	&CurlAgent::Receive ) ;
-	::curl_easy_setopt(curl, CURLOPT_WRITEDATA,		dest ) ;
+	::curl_easy_setopt(curl, CURLOPT_WRITEDATA,		this ) ;
+	m_pimpl->dest = dest ;
 
 	SetHeader( hdr ) ;
 
-//	dest->Clear() ;
 	CURLcode curl_code = ::curl_easy_perform(curl);
 
 	// get the HTTP response code
 	long http_code = 0;
 	::curl_easy_getinfo(curl,	CURLINFO_RESPONSE_CODE, &http_code);
 	Trace( "HTTP response %1%", http_code ) ;
-	
-	// reset the curl buffer to prevent it from touch our "error" buffer
+
+	// reset the curl buffer to prevent it from touching our "error" buffer
 	::curl_easy_setopt(curl,	CURLOPT_ERRORBUFFER, 	0 ) ;
-
+
+	m_pimpl->dest = NULL;
+
 	// only throw for libcurl errors
 	if ( curl_code != CURLE_OK )
 	{
@@ -165,7 +188,7 @@ long CurlAgent::ExecCurl(
 				<< CurlCode( curl_code )
 				<< Url( url )
 				<< CurlErrMsg( error )
-				<< HttpHeader( hdr )
+				<< HttpRequestHeaders( hdr )
 		) ;
 	}
 
@@ -275,6 +298,16 @@ void CurlAgent::SetHeader( const Header& hdr )
 	::curl_easy_setopt( m_pimpl->curl, CURLOPT_HTTPHEADER, curl_hdr ) ;
 }
 
+std::string CurlAgent::LastError() const
+{
+	return m_pimpl->error_data ;
+}
+
+std::string CurlAgent::LastErrorHeaders() const
+{
+	return m_pimpl->error_headers ;
+}
+
 std::string CurlAgent::RedirLocation() const
 {
 	return m_pimpl->location ;

libgrive/src/http/CurlAgent.hh

@@ -70,14 +70,17 @@ public :
 		DataStream			*dest,
 		const Header&		hdr ) ;
 
+	std::string LastError() const ;
+	std::string LastErrorHeaders() const ;
+
 	std::string RedirLocation() const ;
 
 	std::string Escape( const std::string& str ) ;
 	std::string Unescape( const std::string& str ) ;
 
 private :
 	static std::size_t HeaderCallback( void *ptr, size_t size, size_t nmemb, CurlAgent *pthis ) ;
-	static std::size_t Receive( void* ptr, size_t size, size_t nmemb, DataStream *recv ) ;
+	static std::size_t Receive( void* ptr, size_t size, size_t nmemb, CurlAgent *pthis ) ;
 
 	void SetHeader( const Header& hdr ) ;
 	long ExecCurl(

libgrive/src/http/Error.hh

@@ -27,21 +27,24 @@ namespace gr { namespace http {
 struct Error : virtual Exception {} ;
 
 // CURL error code
-typedef boost::error_info<struct CurlCodeTag, int>				CurlCode ;
+typedef boost::error_info<struct CurlCodeTag, int>					CurlCode ;
 
 // CURL error message
-typedef boost::error_info<struct CurlErrMsgTag, std::string>	CurlErrMsg ;
+typedef boost::error_info<struct CurlErrMsgTag, std::string>		CurlErrMsg ;
 
 // URL
-typedef boost::error_info<struct UrlTag, std::string>		Url ;
+typedef boost::error_info<struct UrlTag, std::string>				Url ;
 
-// HTTP headers
-typedef boost::error_info<struct HeaderTag, Header>			HttpHeader ;
+// HTTP request headers
+typedef boost::error_info<struct RequestHeadersTag, Header>			HttpRequestHeaders ;
 
 // HTTP response code
-typedef boost::error_info<struct HttpResponseTag, int>		HttpResponse ;
+typedef boost::error_info<struct ResponseCodeTag, int>				HttpResponseCode ;
+
+// HTTP response headers
+typedef boost::error_info<struct ResponseHeadersTag, std::string>	HttpResponseHeaders ;
 
 // HTTP response body
-typedef boost::error_info<struct HttpResponseStrTag, std::string>	HttpResponseText ;
+typedef boost::error_info<struct ResponseBodyTag, std::string>		HttpResponseText ;
 
 } } // end of namespace

libgrive/src/protocol/AuthAgent.cc

@@ -21,7 +21,6 @@
 
 #include "http/Error.hh"
 #include "http/Header.hh"
-#include "http/XmlResponse.hh"
 #include "util/log/Log.hh"
 #include "util/OS.hh"
 #include "util/File.hh"
@@ -77,9 +76,6 @@ long AuthAgent::Put(
 		keepTrying = CheckRetry( response );
 		if ( keepTrying ) {
 			file->Seek( 0, SEEK_SET );
-			XmlResponse *xmlResponse = dynamic_cast<XmlResponse*>(dest);
-			if( xmlResponse )
-			  xmlResponse->Clear();
 		}
 	}
 
@@ -135,6 +131,16 @@ long AuthAgent::Custom(
 	return CheckHttpResponse(response, url, auth) ;
 }
 
+std::string AuthAgent::LastError() const
+{
+	return m_agent->LastError() ;
+}
+
+std::string AuthAgent::LastErrorHeaders() const
+{
+	return m_agent->LastErrorHeaders() ;
+}
+
 std::string AuthAgent::RedirLocation() const
 {
 	return m_agent->RedirLocation() ;
@@ -152,11 +158,11 @@ std::string AuthAgent::Unescape( const std::string& str )
 
 bool AuthAgent::CheckRetry( long response )
 {
-	// HTTP 500 and 503 should be temperory. just wait a bit and retry
+	// HTTP 500 and 503 should be temporary. just wait a bit and retry
 	if ( response == 500 || response == 503 )
 	{
-		Log( "resquest failed due to temperory error: %1%. retrying in 5 seconds",
-			response, log::warning ) ;
+		Log( "request failed due to temporary error: %1%, body: %2%. retrying in 5 seconds",
+			response, m_agent->LastError(), log::warning ) ;
 
 		os::Sleep( 5 ) ;
 		return true ;
@@ -165,7 +171,7 @@ bool AuthAgent::CheckRetry( long response )
 	// HTTP 401 Unauthorized. the auth token has been expired. refresh it
 	else if ( response == 401 )
 	{
-		Log( "resquest failed due to auth token expired: %1%. refreshing token",
+		Log( "request failed due to auth token expired: %1%. refreshing token",
 			response, log::warning ) ;
 
 		os::Sleep( 5 ) ;
@@ -182,13 +188,15 @@ long AuthAgent::CheckHttpResponse(
 		const http::Header&	hdr  )
 {
 	// throw for other HTTP errors
-	if ( response >= 400 && response < 500 )
+	if ( response >= 400 )
 	{
- 		BOOST_THROW_EXCEPTION(
- 			Error()
-				<< HttpResponse( response )
- 				<< Url( url )
- 				<< HttpHeader( hdr ) ) ;
+		BOOST_THROW_EXCEPTION(
+			Error()
+				<< HttpResponseCode( response )
+				<< HttpResponseHeaders( m_agent->LastErrorHeaders() )
+				<< HttpResponseText( m_agent->LastError() )
+				<< Url( url )
+				<< HttpRequestHeaders( hdr ) ) ;
 	}
 
 	return response ;

libgrive/src/protocol/AuthAgent.hh

@@ -65,6 +65,9 @@ public :
 		DataStream			*dest,
 		const http::Header&	hdr ) ;
 
+	std::string LastError() const ;
+	std::string LastErrorHeaders() const ;
+
 	std::string RedirLocation() const ;
 
 	std::string Escape( const std::string& str ) ;