transfer: cleanup done+excess handling

- add `SingleRequest->download_done` as indicator that
  all download bytes have been received
- remove `stop_reading` bool from readwrite functions
- move excess body handling into client download writer

Closes curl#12371

lib/c-hyper.c

@@ -882,6 +882,7 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done)
      may be parts of the request that is not yet sent, since we can deal with
      the rest of the request in the PERFORM phase. */
   *done = TRUE;
+  Curl_client_cleanup(data);
 
   infof(data, "Time for the Hyper dance");
   memset(h, 0, sizeof(struct hyptransfer));
@@ -892,6 +893,8 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done)
 
   Curl_http_method(data, conn, &method, &httpreq);
 
+  DEBUGASSERT(data->req.bytecount ==  0);
+
   /* setup the authentication headers */
   {
     char *pq = NULL;

lib/http.c

@@ -3996,16 +3996,14 @@ CURLcode Curl_bump_headersize(struct Curl_easy *data,
 CURLcode Curl_http_readwrite_headers(struct Curl_easy *data,
                                      struct connectdata *conn,
                                      const char *buf, size_t blen,
-                                     size_t *pconsumed,
-                                     bool *stop_reading)
+                                     size_t *pconsumed)
 {
   CURLcode result;
   struct SingleRequest *k = &data->req;
   char *headp;
   char *end_ptr;
 
   /* header line within buffer loop */
-  *stop_reading = FALSE;
   *pconsumed = 0;
   do {
     size_t line_length;
@@ -4358,7 +4356,7 @@ CURLcode Curl_http_readwrite_headers(struct Curl_easy *data,
          * out and return home.
          */
         if(data->req.no_body)
-          *stop_reading = TRUE;
+          k->download_done = TRUE;
 #ifndef CURL_DISABLE_RTSP
         else if((conn->handler->protocol & CURLPROTO_RTSP) &&
                 (data->set.rtspreq == RTSPREQ_DESCRIBE) &&
@@ -4367,7 +4365,7 @@ CURLcode Curl_http_readwrite_headers(struct Curl_easy *data,
              absent, a length 0 must be assumed.  It will prevent libcurl from
              hanging on DESCRIBE request that got refused for whatever
              reason */
-          *stop_reading = TRUE;
+          k->download_done = TRUE;
 #endif
 
         /* If max download size is *zero* (nothing) we already have
@@ -4378,12 +4376,7 @@ CURLcode Curl_http_readwrite_headers(struct Curl_easy *data,
         if(0 == k->maxdownload
            && !Curl_conn_is_http2(data, conn, FIRSTSOCKET)
            && !Curl_conn_is_http3(data, conn, FIRSTSOCKET))
-          *stop_reading = TRUE;
-
-        if(*stop_reading) {
-          /* we make sure that this socket isn't read more now */
-          k->keepon &= ~KEEP_RECV;
-        }
+          k->download_done = TRUE;
 
         Curl_debug(data, CURLINFO_HEADER_IN,
                    Curl_dyn_ptr(&data->state.headerb),

lib/http.h

@@ -228,8 +228,7 @@ CURLcode Curl_http_size(struct Curl_easy *data);
 CURLcode Curl_http_readwrite_headers(struct Curl_easy *data,
                                      struct connectdata *conn,
                                      const char *buf, size_t blen,
-                                     size_t *pconsumed,
-                                     bool *stop_reading);
+                                     size_t *pconsumed);
 
 /**
  * Curl_http_output_auth() setups the authentication headers for the

lib/rtsp.c

@@ -793,11 +793,9 @@ static CURLcode rtsp_rtp_readwrite(struct Curl_easy *data,
 
   /* we want to parse headers, do so */
   if(data->req.header && blen) {
-    bool stop_reading;
-
     rtspc->in_header = TRUE;
     result = Curl_http_readwrite_headers(data, conn, buf, blen,
-                                         &consumed, &stop_reading);
+                                         &consumed);
     if(result)
       goto out;
 

lib/sendf.c

@@ -467,7 +467,8 @@ void Curl_client_cleanup(struct Curl_easy *data)
     Curl_dyn_free(&data->state.tempwrite[i].b);
   }
   data->state.tempcount = 0;
-
+  data->req.bytecount = 0;
+  data->req.headerline = 0;
 }
 
 /* Write data using an unencoding writer stack. "nbytes" is not
@@ -525,14 +526,37 @@ static const struct Curl_cwtype cw_client = {
   sizeof(struct Curl_cwriter)
 };
 
+static size_t get_max_body_write_len(struct Curl_easy *data, curl_off_t limit)
+{
+  if(limit != -1) {
+    /* How much more are we allowed to write? */
+    curl_off_t remain_diff;
+    remain_diff = limit - data->req.bytecount;
+    if(remain_diff < 0) {
+      /* already written too much! */
+      return 0;
+    }
+#if SIZEOF_CURL_OFF_T > SIZEOF_SIZE_T
+    else if(remain_diff > SSIZE_T_MAX) {
+      return SIZE_T_MAX;
+    }
+#endif
+    else {
+      return (size_t)remain_diff;
+    }
+  }
+  return SIZE_T_MAX;
+}
+
 /* Download client writer in phase CURL_CW_PROTOCOL that
  * sees the "real" download body data. */
 static CURLcode cw_download_write(struct Curl_easy *data,
                                   struct Curl_cwriter *writer, int type,
                                   const char *buf, size_t nbytes)
 {
   CURLcode result;
-  size_t nwrite;
+  size_t nwrite, excess_len = 0;
+  const char *excess_data = NULL;
 
   if(!(type & CLIENTWRITE_BODY)) {
     if((type & CLIENTWRITE_CONNECT) && data->set.suppress_connect_headers)
@@ -541,22 +565,28 @@ static CURLcode cw_download_write(struct Curl_easy *data,
   }
 
   nwrite = nbytes;
-  data->req.bytecount += nbytes;
-  ++data->req.bodywrites;
-  /* Enforce `max_filesize` also for downloads where we ignore the body.
-   * Also, write body data up to the max size. This ensures that we
-   * always produce the same result, even when buffers vary due to
-   * connection timings. test457 fails in CI randomly otherwise. */
-  if(data->set.max_filesize &&
-     (data->req.bytecount > data->set.max_filesize)) {
-    curl_off_t nexcess;
-    failf(data, "Exceeded the maximum allowed file size "
-          "(%" CURL_FORMAT_CURL_OFF_T ")",
-          data->set.max_filesize);
-    nexcess = data->req.bytecount - data->set.max_filesize;
-    nwrite = (nexcess >= (curl_off_t)nbytes)? 0 : (nbytes - (size_t)nexcess);
+  if(-1 != data->req.maxdownload) {
+    size_t wmax = get_max_body_write_len(data, data->req.maxdownload);
+    if(nwrite > wmax) {
+      excess_len = nbytes - wmax;
+      nwrite = wmax;
+      excess_data = buf + nwrite;
+    }
+
+    if(nwrite == wmax) {
+      data->req.download_done = TRUE;
+    }
+  }
+
+  if(data->set.max_filesize) {
+    size_t wmax = get_max_body_write_len(data, data->set.max_filesize);
+    if(nwrite > wmax) {
+      nwrite = wmax;
+    }
   }
 
+  data->req.bytecount += nwrite;
+  ++data->req.bodywrites;
   if(!data->req.ignorebody && nwrite) {
     result = Curl_cwriter_write(data, writer->next, type, buf, nwrite);
     if(result)
@@ -566,7 +596,44 @@ static CURLcode cw_download_write(struct Curl_easy *data,
   if(result)
     return result;
 
-  return (nwrite == nbytes)? CURLE_OK : CURLE_FILESIZE_EXCEEDED;
+  if(excess_len) {
+    if(data->conn->handler->readwrite) {
+      /* RTSP hack moved from tranfer loop to here */
+      bool readmore = FALSE; /* indicates data is incomplete, need more */
+      size_t consumed = 0;
+      result = data->conn->handler->readwrite(data, data->conn,
+                                              excess_data, excess_len,
+                                              &consumed, &readmore);
+      if(result)
+        return result;
+      DEBUGASSERT(consumed <= excess_len);
+      excess_len -= consumed;
+      if(readmore) {
+        data->req.download_done = FALSE;
+        data->req.keepon |= KEEP_RECV; /* we're not done reading */
+      }
+    }
+    if(excess_len && !data->req.ignorebody) {
+      infof(data,
+            "Excess found writing body:"
+            " excess = %zu"
+            ", size = %" CURL_FORMAT_CURL_OFF_T
+            ", maxdownload = %" CURL_FORMAT_CURL_OFF_T
+            ", bytecount = %" CURL_FORMAT_CURL_OFF_T,
+            excess_len, data->req.size, data->req.maxdownload,
+            data->req.bytecount);
+      connclose(data->conn, "excess found in a read");
+    }
+  }
+  else if(nwrite < nbytes) {
+    failf(data, "Exceeded the maximum allowed file size "
+          "(%" CURL_FORMAT_CURL_OFF_T ") with %"
+          CURL_FORMAT_CURL_OFF_T " bytes",
+          data->set.max_filesize, data->req.bytecount);
+    return CURLE_FILESIZE_EXCEEDED;
+  }
+
+  return CURLE_OK;
 }
 
 static const struct Curl_cwtype cw_download = {