Fixing all of the things. 😩

gizahNL · Apr 10, 2023 · c979acf · c979acf
1 parent 53f9414
commit c979acf
Show file tree

Hide file tree

Showing 14 changed files with 171 additions and 76 deletions.
diff --git a/lib/downloader.cpp b/lib/downloader.cpp
@@ -71,6 +71,14 @@ namespace HTTP{
     return S;
   }
 
+  void Downloader::clean(){
+    H.headerOnly = false;
+    H.Clean();
+    getSocket().close();
+    getSocket().Received().clear();
+    extraHeaders.clear();
+  }
+
   ///Sets an override to use the given socket
   void Downloader::setSocket(Socket::Connection * socketPtr){
     sPtr = socketPtr;
@@ -187,10 +195,7 @@ namespace HTTP{
           if (progressCallback != 0){
             if (!progressCallback()){
               WARN_MSG("Download aborted by callback");
-              H.headerOnly = false;
-              H.Clean();
-              getSocket().close();
-              getSocket().Received().clear();
+              clean();
               return false;
             }
           }
@@ -203,10 +208,7 @@ namespace HTTP{
 
           // If the return status code is invalid, close the socket, wipe all buffers, and return false
           if(!getStatusCode()){
-            H.headerOnly = false;
-            getSocket().close();
-            getSocket().Received().clear();
-            H.Clean();
+            clean();
             return false;
           }
 
@@ -233,10 +235,7 @@ namespace HTTP{
           if (progressCallback != 0){
             if (!progressCallback()){
               WARN_MSG("Download aborted by callback");
-              H.headerOnly = false;
-              H.Clean();
-              getSocket().close();
-              getSocket().Received().clear();
+              clean();
               return false;
             }
           }
@@ -246,20 +245,17 @@ namespace HTTP{
           }
         }
       }
-      H.headerOnly = false;
 
       if (getSocket()){
         FAIL_MSG("Timeout while retrieving %s (%zu/%" PRIu32 ")", link.getUrl().c_str(), loop, retryCount);
-        getSocket().close();
       }else{
         if (loop > 1){
           INFO_MSG("Lost connection while retrieving %s (%zu/%" PRIu32 ")", link.getUrl().c_str(), loop, retryCount);
         }else{
           MEDIUM_MSG("Lost connection while retrieving %s (%zu/%" PRIu32 ")", link.getUrl().c_str(), loop, retryCount);
         }
       }
-      H.Clean();
-      getSocket().Received().clear();
+      clean();
       Util::sleep(100); // wait a bit before retrying
     }
     FAIL_MSG("Could not retrieve %s", link.getUrl().c_str());
@@ -309,6 +305,7 @@ namespace HTTP{
     nbMaxRecursiveDepth = maxRecursiveDepth;
     nbLoop = retryCount + 1; // max 5 attempts
     isComplete = false;
+    extraHeaders.erase("Range");
     doRequest(nbLink);
     nbReqTime = Util::bootSecs();
     nbLastOff = getSocket().dataDown();
@@ -343,38 +340,48 @@ namespace HTTP{
         }
 
         if (H.hasHeader("Accept-Ranges") && getHeader("Accept-Ranges").size() > 0){
-          getRangeNonBlocking(nbLink, H.currentLength, 0, cb);
-          return true;
+          getRangeNonBlocking(nbLink, cb.getDataCallbackPos(), 0, cb);
+          continue;
         }else{
           doRequest(nbLink);
         }
 
-        if (!getSocket()){
-          WARN_MSG("Aborting download: could not open connection");
-          return true;
-        }
         nbReqTime = Util::bootSecs();
         nbLastOff = getSocket().dataDown();
       }
 
-      if (Util::bootSecs() >= nbReqTime + dataTimeout){
-        FAIL_MSG("Timeout while retrieving %s (%zu/%" PRIu32 ")", nbLink.getUrl().c_str(),
-                 retryCount - nbLoop + 1, retryCount);
-        getSocket().close();
-        return false; // because we may have retries left
-      }
-
-      // No data? Wait for a second or so.
+      // No data? Return false to indicate retry later.
       if (!getSocket().spool() && getSocket()){
         if (progressCallback != 0){
           if (!progressCallback()){
             WARN_MSG("Download aborted by callback");
             return true;
           }
         }
+        if (Util::bootSecs() >= nbReqTime + dataTimeout){
+          FAIL_MSG("Timeout while retrieving %s (%zu/%" PRIu32 ")", nbLink.getUrl().c_str(),
+                   retryCount - nbLoop + 1, retryCount);
+          getSocket().close();
+          return false; // Retry on next attempt
+        }
         return false;
       }
-      // Data! Check if we can parse it...
+
+      // Reset the data timeout
+      if (nbReqTime != Util::bootSecs()){
+        if (progressCallback != 0){
+          if (!progressCallback()){
+            WARN_MSG("Download aborted by callback");
+            return true;
+          }
+        }
+        if (getSocket().dataDown() > nbLastOff + 25600){
+          nbReqTime = Util::bootSecs();
+          nbLastOff = getSocket().dataDown();
+        }
+      }
+
+      //Attempt to parse the data we received
       if (H.Read(getSocket(), cb)){
         if (shouldContinue()){
           if (nbMaxRecursiveDepth == 0){
@@ -393,21 +400,7 @@ namespace HTTP{
         isComplete = true; // Success
         return true;
       }
-      // reset the data timeout
-      if (nbReqTime != Util::bootSecs()){
-        if (progressCallback != 0){
-          if (!progressCallback()){
-            WARN_MSG("Download aborted by callback");
-            return true;
-          }
-        }
-        if (getSocket().dataDown() > nbLastOff + 25600){
-          nbReqTime = Util::bootSecs();
-          nbLastOff = getSocket().dataDown();
-        }
-      }
     }
-    WARN_MSG("Invalid connection state for HTTP request");
     return false; // we should never get here
   }
 

diff --git a/lib/downloader.h b/lib/downloader.h
@@ -45,6 +45,7 @@ namespace HTTP{
     Parser &getHTTP();
     Socket::Connection &getSocket();
     const Socket::Connection &getSocket() const;
+    void clean();
     void setSocket(Socket::Connection * socketPtr);
     uint32_t retryCount, dataTimeout;
     bool isProxied() const;

diff --git a/lib/http_parser.h b/lib/http_parser.h
@@ -17,7 +17,7 @@ namespace HTTP{
   void parseVars(const std::string &data, std::map<std::string, std::string> &storage, const std::string & separator = "&");
 
   /// Simple class for reading and writing HTTP 1.0 and 1.1.
-  class Parser : public Util::DataCallback{
+  class Parser{
   public:
     Parser();
     bool Read(Socket::Connection &conn, Util::DataCallback &cb = Util::defaultDataCallback);

diff --git a/lib/urireader.cpp b/lib/urireader.cpp
@@ -65,13 +65,17 @@ namespace HTTP{
     return url;
   }
 
+  HTTP::URL localURIResolver(){
+    char workDir[512];
+    getcwd(workDir, 512);
+    return HTTP::URL(std::string("file://") + workDir + "/");
+  }
 
   void URIReader::init(){
     handle = -1;
     mapped = 0;
-    char workDir[512];
-    getcwd(workDir, 512);
-    myURI = HTTP::URL(std::string("file://") + workDir + "/");
+    myURI = localURIResolver();
+    originalUrl = myURI;
     cbProgress = 0;
     minLen = 1;
     maxLen = std::string::npos;
@@ -97,11 +101,13 @@ namespace HTTP{
     open(reluri);
   }
 
-  bool URIReader::open(const std::string &reluri){return open(myURI.link(reluri));}
+  bool URIReader::open(const std::string &reluri){return open(originalUrl.link(reluri));}
 
   /// Internal callback function, used to buffer data.
   void URIReader::dataCallback(const char *ptr, size_t size){allData.append(ptr, size);}
 
+  size_t URIReader::getDataCallbackPos() const{return allData.size();}
+
   bool URIReader::open(const HTTP::URL &uri){
     close();
     myURI = uri;
@@ -247,14 +253,13 @@ namespace HTTP{
 
     //HTTP-based needs to do a range request
     if (stateType == HTTP::HTTP && supportRangeRequest){
-      downer.getSocket().close();
-      downer.getSocket().Received().clear();
+      downer.clean();
+      curPos = pos;
       injectHeaders(originalUrl, "GET", downer);
-      if (!downer.getRangeNonBlocking(myURI.getUrl(), pos, 0)){
+      if (!downer.getRangeNonBlocking(myURI, pos, 0)){
         FAIL_MSG("Error making range request");
         return false;
       }
-      curPos = pos;
       return true;
     }
     return false;
@@ -354,9 +359,7 @@ namespace HTTP{
     allData.truncate(0);
     bufPos = 0;
     // Close downloader socket if open
-    downer.getSocket().close();
-    downer.getSocket().Received().clear();
-    downer.getHTTP().Clean();
+    downer.clean();
     // Unmap file if mapped
     if (mapped){
       munmap(mapped, totalSize);
@@ -408,7 +411,7 @@ namespace HTTP{
 
   uint64_t URIReader::getPos(){return curPos;}
 
-  const HTTP::URL &URIReader::getURI() const{return myURI;}
+  const HTTP::URL &URIReader::getURI() const{return originalUrl;}
 
   size_t URIReader::getSize() const{return totalSize;}
 

diff --git a/lib/urireader.h b/lib/urireader.h
@@ -60,7 +60,8 @@ namespace HTTP{
     size_t getSize() const; ///< Returns the size of the currently open URI, if known. Returns std::string::npos if unknown size.
 
     void (*httpBodyCallback)(const char *ptr, size_t size);
-    void dataCallback(const char *ptr, size_t size);
+    virtual void dataCallback(const char *ptr, size_t size);
+    virtual size_t getDataCallbackPos() const;
 
     std::string userAgentOverride;
 
@@ -86,4 +87,6 @@ namespace HTTP{
     HTTP::Downloader downer; ///< For HTTP(S)-based URIs, the Downloader instance used for the download.
     void init();
   };
+
+  HTTP::URL localURIResolver();
 }// namespace HTTP
diff --git a/lib/url.cpp b/lib/url.cpp
@@ -345,3 +345,13 @@ HTTP::URL HTTP::URL::link(const std::string &l) const{
   DONTEVEN_MSG("Relative link: %s+%s", base.c_str(), l.c_str());
   return URL(base + l);
 }
+
+/// Returns true if the URL matches, ignoring username, password and fragment
+bool HTTP::URL::operator==(const URL& rhs) const{
+  return (host == rhs.host && getPort() == rhs.getPort() && protocol == rhs.protocol && path == rhs.path && args == rhs.args);
+}
+
+/// Returns false if the URL matches, ignoring username, password and fragment.
+/// Simply calls == internally, and negates the result.
+bool HTTP::URL::operator!=(const URL& rhs) const{return !(*this == rhs);}
+
diff --git a/lib/url.h b/lib/url.h
@@ -34,6 +34,8 @@ namespace HTTP{
     std::string pass; ///< Password, if it was present
     URL link(const std::string &l) const;
     bool IPv6Addr;
+    bool operator==(const URL& rhs) const;
+    bool operator!=(const URL& rhs) const;
   };
 
 }// namespace HTTP
diff --git a/lib/util.h b/lib/util.h
@@ -30,6 +30,7 @@ namespace Util{
     virtual void dataCallback(const char *ptr, size_t size){
       INFO_MSG("default callback, size: %zu", size);
     }
+    virtual size_t getDataCallbackPos() const{return 0;}
     virtual ~DataCallback(){};
   };
 

diff --git a/src/input/input.cpp b/src/input/input.cpp
@@ -58,7 +58,7 @@ namespace Mist{
       if (i == key){
         keyLoadPriority[trackKey(track, pageNumber)] += 10000;
       }else{
-        keyLoadPriority[trackKey(track, pageNumber)] += 1000 - (key - i);
+        keyLoadPriority[trackKey(track, pageNumber)] += 1000 - (i - key);
       }
       uint64_t cnt = tPages.getInt("keycount", pageIdx);
       if (pageNumber + cnt <= i){return;}
@@ -73,6 +73,7 @@ namespace Mist{
     std::multimap<uint64_t, trackKey> reverse;
     for (std::map<trackKey, uint64_t>::iterator i = keyLoadPriority.begin(); i != keyLoadPriority.end(); ++i){
       reverse.insert(std::pair<uint64_t, trackKey>(i->second, i->first));
+      VERYHIGH_MSG("Key priority for %zu:%zu = %" PRIu64, i->first.track, i->first.key, i->second);
     }
     uint64_t timer = Util::bootMS();
     for (std::multimap<uint64_t, trackKey>::reverse_iterator i = reverse.rbegin(); i != reverse.rend() && Util::bootMS() < timer + 500; ++i){
@@ -1543,12 +1544,21 @@ namespace Mist{
     }
     bufferFinalize(idx, page);
     bufferTimer = Util::bootMS() - bufferTimer;
-    INFO_MSG("Track %zu, page %" PRIu32 " (%" PRIu64 " - %" PRIu64 " ms) buffered in %" PRIu64 "ms",
-             idx, pageNumber, tPages.getInt("firsttime", pageIdx), thisTime, bufferTimer);
-    INFO_MSG("  (%" PRIu32 "/%" PRIu64 " parts, %" PRIu64 " bytes)", packCounter,
-             tPages.getInt("parts", pageIdx), byteCounter);
-    pageCounter[idx][pageNumber] = Util::bootSecs();
-    return true;
+    if (packCounter != tPages.getInt("parts", pageIdx)){
+      FAIL_MSG("Track %zu, page %" PRIu32 " (%" PRIu64 " - %" PRIu64 " ms) NOT FULLY buffered in %" PRIu64 "ms",
+               idx, pageNumber, tPages.getInt("firsttime", pageIdx), thisTime, bufferTimer);
+      INFO_MSG("  (%" PRIu32 "/%" PRIu64 " parts, %" PRIu64 " bytes)", packCounter,
+               tPages.getInt("parts", pageIdx), byteCounter);
+      pageCounter[idx][pageNumber] = Util::bootSecs();
+      return false;
+    }else{
+      INFO_MSG("Track %zu, page %" PRIu32 " (%" PRIu64 " - %" PRIu64 " ms) buffered in %" PRIu64 "ms",
+               idx, pageNumber, tPages.getInt("firsttime", pageIdx), thisTime, bufferTimer);
+      INFO_MSG("  (%" PRIu32 "/%" PRIu64 " parts, %" PRIu64 " bytes)", packCounter,
+               tPages.getInt("parts", pageIdx), byteCounter);
+      pageCounter[idx][pageNumber] = Util::bootSecs();
+      return true;
+    }
   }
 
   bool Input::atKeyFrame(){