Reworked download/upload statistics calculation

The old implementation calculates download/upload statistics for a RequestGroup by summing up all PeerStat objects. For global statistics, those are summed together. This clearly incurs runtime penalty and we introduced some kind of caching which updates statistics every 250ms but it did not work right. This change removes all these aggregation code, and instead makes RequestGroup and RequestGroupMan objects hold NetStat object and download/upload bytes are directly calculated by thier own NetStat. This is far more simplar than the old way and less runtime penalty and brings more accuracy.
whxjf · Oct 25, 2012 · 0ecfa19 · 0ecfa19
1 parent 46bdaf0
commit 0ecfa19
Show file tree

Hide file tree

Showing 38 changed files with 434 additions and 340 deletions.
diff --git a/src/AbstractCommand.cc b/src/AbstractCommand.cc
@@ -431,7 +431,7 @@ void AbstractCommand::onAbort() {
     // limitation of current implementation.
     if(!getOption()->getAsBool(PREF_ALWAYS_RESUME) &&
        fileEntry_ &&
-       getSegmentMan()->calculateSessionDownloadLength() == 0 &&
+       getDownloadContext()->getNetStat().getSessionDownloadLength() == 0 &&
        !requestGroup_->p2pInvolved() &&
        getDownloadContext()->getFileEntries().size() == 1) {
       const int maxTries = getOption()->getAsInt(PREF_MAX_RESUME_FAILURE_TRIES);

diff --git a/src/BtPieceMessage.cc b/src/BtPieceMessage.cc
@@ -100,6 +100,7 @@ void BtPieceMessage::doReceivedAction()
   RequestSlot slot = getBtMessageDispatcher()->getOutstandingRequest
     (index_, begin_, blockLength_);
   getPeer()->updateDownloadLength(blockLength_);
+  downloadContext_->updateDownloadLength(blockLength_);
   if(!RequestSlot::isNull(slot)) {
     getPeer()->snubbing(false);
     SharedHandle<Piece> piece = getPieceStorage()->getPiece(index_);
@@ -188,6 +189,7 @@ void BtPieceMessage::send()
   }
   writtenLength = getPeerConnection()->sendPendingData();
   getPeer()->updateUploadLength(writtenLength);
+  downloadContext_->updateUploadLength(writtenLength);
   setSendingInProgress(!getPeerConnection()->sendBufferIsEmpty());
 }
 

diff --git a/src/BtSetup.cc b/src/BtSetup.cc
@@ -171,7 +171,7 @@ void BtSetup::setup(std::vector<Command*>& commands,
           (new ShareRatioSeedCriteria(option->getAsDouble(PREF_SEED_RATIO),
                                       requestGroup->getDownloadContext()));
         cri->setPieceStorage(pieceStorage);
-        cri->setPeerStorage(peerStorage);
+        cri->setBtRuntime(btRuntime);
 
         unionCri->addSeedCriteria(cri);
       }

diff --git a/src/ConsoleStatCalc.cc b/src/ConsoleStatCalc.cc
@@ -304,10 +304,8 @@ ConsoleStatCalc::calculateStat(const DownloadEngine* e)
 
   if(e->getRequestGroupMan()->countRequestGroup() > 1 &&
      !e->getRequestGroupMan()->downloadFinished()) {
-    TransferStat stat = e->getRequestGroupMan()->calculateStat();
-    o << " "
-      << "[TOTAL SPD:"
-      << sizeFormatter(stat.getDownloadSpeed()) << "Bs" << "]";
+    int spd = e->getRequestGroupMan()->getNetStat().calculateDownloadSpeed();
+    o << " [TOTAL SPD:" << sizeFormatter(spd) << "Bs" << "]";
   }
 
   {

diff --git a/src/DefaultBtAnnounce.cc b/src/DefaultBtAnnounce.cc
@@ -141,7 +141,7 @@ std::string DefaultBtAnnounce::getAnnounceUrl() {
   if(!btRuntime_->lessThanMinPeers() || btRuntime_->isHalt()) {
     numWant = 0;
   }
-  TransferStat stat = peerStorage_->calculateStat();
+  NetStat& stat = downloadContext_->getNetStat();
   int64_t left =
     pieceStorage_->getTotalLength()-pieceStorage_->getCompletedLength();
   // Use last 8 bytes of peer ID as a key

diff --git a/src/DefaultBtInteractive.cc b/src/DefaultBtInteractive.cc
@@ -305,8 +305,6 @@ size_t DefaultBtInteractive::receiveMessages() {
       }
       break;
     case BtPieceMessage::ID:
-      peerStorage_->updateTransferStatFor(peer_);
-      // pass through
     case BtRequestMessage::ID:
       inactiveTimer_ = global::wallclock();
       break;

diff --git a/src/DefaultBtMessageDispatcher.cc b/src/DefaultBtMessageDispatcher.cc
@@ -100,9 +100,6 @@ void DefaultBtMessageDispatcher::sendMessages() {
       }
     }
     msg->send();
-    if(msg->isUploading()) {
-      peerStorage_->updateTransferStatFor(peer_);
-    }
     if(msg->isSendingInProgress()) {
       messageQueue_.push_front(msg);
       break;

diff --git a/src/DefaultBtProgressInfoFile.cc b/src/DefaultBtProgressInfoFile.cc
@@ -159,8 +159,8 @@ void DefaultBtProgressInfoFile::save()
     uint64_t uploadLengthNL = 0;
 #ifdef ENABLE_BITTORRENT
     if(torrentDownload) {
-      TransferStat stat = peerStorage_->calculateStat();
-      uploadLengthNL = hton64(stat.getAllTimeUploadLength());
+      uploadLengthNL = hton64(btRuntime_->getUploadLengthAtStartup()+
+                              dctx_->getNetStat().getSessionUploadLength());
     }
 #endif // ENABLE_BITTORRENT
     WRITE_CHECK(fp, &uploadLengthNL, sizeof(uploadLengthNL));

diff --git a/src/DefaultPeerStorage.cc b/src/DefaultPeerStorage.cc
@@ -60,8 +60,6 @@ const size_t MAX_PEER_LIST_UPDATE = 100;
 
 DefaultPeerStorage::DefaultPeerStorage()
   : maxPeerListSize_(MAX_PEER_LIST_SIZE),
-    removedPeerSessionDownloadLength_(0LL),
-    removedPeerSessionUploadLength_(0LL),
     seederStateChoke_(new BtSeederStateChoke()),
     leecherStateChoke_(new BtLeecherStateChoke()),
     lastTransferStatMapUpdated_(0)
@@ -241,76 +239,6 @@ void DefaultPeerStorage::getActivePeers
   std::for_each(peers_.begin(), peers_.end(), CollectActivePeer(activePeers));
 }
 
-namespace {
-TransferStat calculateStatFor(const SharedHandle<Peer>& peer)
-{
-  TransferStat s;
-  s.downloadSpeed = peer->calculateDownloadSpeed();
-  s.uploadSpeed = peer->calculateUploadSpeed();
-  s.sessionDownloadLength = peer->getSessionDownloadLength();
-  s.sessionUploadLength = peer->getSessionUploadLength();
-  return s;
-}
-} // namespace
-
-TransferStat DefaultPeerStorage::calculateStat()
-{
-  TransferStat stat;
-  if(lastTransferStatMapUpdated_.differenceInMillis(global::wallclock())+
-     A2_DELTA_MILLIS >= 250) {
-    A2_LOG_DEBUG("Updating TransferStat of PeerStorage");
-    lastTransferStatMapUpdated_ = global::wallclock();
-    peerTransferStatMap_.clear();
-    std::vector<SharedHandle<Peer> > activePeers;
-    getActivePeers(activePeers);
-    for(std::vector<SharedHandle<Peer> >::const_iterator i =
-          activePeers.begin(), eoi = activePeers.end(); i != eoi; ++i) {
-      TransferStat s;
-      s.downloadSpeed = (*i)->calculateDownloadSpeed();
-      s.uploadSpeed = (*i)->calculateUploadSpeed();
-      s.sessionDownloadLength = (*i)->getSessionDownloadLength();
-      s.sessionUploadLength = (*i)->getSessionUploadLength();
-
-      peerTransferStatMap_[(*i)->getID()] = calculateStatFor(*i);
-      stat += s;
-    }
-    cachedTransferStat_ = stat;
-  } else {
-    stat = cachedTransferStat_;
-  }
-  stat.sessionDownloadLength += removedPeerSessionDownloadLength_;
-  stat.sessionUploadLength += removedPeerSessionUploadLength_;
-  stat.setAllTimeUploadLength(btRuntime_->getUploadLengthAtStartup()+
-                              stat.getSessionUploadLength());
-  return stat;
-}
-
-void DefaultPeerStorage::updateTransferStatFor(const SharedHandle<Peer>& peer)
-{
-  A2_LOG_DEBUG(fmt("Updating TransferStat for peer %s", peer->getID().c_str()));
-  std::map<std::string, TransferStat>::iterator itr =
-    peerTransferStatMap_.find(peer->getID());
-  if(itr == peerTransferStatMap_.end()) {
-    return;
-  }
-  cachedTransferStat_ -= (*itr).second;
-  TransferStat s = calculateStatFor(peer);
-  cachedTransferStat_ += s;
-  (*itr).second = s;
-}
-
-TransferStat DefaultPeerStorage::getTransferStatFor
-(const SharedHandle<Peer>& peer)
-{
-  std::map<std::string, TransferStat>::const_iterator itr =
-    peerTransferStatMap_.find(peer->getID());
-  if(itr == peerTransferStatMap_.end()) {
-    return TransferStat();
-  } else {
-    return (*itr).second;
-  }
-}
-
 bool DefaultPeerStorage::isBadPeer(const std::string& ipaddr)
 {
   std::map<std::string, time_t>::iterator i = badPeers_.find(ipaddr);
@@ -365,11 +293,6 @@ void DefaultPeerStorage::onErasingPeer(const SharedHandle<Peer>& peer) {}
 void DefaultPeerStorage::onReturningPeer(const SharedHandle<Peer>& peer)
 {
   if(peer->isActive()) {
-    TransferStat removedStat(calculateStatFor(peer));
-    removedPeerSessionDownloadLength_ += removedStat.getSessionDownloadLength();
-    removedPeerSessionUploadLength_ += removedStat.getSessionUploadLength();
-    cachedTransferStat_ -= removedStat;
-
     if(peer->isDisconnectedGracefully() && !peer->isIncomingPeer()) {
       peer->startBadCondition();
       addDroppedPeer(peer);

diff --git a/src/DefaultPeerStorage.h b/src/DefaultPeerStorage.h
@@ -56,18 +56,12 @@ class DefaultPeerStorage : public PeerStorage {
   size_t maxPeerListSize_;
   std::deque<SharedHandle<Peer> > peers_;
   std::deque<SharedHandle<Peer> > droppedPeers_;
-  int64_t removedPeerSessionDownloadLength_;
-  int64_t removedPeerSessionUploadLength_;
 
   BtSeederStateChoke* seederStateChoke_;
   BtLeecherStateChoke* leecherStateChoke_;
 
-  std::map<std::string, TransferStat> peerTransferStatMap_;
-
   Timer lastTransferStatMapUpdated_;
 
-  TransferStat cachedTransferStat_;
-
   std::map<std::string, time_t> badPeers_;
   Timer lastBadPeerCleaned_;
 
@@ -97,12 +91,6 @@ class DefaultPeerStorage : public PeerStorage {
 
   virtual void getActivePeers(std::vector<SharedHandle<Peer> >& peers);
 
-  virtual TransferStat calculateStat();
-
-  virtual void updateTransferStatFor(const SharedHandle<Peer>& peer);
-
-  virtual TransferStat getTransferStatFor(const SharedHandle<Peer>& peer);
-
   virtual bool isBadPeer(const std::string& ipaddr);
 
   virtual void addBadPeer(const std::string& ipaddr);

diff --git a/src/DownloadCommand.cc b/src/DownloadCommand.cc
@@ -176,6 +176,7 @@ bool DownloadCommand::executeInternal() {
     }
     getSocketRecvBuffer()->shiftBuffer(bufSize);
     peerStat_->updateDownloadLength(bufSize);
+    getDownloadContext()->updateDownloadLength(bufSize);
   }
   bool segmentPartComplete = false;
   // Note that GrowSegment::complete() always returns false.

diff --git a/src/DownloadContext.cc b/src/DownloadContext.cc
@@ -43,6 +43,7 @@
 #include "DlAbortEx.h"
 #include "a2functional.h"
 #include "Signature.h"
+#include "RequestGroupMan.h"
 
 namespace aria2 {
 
@@ -52,8 +53,7 @@ DownloadContext::DownloadContext():
   knowsTotalLength_(true),
   ownerRequestGroup_(0),
   attrs_(MAX_CTX_ATTR),
-  downloadStartTime_(0),
-  downloadStopTime_(downloadStartTime_),
+  downloadStopTime_(0),
   acceptMetalink_(true) {}
 
 DownloadContext::DownloadContext(int32_t pieceLength,
@@ -64,7 +64,6 @@ DownloadContext::DownloadContext(int32_t pieceLength,
   knowsTotalLength_(true),
   ownerRequestGroup_(0),
   attrs_(MAX_CTX_ATTR),
-  downloadStartTime_(0),
   downloadStopTime_(0),
   acceptMetalink_(true)
 {
@@ -76,23 +75,20 @@ DownloadContext::~DownloadContext() {}
 
 void DownloadContext::resetDownloadStartTime()
 {
-  downloadStartTime_ = global::wallclock();
   downloadStopTime_.reset(0);
+  netStat_.downloadStart();
 }
 
 void DownloadContext::resetDownloadStopTime()
 {
   downloadStopTime_ = global::wallclock();
+  netStat_.downloadStop();
 }
 
 int64_t DownloadContext::calculateSessionTime() const
 {
-  if(downloadStopTime_ > downloadStartTime_) {
-    return
-      downloadStartTime_.differenceInMillis(downloadStopTime_);
-  } else {
-    return 0;
-  }
+  const Timer& startTime = netStat_.getDownloadStartTime();
+  return startTime.differenceInMillis(downloadStopTime_);
 }
 
 SharedHandle<FileEntry>
@@ -283,4 +279,22 @@ void DownloadContext::setSignature(const SharedHandle<Signature>& signature)
   signature_ = signature;
 }
 
+void DownloadContext::updateDownloadLength(size_t bytes)
+{
+  netStat_.updateDownloadLength(bytes);
+  RequestGroupMan* rgman = ownerRequestGroup_->getRequestGroupMan();
+  if(rgman) {
+    rgman->getNetStat().updateDownloadLength(bytes);
+  }
+}
+
+void DownloadContext::updateUploadLength(size_t bytes)
+{
+  netStat_.updateUploadLength(bytes);
+  RequestGroupMan* rgman = ownerRequestGroup_->getRequestGroupMan();
+  if(rgman) {
+    rgman->getNetStat().updateUploadLength(bytes);
+  }
+}
+
 } // namespace aria2
diff --git a/src/DownloadContext.h b/src/DownloadContext.h
@@ -47,6 +47,7 @@
 #include "ValueBase.h"
 #include "SegList.h"
 #include "ContextAttribute.h"
+#include "NetStat.h"
 
 namespace aria2 {
 
@@ -79,7 +80,7 @@ class DownloadContext
 
   std::vector<SharedHandle<ContextAttribute> > attrs_;
 
-  Timer downloadStartTime_;
+  NetStat netStat_;
 
   Timer downloadStopTime_;
 
@@ -234,6 +235,19 @@ class DownloadContext
   {
     return acceptMetalink_;
   }
+
+  NetStat& getNetStat()
+  {
+    return netStat_;
+  }
+
+  // This method also updates global download length held by
+  // RequestGroupMan via getOwnerRequestGroup().
+  void updateDownloadLength(size_t bytes);
+
+  // This method also updates global upload length held by
+  // RequestGroupMan via getOwnerRequestGroup().
+  void updateUploadLength(size_t bytes);
 };
 
 } // namespace aria2

diff --git a/src/DownloadEngine.cc b/src/DownloadEngine.cc
@@ -220,7 +220,6 @@ void DownloadEngine::onEndOfRun()
 
 void DownloadEngine::afterEachIteration()
 {
-  requestGroupMan_->calculateStat();
   if(global::globalHaltRequested == 1) {
     A2_LOG_NOTICE(_("Shutdown sequence commencing..."
                     " Press Ctrl-C again for emergency shutdown."));

diff --git a/src/Makefile.am b/src/Makefile.am
@@ -58,6 +58,7 @@ SRCS =  Socket.h\
 	DownloadEngineFactory.cc DownloadEngineFactory.h\
 	SpeedCalc.cc SpeedCalc.h\
 	PeerStat.cc PeerStat.h\
+	NetStat.cc NetStat.h\
 	BitfieldMan.cc BitfieldMan.h\
 	Randomizer.h\
 	SimpleRandomizer.cc SimpleRandomizer.h\

diff --git a/src/MultiUrlRequestInfo.cc b/src/MultiUrlRequestInfo.cc
@@ -239,6 +239,7 @@ error_code::Value MultiUrlRequestInfo::execute()
     util::setGlobalSignalHandler(SIGINT, handler, 0);
     util::setGlobalSignalHandler(SIGTERM, handler, 0);
 
+    e->getRequestGroupMan()->getNetStat().downloadStart();
     e->run();
 
     if(!option_->blank(PREF_SAVE_COOKIES)) {