2010-07-14 Tatsuhiro Tsujikawa <[email protected]>

	FeedbackURISelector now tries to chooses URI which is not used in
	aria2 globally. If it is possible, it may return used URI.
	* src/AdaptiveURISelector.cc
	* src/AdaptiveURISelector.h
	* src/CreateRequestCommand.cc
	* src/FeedbackURISelector.cc
	* src/FeedbackURISelector.h
	* src/FileEntry.cc
	* src/FileEntry.h
	* src/InOrderURISelector.cc
	* src/InOrderURISelector.h
	* src/RequestGroupMan.cc
	* src/RequestGroupMan.h
	* src/URISelector.h
	* test/FeedbackURISelectorTest.cc
	* test/InOrderURISelectorTest.cc

ChangeLog

@@ -1,3 +1,22 @@
+2010-07-14  Tatsuhiro Tsujikawa  <[email protected]>
+
+	FeedbackURISelector now tries to chooses URI which is not used in
+	aria2 globally. If it is possible, it may return used URI.
+	* src/AdaptiveURISelector.cc
+	* src/AdaptiveURISelector.h
+	* src/CreateRequestCommand.cc
+	* src/FeedbackURISelector.cc
+	* src/FeedbackURISelector.h
+	* src/FileEntry.cc
+	* src/FileEntry.h
+	* src/InOrderURISelector.cc
+	* src/InOrderURISelector.h
+	* src/RequestGroupMan.cc
+	* src/RequestGroupMan.h
+	* src/URISelector.h
+	* test/FeedbackURISelectorTest.cc
+	* test/InOrderURISelectorTest.cc
+
 2010-07-14  Tatsuhiro Tsujikawa  <[email protected]>
 
 	Updated doc for options.

src/AdaptiveURISelector.cc

@@ -72,7 +72,8 @@ AdaptiveURISelector::AdaptiveURISelector
 
 AdaptiveURISelector::~AdaptiveURISelector() {}
 
-std::string AdaptiveURISelector::select(FileEntry* fileEntry)
+std::string AdaptiveURISelector::select
+(FileEntry* fileEntry, const std::vector<std::string>& usedHosts)
 {
   if(logger_->debug()) {
     logger_->debug("AdaptiveURISelector: called %d",

src/AdaptiveURISelector.h

@@ -78,7 +78,9 @@ public:
 
   virtual ~AdaptiveURISelector();
 
-  virtual std::string select(FileEntry* fileEntry);
+  virtual std::string select
+  (FileEntry* fileEntry, const std::vector<std::string>& usedHosts);
+
   virtual void tuneDownloadCommand(const std::deque<std::string>& uris,
                                    DownloadCommand* command);
 

src/CreateRequestCommand.cc

@@ -33,7 +33,6 @@
  */
 /* copyright --> */
 #include "CreateRequestCommand.h"
-
 #include "InitiateConnectionCommandFactory.h"
 #include "RequestGroup.h"
 #include "Segment.h"
@@ -74,9 +73,12 @@ bool CreateRequestCommand::executeInternal()
     setFileEntry(getDownloadContext()->findFileEntryByOffset
                  (getSegments().front()->getPositionToWrite()));
   }
+  std::vector<std::string> usedHosts;
+  getDownloadEngine()->getRequestGroupMan()->getUsedHosts(usedHosts);
   setRequest
     (getFileEntry()->getRequest(getRequestGroup()->getURISelector(),
                                 getOption()->getAsBool(PREF_REUSE_URI),
+                                usedHosts,
                                 getOption()->get(PREF_REFERER),
                                 // Don't use HEAD request when file
                                 // size is known.

src/FeedbackURISelector.cc

@@ -60,52 +60,70 @@ public:
   }
 };
 
-std::string FeedbackURISelector::select(FileEntry* fileEntry)
+std::string FeedbackURISelector::select
+(FileEntry* fileEntry, const std::vector<std::string>& usedHosts)
 {
-  std::deque<std::string>& uris = fileEntry->getRemainingUris();
-  if(uris.empty()) {
+  if(fileEntry->getRemainingUris().empty()) {
     return A2STR::NIL;
   }
-  // Use first 10 URIs to introduce some randomness.
-  const int NUM_URI = 10;
+  // Select URI with usedHosts first. If no URI is selected, then do
+  // it again without usedHosts.
+  std::string uri = selectInternal(fileEntry->getRemainingUris(), usedHosts);
+  if(uri.empty()) {
+    uri = selectInternal
+      (fileEntry->getRemainingUris(), std::vector<std::string>());
+  } 
+  if(!uri.empty()) {
+    std::deque<std::string>& uris = fileEntry->getRemainingUris();
+    uris.erase(std::find(uris.begin(), uris.end(), uri));
+  }
+  return uri;
+}
+
+std::string FeedbackURISelector::selectInternal
+(const std::deque<std::string>& uris,
+ const std::vector<std::string>& usedHosts)
+{
+  // Use first 10 good URIs to introduce some randomness.
+  const size_t NUM_URI = 10;
   // Ignore low speed server
   const unsigned int SPEED_THRESHOLD = 20*1024;
-  size_t max = std::min(uris.size(), static_cast<size_t>(NUM_URI));
-  std::deque<std::string>::iterator urisLast = uris.begin()+max;
-  std::deque<std::pair<SharedHandle<ServerStat>, std::string> > cands;
-  for(std::deque<std::string>::iterator i = uris.begin(), eoi = urisLast;
-      i != eoi; ++i) {
+  std::vector<std::pair<SharedHandle<ServerStat>, std::string> > fastCands;
+  std::vector<std::string> normCands;
+  for(std::deque<std::string>::const_iterator i = uris.begin(),
+        eoi = uris.end(); i != eoi && fastCands.size() < NUM_URI; ++i) {
     Request r;
     r.setUri(*i);
-    SharedHandle<ServerStat> ss = serverStatMan_->find(r.getHost(),
-                                                       r.getProtocol());
+    if(std::find(usedHosts.begin(), usedHosts.end(), r.getHost())
+       != usedHosts.end()) {
+      continue;
+    }
+    SharedHandle<ServerStat> ss =
+      serverStatMan_->find(r.getHost(), r.getProtocol());
     if(!ss.isNull() && ss->isOK() && ss->getDownloadSpeed() > SPEED_THRESHOLD) {
-      cands.push_back(std::pair<SharedHandle<ServerStat>, std::string>(ss, *i));
+      fastCands.push_back(std::make_pair(ss, *i));
+    }
+    if(ss.isNull() || ss->isOK()) {
+      normCands.push_back(*i);
     }
   }
-  if(cands.empty()) {
-    for(std::deque<std::string>::iterator i = uris.begin(), eoi = uris.end();
-        i != eoi; ++i) {
-      Request r;
-      r.setUri(*i);
-      SharedHandle<ServerStat> ss = serverStatMan_->find(r.getHost(),
-                                                         r.getProtocol());
-      // Skip ERROR state URI
-      if(ss.isNull() || ss->isOK()) {
-        std::string nextURI = *i;
-        uris.erase(uris.begin(), i+1);
-        return nextURI;
+  if(fastCands.empty()) {
+    if(normCands.empty()) {
+      if(usedHosts.empty()) {
+        // All URIs are inspected but aria2 cannot find usable one.
+        // Return first URI anyway in this case.
+        return uris.front();
+      } else {
+        // If usedHosts is not empty, there is a possibility it
+        // includes usable host.
+        return A2STR::NIL;
       }
+    } else {
+      return normCands.front();
     }
-    // All URIs are inspected but aria2 cannot find usable one.
-    // Return first URI anyway in this case.
-    std::string nextURI = uris.front();
-    uris.pop_front();
-    return nextURI;
   } else {
-    std::sort(cands.begin(), cands.end(), ServerStatFaster());
-    uris.erase(std::find(uris.begin(), uris.end(), cands.front().second));
-    return cands.front().second;
+    std::sort(fastCands.begin(), fastCands.end(), ServerStatFaster());
+    return fastCands.front().second;
   }
 }
 

src/FeedbackURISelector.h

@@ -45,12 +45,16 @@ class FeedbackURISelector:public URISelector {
 private:
   SharedHandle<ServerStatMan> serverStatMan_;
 
+  std::string selectInternal
+  (const std::deque<std::string>& uris,
+   const std::vector<std::string>& usedHosts);
 public:
   FeedbackURISelector(const SharedHandle<ServerStatMan>& serverStatMan);
 
   virtual ~FeedbackURISelector();
 
-  virtual std::string select(FileEntry* fileEntry);
+  virtual std::string select
+  (FileEntry* fileEntry, const std::vector<std::string>& ignoreHosts);
 };
 
 } // namespace aria2

src/FileEntry.cc

@@ -102,11 +102,6 @@ void FileEntry::getUris(std::vector<std::string>& uris) const
   uris.insert(uris.end(), uris_.begin(), uris_.end());
 }
 
-std::string FileEntry::selectUri(const SharedHandle<URISelector>& uriSelector)
-{
-  return uriSelector->select(this);
-}
-
 template<typename InputIterator>
 static size_t countInFlightHost(InputIterator first, InputIterator last,
                                 const std::string& hostname)
@@ -126,51 +121,81 @@ SharedHandle<Request>
 FileEntry::getRequest
 (const SharedHandle<URISelector>& selector,
  bool uriReuse,
+ const std::vector<std::string>& usedHosts,
  const std::string& referer,
  const std::string& method)
 {
   SharedHandle<Request> req;
-  if(requestPool_.empty()) {
-    for(int g = 0; g < 2; ++g) {
-      std::vector<std::string> pending;
-      std::vector<std::string> ignoreHost;
-      while(1) {
-        std::string uri = selector->select(this);
-        if(uri.empty()) {
-          break;
-        }
-        req.reset(new Request());
-        if(req->setUri(uri)) {
-          if(countInFlightHost(inFlightRequests_.begin(),
-                               inFlightRequests_.end(),
-                               req->getHost()) >= maxConnectionPerServer_) {
-            pending.push_back(uri);
-            ignoreHost.push_back(req->getHost());
-            req.reset();
-            continue;
-          }
-          req->setReferer(referer);
-          req->setMethod(method);
-          spentUris_.push_back(uri);
-          inFlightRequests_.push_back(req);
-          break;
-        } else {
+  Request r;
+  if(!requestPool_.empty()) {
+    for(std::deque<SharedHandle<Request> >::iterator i = requestPool_.begin(),
+          eoi = requestPool_.end(); i != eoi; ++i) {
+      r.setUri((*i)->getUri());
+      if(std::find(usedHosts.begin(), usedHosts.end(), r.getHost()) !=
+         usedHosts.end()) {
+        continue;
+      }
+      if(countInFlightHost(inFlightRequests_.begin(), inFlightRequests_.end(),
+                           r.getHost()) >= maxConnectionPerServer_) {
+        continue;
+      }
+      req = *i;
+      requestPool_.erase(i);
+      inFlightRequests_.push_back(req);
+      return req;
+    }
+  }
+
+  for(int g = 0; g < 2; ++g) {
+    std::vector<std::string> pending;
+    std::vector<std::string> ignoreHost;
+    while(1) {
+      std::string uri = selector->select(this, usedHosts);
+      if(uri.empty()) {
+        break;
+      }
+      req.reset(new Request());
+      if(req->setUri(uri)) {
+        if(countInFlightHost(inFlightRequests_.begin(),
+                             inFlightRequests_.end(),
+                             req->getHost()) >= maxConnectionPerServer_) {
+          pending.push_back(uri);
+          ignoreHost.push_back(req->getHost());
           req.reset();
+          continue;
         }
-      }
-      uris_.insert(uris_.begin(), pending.begin(), pending.end());
-      // TODO UriReuse is performed only when PREF_REUSE_URI is true.
-      if(g == 0 && uriReuse && req.isNull() && uris_.size() == pending.size()) {
-        // Reuse URIs other than ones in pending
-        reuseUri(ignoreHost);
-      } else {
+        req->setReferer(referer);
+        req->setMethod(method);
+        spentUris_.push_back(uri);
+        inFlightRequests_.push_back(req);
         break;
+      } else {
+        req.reset();
       }
     }
-  } else {
-    req = requestPool_.front();
-    requestPool_.pop_front();
-    inFlightRequests_.push_back(req);
+    uris_.insert(uris_.begin(), pending.begin(), pending.end());
+    // TODO UriReuse is performed only when PREF_REUSE_URI is true.
+    if(g == 0 && uriReuse && req.isNull() && uris_.size() == pending.size()) {
+      // Reuse URIs other than ones in pending
+      reuseUri(ignoreHost);
+    } else {
+      break;
+    }
+  }
+  if(req.isNull()) {
+    Request r;
+    for(std::deque<SharedHandle<Request> >::iterator i = requestPool_.begin(),
+          eoi = requestPool_.end(); i != eoi; ++i) {
+      r.setUri((*i)->getUri());
+      if(countInFlightHost(inFlightRequests_.begin(), inFlightRequests_.end(),
+                           r.getHost()) >= maxConnectionPerServer_) {
+        continue;
+      }
+      req = *i;
+      requestPool_.erase(i);
+      inFlightRequests_.push_back(req);
+      return req;
+    }
   }
   return req;
 }

src/FileEntry.h

@@ -154,8 +154,6 @@ public:
 
   const std::string& getContentType() const { return contentType_; }
 
-  std::string selectUri(const SharedHandle<URISelector>& uriSelector);
-
   // If pooled Request object is available, one of them is removed
   // from the pool and returned.  If pool is empty, then select URI
   // using selectUri(selector) and construct Request object using it
@@ -170,6 +168,7 @@ public:
   SharedHandle<Request> getRequest
   (const SharedHandle<URISelector>& selector,
    bool uriReuse = true,
+   const std::vector<std::string>& usedHosts = std::vector<std::string>(),
    const std::string& referer = A2STR::NIL,
    const std::string& method = Request::METHOD_GET);
 

src/InOrderURISelector.cc

@@ -42,7 +42,8 @@ InOrderURISelector::InOrderURISelector() {}
 
 InOrderURISelector::~InOrderURISelector() {}
 
-std::string InOrderURISelector::select(FileEntry* fileEntry)
+std::string InOrderURISelector::select
+(FileEntry* fileEntry, const std::vector<std::string>& usedHosts)
 {
   std::deque<std::string>& uris = fileEntry->getRemainingUris();
   if(uris.empty()) {

src/InOrderURISelector.h

@@ -44,7 +44,8 @@ public:
 
   virtual ~InOrderURISelector();
 
-  virtual std::string select(FileEntry* fileEntry);
+  virtual std::string select
+  (FileEntry* fileEntry, const std::vector<std::string>& usedHosts);
 };
 
 } // namespace aria2

src/RequestGroupMan.cc

@@ -896,4 +896,20 @@ bool RequestGroupMan::doesOverallUploadSpeedExceed()
     maxOverallUploadSpeedLimit_ < calculateStat().getUploadSpeed();
 }
 
+void RequestGroupMan::getUsedHosts(std::vector<std::string>& usedHosts)
+{
+  Request r;
+  for(std::deque<SharedHandle<RequestGroup> >::const_iterator i =
+        requestGroups_.begin(), eoi = requestGroups_.end(); i != eoi; ++i) {
+    const std::deque<SharedHandle<Request> >& inFlightReqs =
+      (*i)->getDownloadContext()->getFirstFileEntry()->getInFlightRequests();
+    for(std::deque<SharedHandle<Request> >::const_iterator j =
+          inFlightReqs.begin(), eoj = inFlightReqs.end(); j != eoj; ++j) {
+      if(r.setUri((*j)->getUri())) {
+        usedHosts.push_back(r.getHost());
+      }
+    }
+  }
+}
+
 } // namespace aria2

src/RequestGroupMan.h

@@ -273,6 +273,8 @@ public:
   {
     return queueCheck_;
   }
+
+  void getUsedHosts(std::vector<std::string>& usedHosts);
 };
 
 typedef SharedHandle<RequestGroupMan> RequestGroupManHandle;

src/URISelector.h

@@ -35,7 +35,9 @@
 #ifndef _D_URI_SELECTOR_H_
 #define _D_URI_SELECTOR_H_
 #include "common.h"
+
 #include <string>
+#include <vector>
 #include <deque>
 
 namespace aria2 {
@@ -47,7 +49,8 @@ class URISelector {
 public:
   virtual ~URISelector() {}
 
-  virtual std::string select(FileEntry* fileEntry) = 0;
+  virtual std::string select
+  (FileEntry* fileEntry, const std::vector<std::string>& usedHosts) = 0;
 
   virtual void tuneDownloadCommand(const std::deque<std::string>& uris,
                                    DownloadCommand* command) {};

test/FeedbackURISelectorTest.cc

@@ -16,6 +16,7 @@ class FeedbackURISelectorTest:public CppUnit::TestFixture {
   CPPUNIT_TEST_SUITE(FeedbackURISelectorTest);
   CPPUNIT_TEST(testSelect_withoutServerStat);
   CPPUNIT_TEST(testSelect);
+  CPPUNIT_TEST(testSelect_withUsedHosts);
   CPPUNIT_TEST(testSelect_skipErrorHost);
   CPPUNIT_TEST_SUITE_END();
 private:
@@ -48,6 +49,8 @@ public:
   
   void testSelect();
 
+  void testSelect_withUsedHosts();
+
   void testSelect_skipErrorHost();
 };
 
@@ -56,8 +59,9 @@ CPPUNIT_TEST_SUITE_REGISTRATION(FeedbackURISelectorTest);
 
 void FeedbackURISelectorTest::testSelect_withoutServerStat()
 {
+  std::vector<std::string> usedHosts;
   // Without ServerStat, selector returns first URI
-  std::string uri = sel->select(&fileEntry_);
+  std::string uri = sel->select(&fileEntry_, usedHosts);
   CPPUNIT_ASSERT_EQUAL(std::string("http://alpha/file"), uri);
   CPPUNIT_ASSERT_EQUAL((size_t)2, fileEntry_.getRemainingUris().size());
 }
@@ -71,18 +75,45 @@ void FeedbackURISelectorTest::testSelect()
   SharedHandle<ServerStat> alphaHTTP(new ServerStat("alpha", "http"));
   alphaHTTP->updateDownloadSpeed(180000);
   alphaHTTP->setError();
+  std::vector<std::string> usedHosts;
 
   ssm->add(bravo);
   ssm->add(alphaFTP);
   ssm->add(alphaHTTP);
 
   CPPUNIT_ASSERT_EQUAL(std::string("http://bravo/file"),
-                       sel->select(&fileEntry_));
+                       sel->select(&fileEntry_, usedHosts));
   CPPUNIT_ASSERT_EQUAL((size_t)2, fileEntry_.getRemainingUris().size());
   
   CPPUNIT_ASSERT_EQUAL(std::string("ftp://alpha/file"),
-                       sel->select(&fileEntry_));
+                       sel->select(&fileEntry_, usedHosts));
+  CPPUNIT_ASSERT_EQUAL((size_t)1, fileEntry_.getRemainingUris().size());
+}
+
+void FeedbackURISelectorTest::testSelect_withUsedHosts()
+{
+  SharedHandle<ServerStat> bravo(new ServerStat("bravo", "http"));
+  bravo->updateDownloadSpeed(100000);
+  SharedHandle<ServerStat> alphaHTTP(new ServerStat("alpha", "http"));
+  alphaHTTP->updateDownloadSpeed(180000);
+  alphaHTTP->setError();
+  std::vector<std::string> usedHosts;
+  usedHosts.push_back("bravo");
+
+  ssm->add(bravo);
+  ssm->add(alphaHTTP);
+
+  CPPUNIT_ASSERT_EQUAL(std::string("ftp://alpha/file"),
+                       sel->select(&fileEntry_, usedHosts));
+  CPPUNIT_ASSERT_EQUAL((size_t)2, fileEntry_.getRemainingUris().size());
+
+  CPPUNIT_ASSERT_EQUAL(std::string("http://bravo/file"),
+                       sel->select(&fileEntry_, usedHosts));
   CPPUNIT_ASSERT_EQUAL((size_t)1, fileEntry_.getRemainingUris().size());
+
+  CPPUNIT_ASSERT_EQUAL(std::string("http://alpha/file"),
+                       sel->select(&fileEntry_, usedHosts));
+  CPPUNIT_ASSERT_EQUAL((size_t)0, fileEntry_.getRemainingUris().size());
 }
 
 void FeedbackURISelectorTest::testSelect_skipErrorHost()
@@ -91,14 +122,14 @@ void FeedbackURISelectorTest::testSelect_skipErrorHost()
   alphaHTTP->setError();
   SharedHandle<ServerStat> alphaFTP(new ServerStat("alpha", "ftp"));
   alphaFTP->setError();
+  std::vector<std::string> usedHosts;
 
   ssm->add(alphaHTTP);
   ssm->add(alphaFTP);
 
-  // See error URIs are removed from URI List.
   CPPUNIT_ASSERT_EQUAL(std::string("http://bravo/file"),
-                       sel->select(&fileEntry_));
-  CPPUNIT_ASSERT_EQUAL((size_t)0, fileEntry_.getRemainingUris().size());
+                       sel->select(&fileEntry_, usedHosts));
+  CPPUNIT_ASSERT_EQUAL((size_t)2, fileEntry_.getRemainingUris().size());
 }
 
 } // namespace aria2

test/InOrderURISelectorTest.cc

@@ -45,13 +45,14 @@ CPPUNIT_TEST_SUITE_REGISTRATION(InOrderURISelectorTest);
 
 void InOrderURISelectorTest::testSelect()
 {
+  std::vector<std::string> usedHosts;
   CPPUNIT_ASSERT_EQUAL(std::string("http://alpha/file"),
-                       sel->select(&fileEntry_));
+                       sel->select(&fileEntry_, usedHosts));
   CPPUNIT_ASSERT_EQUAL(std::string("ftp://alpha/file"),
-                       sel->select(&fileEntry_));
+                       sel->select(&fileEntry_, usedHosts));
   CPPUNIT_ASSERT_EQUAL(std::string("http://bravo/file"),
-                       sel->select(&fileEntry_));
-  CPPUNIT_ASSERT_EQUAL(std::string(""), sel->select(&fileEntry_));
+                       sel->select(&fileEntry_, usedHosts));
+  CPPUNIT_ASSERT_EQUAL(std::string(""), sel->select(&fileEntry_, usedHosts));
 }
 
 } // namespace aria2