2007-11-12 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>

	Implemented checksum validation feature(1 checksum for each 
file)
	The validation takes place after the download.
	* src/PieceHashCheckIntegrityEntry.{h, cc}: New class.
	* src/IteratableChecksumValidator.{h, cc}: Rewritten.
	* src/CheckIntegrityCommand.cc: Changed log message.
	* src/Metalink2RequestGroup.cc: Set checksum to
	SingleFileDownloadContext.
	* src/StreamCheckIntegrityEntry.{h, cc}: Now derived from
	PieceHashCheckIntegrity class.
	* src/BtCheckIntegrityEntry.{h, cc}: Now derived from
	PieceHashCheckIntegrity class.
	* src/ChecksumCheckIntegrityEntry.{h, cc}: New class.
	* src/IteratableValidator.h: New class.
	* src/message.h
	* src/CheckIntegrityEntry.{h, cc}
	* src/IteratableChunkChecksumValidator.{h, cc}
	* src/SingleFileDownloadContext.h
	* src/DownloadCommand.cc
	
	--allow-overwrite=true is no longer needed to check file 
integrity
	before download in BitTorrent download.
	* src/RequestGroup.cc (getInitialCommand)

	Removed RequestGroup from queue when 
RequestGroup::getInitialCommand()
	throws exception.
	* src/RequestGroupMan.cc (getInitialCommands)

ChangeLog

@@ -1,3 +1,32 @@
+2007-11-12  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
+
+	Implemented checksum validation feature(1 checksum for each file)
+	The validation takes place after the download.
+	* src/PieceHashCheckIntegrityEntry.{h, cc}: New class.
+	* src/IteratableChecksumValidator.{h, cc}: Rewritten.
+	* src/CheckIntegrityCommand.cc: Changed log message.
+	* src/Metalink2RequestGroup.cc: Set checksum to
+	SingleFileDownloadContext.
+	* src/StreamCheckIntegrityEntry.{h, cc}: Now derived from
+	PieceHashCheckIntegrity class.
+	* src/BtCheckIntegrityEntry.{h, cc}: Now derived from
+	PieceHashCheckIntegrity class.
+	* src/ChecksumCheckIntegrityEntry.{h, cc}: New class.
+	* src/IteratableValidator.h: New class.
+	* src/message.h
+	* src/CheckIntegrityEntry.{h, cc}
+	* src/IteratableChunkChecksumValidator.{h, cc}
+	* src/SingleFileDownloadContext.h
+	* src/DownloadCommand.cc
+	
+	--allow-overwrite=true is no longer needed to check file integrity
+	before download in BitTorrent download.
+	* src/RequestGroup.cc (getInitialCommand)
+
+	Removed RequestGroup from queue when RequestGroup::getInitialCommand()
+	throws exception.
+	* src/RequestGroupMan.cc (getInitialCommands)
+	
 2007-11-11  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
 
 	urlencode the given url inside Request::parseUrl(...)

TODO

@@ -56,7 +56,6 @@
   FatalException .... Program should abort.
 
 -- remaining features to be implemented for 0.12.0 release
-* Reimplement ChecksumCommand(validation using 1 checksum for 1 file)
 * Implement duplicate download checking in Bt
 * improve --metalink-location field
 * Piece length conversion when loading file

src/AnnounceList.cc

@@ -103,6 +103,7 @@ void AnnounceList::announceSuccess() {
   }
 }
 
+// TODO if currentTier reaches tiers.end(), then getAllTierTried() returns true
 void AnnounceList::announceFailure() {
   if(currentTrackerInitialized) {
     currentTracker++;

src/BtCheckIntegrityEntry.cc

@@ -40,10 +40,9 @@
 #include "PieceStorage.h"
 #include "DownloadEngine.h"
 #include "FileAllocationMan.h"
-#include "DiskAdaptor.h"
 
 BtCheckIntegrityEntry::BtCheckIntegrityEntry(RequestGroup* requestGroup):
-  CheckIntegrityEntry(requestGroup, 0) {}
+  PieceHashCheckIntegrityEntry(requestGroup, 0) {}
 
 BtCheckIntegrityEntry::~BtCheckIntegrityEntry() {}
 

src/BtCheckIntegrityEntry.h

@@ -35,9 +35,9 @@
 #ifndef _D_BT_CHECK_INTEGRITY_ENTRY_H_
 #define _D_BT_CHECK_INTEGRITY_ENTRY_H_
 
-#include "CheckIntegrityEntry.h"
+#include "PieceHashCheckIntegrityEntry.h"
 
-class BtCheckIntegrityEntry : public CheckIntegrityEntry {
+class BtCheckIntegrityEntry : public PieceHashCheckIntegrityEntry {
 public:
   BtCheckIntegrityEntry(RequestGroup* requestGroup);
 

src/CheckIntegrityCommand.cc

@@ -62,12 +62,13 @@ bool CheckIntegrityCommand::executeInternal()
   }
   _entry->validateChunk();
   if(_entry->finished()) {
-    _entry->updatePieceStorage();
     if(_requestGroup->downloadFinished()) {
-      logger->notice(MSG_DOWNLOAD_ALREADY_COMPLETED, _requestGroup->getGID(),
+      logger->notice(MSG_VERIFICATION_SUCCESSFUL,
 		     _requestGroup->getFilePath().c_str());
       _e->addCommand(_entry->onDownloadFinished(_e));
     } else {
+      logger->error(MSG_VERIFICATION_FAILED,
+		    _requestGroup->getFilePath().c_str());
       _e->addCommand(_entry->onDownloadIncomplete(_e));
     }
     return true;

src/CheckIntegrityEntry.cc

@@ -33,12 +33,9 @@
  */
 /* copyright --> */
 #include "CheckIntegrityEntry.h"
-#include "DlAbortEx.h"
 #include "Command.h"
 #include "RequestGroup.h"
-#include "IteratableChunkChecksumValidator.h"
-#include "DownloadContext.h"
-#include "DownloadEngine.h"
+#include "IteratableValidator.h"
 
 CheckIntegrityEntry::CheckIntegrityEntry(RequestGroup* requestGroup,
 					 Command* nextCommand):
@@ -75,23 +72,3 @@ bool CheckIntegrityEntry::finished()
 {
   return _validator->finished();
 }
-
-bool CheckIntegrityEntry::isValidationReady()
-{
-  DownloadContextHandle dctx = _requestGroup->getDownloadContext();
-  return dctx->getPieceHashes().size() > 0 &&
-    dctx->getPieceHashes().size() == (uint32_t)dctx->getNumPieces();
-}
-
-void CheckIntegrityEntry::initValidator()
-{
-  IteratableChunkChecksumValidatorHandle validator =
-    new IteratableChunkChecksumValidator(_requestGroup->getDownloadContext(),
-					 _requestGroup->getPieceStorage());
-  _validator = validator;
-}
-
-void CheckIntegrityEntry::updatePieceStorage()
-{
-  _validator->updatePieceStorage();
-}

src/CheckIntegrityEntry.h

@@ -37,16 +37,16 @@
 
 #include "RequestGroupEntry.h"
 
-class IteratableChunkChecksumValidator;
-extern typedef SharedHandle<IteratableChunkChecksumValidator> IteratableChunkChecksumValidatorHandle;
+class IteratableValidator;
+extern typedef SharedHandle<IteratableValidator> IteratableValidatorHandle;
 class Command;
 extern typedef deque<Command*> Commands;
 class DownloadEngine;
 
 class CheckIntegrityEntry : public RequestGroupEntry,
 			    public ProgressAwareEntry {
-private:
-  IteratableChunkChecksumValidatorHandle _validator;
+protected:
+  IteratableValidatorHandle _validator;
 public:
   CheckIntegrityEntry(RequestGroup* requestGroup, Command* nextCommand = 0);
 
@@ -56,15 +56,13 @@ public:
 
   virtual int64_t getCurrentLength();
 
-  virtual bool finished();
-
-  bool isValidationReady();
+  virtual void validateChunk();
 
-  void initValidator();
+  virtual bool finished();
 
-  void validateChunk();
+  virtual bool isValidationReady() = 0;
 
-  void updatePieceStorage();
+  virtual void initValidator() = 0;
 
   virtual Commands onDownloadFinished(DownloadEngine* e) = 0;
 

src/ChecksumCommand.cc → src/ChecksumCheckIntegrityEntry.cc

@@ -32,52 +32,40 @@
  * files in the program, then also delete it here.
  */
 /* copyright --> */
-#include "ChecksumCommand.h"
-#include "DlAbortEx.h"
-#include "message.h"
+#include "ChecksumCheckIntegrityEntry.h"
+#include "RequestGroup.h"
+#include "Command.h"
+#include "SingleFileDownloadContext.h"
+#include "IteratableChecksumValidator.h"
+#include "DownloadEngine.h"
+#include "PieceStorage.h"
 
-void ChecksumCommand::initValidator()
+ChecksumCheckIntegrityEntry::ChecksumCheckIntegrityEntry(RequestGroup* requestGroup, Command* nextCommand):
+  CheckIntegrityEntry(requestGroup, nextCommand) {}
+
+ChecksumCheckIntegrityEntry::~ChecksumCheckIntegrityEntry() {}
+
+bool ChecksumCheckIntegrityEntry::isValidationReady()
 {
-  _validator = new IteratableChecksumValidator();
-  // TODO checksum will be held by DownloadContext
-  _validator->setChecksum(0);
-  //_validator->setDiskWriter(new DiskAdaptorWriter(_requestGroup->getSegmentMan()->getDiskAdaptor()));
-  // TODO we should use PieceStorage instead of BitfieldMan
-  //_validator->setBitfield(_requestGroup->getSegmentMan()->getBitfield());
-  if(!_validator->canValidate()) {
-    // insufficient checksums.
-    throw new DlAbortEx(EX_INSUFFICIENT_CHECKSUM);
-  }
-  _validator->init();
+  SingleFileDownloadContextHandle dctx = _requestGroup->getDownloadContext();
+  return !dctx.isNull() && dctx->getChecksum().size() > 0 &&
+    dctx->getChecksumHashAlgo().size() > 0;
 }
 
-bool ChecksumCommand::executeInternal()
+void ChecksumCheckIntegrityEntry::initValidator()
 {
-  if(_e->isHaltRequested()) {
-    return true;
-  }
-  _validator->validateChunk();
-  if(_validator->finished()) {
-    if(_requestGroup->downloadFinished()) {
-      logger->notice(MSG_GOOD_CHECKSUM, cuid, _requestGroup->getFilePath().c_str());
-      return true;
-    } else {
-      logger->error(MSG_BAD_CHECKSUM, cuid, _requestGroup->getFilePath().c_str());
-      return true;
-    }
-  } else {
-    _e->commands.push_back(this);
-    return false;
-  }
+  _validator = new IteratableChecksumValidator(_requestGroup->getDownloadContext(),
+					       _requestGroup->getPieceStorage());
+  _validator->init();
+}
 
+Commands ChecksumCheckIntegrityEntry::onDownloadFinished(DownloadEngine* e)
+{
+  return Commands();
 }
 
-bool ChecksumCommand::handleException(Exception* e)
+Commands ChecksumCheckIntegrityEntry::onDownloadIncomplete(DownloadEngine* e)
 {
-  logger->error(MSG_FILE_VALIDATION_FAILURE, e, cuid);
-  logger->error(MSG_DOWNLOAD_NOT_COMPLETE, cuid, _requestGroup->getFilePath().c_str());
-  // TODO We need to set bitfield back to the state when validation begun.
-  // The one of the solution is having a copy of bitfield before settting its
-  // all bit to 1. If exception is thrown, then assign the copy to the bitfield.
-  return true;
+  return Commands();
 }
+

src/ChecksumCommand.h → src/ChecksumCheckIntegrityEntry.h

@@ -32,35 +32,25 @@
  * files in the program, then also delete it here.
  */
 /* copyright --> */
-#ifndef _D_CHECKSUM_COMMAND_H_
-#define _D_CHECKSUM_COMMAND_H_
+#ifndef _D_CHECKSUM_CHECK_INTEGRITY_ENTRY_H_
+#define _D_CHECKSUM_CHECK_INTEGRITY_ENTRY_H_
 
-#include "RealtimeCommand.h"
-#include "IteratableChecksumValidator.h"
+#include "CheckIntegrityEntry.h"
 
-class ChecksumCommand : public RealtimeCommand
+class ChecksumCheckIntegrityEntry:public CheckIntegrityEntry
 {
-private:
-  IteratableChecksumValidatorHandle _validator;
 public:
-  ChecksumCommand(int cuid, RequestGroup* requestGroup, DownloadEngine* e):
-    RealtimeCommand(cuid, requestGroup, e),
-    _validator(0)
-  {
-    _requestGroup->increaseNumCommand();
-  }
+  ChecksumCheckIntegrityEntry(RequestGroup* requestGroup, Command* nextCommand = 0);
 
-  virtual ~ChecksumCommand()
-  {
-    _requestGroup->decreaseNumCommand();
-  }
+  virtual ~ChecksumCheckIntegrityEntry();
 
-  void initValidator();
+  virtual bool isValidationReady();
 
-  virtual bool executeInternal();
+  virtual void initValidator();
 
-  virtual bool handleException(Exception* e);
-};
+  virtual Commands onDownloadFinished(DownloadEngine* e);
 
+  virtual Commands onDownloadIncomplete(DownloadEngine* e);
+};
 
-#endif // _D_CHECKSUM_COMMAND_H_
+#endif // _D_CHECKSUM_CHECK_INTEGRITY_ENTRY_H_

src/DownloadCommand.cc

@@ -48,6 +48,9 @@
 #include "PieceStorage.h"
 #include "Option.h"
 #include "HttpRequestCommand.h"
+#include "ChecksumCheckIntegrityEntry.h"
+#include "CheckIntegrityCommand.h"
+#include "CUIDCounter.h"
 #ifdef ENABLE_MESSAGE_DIGEST
 #include "MessageDigestHelper.h"
 #endif // ENABLE_MESSAGE_DIGEST
@@ -145,17 +148,15 @@ bool DownloadCommand::executeInternal() {
 
 bool DownloadCommand::prepareForNextSegment() {
   if(_requestGroup->downloadFinished()) {
-    // TODO According to the current plan, checksum is held by DownloadContext.
-    /*
 #ifdef ENABLE_MESSAGE_DIGEST
-    if(!_requestGroup->getChecksum().isNull() &&
-       !_requestGroup->getChecksum()->isEmpty()) {
-      ChecksumCommand* command = new ChecksumCommand(cuid, _requestGroup, e);
-      command->initValidator();
+    CheckIntegrityEntryHandle entry = new ChecksumCheckIntegrityEntry(_requestGroup);
+    if(entry->isValidationReady()) {
+      entry->initValidator();
+      CheckIntegrityCommand* command =
+	new CheckIntegrityCommand(CUIDCounterSingletonHolder::instance()->newID(), _requestGroup, e, entry);
       e->commands.push_back(command);
     }
 #endif // ENABLE_MESSAGE_DIGEST
-    */
     return true;
   } else {
     SegmentHandle tempSegment = _segments.front();

src/IteratableChecksumValidator.cc

@@ -35,48 +35,59 @@
 #include "IteratableChecksumValidator.h"
 #include "Util.h"
 #include "message.h"
+#include "SingleFileDownloadContext.h"
+#include "PieceStorage.h"
+#include "messageDigest.h"
+#include "LogFactory.h"
+#include "Logger.h"
+#include "DiskAdaptor.h"
+#include "BitfieldMan.h"
 
 #define BUFSIZE 16*1024
 
+IteratableChecksumValidator::IteratableChecksumValidator(const SingleFileDownloadContextHandle& dctx,
+							 const PieceStorageHandle& pieceStorage):
+  _dctx(dctx),
+  _pieceStorage(pieceStorage),
+  _currentOffset(0),
+  _ctx(0),
+  _logger(LogFactory::getInstance()) {}
+
+IteratableChecksumValidator::~IteratableChecksumValidator() {}
+
 void IteratableChecksumValidator::validateChunk()
 {
   if(!finished()) {
-    
     unsigned char data[BUFSIZE];
-
-    int32_t size = _diskWriter->readData(data, sizeof(data), _currentOffset);
-
-    _ctx->digestUpdate(data, size);
-    _currentOffset += sizeof(data);    
-
+    int32_t length = _pieceStorage->getDiskAdaptor()->readData(data, sizeof(data), _currentOffset);
+    _ctx->digestUpdate(data, length);
+    _currentOffset += length;
     if(finished()) {
-      unsigned char* digest = new unsigned char[_ctx->digestLength()];
-      try {
-	_ctx->digestFinal(digest);
-	if(_checksum->getMessageDigest() != Util::toHex(digest, _ctx->digestLength())) {
-	  _bitfield->clearAllBit();
-	}
-	delete [] digest;
-      } catch(...) {
-	delete [] digest;
-	throw;
+      string actualChecksum = Util::toHex((const unsigned char*)_ctx->digestFinal().c_str(), _ctx->digestLength());
+      if(_dctx->getChecksum() == actualChecksum) {
+	_pieceStorage->markAllPiecesDone();
+      } else {
+	BitfieldMan bitfield(_dctx->getPieceLength(), _dctx->getTotalLength());
+	_pieceStorage->setBitfield(bitfield.getBitfield(), bitfield.getBitfieldLength());
       }
     }
   }
 }
 
-bool IteratableChecksumValidator::canValidate() const
+bool IteratableChecksumValidator::finished() const
 {
-  // We assume file is already opened using DiskWriter::open or openExistingFile.
-  return !_checksum.isNull() && !_checksum->isEmpty();
+  return _currentOffset >= _dctx->getTotalLength();
+}
+
+int64_t IteratableChecksumValidator::getTotalLength() const
+{
+  return _dctx->getTotalLength();
 }
 
 void IteratableChecksumValidator::init()
 {
-  _bitfield->setAllBit();
   _currentOffset = 0;
-
   _ctx = new MessageDigestContext();
-  _ctx->trySetAlgo(_checksum->getAlgo());
+  _ctx->trySetAlgo(_dctx->getChecksumHashAlgo());
   _ctx->digestInit();
 }

src/IteratableChecksumValidator.h

@@ -35,57 +35,47 @@
 #ifndef _D_ITERATABLE_CHECKSUM_VALIDATOR_H_
 #define _D_ITERATABLE_CHECKSUM_VALIDATOR_H_
 
-#include "common.h"
-#include "BitfieldMan.h"
-#include "Checksum.h"
-#include "DiskWriter.h"
-#include "LogFactory.h"
-#include "messageDigest.h"
-
-class IteratableChecksumValidator
+#include "IteratableValidator.h"
+
+class SingleFileDownloadContext;
+extern typedef SharedHandle<SingleFileDownloadContext> SingleFileDownloadContextHandle;
+class PieceStorage;
+extern typedef SharedHandle<PieceStorage> PieceStorageHandle;
+class Logger;
+class MessageDigestContext;
+extern typedef SharedHandle<MessageDigestContext> MessageDigestContextHandle;
+
+class IteratableChecksumValidator:public IteratableValidator
 {
 private:
-  DiskWriterHandle _diskWriter;
-  BitfieldMan* _bitfield;
-  int64_t _currentOffset;
-  ChecksumHandle _checksum;
-  const Logger* logger;
-  MessageDigestContextHandle _ctx;
+  SingleFileDownloadContextHandle _dctx;
 
-  string calculateActualChecksum();
-public:
-  IteratableChecksumValidator():_diskWriter(0), _bitfield(0), _currentOffset(0), _checksum(0), logger(LogFactory::getInstance()), _ctx(0) {}
+  PieceStorageHandle _pieceStorage;
 
-  bool canValidate() const;
+  int64_t _currentOffset;
 
-  void init();
+  MessageDigestContextHandle _ctx;
 
-  void validateChunk();
+  const Logger* _logger;
 
-  bool finished() const
-  {
-    return _currentOffset >= _bitfield->getTotalLength();
-  }
+public:
+  IteratableChecksumValidator(const SingleFileDownloadContextHandle& dctx,
+			      const PieceStorageHandle& pieceStorage);
 
-  void setDiskWriter(const DiskWriterHandle& diskWriter)
-  {
-    _diskWriter = diskWriter;
-  }
+  virtual ~IteratableChecksumValidator();
 
-  void setBitfield(BitfieldMan* bitfield)
-  {
-    _bitfield = bitfield;
-  }
+  virtual void init();
 
-  void setChecksum(const ChecksumHandle& checksum)
-  {
-    _checksum = checksum;
-  }
+  virtual void validateChunk();
 
-  int64_t getCurrentOffset() const
+  virtual bool finished() const;
+
+  virtual int64_t getCurrentOffset() const
   {
     return _currentOffset;
   }
+
+  virtual int64_t getTotalLength() const;
 };
 
 typedef SharedHandle<IteratableChecksumValidator> IteratableChecksumValidatorHandle;

src/IteratableChunkChecksumValidator.cc

@@ -38,6 +38,23 @@
 #include "MessageDigestHelper.h"
 #include "DiskAdaptor.h"
 #include "RecoverableException.h"
+#include "DownloadContext.h"
+#include "PieceStorage.h"
+#include "BitfieldMan.h"
+#include "LogFactory.h"
+#include "Logger.h"
+
+IteratableChunkChecksumValidator::
+IteratableChunkChecksumValidator(const DownloadContextHandle& dctx,
+				 const PieceStorageHandle& pieceStorage):
+  _dctx(dctx),
+  _pieceStorage(pieceStorage),
+  _bitfield(new BitfieldMan(_dctx->getPieceLength(), _dctx->getTotalLength())),
+  _currentIndex(0),
+  _logger(LogFactory::getInstance()) {}
+
+IteratableChunkChecksumValidator::~IteratableChunkChecksumValidator() {}
+
 
 void IteratableChunkChecksumValidator::validateChunk()
 {
@@ -63,6 +80,9 @@ void IteratableChunkChecksumValidator::validateChunk()
       _bitfield->unsetBit(_currentIndex);
     }
     _currentIndex++;
+    if(finished()) {
+      _pieceStorage->setBitfield(_bitfield->getBitfield(), _bitfield->getBitfieldLength());
+    }
   }
 }
 
@@ -87,7 +107,17 @@ void IteratableChunkChecksumValidator::init()
   _currentIndex = 0;
 }
 
-void IteratableChunkChecksumValidator::updatePieceStorage()
+bool IteratableChunkChecksumValidator::finished() const
+{
+  return _currentIndex >= (uint32_t)_dctx->getNumPieces();
+}
+
+int64_t IteratableChunkChecksumValidator::getCurrentOffset() const
+{
+  return (int64_t)_currentIndex*_dctx->getPieceLength();
+}
+
+int64_t IteratableChunkChecksumValidator::getTotalLength() const
 {
-  _pieceStorage->setBitfield(_bitfield->getBitfield(), _bitfield->getBitfieldLength());
+  return _dctx->getTotalLength();
 }

src/IteratableChunkChecksumValidator.h

@@ -35,13 +35,16 @@
 #ifndef _D_ITERATABLE_CHUNK_CHECKSUM_VALIDATOR_H_
 #define _D_ITERATABLE_CHUNK_CHECKSUM_VALIDATOR_H_
 
-#include "common.h"
-#include "DownloadContext.h"
-#include "PieceStorage.h"
-#include "BitfieldMan.h"
-#include "LogFactory.h"
+#include "IteratableValidator.h"
 
-class IteratableChunkChecksumValidator
+class DownloadContext;
+extern typedef SharedHandle<DownloadContext> DownloadContextHandle;
+class PieceStorage;
+extern typedef SharedHandle<PieceStorage> PieceStorageHandle;
+class BitfieldMan;
+class Logger;
+
+class IteratableChunkChecksumValidator:public IteratableValidator
 {
 private:
   DownloadContextHandle _dctx;
@@ -53,33 +56,19 @@ private:
   string calculateActualChecksum();
 public:
   IteratableChunkChecksumValidator(const DownloadContextHandle& dctx,
-				   const PieceStorageHandle& pieceStorage):
-    _dctx(dctx),
-    _pieceStorage(pieceStorage),
-    _bitfield(new BitfieldMan(_dctx->getPieceLength(), _dctx->getTotalLength())),
-    _currentIndex(0),
-    _logger(LogFactory::getInstance()) {}
+				   const PieceStorageHandle& pieceStorage);
 
-  void init();
+  virtual ~IteratableChunkChecksumValidator();
 
-  void validateChunk();
+  virtual void init();
 
-  bool finished() const
-  {
-    return _currentIndex >= (uint32_t)_dctx->getNumPieces();
-  }
+  virtual void validateChunk();
 
-  int64_t getCurrentOffset() const
-  {
-    return ((int64_t)_currentIndex)*_dctx->getPieceLength();
-  }
+  virtual bool finished() const;
 
-  int64_t getTotalLength() const
-  {
-    return _dctx->getTotalLength();
-  }
+  virtual int64_t getCurrentOffset() const;
 
-  void updatePieceStorage();
+  virtual int64_t getTotalLength() const;
 };
 
 typedef SharedHandle<IteratableChunkChecksumValidator> IteratableChunkChecksumValidatorHandle;

src/IteratableValidator.h

@@ -0,0 +1,66 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2006 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#ifndef _D_ITERATABLE_VALIDATOR_H_
+#define _D_ITERATABLE_VALIDATOR_H_
+
+#include "common.h"
+
+/**
+ * This class provides the interface to validate files.
+ *
+ * Be aware to call init() before first validateChunk() call.
+ * Then, call validateChunk() until finished() returns true.
+ * The progress information is available using getCurrentOffset() and
+ * getTotalLength().
+ */
+class IteratableValidator
+{
+public:
+  virtual ~IteratableValidator() {}
+
+  virtual void init() = 0;
+
+  virtual void validateChunk() = 0;
+
+  virtual bool finished() const = 0;
+
+  virtual int64_t getCurrentOffset() const = 0;
+
+  virtual int64_t getTotalLength() const = 0;
+};
+
+typedef SharedHandle<IteratableValidator> IteratableValidatorHandle;
+
+#endif // _D_ITERATABLE_VALIDATOR_H_

src/Makefile.am

@@ -131,10 +131,12 @@ SRCS =  Socket.h\
 if ENABLE_MESSAGE_DIGEST
 SRCS += IteratableChunkChecksumValidator.cc IteratableChunkChecksumValidator.h\
 	IteratableChecksumValidator.cc IteratableChecksumValidator.h\
-	ChecksumCommand.cc ChecksumCommand.h\
+	IteratableValidator.h\
 	CheckIntegrityCommand.cc CheckIntegrityCommand.h\
 	CheckIntegrityEntry.cc CheckIntegrityEntry.h\
+	PieceHashCheckIntegrityEntry.cc PieceHashCheckIntegrityEntry.h\
 	StreamCheckIntegrityEntry.cc StreamCheckIntegrityEntry.h\
+	ChecksumCheckIntegrityEntry.cc ChecksumCheckIntegrityEntry.h\
 	CheckIntegrityMan.cc CheckIntegrityMan.h\
 	messageDigest.cc messageDigest.h\
 	MessageDigestHelper.cc MessageDigestHelper.h

src/Makefile.in

@@ -42,10 +42,12 @@ bin_PROGRAMS = aria2c$(EXEEXT)
 #	debug_new.cpp
 @ENABLE_MESSAGE_DIGEST_TRUE@am__append_1 = IteratableChunkChecksumValidator.cc IteratableChunkChecksumValidator.h\
 @ENABLE_MESSAGE_DIGEST_TRUE@	IteratableChecksumValidator.cc IteratableChecksumValidator.h\
-@ENABLE_MESSAGE_DIGEST_TRUE@	ChecksumCommand.cc ChecksumCommand.h\
+@ENABLE_MESSAGE_DIGEST_TRUE@	IteratableValidator.h\
 @ENABLE_MESSAGE_DIGEST_TRUE@	CheckIntegrityCommand.cc CheckIntegrityCommand.h\
 @ENABLE_MESSAGE_DIGEST_TRUE@	CheckIntegrityEntry.cc CheckIntegrityEntry.h\
+@ENABLE_MESSAGE_DIGEST_TRUE@	PieceHashCheckIntegrityEntry.cc PieceHashCheckIntegrityEntry.h\
 @ENABLE_MESSAGE_DIGEST_TRUE@	StreamCheckIntegrityEntry.cc StreamCheckIntegrityEntry.h\
+@ENABLE_MESSAGE_DIGEST_TRUE@	ChecksumCheckIntegrityEntry.cc ChecksumCheckIntegrityEntry.h\
 @ENABLE_MESSAGE_DIGEST_TRUE@	CheckIntegrityMan.cc CheckIntegrityMan.h\
 @ENABLE_MESSAGE_DIGEST_TRUE@	messageDigest.cc messageDigest.h\
 @ENABLE_MESSAGE_DIGEST_TRUE@	MessageDigestHelper.cc MessageDigestHelper.h
@@ -272,10 +274,12 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \
 	Piece.cc Piece.h IteratableChunkChecksumValidator.cc \
 	IteratableChunkChecksumValidator.h \
 	IteratableChecksumValidator.cc IteratableChecksumValidator.h \
-	ChecksumCommand.cc ChecksumCommand.h CheckIntegrityCommand.cc \
+	IteratableValidator.h CheckIntegrityCommand.cc \
 	CheckIntegrityCommand.h CheckIntegrityEntry.cc \
-	CheckIntegrityEntry.h StreamCheckIntegrityEntry.cc \
-	StreamCheckIntegrityEntry.h CheckIntegrityMan.cc \
+	CheckIntegrityEntry.h PieceHashCheckIntegrityEntry.cc \
+	PieceHashCheckIntegrityEntry.h StreamCheckIntegrityEntry.cc \
+	StreamCheckIntegrityEntry.h ChecksumCheckIntegrityEntry.cc \
+	ChecksumCheckIntegrityEntry.h CheckIntegrityMan.cc \
 	CheckIntegrityMan.h messageDigest.cc messageDigest.h \
 	MessageDigestHelper.cc MessageDigestHelper.h MetaEntry.h \
 	Data.cc Data.h Dictionary.cc Dictionary.h List.cc List.h \
@@ -352,10 +356,11 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \
 	timegm.h
 @ENABLE_MESSAGE_DIGEST_TRUE@am__objects_1 = IteratableChunkChecksumValidator.$(OBJEXT) \
 @ENABLE_MESSAGE_DIGEST_TRUE@	IteratableChecksumValidator.$(OBJEXT) \
-@ENABLE_MESSAGE_DIGEST_TRUE@	ChecksumCommand.$(OBJEXT) \
 @ENABLE_MESSAGE_DIGEST_TRUE@	CheckIntegrityCommand.$(OBJEXT) \
 @ENABLE_MESSAGE_DIGEST_TRUE@	CheckIntegrityEntry.$(OBJEXT) \
+@ENABLE_MESSAGE_DIGEST_TRUE@	PieceHashCheckIntegrityEntry.$(OBJEXT) \
 @ENABLE_MESSAGE_DIGEST_TRUE@	StreamCheckIntegrityEntry.$(OBJEXT) \
+@ENABLE_MESSAGE_DIGEST_TRUE@	ChecksumCheckIntegrityEntry.$(OBJEXT) \
 @ENABLE_MESSAGE_DIGEST_TRUE@	CheckIntegrityMan.$(OBJEXT) \
 @ENABLE_MESSAGE_DIGEST_TRUE@	messageDigest.$(OBJEXT) \
 @ENABLE_MESSAGE_DIGEST_TRUE@	MessageDigestHelper.$(OBJEXT)
@@ -893,7 +898,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CheckIntegrityCommand.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CheckIntegrityEntry.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CheckIntegrityMan.Po@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ChecksumCommand.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ChecksumCheckIntegrityEntry.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ChunkedEncoding.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Command.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CompactPeerListProcessor.Po@am__quote@
@@ -988,6 +993,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PeerMessageUtil.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PeerReceiveHandshakeCommand.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Piece.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PieceHashCheckIntegrityEntry.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PiecedSegment.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Platform.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PostDownloadHandler.Po@am__quote@

src/Metalink2RequestGroup.cc

@@ -162,12 +162,15 @@ RequestGroups Metalink2RequestGroup::generate(const string& metalinkFile)
 				    "",
 				    entry->file->getPath());
     dctx->setDir(_option->get(PREF_DIR));
-    if(!entry->chunkChecksum.isNull()) {
+    if(entry->chunkChecksum.isNull()) {
+      if(!entry->checksum.isNull()) {
+	dctx->setChecksum(entry->checksum->getMessageDigest());
+	dctx->setChecksumHashAlgo(entry->checksum->getAlgo());
+      }
+    } else {
       dctx->setPieceHashes(entry->chunkChecksum->getChecksums());
       dctx->setPieceHashAlgo(entry->chunkChecksum->getAlgo());
     }
-    // TODO set checksum here to DownloadContext
-    // * hash and hash algorithm
 
     rg->setDownloadContext(dctx);
     rg->setHintTotalLength(entry->getLength());

src/PieceHashCheckIntegrityEntry.cc

@@ -0,0 +1,61 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2006 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#include "PieceHashCheckIntegrityEntry.h"
+#include "Command.h"
+#include "RequestGroup.h"
+#include "IteratableChunkChecksumValidator.h"
+#include "DownloadContext.h"
+#include "PieceStorage.h"
+
+PieceHashCheckIntegrityEntry::PieceHashCheckIntegrityEntry(RequestGroup* requestGroup,
+							   Command* nextCommand):
+  CheckIntegrityEntry(requestGroup, nextCommand) {}
+
+PieceHashCheckIntegrityEntry::~PieceHashCheckIntegrityEntry() {}
+
+bool PieceHashCheckIntegrityEntry::isValidationReady()
+{
+  DownloadContextHandle dctx = _requestGroup->getDownloadContext();
+  return dctx->getPieceHashes().size() > 0 &&
+    dctx->getPieceHashes().size() == (uint32_t)dctx->getNumPieces();
+}
+
+void PieceHashCheckIntegrityEntry::initValidator()
+{
+  IteratableChunkChecksumValidatorHandle validator =
+    new IteratableChunkChecksumValidator(_requestGroup->getDownloadContext(),
+					 _requestGroup->getPieceStorage());
+  _validator = validator;
+}

src/PieceHashCheckIntegrityEntry.h

@@ -0,0 +1,52 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2006 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#ifndef _D_PIECE_HASH_CHECK_INTEGRITY_ENTRY_H_
+#define _D_PIECE_HASH_CHECK_INTEGRITY_ENTRY_H_
+
+#include "CheckIntegrityEntry.h"
+
+class PieceHashCheckIntegrityEntry : public CheckIntegrityEntry
+{
+public:
+  PieceHashCheckIntegrityEntry(RequestGroup* requestGroup, Command* nextCommand = 0);
+
+  virtual ~PieceHashCheckIntegrityEntry();
+
+  virtual bool isValidationReady();
+
+  virtual void initValidator();
+};
+
+#endif // _D_PIECE_HASH_CHECK_INTEGRITY_ENTRY_H_

src/RequestGroup.cc

@@ -190,7 +190,8 @@ Commands RequestGroup::createInitialCommand(DownloadEngine* e)
 	_pieceStorage->getDiskAdaptor()->openFile();
       } else {
 	if(_pieceStorage->getDiskAdaptor()->fileExists()) {
-	  if(_option->get(PREF_ALLOW_OVERWRITE) != V_TRUE) {
+	  if(_option->get(PREF_CHECK_INTEGRITY) != V_TRUE &&
+	     _option->get(PREF_ALLOW_OVERWRITE) != V_TRUE) {
 	    _logger->error(MSG_FILE_ALREADY_EXISTS,
 			   getFilePath().c_str(),
 			   progressInfoFile->getFilename().c_str());

src/RequestGroupMan.cc

@@ -178,6 +178,7 @@ Commands RequestGroupMan::getInitialCommands(DownloadEngine* e)
       } catch(RecoverableException* e) {
 	_logger->error(EX_EXCEPTION_CAUGHT, e);
 	delete e;
+	itr = _requestGroups.erase(itr);
       }
     } else {
       _reservedGroups.push_front((*itr));

src/SingleFileDownloadContext.h

@@ -56,6 +56,9 @@ private:
   Strings _pieceHashes;
   string _pieceHashAlgo;
 
+  string _checksum;
+  string _checksumHashAlgo;
+
   void updateFileEntry()
   {
     if(_ufilename != "") {
@@ -136,11 +139,31 @@ public:
     return _pieceHashAlgo;
   }
 
+  const string& getChecksumHashAlgo() const
+  {
+    return _checksumHashAlgo;
+  }
+
+  const string& getChecksum() const
+  {
+    return _checksum;
+  }
+
   void setPieceHashes(const Strings& pieceHashes)
   {
     _pieceHashes = pieceHashes;
   }
   
+  void setChecksumHashAlgo(const string& algo)
+  {
+    _checksumHashAlgo = algo;
+  }
+
+  void setChecksum(const string& checksum)
+  {
+    _checksum = checksum;
+  }
+
   void setFilename(const string& filename)
   {
     _filename = filename;

src/StreamCheckIntegrityEntry.cc

@@ -40,13 +40,12 @@
 #include "CUIDCounter.h"
 #include "PieceStorage.h"
 #include "FileAllocationMan.h"
-#include "DiskAdaptor.h"
 
 StreamCheckIntegrityEntry::StreamCheckIntegrityEntry(const RequestHandle& currentRequest,
 						     RequestGroup* requestGroup,
 						     Command* nextCommand):
-  CheckIntegrityEntry(requestGroup, nextCommand),
-    _currentRequest(currentRequest)
+  PieceHashCheckIntegrityEntry(requestGroup, nextCommand),
+  _currentRequest(currentRequest)
 {}
 
 StreamCheckIntegrityEntry::~StreamCheckIntegrityEntry() {}

src/StreamCheckIntegrityEntry.h

@@ -35,17 +35,13 @@
 #ifndef _D_STREAM_CHECK_INTEGRITY_ENTRY_H_
 #define _D_STREAM_CHECK_INTEGRITY_ENTRY_H_
 
-#include "CheckIntegrityEntry.h"
+#include "PieceHashCheckIntegrityEntry.h"
 #include "TimeA2.h"
 
 class Request;
 typedef SharedHandle<Request> RequestHandle;
-class Command;
-typedef deque<Command*> Commands;
-class RequestGroup;
-class DownloadEngine;
 
-class StreamCheckIntegrityEntry:public CheckIntegrityEntry
+class StreamCheckIntegrityEntry:public PieceHashCheckIntegrityEntry
 {
 private:
   RequestHandle _currentRequest;

src/UrlRequestInfo.cc

@@ -1,225 +0,0 @@
-/* <!-- copyright */
-/*
- * aria2 - The high speed download utility
- *
- * Copyright (C) 2006 Tatsuhiro Tsujikawa
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
- *
- * In addition, as a special exception, the copyright holders give
- * permission to link the code of portions of this program with the
- * OpenSSL library under certain conditions as described in each
- * individual source file, and distribute linked combinations
- * including the two.
- * You must obey the GNU General Public License in all respects
- * for all of the code used other than OpenSSL.  If you modify
- * file(s) with this exception, you may extend this exception to your
- * version of the file(s), but you are not obligated to do so.  If you
- * do not wish to do so, delete this exception statement from your
- * version.  If you delete this exception statement from all source
- * files in the program, then also delete it here.
- */
-/* copyright --> */
-#include "UrlRequestInfo.h"
-#include "TorrentRequestInfo.h"
-#include "MetalinkRequestInfo.h"
-#include "prefs.h"
-#include "DownloadEngineFactory.h"
-#include "RecoverableException.h"
-#include "FatalException.h"
-#include "message.h"
-#include "RequestFactory.h"
-#include "GlowFileAllocator.h"
-#include "File.h"
-#include "DefaultDiskWriter.h"
-#include "DlAbortEx.h"
-#include "DNSCache.h"
-
-std::ostream& operator<<(std::ostream& o, const HeadResult& hr) {
-  o << "filename = " << hr.filename << ", " << "totalLength = " << hr.totalLength;
-  return o;
-}
-
-extern volatile sig_atomic_t haltRequested;
-
-void UrlRequestInfo::adjustRequestSize(Requests& requests,
-				       Requests& reserved,
-				       int maxConnections) const
-{
-  if(maxConnections > 0 && (int)requests.size() > maxConnections) {
-    copy(requests.begin()+maxConnections, requests.end(),
-	 back_inserter(reserved));
-    //insert_iterator<Requests>(reserved, reserved.end()));
-    requests.erase(requests.begin()+maxConnections, requests.end());
-  }
-}
-
-RequestInfo* UrlRequestInfo::createNextRequestInfo() const
-{
-#ifdef ENABLE_BITTORRENT
-  if(op->getAsBool(PREF_FOLLOW_TORRENT) &&
-     Util::endsWith(fileInfo.filename, ".torrent")) {
-    return new TorrentRequestInfo(fileInfo.filename, op);
-  } else
-#endif // ENABLE_BITTORRENT
-#ifdef ENABLE_METALINK
-    if(op->getAsBool(PREF_FOLLOW_METALINK) &&
-       Util::endsWith(fileInfo.filename, ".metalink")) {
-      return new MetalinkRequestInfo(fileInfo.filename, op);
-    } else
-#endif // ENABLE_METALINK
-      {
-	return 0;
-      }
-}
-
-void handler(int signal) {
-  haltRequested = true;
-}
-/*
-class CreateRequest {
-private:
-  Requests* requestsPtr;
-  string referer;
-  int split;
-  string method;
-public:
-  CreateRequest(Requests* requestsPtr,
-		const string& referer,
-		int split,
-		const string& method = Request::METHOD_GET)
-    :requestsPtr(requestsPtr),
-     referer(referer),
-     split(split),
-     method(method)
-  {}
-
-  void operator()(const string& url) {
-    for(int s = 1; s <= split; s++) {
-      RequestHandle req = RequestFactorySingletonHolder::instance()->createRequest();
-      req->setReferer(referer);
-      req->setMethod(method);
-      if(req->setUrl(url)) {
-	requestsPtr->push_back(req);
-      } else {
-	fprintf(stderr, _("Unrecognized URL or unsupported protocol: %s\n"),
-		req->getUrl().c_str());
-      }
-    }
-  }
-};
-*/
-void UrlRequestInfo::printUrls(const Strings& urls) const {
-  for(Strings::const_iterator itr = urls.begin(); itr != urls.end(); itr++) {
-    logger->info("Adding URL: %s", itr->c_str());
-  }
-}
-/*
-HeadResultHandle UrlRequestInfo::getHeadResult() {
-  Requests requests;
-  for_each(urls.begin(), urls.end(),
-	   CreateRequest(&requests,
-			 op->get(PREF_REFERER),
-			 1,
-			 Request::METHOD_HEAD));
-  if(requests.size() == 0) {
-    return 0;
-  }
-  Requests reserved(requests.begin()+1, requests.end());
-  requests.erase(requests.begin()+1, requests.end());
-
-  SharedHandle<ConsoleDownloadEngine> e(DownloadEngineFactory::newConsoleEngine(op, requests, reserved));
-
-  HeadResultHandle hr = 0;
-  
-  e->run();
-  hr = new HeadResult();
-  hr->filename = e->segmentMan->filename;
-  hr->totalLength = e->segmentMan->totalSize;
-
-  return hr;
-}
-*/
-
-RequestInfos UrlRequestInfo::execute() {
-  {
-    DNSCacheHandle dnsCache = new SimpleDNSCache();
-    DNSCacheSingletonHolder::instance(dnsCache);
-  }
-
-  RequestInfo* next = 0;
-
-  try {
-    RequestGroups requestGroups;
-
-    Strings urls;
-    urls.push_back("http://localhost/~tujikawa/linux-2.6.6.tar.bz2");    
-    RequestGroupHandle rg1 = new RequestGroup(urls, op);
-
-    Strings urls2;
-    urls2.push_back("http://localhost/~tujikawa/linux-2.6.19.1.tar.bz2");
-    RequestGroupHandle rg2 = new RequestGroup(urls2, op);
-
-    requestGroups.push_back(rg1);
-    requestGroups.push_back(rg2);
-    
-    SharedHandle<ConsoleDownloadEngine> e(DownloadEngineFactory::newConsoleEngine(op, requestGroups));
-
-
-    Strings reservedUrls1;
-    reservedUrls1.push_back("http://localhost/~tujikawa/linux-2.6.1.tar.bz2");
-
-    RequestGroupHandle rrg1 = new RequestGroup(reservedUrls1, op);
-
-    e->_requestGroupMan->addReservedGroup(rrg1);
-    
-    e->fillCommand();
-
-
-
-
-    // The number of simultaneous download is specified by PREF_MAX_CONCURRENT_DOWNLOADS.
-    // The remaining urls are queued into FillRequestGroupCommand.
-    // It observes the number of simultaneous downloads and if it is under
-    // the limit, it adds RequestGroup object from its queue to DownloadEngine.
-    // This is done every 1 second. At the same time, it removes finished/error
-    // RequestGroup from DownloadEngine.
-
-    Util::setGlobalSignalHandler(SIGINT, handler, 0);
-    Util::setGlobalSignalHandler(SIGTERM, handler, 0);
-    
-    e->run();
-    
-    if(e->_requestGroupMan->downloadFinished()) {
-      next = createNextRequestInfo();
-    } else {
-      e->_requestGroupMan->save();
-      e->_requestGroupMan->closeFile();
-      printDownloadAbortMessage();
-    }
-  } catch(RecoverableException *ex) {
-    logger->error("Exception caught", ex);
-    delete ex;
-    fail = true;
-  }
-  RequestInfos nextReqInfos;
-  if(next) {
-    nextReqInfos.push_front(next);
-  }
-  Util::setGlobalSignalHandler(SIGINT, SIG_DFL, 0);
-  Util::setGlobalSignalHandler(SIGTERM, SIG_DFL, 0);
-  
-  return nextReqInfos;
-}

src/UrlRequestInfo.h

@@ -1,117 +0,0 @@
-/* <!-- copyright */
-/*
- * aria2 - The high speed download utility
- *
- * Copyright (C) 2006 Tatsuhiro Tsujikawa
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
- *
- * In addition, as a special exception, the copyright holders give
- * permission to link the code of portions of this program with the
- * OpenSSL library under certain conditions as described in each
- * individual source file, and distribute linked combinations
- * including the two.
- * You must obey the GNU General Public License in all respects
- * for all of the code used other than OpenSSL.  If you modify
- * file(s) with this exception, you may extend this exception to your
- * version of the file(s), but you are not obligated to do so.  If you
- * do not wish to do so, delete this exception statement from your
- * version.  If you delete this exception statement from all source
- * files in the program, then also delete it here.
- */
-/* copyright --> */
-#ifndef _D_URL_REQUEST_INFO_H_
-#define _D_URL_REQUEST_INFO_H_
-
-#include "RequestInfo.h"
-
-class HeadResult {
-public:
-  string filename;
-  int64_t totalLength;
-public:
-  HeadResult():totalLength(0) {}
-};
-
-std::ostream& operator<<(std::ostream& o, const HeadResult& hr);
-
-typedef SharedHandle<HeadResult> HeadResultHandle;
-
-class UrlRequestInfo : public RequestInfo {
-private:
-  Strings urls;
-  int32_t maxConnections;
-  string _filename;
-  int64_t _totalLength;
-#ifdef ENABLE_MESSAGE_DIGEST
-  MessageDigestContext::DigestAlgo digestAlgo;
-  int32_t chunkChecksumLength;
-  Strings chunkChecksums;
-#endif // ENABLE_MESSAGE_DIGEST
-
-  RequestInfo* createNextRequestInfo() const;
-  void adjustRequestSize(Requests& requests,
-			 Requests& reserved,
-			 int32_t maxConnections) const;
-  void printUrls(const Strings& urls) const;
-  HeadResultHandle getHeadResult();
-public:
-  UrlRequestInfo(const Strings& urls, int32_t maxConnections, Option* op):
-    RequestInfo(op),
-    urls(urls),
-    maxConnections(maxConnections),
-    _totalLength(0)
-#ifdef ENABLE_MESSAGE_DIGEST
-    ,
-    digestAlgo(DIGEST_ALGO_SHA1),
-    chunkChecksumLength(0)
-#endif // ENABLE_MESSAGE_DIGEST
-    {}
-
-  virtual ~UrlRequestInfo() {}
-
-  virtual RequestInfos execute();
-
-#ifdef ENABLE_MESSAGE_DIGEST
-  void setDigestAlgo(const MessageDigestContext::DigestAlgo& algo) {
-    this->digestAlgo = algo;
-  }
-#endif // ENABLE_MESSAGE_DIGEST
-
-#ifdef ENABLE_MESSAGE_DIGEST
-  void setChunkChecksumLength(int32_t chunkChecksumLength) {
-    this->chunkChecksumLength = chunkChecksumLength;
-  }
-#endif // ENABLE_MESSAGE_DIGEST
-
-#ifdef ENABLE_MESSAGE_DIGEST
-  void setChunkChecksums(const Strings& chunkChecksums) {
-    this->chunkChecksums = chunkChecksums;
-  }
-#endif // ENABLE_MESSAGE_DIGEST
-
-  void setTotalLength(int64_t totalLength)
-  {
-    _totalLength = totalLength;
-  }
-
-  void setFilename(const string& filename)
-  {
-    _filename = filename;
-  }
-};
-
-typedef SharedHandle<UrlRequestInfo> UrlRequestInfoHandle;
-#endif // _D_URL_REQUEST_INFO_H_

src/message.h

@@ -124,6 +124,8 @@
 #define MSG_LOGGING_STARTED _("Logging started.")
 #define MSG_URI_REQUIRED _("Specify at least one URL.")
 #define MSG_DAEMON_FAILED _("daemon failed.")
+#define MSG_VERIFICATION_SUCCESSFUL _("Verification finished successfully. file=%s")
+#define MSG_VERIFICATION_FAILED _("Checksum error detected. file=%s")
 
 #define EX_TIME_OUT _("Timeout.")
 #define EX_INVALID_CHUNK_SIZE _("Invalid chunk size.")

test/IteratableChecksumValidatorTest.cc

@@ -1,5 +1,8 @@
 #include "IteratableChecksumValidator.h"
-#include "DefaultDiskWriter.h"
+#include "SingleFileDownloadContext.h"
+#include "DefaultPieceStorage.h"
+#include "Option.h"
+#include "DiskAdaptor.h"
 #include <cppunit/extensions/HelperMacros.h>
 
 using namespace std;
@@ -8,7 +11,7 @@ class IteratableChecksumValidatorTest:public CppUnit::TestFixture {
 
   CPPUNIT_TEST_SUITE(IteratableChecksumValidatorTest);
   CPPUNIT_TEST(testValidate);
-  CPPUNIT_TEST(testValidate2);
+  CPPUNIT_TEST(testValidate_fail);
   CPPUNIT_TEST_SUITE_END();
 private:
 
@@ -17,103 +20,47 @@ public:
   }
 
   void testValidate();
-  void testValidate2();
+  void testValidate_fail();
 };
 
 
 CPPUNIT_TEST_SUITE_REGISTRATION( IteratableChecksumValidatorTest );
 
 void IteratableChecksumValidatorTest::testValidate() {
-  BitfieldMan bitfieldMan(100, 250);
-  bitfieldMan.setAllBit();
-
-  ChecksumHandle checksum = new Checksum("sha1",
-					 "898a81b8e0181280ae2ee1b81e269196d91e869a");
-
-  DefaultDiskWriterHandle diskWriter = new DefaultDiskWriter();
-  diskWriter->openExistingFile("chunkChecksumTestFile250.txt");
-
-  IteratableChecksumValidator validator;
-  validator.setDiskWriter(diskWriter);
-  validator.setBitfield(&bitfieldMan);
-  validator.setChecksum(checksum);
-
+  Option option;
+  SingleFileDownloadContextHandle dctx =
+    new SingleFileDownloadContext(100, 250, "chunkChecksumTestFile250.txt");
+  dctx->setChecksum("898a81b8e0181280ae2ee1b81e269196d91e869a");
+  dctx->setChecksumHashAlgo("sha1");
+  DefaultPieceStorageHandle ps = new DefaultPieceStorage(dctx, &option);
+  ps->initStorage();
+  ps->getDiskAdaptor()->openFile();
+
+  IteratableChecksumValidator validator(dctx, ps);
   validator.init();
   while(!validator.finished()) {
     validator.validateChunk();
   }
-  CPPUNIT_ASSERT(bitfieldMan.isAllBitSet());
-}
-
-void IteratableChecksumValidatorTest::testValidate2() {
-  BitfieldMan bitfieldMan(100, 250);
-  bitfieldMan.setAllBit();
 
-  ChecksumHandle checksum = new Checksum("sha1", "ffffffffffffffffffffffffffffffffffffffff");
-
-  DefaultDiskWriterHandle diskWriter = new DefaultDiskWriter();
-  diskWriter->openExistingFile("chunkChecksumTestFile250.txt");
-
-  IteratableChecksumValidator validator;
-  validator.setDiskWriter(diskWriter);
-  validator.setBitfield(&bitfieldMan);
-  validator.setChecksum(checksum);
+  CPPUNIT_ASSERT(ps->downloadFinished());
+}
 
+void IteratableChecksumValidatorTest::testValidate_fail() {
+  Option option;
+  SingleFileDownloadContextHandle dctx =
+    new SingleFileDownloadContext(100, 250, "chunkChecksumTestFile250.txt");
+  dctx->setChecksum(string(40, '0')); // set wrong checksum
+  dctx->setChecksumHashAlgo("sha1");
+  DefaultPieceStorageHandle ps = new DefaultPieceStorage(dctx, &option);
+  ps->initStorage();
+  ps->getDiskAdaptor()->openFile();
+
+  IteratableChecksumValidator validator(dctx, ps);
   validator.init();
+  
   while(!validator.finished()) {
     validator.validateChunk();
   }
-  CPPUNIT_ASSERT(!bitfieldMan.isAllBitSet());
-}
-/*
-void IteratableChecksumValidatorTest::testValidate3() {
-  BitfieldMan bitfieldMan(50, 250);
-  bitfieldMan.setAllBit();
-  Strings checksums;
-  checksums.push_back("898a81b8e0181280ae2ee1b81e269196d91e869a");
-
-  DefaultDiskWriterHandle diskWriter = new DefaultDiskWriter();
-  diskWriter->openExistingFile("chunkChecksumTestFile250.txt");
-
-  IteratableChecksumValidator validator;
-  validator.setDiskWriter(diskWriter);
-
-  validator.validate(&bitfieldMan, checksums, 250);
-
-  CPPUNIT_ASSERT(bitfieldMan.isAllBitSet());
-
-  checksums[0] = "ffffffffffffffffffffffffffffffffffffffff";
-
-  validator.validate(&bitfieldMan, checksums, 250);
-
-  CPPUNIT_ASSERT(!bitfieldMan.isBitSet(0));
-  CPPUNIT_ASSERT(!bitfieldMan.isBitSet(1));
-  CPPUNIT_ASSERT(!bitfieldMan.isBitSet(2));
-  CPPUNIT_ASSERT(!bitfieldMan.isBitSet(3));
-  CPPUNIT_ASSERT(!bitfieldMan.isBitSet(4));
-}
-
-void IteratableChecksumValidatorTest::testValidate4() {
-  BitfieldMan bitfieldMan(70, 250);
-  bitfieldMan.setAllBit();
-  Strings checksums(&csArray[0], &csArray[3]);
-
-  DefaultDiskWriterHandle diskWriter = new DefaultDiskWriter();
-  diskWriter->openExistingFile("chunkChecksumTestFile250.txt");
-
-  IteratableChecksumValidator validator;
-  validator.setDiskWriter(diskWriter);
-
-  validator.validate(&bitfieldMan, checksums, 100);
-
-  CPPUNIT_ASSERT(bitfieldMan.isAllBitSet());
-
-  checksums[1] = "ffffffffffffffffffffffffffffffffffffffff";
-  validator.validate(&bitfieldMan, checksums, 100);
 
-  CPPUNIT_ASSERT(bitfieldMan.isBitSet(0));
-  CPPUNIT_ASSERT(!bitfieldMan.isBitSet(1));
-  CPPUNIT_ASSERT(!bitfieldMan.isBitSet(2));
-  CPPUNIT_ASSERT(bitfieldMan.isBitSet(3));
+  CPPUNIT_ASSERT(!ps->downloadFinished());
 }
-*/

test/IteratableChunkChecksumValidatorTest.cc

@@ -47,7 +47,6 @@ void IteratableChunkChecksumValidatorTest::testValidate() {
   CPPUNIT_ASSERT(!validator.finished());
   validator.validateChunk();
   CPPUNIT_ASSERT(validator.finished());
-  validator.updatePieceStorage();
   CPPUNIT_ASSERT(ps->downloadFinished());
 
   // make the test fail
@@ -60,7 +59,6 @@ void IteratableChunkChecksumValidatorTest::testValidate() {
   while(!validator.finished()) {
     validator.validateChunk();
   }
-  validator.updatePieceStorage();
   CPPUNIT_ASSERT(ps->hasPiece(0));
   CPPUNIT_ASSERT(!ps->hasPiece(1));
   CPPUNIT_ASSERT(ps->hasPiece(2));

test/Makefile.am

@@ -46,7 +46,8 @@ aria2c_SOURCES = AllTest.cc\
 
 if ENABLE_MESSAGE_DIGEST
 aria2c_SOURCES += MessageDigestHelperTest.cc\
-	IteratableChunkChecksumValidatorTest.cc
+	IteratableChunkChecksumValidatorTest.cc\
+	IteratableChecksumValidatorTest.cc
 endif # ENABLE_MESSAGE_DIGEST
 
 if ENABLE_BITTORRENT

test/Makefile.in

@@ -38,7 +38,8 @@ host_triplet = @host@
 target_triplet = @target@
 check_PROGRAMS = $(am__EXEEXT_1)
 @ENABLE_MESSAGE_DIGEST_TRUE@am__append_1 = MessageDigestHelperTest.cc\
-@ENABLE_MESSAGE_DIGEST_TRUE@	IteratableChunkChecksumValidatorTest.cc
+@ENABLE_MESSAGE_DIGEST_TRUE@	IteratableChunkChecksumValidatorTest.cc\
+@ENABLE_MESSAGE_DIGEST_TRUE@	IteratableChecksumValidatorTest.cc
 
 @ENABLE_BITTORRENT_TRUE@am__append_2 = BtAllowedFastMessageTest.cc\
 @ENABLE_BITTORRENT_TRUE@	BtBitfieldMessageTest.cc\
@@ -132,16 +133,17 @@ am__aria2c_SOURCES_DIST = AllTest.cc a2functionalTest.cc \
 	DefaultDiskWriterTest.cc FeatureConfigTest.cc SpeedCalcTest.cc \
 	FixedNumberRandomizer.h MessageDigestHelperTest.cc \
 	IteratableChunkChecksumValidatorTest.cc \
-	BtAllowedFastMessageTest.cc BtBitfieldMessageTest.cc \
-	BtCancelMessageTest.cc BtChokeMessageTest.cc \
-	BtHandshakeMessageTest.cc BtHaveAllMessageTest.cc \
-	BtHaveMessageTest.cc BtHaveNoneMessageTest.cc \
-	BtInterestedMessageTest.cc BtKeepAliveMessageTest.cc \
-	BtNotInterestedMessageTest.cc BtPieceMessageTest.cc \
-	BtPortMessageTest.cc BtRejectMessageTest.cc \
-	BtRequestMessageTest.cc BtSuggestPieceMessageTest.cc \
-	BtUnchokeMessageTest.cc DefaultBtAnnounceTest.cc \
-	DefaultBtContextTest.cc DefaultBtMessageDispatcherTest.cc \
+	IteratableChecksumValidatorTest.cc BtAllowedFastMessageTest.cc \
+	BtBitfieldMessageTest.cc BtCancelMessageTest.cc \
+	BtChokeMessageTest.cc BtHandshakeMessageTest.cc \
+	BtHaveAllMessageTest.cc BtHaveMessageTest.cc \
+	BtHaveNoneMessageTest.cc BtInterestedMessageTest.cc \
+	BtKeepAliveMessageTest.cc BtNotInterestedMessageTest.cc \
+	BtPieceMessageTest.cc BtPortMessageTest.cc \
+	BtRejectMessageTest.cc BtRequestMessageTest.cc \
+	BtSuggestPieceMessageTest.cc BtUnchokeMessageTest.cc \
+	DefaultBtAnnounceTest.cc DefaultBtContextTest.cc \
+	DefaultBtMessageDispatcherTest.cc \
 	DefaultBtRequestFactoryTest.cc MockBtMessage.h \
 	MockBtMessageDispatcher.h MockBtMessageFactory.h \
 	ShaVisitorTest.cc DefaultPeerListProcessorTest.cc \
@@ -157,7 +159,8 @@ am__aria2c_SOURCES_DIST = AllTest.cc a2functionalTest.cc \
 	MetalinkPostDownloadHandlerTest.cc MetalinkHelperTest.cc
 @ENABLE_MESSAGE_DIGEST_TRUE@am__objects_1 =  \
 @ENABLE_MESSAGE_DIGEST_TRUE@	MessageDigestHelperTest.$(OBJEXT) \
-@ENABLE_MESSAGE_DIGEST_TRUE@	IteratableChunkChecksumValidatorTest.$(OBJEXT)
+@ENABLE_MESSAGE_DIGEST_TRUE@	IteratableChunkChecksumValidatorTest.$(OBJEXT) \
+@ENABLE_MESSAGE_DIGEST_TRUE@	IteratableChecksumValidatorTest.$(OBJEXT)
 @ENABLE_BITTORRENT_TRUE@am__objects_2 =  \
 @ENABLE_BITTORRENT_TRUE@	BtAllowedFastMessageTest.$(OBJEXT) \
 @ENABLE_BITTORRENT_TRUE@	BtBitfieldMessageTest.$(OBJEXT) \
@@ -550,6 +553,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpHeaderTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpRequestTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpResponseTest.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/IteratableChecksumValidatorTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/IteratableChunkChecksumValidatorTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ListTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MessageDigestHelperTest.Po@am__quote@

test/Metalink2RequestGroupTest.cc

@@ -39,6 +39,9 @@ void Metalink2RequestGroupTest::testGenerate()
     SingleFileDownloadContextHandle dctx = rg->getDownloadContext();
     CPPUNIT_ASSERT(!dctx.isNull());
     CPPUNIT_ASSERT_EQUAL((int64_t)0, dctx->getTotalLength());
+    CPPUNIT_ASSERT_EQUAL(string("sha1"), dctx->getChecksumHashAlgo());
+    CPPUNIT_ASSERT_EQUAL(string("a96cf3f0266b91d87d5124cf94326422800b627d"),
+			 dctx->getChecksum());
   }
   // second file
   {
@@ -50,6 +53,8 @@ void Metalink2RequestGroupTest::testGenerate()
     CPPUNIT_ASSERT_EQUAL(string("sha1"), dctx->getPieceHashAlgo());
     CPPUNIT_ASSERT_EQUAL((size_t)2, dctx->getPieceHashes().size());
     CPPUNIT_ASSERT_EQUAL((int32_t)262144, dctx->getPieceLength());
+    CPPUNIT_ASSERT_EQUAL(string(""), dctx->getChecksumHashAlgo());
+    CPPUNIT_ASSERT_EQUAL(string(""), dctx->getChecksum());
   }
 
   // fifth file <- downloading .torrent file