table: dead Nonce list

refs #1953

Change-Id: I65eb2346716dd47bcf1850c832e37e5354042fd0

.gitignore

@@ -5,3 +5,4 @@
 **/*.pyc
 build/
 VERSION
+unit-tests.conf

daemon/table/dead-nonce-list.cpp

@@ -0,0 +1,169 @@
+/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
+/**
+ * Copyright (c) 2014,  Regents of the University of California,
+ *                      Arizona Board of Regents,
+ *                      Colorado State University,
+ *                      University Pierre & Marie Curie, Sorbonne University,
+ *                      Washington University in St. Louis,
+ *                      Beijing Institute of Technology,
+ *                      The University of Memphis
+ *
+ * This file is part of NFD (Named Data Networking Forwarding Daemon).
+ * See AUTHORS.md for complete list of NFD authors and contributors.
+ *
+ * NFD is free software: you can redistribute it and/or modify it under the terms
+ * of the GNU General Public License as published by the Free Software Foundation,
+ * either version 3 of the License, or (at your option) any later version.
+ *
+ * NFD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+ * PURPOSE.  See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * NFD, e.g., in COPYING.md file.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "dead-nonce-list.hpp"
+#include "core/city-hash.hpp"
+#include "core/logger.hpp"
+
+NFD_LOG_INIT("DeadNonceList");
+
+namespace nfd {
+
+const time::nanoseconds DeadNonceList::DEFAULT_LIFETIME = time::seconds(6);
+const time::nanoseconds DeadNonceList::MIN_LIFETIME = time::milliseconds(1);
+const size_t DeadNonceList::INITIAL_CAPACITY = (1 << 7);
+const size_t DeadNonceList::MIN_CAPACITY = (1 << 3);
+const size_t DeadNonceList::MAX_CAPACITY = (1 << 24);
+const DeadNonceList::Entry DeadNonceList::MARK = 0;
+const size_t DeadNonceList::EXPECTED_MARK_COUNT = 5;
+const double DeadNonceList::CAPACITY_UP = 1.2;
+const double DeadNonceList::CAPACITY_DOWN = 0.9;
+const size_t DeadNonceList::EVICT_LIMIT = (1 << 6);
+
+DeadNonceList::DeadNonceList(const time::nanoseconds& lifetime)
+  : m_lifetime(lifetime)
+  , m_queue(m_index.get<0>())
+  , m_ht(m_index.get<1>())
+  , m_capacity(INITIAL_CAPACITY)
+  , m_markInterval(m_lifetime / EXPECTED_MARK_COUNT)
+  , m_adjustCapacityInterval(m_lifetime)
+{
+  if (m_lifetime < MIN_LIFETIME) {
+    throw std::invalid_argument("lifetime is less than MIN_LIFETIME");
+  }
+
+  for (size_t i = 0; i < EXPECTED_MARK_COUNT; ++i) {
+    m_queue.push_back(MARK);
+  }
+
+  m_markEvent = scheduler::schedule(m_markInterval, bind(&DeadNonceList::mark, this));
+  m_adjustCapacityEvent = scheduler::schedule(m_adjustCapacityInterval,
+                                              bind(&DeadNonceList::adjustCapacity, this));
+}
+
+DeadNonceList::~DeadNonceList()
+{
+  scheduler::cancel(m_markEvent);
+  scheduler::cancel(m_adjustCapacityEvent);
+
+  BOOST_ASSERT(MIN_LIFETIME <= DEFAULT_LIFETIME);
+  BOOST_ASSERT(INITIAL_CAPACITY >= MIN_CAPACITY);
+  BOOST_ASSERT(INITIAL_CAPACITY <= MAX_CAPACITY);
+  BOOST_ASSERT(static_cast<size_t>(MIN_CAPACITY * CAPACITY_UP) > MIN_CAPACITY);
+  BOOST_ASSERT(static_cast<size_t>(MAX_CAPACITY * CAPACITY_DOWN) < MAX_CAPACITY);
+  BOOST_ASSERT(CAPACITY_UP > 1.0);
+  BOOST_ASSERT(CAPACITY_DOWN < 1.0);
+  BOOST_ASSERT(EVICT_LIMIT >= 1);
+}
+
+size_t
+DeadNonceList::size() const
+{
+  return m_queue.size() - this->countMarks();
+}
+
+bool
+DeadNonceList::has(const Name& name, uint32_t nonce) const
+{
+  Entry entry = DeadNonceList::makeEntry(name, nonce);
+  return m_ht.find(entry) != m_ht.end();
+}
+
+void
+DeadNonceList::add(const Name& name, uint32_t nonce)
+{
+  Entry entry = this->makeEntry(name, nonce);
+  m_queue.push_back(entry);
+
+  this->evictEntries();
+}
+
+DeadNonceList::Entry
+DeadNonceList::makeEntry(const Name& name, uint32_t nonce)
+{
+  Block nameWire = name.wireEncode();
+  return CityHash64WithSeed(reinterpret_cast<const char*>(nameWire.wire()), nameWire.size(),
+                            static_cast<uint64_t>(nonce));
+}
+
+size_t
+DeadNonceList::countMarks() const
+{
+  return m_ht.count(MARK);
+}
+
+void
+DeadNonceList::mark()
+{
+  m_queue.push_back(MARK);
+  size_t nMarks = this->countMarks();
+  m_actualMarkCounts.insert(nMarks);
+
+  NFD_LOG_DEBUG("mark nMarks=" << nMarks);
+
+  scheduler::schedule(m_markInterval, bind(&DeadNonceList::mark, this));
+}
+
+void
+DeadNonceList::adjustCapacity()
+{
+  std::pair<std::multiset<size_t>::iterator, std::multiset<size_t>::iterator> equalRange =
+    m_actualMarkCounts.equal_range(EXPECTED_MARK_COUNT);
+
+  if (equalRange.second == m_actualMarkCounts.begin()) {
+    // all counts are above expected count, adjust down
+    m_capacity = std::max(MIN_CAPACITY,
+                          static_cast<size_t>(m_capacity * CAPACITY_DOWN));
+    NFD_LOG_DEBUG("adjustCapacity DOWN capacity=" << m_capacity);
+  }
+  else if (equalRange.first == m_actualMarkCounts.end()) {
+    // all counts are below expected count, adjust up
+    m_capacity = std::min(MAX_CAPACITY,
+                          static_cast<size_t>(m_capacity * CAPACITY_UP));
+    NFD_LOG_DEBUG("adjustCapacity UP capacity=" << m_capacity);
+  }
+
+  m_actualMarkCounts.clear();
+
+  this->evictEntries();
+
+  m_adjustCapacityEvent = scheduler::schedule(m_adjustCapacityInterval,
+                                              bind(&DeadNonceList::adjustCapacity, this));
+}
+
+void
+DeadNonceList::evictEntries()
+{
+  ssize_t nOverCapacity = m_queue.size() - m_capacity;
+  if (nOverCapacity <= 0) // not over capacity
+    return;
+
+  for (ssize_t nEvict = std::min<ssize_t>(nOverCapacity, EVICT_LIMIT); nEvict > 0; --nEvict) {
+    m_queue.erase(m_queue.begin());
+  }
+  BOOST_ASSERT(m_queue.size() >= m_capacity);
+}
+
+} // namespace nfd

daemon/table/dead-nonce-list.hpp

@@ -0,0 +1,211 @@
+/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
+/**
+ * Copyright (c) 2014,  Regents of the University of California,
+ *                      Arizona Board of Regents,
+ *                      Colorado State University,
+ *                      University Pierre & Marie Curie, Sorbonne University,
+ *                      Washington University in St. Louis,
+ *                      Beijing Institute of Technology,
+ *                      The University of Memphis
+ *
+ * This file is part of NFD (Named Data Networking Forwarding Daemon).
+ * See AUTHORS.md for complete list of NFD authors and contributors.
+ *
+ * NFD is free software: you can redistribute it and/or modify it under the terms
+ * of the GNU General Public License as published by the Free Software Foundation,
+ * either version 3 of the License, or (at your option) any later version.
+ *
+ * NFD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+ * PURPOSE.  See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * NFD, e.g., in COPYING.md file.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef NFD_DAEMON_TABLE_DEAD_NONCE_LIST_HPP
+#define NFD_DAEMON_TABLE_DEAD_NONCE_LIST_HPP
+
+#include "common.hpp"
+#include <boost/multi_index_container.hpp>
+#include <boost/multi_index/sequenced_index.hpp>
+#include <boost/multi_index/hashed_index.hpp>
+#include "core/scheduler.hpp"
+
+namespace nfd {
+
+/** \brief represents the Dead Nonce list
+ *
+ *  The Dead Nonce List is a global table that supplements PIT for loop detection.
+ *  When a Nonce is erased (dead) from PIT entry, the Nonce and the Interest Name is added to
+ *  Dead Nonce List, and kept for a duration in which most loops are expected to have occured.
+ *
+ *  To reduce memory usage, the Interest Name and Nonce are stored as a 64-bit hash.
+ *  There could be false positives (non-looping Interest could be considered looping),
+ *  but the probability is small, and the error is recoverable when consumer retransmits
+ *  with a different Nonce.
+ *
+ *  To reduce memory usage, entries do not have associated timestamps. Instead,
+ *  lifetime of entries is controlled by dynamically adjusting the capacity of the container.
+ *  At fixed intervals, the MARK, an entry with a special value, is inserted into the container.
+ *  The number of MARKs stored in the container reflects the lifetime of entries,
+ *  because MARKs are inserted at fixed intervals.
+ */
+class DeadNonceList : noncopyable
+{
+public:
+  /** \brief constructs the Dead Nonce List
+   *  \param lifetime duration of the expected lifetime of each nonce,
+   *         must be no less than MIN_LIFETIME.
+   *         This should be set to the duration in which most loops would have occured.
+   *         A loop cannot be detected if delay of the cycle is greater than lifetime.
+   *  \throw std::invalid_argument if lifetime is less than MIN_LIFETIME
+   */
+  explicit
+  DeadNonceList(const time::nanoseconds& lifetime = DEFAULT_LIFETIME);
+
+  ~DeadNonceList();
+
+  /** \brief determines if name+nonce exists
+   *  \return true if name+nonce exists
+   */
+  bool
+  has(const Name& name, uint32_t nonce) const;
+
+  /** \brief records name+nonce
+   */
+  void
+  add(const Name& name, uint32_t nonce);
+
+  /** \return number of stored Nonces
+   *  \note The return value does not contain non-Nonce entries in the index, if any.
+   */
+  size_t
+  size() const;
+
+private: // Entry and Index
+  typedef uint64_t Entry;
+
+  static Entry
+  makeEntry(const Name& name, uint32_t nonce);
+
+  typedef boost::multi_index_container<
+    Entry,
+    boost::multi_index::indexed_by<
+      boost::multi_index::sequenced<>,
+      boost::multi_index::hashed_non_unique<
+        boost::multi_index::identity<Entry>
+      >
+    >
+  > Index;
+
+  typedef Index::nth_index<0>::type Queue;
+  typedef Index::nth_index<1>::type Hashtable;
+
+private: // actual lifetime estimation and capacity control
+  /** \return number of MARKs in the index
+   */
+  size_t
+  countMarks() const;
+
+  /** \brief add a MARK, then record number of MARKs in m_actualMarkCounts
+   */
+  void
+  mark();
+
+  /** \brief adjust capacity according to m_actualMarkCounts
+   *
+   *  If all counts are above EXPECTED_MARK_COUNT, reduce capacity to m_capacity * CAPACITY_DOWN.
+   *  If all counts are below EXPECTED_MARK_COUNT, increase capacity to m_capacity * CAPACITY_UP.
+   */
+  void
+  adjustCapacity();
+
+  /** \brief evict some entries if index is over capacity
+   */
+  void
+  evictEntries();
+
+public:
+  /// default entry lifetime
+  static const time::nanoseconds DEFAULT_LIFETIME;
+
+  /// minimum entry lifetime
+  static const time::nanoseconds MIN_LIFETIME;
+
+private:
+  time::nanoseconds m_lifetime;
+  Index m_index;
+  Queue& m_queue;
+  Hashtable& m_ht;
+
+PUBLIC_WITH_TESTS_ELSE_PRIVATE: // actual lifetime estimation and capacity control
+
+  // ---- current capacity and hard limits
+
+  /** \brief current capacity of index
+   *
+   *  The index size is maintained to be near this capacity.
+   *
+   *  The capacity is adjusted so that every Entry is expected to be kept for m_lifetime.
+   *  This is achieved by mark() and adjustCapacity().
+   */
+  size_t m_capacity;
+
+  static const size_t INITIAL_CAPACITY;
+
+  /** \brief minimum capacity
+   *
+   *  This is to ensure correct algorithm operations.
+   */
+  static const size_t MIN_CAPACITY;
+
+  /** \brief maximum capacity
+   *
+   *  This is to limit memory usage.
+   */
+  static const size_t MAX_CAPACITY;
+
+  // ---- actual entry lifetime estimation
+
+  /** \brief the MARK for capacity
+   *
+   *  The MARK doesn't have a distinct type.
+   *  Entry is a hash. The hash function should have non-invertible property,
+   *  so it's unlikely for a usual Entry to have collision with the MARK.
+   */
+  static const Entry MARK;
+
+  /** \brief expected number of MARKs in the index
+   */
+  static const size_t EXPECTED_MARK_COUNT;
+
+  /** \brief number of MARKs in the index after each MARK insertion
+   *
+   *  adjustCapacity uses this to determine whether and how to adjust capcity,
+   *  and then clears this list.
+   */
+  std::multiset<size_t> m_actualMarkCounts;
+
+  time::nanoseconds m_markInterval;
+
+  scheduler::EventId m_markEvent;
+
+  // ---- capacity adjustments
+
+  static const double CAPACITY_UP;
+
+  static const double CAPACITY_DOWN;
+
+  time::nanoseconds m_adjustCapacityInterval;
+
+  scheduler::EventId m_adjustCapacityEvent;
+
+  /** \brief maximum number of entries to evict at each operation if index is over capacity
+   */
+  static const size_t EVICT_LIMIT;
+};
+
+} // namespace nfd
+
+#endif // NFD_DAEMON_TABLE_DEAD_NONCE_LIST_HPP