lsdb: use ndn-cxx's Segmenter

This commit also makes lld the preferred linker on Linux

Change-Id: I178ca7d28ad21d8a1e3f3daf99f80206e8b1c54e

.github/workflows/ci.yml

@@ -52,6 +52,8 @@ jobs:
         include:
           - os: macos-12
             xcode: '13.4'
+          - os: macos-12
+            xcode: '14.1'
     steps:
       - name: Set up Xcode
         uses: maxim-lobanov/setup-xcode@v1

.github/workflows/docs.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-12, ubuntu-20.04]
+        os: [macos-latest, ubuntu-latest]
     env:
       JOB_NAME: Docs
     steps:
@@ -27,6 +27,8 @@ jobs:
           ./.jenkins
       - name: Build documentation
         run: |
+          pybindir=$(python3 -c 'import sysconfig; print(sysconfig.get_path("scripts", "posix_user"))')
+          export PATH="${pybindir}${PATH:+:}${PATH}"
           ./waf --color=yes configure
           ./waf --color=yes build --targets=version.hpp
           ./waf --color=yes build --targets=manpages

.jenkins

@@ -16,6 +16,7 @@ case $(uname) in
         # Emulate a subset of os-release(5)
         export ID=macos
         export VERSION_ID=$(sw_vers -productVersion)
+        export PATH="/usr/local/bin${PATH:+:}${PATH}"
         if [[ -x /opt/homebrew/bin/brew ]]; then
             eval "$(/opt/homebrew/bin/brew shellenv)"
         elif [[ -x /usr/local/bin/brew ]]; then
@@ -25,7 +26,11 @@ case $(uname) in
 esac
 
 export CACHE_DIR=${CACHE_DIR:-/tmp}
-[[ $JOB_NAME == *"code-coverage" ]] && export DISABLE_ASAN=yes
+
+if [[ $JOB_NAME == *"code-coverage" ]]; then
+    export DISABLE_ASAN=yes
+    export DISABLE_HEADERS_CHECK=yes
+fi
 
 for file in .jenkins.d/*; do
     [[ -f $file && -x $file ]] || continue

.jenkins.d/00-deps.sh

@@ -25,20 +25,14 @@ if [[ $ID == macos ]]; then
     fi
     brew update
     brew install --formula "${FORMULAE[@]}"
-
-    if (( ${#PIP_PKGS[@]} )); then
-        pip3 install --upgrade --upgrade-strategy=eager "${PIP_PKGS[@]}"
-    fi
-
 elif [[ $ID_LIKE == *debian* ]]; then
     sudo apt-get -qq update
     sudo apt-get -qy install "${APT_PKGS[@]}"
-
-    if (( ${#PIP_PKGS[@]} )); then
-        pip3 install --user --upgrade --upgrade-strategy=eager "${PIP_PKGS[@]}"
-    fi
-
 elif [[ $ID_LIKE == *fedora* ]]; then
-    sudo dnf -y install gcc-c++ libasan pkgconf-pkg-config python3 \
+    sudo dnf -y install gcc-c++ libasan lld pkgconf-pkg-config python3 \
                         boost-devel openssl-devel sqlite-devel
 fi
+
+if (( ${#PIP_PKGS[@]} )); then
+    pip3 install --user --upgrade --upgrade-strategy=eager "${PIP_PKGS[@]}"
+fi

.waf-tools/default-compiler-flags.py

@@ -137,9 +137,7 @@ class GccBasicFlags(CompilerFlags):
     def getGeneralFlags(self, conf):
         flags = super(GccBasicFlags, self).getGeneralFlags(conf)
         flags['CXXFLAGS'] += ['-std=c++17']
-        if Utils.unversioned_sys_platform() == 'linux':
-            flags['LINKFLAGS'] += ['-fuse-ld=gold']
-        elif Utils.unversioned_sys_platform() == 'freebsd':
+        if Utils.unversioned_sys_platform() != 'darwin':
             flags['LINKFLAGS'] += ['-fuse-ld=lld']
         return flags
 

src/lsdb.cpp

@@ -51,7 +51,8 @@ Lsdb::Lsdb(ndn::Face& face, ndn::KeyChain& keyChain, ConfParameter& confParam)
         lsaInterest.appendNumber(sequenceNumber);
         expressInterest(lsaInterest, 0, incomingFaceId);
       }))
-  , m_segmentPublisher(m_face, keyChain)
+  , m_segmenter(keyChain, m_confParam.getSigningInfo())
+  , m_segmentFifo(100)
   , m_isBuildAdjLsaScheduled(false)
   , m_adjBuildCount(0)
 {
@@ -74,6 +75,13 @@ Lsdb::Lsdb(ndn::Face& face, ndn::KeyChain& keyChain, ConfParameter& confParam)
   }
 }
 
+Lsdb::~Lsdb()
+{
+  for (const auto& fetcher : m_fetchers) {
+    fetcher->stop();
+  }
+}
+
 void
 Lsdb::buildAndInstallOwnNameLsa()
 {
@@ -110,7 +118,7 @@ Lsdb::scheduleAdjLsaBuild()
 
   if (m_confParam.getHyperbolicState() == HYPERBOLIC_STATE_ON) {
     // Don't build adjacency LSAs in hyperbolic routing
-    NLSR_LOG_DEBUG("Adjacency LSA not built. Currently in hyperbolic routing state.");
+    NLSR_LOG_DEBUG("Adjacency LSA not built while in hyperbolic routing state");
     return;
   }
 
@@ -152,12 +160,15 @@ Lsdb::processInterest(const ndn::Name& name, const ndn::Interest& interest)
 
   if (interestName[-2].isVersion()) {
     // Interest for particular segment
-    if (m_segmentPublisher.replyFromStore(interestName)) {
-      NLSR_LOG_TRACE("Reply from SegmentPublisher storage");
+    auto data = m_segmentFifo.find(interestName);
+    if (data) {
+      NLSR_LOG_TRACE("Replying from FIFO buffer");
+      m_face.put(*data);
       return;
     }
+
     // Remove version and segment
-    interestName = interestName.getSubName(0, interestName.size() - 2);
+    interestName = interestName.getPrefix(-2);
     NLSR_LOG_TRACE("Interest w/o segment and version: " << interestName);
   }
 
@@ -191,8 +202,8 @@ Lsdb::processInterest(const ndn::Name& name, const ndn::Interest& interest)
     }
   }
   // else the interest is for other router's LSA, serve signed data from LsaSegmentStorage
-  else if (auto lsaSegment = m_lsaStorage.find(interest)) {
-    NLSR_LOG_TRACE("Found data in lsa storage. Sending the data for " << interest.getName());
+  else if (auto lsaSegment = m_lsaStorage.find(interest); lsaSegment) {
+    NLSR_LOG_TRACE("Found data in lsa storage. Sending data for " << interest.getName());
     m_face.put(*lsaSegment);
   }
 }
@@ -202,17 +213,32 @@ Lsdb::processInterestForLsa(const ndn::Interest& interest, const ndn::Name& orig
                             Lsa::Type lsaType, uint64_t seqNo)
 {
   NLSR_LOG_DEBUG(interest << " received for " << lsaType);
-  if (auto lsaPtr = findLsa(originRouter, lsaType)) {
-    NLSR_LOG_TRACE("Verifying SeqNo for " << lsaType << " is same as requested.");
+
+  if (auto lsaPtr = findLsa(originRouter, lsaType); lsaPtr) {
+    NLSR_LOG_TRACE("Verifying SeqNo for " << lsaType << " is same as requested");
     if (lsaPtr->getSeqNo() == seqNo) {
-      m_segmentPublisher.publish(interest.getName(), interest.getName(), lsaPtr->wireEncode(),
-                                 m_lsaRefreshTime, m_confParam.getSigningInfo());
+      auto segments = m_segmenter.segment(lsaPtr->wireEncode(),
+                                          ndn::Name(interest.getName()).appendVersion(),
+                                          ndn::MAX_NDN_PACKET_SIZE / 2, m_lsaRefreshTime);
+      for (const auto& data : segments) {
+        m_segmentFifo.insert(*data, m_lsaRefreshTime);
+        m_scheduler.schedule(m_lsaRefreshTime,
+                             [this, name = data->getName()] { m_segmentFifo.erase(name); });
+      }
+
+      uint64_t segNum = 0;
+      if (interest.getName()[-1].isSegment()) {
+        segNum = interest.getName()[-1].toSegment();
+      }
+      if (segNum < segments.size()) {
+        m_face.put(*segments[segNum]);
+      }
       incrementDataSentStats(lsaType);
       return true;
     }
   }
   else {
-    NLSR_LOG_TRACE(interest << "  was not found in our LSDB");
+    NLSR_LOG_TRACE(interest << " was not found in our LSDB");
   }
   return false;
 }
@@ -358,7 +384,7 @@ Lsdb::expireOrRefreshLsa(std::shared_ptr<Lsa> lsa)
     // If its seq no is the one we are expecting.
     if (lsaPtr->getSeqNo() == lsa->getSeqNo()) {
       if (lsaPtr->getOriginRouter() == m_thisRouterPrefix) {
-        NLSR_LOG_DEBUG("Own " << lsaPtr->getType() << " LSA, so refreshing it.");
+        NLSR_LOG_DEBUG("Own " << lsaPtr->getType() << " LSA, so refreshing it");
         NLSR_LOG_DEBUG("Current LSA:");
         NLSR_LOG_DEBUG(lsaPtr->toString());
         lsaPtr->setSeqNo(lsaPtr->getSeqNo() + 1);
@@ -429,10 +455,9 @@ Lsdb::expressInterest(const ndn::Name& interestName, uint32_t timeoutCount, uint
     // If we don't do this IMS throws: std::bad_weak_ptr: bad_weak_ptr
     auto lsaSegment = std::make_shared<const ndn::Data>(data);
     m_lsaStorage.insert(*lsaSegment);
-    const ndn::Name& segmentName = lsaSegment->getName();
     // Schedule deletion of the segment
     m_scheduler.schedule(ndn::time::seconds(LSA_REFRESH_TIME_DEFAULT),
-                         [this, segmentName] { m_lsaStorage.erase(segmentName); });
+                         [this, name = lsaSegment->getName()] { m_lsaStorage.erase(name); });
   });
 
   fetcher->onComplete.connect([=] (const ndn::ConstBufferPtr& bufferPtr) {
@@ -491,7 +516,7 @@ Lsdb::afterFetchLsa(const ndn::ConstBufferPtr& bufferPtr, const ndn::Name& inter
   }
   else if (seqNo > m_highestSeqNo[lsaName]) {
     m_highestSeqNo[lsaName] = seqNo;
-    NLSR_LOG_TRACE("SeqNo for LSA(name): " << interestName << "  updated");
+    NLSR_LOG_TRACE("SeqNo for LSA(name): " << interestName << " updated");
   }
   else if (seqNo < m_highestSeqNo[lsaName]) {
     return;

src/lsdb.hpp

@@ -22,27 +22,27 @@
 #ifndef NLSR_LSDB_HPP
 #define NLSR_LSDB_HPP
 
+#include "communication/sync-logic-handler.hpp"
 #include "conf-parameter.hpp"
 #include "lsa/lsa.hpp"
 #include "lsa/name-lsa.hpp"
 #include "lsa/coordinate-lsa.hpp"
 #include "lsa/adj-lsa.hpp"
 #include "sequencing-manager.hpp"
-#include "test-access-control.hpp"
-#include "communication/sync-logic-handler.hpp"
 #include "statistics.hpp"
+#include "test-access-control.hpp"
 
+#include <ndn-cxx/ims/in-memory-storage-fifo.hpp>
+#include <ndn-cxx/ims/in-memory-storage-persistent.hpp>
 #include <ndn-cxx/security/key-chain.hpp>
+#include <ndn-cxx/util/segmenter.hpp>
+#include <ndn-cxx/util/segment-fetcher.hpp>
 #include <ndn-cxx/util/signal.hpp>
 #include <ndn-cxx/util/time.hpp>
-#include <ndn-cxx/util/segment-fetcher.hpp>
-#include <ndn-cxx/ims/in-memory-storage-persistent.hpp>
 
 #include <boost/multi_index_container.hpp>
-#include <boost/multi_index/hashed_index.hpp>
 #include <boost/multi_index/composite_key.hpp>
-
-#include <PSync/segment-publisher.hpp>
+#include <boost/multi_index/hashed_index.hpp>
 
 namespace nlsr {
 
@@ -61,12 +61,7 @@ class Lsdb
 public:
   Lsdb(ndn::Face& face, ndn::KeyChain& keyChain, ConfParameter& confParam);
 
-  ~Lsdb()
-  {
-    for (const auto& sp : m_fetchers) {
-      sp->stop();
-    }
-  }
+  ~Lsdb();
 
   /*! \brief Returns whether the LSDB contains some LSA.
    */
@@ -335,7 +330,6 @@ public:
 PUBLIC_WITH_TESTS_ELSE_PRIVATE:
   ndn::Face& m_face;
   ndn::Scheduler m_scheduler;
-
   ConfParameter& m_confParam;
 
   SyncLogicHandler m_sync;
@@ -355,7 +349,8 @@ PUBLIC_WITH_TESTS_ELSE_PRIVATE:
   ndn::util::signal::ScopedConnection m_onNewLsaConnection;
 
   std::set<std::shared_ptr<ndn::util::SegmentFetcher>> m_fetchers;
-  psync::SegmentPublisher m_segmentPublisher;
+  ndn::util::Segmenter m_segmenter;
+  ndn::InMemoryStorageFifo m_segmentFifo;
 
   bool m_isBuildAdjLsaScheduled;
   int64_t m_adjBuildCount;

tests/test-lsa-segment-storage.cpp

@@ -20,12 +20,13 @@
  */
 
 #include "nlsr.hpp"
-#include "name-prefix-list.hpp"
-// #include "lsa.hpp"
 
 #include "tests/io-key-chain-fixture.hpp"
 #include "tests/test-common.hpp"
 
+#include <ndn-cxx/ims/in-memory-storage-fifo.hpp>
+#include <ndn-cxx/util/segmenter.hpp>
+
 namespace nlsr {
 namespace test {
 
@@ -33,16 +34,11 @@ class LsaSegmentStorageFixture : public IoKeyChainFixture
 {
 public:
   LsaSegmentStorageFixture()
-    : face(m_io, m_keyChain, {true, true})
-    , conf(face, m_keyChain)
-    , confProcessor(conf)
-    , nlsr(face, m_keyChain, conf)
-    , lsdb(nlsr.m_lsdb)
-    , segmentPublisher(face, m_keyChain)
-    , numValidationSignal(0)
-    , afterSegmentValidatedConnection(lsdb.afterSegmentValidatedSignal.connect(
-                                      [this] (const ndn::Data& data) { ++numValidationSignal; }))
   {
+    afterSegmentValidatedConn = lsdb.afterSegmentValidatedSignal.connect([this] (auto&&...) {
+      ++numValidationSignal;
+    });
+
     std::string config = R"CONF(
                             trust-anchor
                               {
@@ -56,44 +52,47 @@ public:
   }
 
   void
-  makeLsaContent(const ndn::Name& interestName, int numNames = 1000)
+  makeLsaContent(ndn::Name interestName, int numNames = 1000)
   {
+    const ndn::time::seconds refreshTime{LSA_REFRESH_TIME_DEFAULT};
+
     NamePrefixList npl1;
     for (int i = 0; i < numNames; ++i) {
       npl1.insert("name1-" + std::to_string(i));
     }
     NameLsa nameLsa("/ndn/other-site/%C1.Router/other-router", 12,
-                    ndn::time::system_clock::now() + ndn::time::seconds(LSA_REFRESH_TIME_DEFAULT),
-                    npl1);
-    segmentPublisher.publish(interestName, interestName, nameLsa.wireEncode(),
-                             ndn::time::seconds(LSA_REFRESH_TIME_DEFAULT));
+                    ndn::time::system_clock::now() + refreshTime, npl1);
+
+    interestName.appendVersion();
+    ndn::util::Segmenter segmenter(m_keyChain, ndn::security::SigningInfo());
+    auto segments = segmenter.segment(nameLsa.wireEncode(), interestName,
+                                      ndn::MAX_NDN_PACKET_SIZE / 2, refreshTime);
+    for (const auto& seg : segments) {
+      ims.insert(*seg, refreshTime);
+    }
   }
 
   void
-  receiveLsaInterest(const ndn::Name& baseInterestName, uint64_t segmentNo,
-                     bool isSegmentZero)
+  sendReplies()
   {
-    if (isSegmentZero) {
-      lsdb.processInterest(ndn::Name(), ndn::Interest(baseInterestName));
-    }
-    else {
-      ndn::Name nextInterestName(baseInterestName);
-      nextInterestName.appendSegment(segmentNo);
-      lsdb.processInterest(ndn::Name(), ndn::Interest(nextInterestName));
-      advanceClocks(ndn::time::milliseconds(1), 10);
+    for (const auto& interest : face.sentInterests) {
+      if (auto data = ims.find(interest.getName()); data) {
+        face.put(*data);
+      }
     }
+    face.sentInterests.clear();
   }
 
 public:
-  ndn::util::DummyClientFace face;
-  ConfParameter conf;
-  DummyConfFileProcessor confProcessor;
-  Nlsr nlsr;
-  Lsdb& lsdb;
-  psync::SegmentPublisher segmentPublisher;
+  ndn::util::DummyClientFace face{m_io, m_keyChain, {true, true}};
+  ConfParameter conf{face, m_keyChain};
+  DummyConfFileProcessor confProcessor{conf};
+  Nlsr nlsr{face, m_keyChain, conf};
+  Lsdb& lsdb{nlsr.m_lsdb};
+  ndn::InMemoryStorageFifo ims{100};
 
-  int numValidationSignal;
-  ndn::util::signal::ScopedConnection afterSegmentValidatedConnection;
+  int numValidationSignal = 0;
+  ndn::util::signal::ScopedConnection afterSegmentValidatedConn;
 };
 
 BOOST_FIXTURE_TEST_SUITE(TestLsaSegmentStorage, LsaSegmentStorageFixture)
@@ -107,18 +106,19 @@ BOOST_AUTO_TEST_CASE(Basic)
   advanceClocks(ndn::time::milliseconds(10));
 
   makeLsaContent(lsaInterestName);
-  advanceClocks(ndn::time::milliseconds(10));
+  BOOST_CHECK_EQUAL(ims.size(), 3);
 
-  for (const auto& interest : face.sentInterests) {
-    segmentPublisher.replyFromStore(interest.getName());
-    advanceClocks(ndn::time::milliseconds(10));
-  }
+  // 1st segment
+  sendReplies();
+  advanceClocks(ndn::time::milliseconds(10));
+  // 2nd and 3rd segments
+  sendReplies();
+  advanceClocks(ndn::time::milliseconds(10));
 
   // 3 data segments should be in the storage
   BOOST_CHECK_EQUAL(lsdb.m_lsaStorage.size(), 3);
   BOOST_CHECK_EQUAL(numValidationSignal, 3);
   numValidationSignal = 0;
-  face.sentInterests.clear();
 
   // Remove older LSA from storage upon receiving that of higher sequence
   ndn::Name lsaInterestName2("/ndn/NLSR/LSA/other-site/%C1.Router/other-router/NAME");
@@ -127,7 +127,9 @@ BOOST_AUTO_TEST_CASE(Basic)
   advanceClocks(ndn::time::milliseconds(10));
 
   makeLsaContent(lsaInterestName2, 1);
+  sendReplies();
   advanceClocks(ndn::time::milliseconds(10));
+
   // Should have cleared all the three segments for the previous interest w/ seq 12
   // And add one segment for this sequence 13
   BOOST_CHECK_EQUAL(lsdb.m_lsaStorage.size(), 1);