diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3495a8f..b551e45 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,8 @@ jobs: include: - os: macos-12 xcode: '13.4' + - os: macos-12 + xcode: '14.1' steps: - name: Set up Xcode uses: maxim-lobanov/setup-xcode@v1 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d5fcfb0..fd27188 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - os: [macos-12, ubuntu-20.04] + os: [macos-latest, ubuntu-latest] env: JOB_NAME: Docs steps: @@ -27,5 +27,7 @@ jobs: ./.jenkins - name: Build documentation run: | + pybindir=$(python3 -c 'import sysconfig; print(sysconfig.get_path("scripts", "posix_user"))') + export PATH="${pybindir}${PATH:+:}${PATH}" ./waf --color=yes configure ./waf --color=yes docs diff --git a/.jenkins b/.jenkins index ee16e29..52b94c5 100755 --- a/.jenkins +++ b/.jenkins @@ -16,6 +16,7 @@ case $(uname) in # Emulate a subset of os-release(5) export ID=macos export VERSION_ID=$(sw_vers -productVersion) + export PATH="/usr/local/bin${PATH:+:}${PATH}" if [[ -x /opt/homebrew/bin/brew ]]; then eval "$(/opt/homebrew/bin/brew shellenv)" elif [[ -x /usr/local/bin/brew ]]; then diff --git a/.jenkins.d/00-deps.sh b/.jenkins.d/00-deps.sh index 12b5de3..9b1e073 100755 --- a/.jenkins.d/00-deps.sh +++ b/.jenkins.d/00-deps.sh @@ -25,20 +25,14 @@ if [[ $ID == macos ]]; then fi brew update brew install --formula "${FORMULAE[@]}" - - if (( ${#PIP_PKGS[@]} )); then - pip3 install --upgrade --upgrade-strategy=eager "${PIP_PKGS[@]}" - fi - elif [[ $ID_LIKE == *debian* ]]; then sudo apt-get -qq update sudo apt-get -qy install "${APT_PKGS[@]}" - - if (( ${#PIP_PKGS[@]} )); then - pip3 install --user --upgrade --upgrade-strategy=eager "${PIP_PKGS[@]}" - fi - elif [[ $ID_LIKE == *fedora* ]]; then - sudo dnf -y install gcc-c++ libasan pkgconf-pkg-config python3 \ + sudo dnf -y install gcc-c++ libasan lld pkgconf-pkg-config python3 \ boost-devel openssl-devel sqlite-devel fi + +if (( ${#PIP_PKGS[@]} )); then + pip3 install --user --upgrade --upgrade-strategy=eager "${PIP_PKGS[@]}" +fi diff --git a/.waf-tools/default-compiler-flags.py b/.waf-tools/default-compiler-flags.py index 3ba2dc3..4debb4e 100644 --- a/.waf-tools/default-compiler-flags.py +++ b/.waf-tools/default-compiler-flags.py @@ -137,9 +137,7 @@ class GccBasicFlags(CompilerFlags): def getGeneralFlags(self, conf): flags = super(GccBasicFlags, self).getGeneralFlags(conf) flags['CXXFLAGS'] += ['-std=c++17'] - if Utils.unversioned_sys_platform() == 'linux': - flags['LINKFLAGS'] += ['-fuse-ld=gold'] - elif Utils.unversioned_sys_platform() == 'freebsd': + if Utils.unversioned_sys_platform() != 'darwin': flags['LINKFLAGS'] += ['-fuse-ld=lld'] return flags diff --git a/PSync/full-producer.cpp b/PSync/full-producer.cpp index baa50e4..d14cae2 100644 --- a/PSync/full-producer.cpp +++ b/PSync/full-producer.cpp @@ -266,14 +266,14 @@ FullProducer::sendSyncData(const ndn::Name& name, const ndn::Block& block) NDN_LOG_DEBUG("Sending sync Data"); // Send data after removing pending sync interest on face - m_segmentPublisher.publish(name, dataName, content, m_syncReplyFreshness); + m_segmentPublisher.publish(name, dataName, *content, m_syncReplyFreshness); NDN_LOG_TRACE("Renewing sync interest"); sendSyncInterest(); } else { NDN_LOG_DEBUG("Sending sync Data"); - m_segmentPublisher.publish(name, dataName, content, m_syncReplyFreshness); + m_segmentPublisher.publish(name, dataName, *content, m_syncReplyFreshness); } } diff --git a/PSync/segment-publisher.cpp b/PSync/segment-publisher.cpp index 5d42b88..ed20646 100644 --- a/PSync/segment-publisher.cpp +++ b/PSync/segment-publisher.cpp @@ -19,79 +19,37 @@ #include "PSync/segment-publisher.hpp" -#include - namespace psync { -SegmentPublisher::SegmentPublisher(ndn::Face& face, ndn::KeyChain& keyChain, size_t imsLimit) +SegmentPublisher::SegmentPublisher(ndn::Face& face, ndn::KeyChain& keyChain, + const ndn::security::SigningInfo& signingInfo, size_t imsLimit) : m_face(face) , m_scheduler(m_face.getIoService()) - , m_keyChain(keyChain) + , m_segmenter(keyChain, signingInfo) , m_ims(imsLimit) { } void SegmentPublisher::publish(const ndn::Name& interestName, const ndn::Name& dataName, - const ndn::Block& block, ndn::time::milliseconds freshness, - const ndn::security::SigningInfo& signingInfo) + ndn::span buffer, ndn::time::milliseconds freshness) { - auto buf = std::make_shared(block.begin(), block.end()); - publish(interestName, dataName, buf, freshness, signingInfo); -} + auto segments = m_segmenter.segment(buffer, ndn::Name(dataName).appendVersion(), + ndn::MAX_NDN_PACKET_SIZE >> 1, freshness); + for (const auto& data : segments) { + m_ims.insert(*data, freshness); + m_scheduler.schedule(freshness, [this, name = data->getName()] { m_ims.erase(name); }); + } -void -SegmentPublisher::publish(const ndn::Name& interestName, const ndn::Name& dataName, - const ndn::ConstBufferPtr& buffer, - ndn::time::milliseconds freshness, - const ndn::security::SigningInfo& signingInfo) -{ + // Put on face only the segment which has a pending interest, + // otherwise the segment is unsolicited uint64_t interestSegment = 0; if (interestName[-1].isSegment()) { interestSegment = interestName[-1].toSegment(); } - - const uint8_t* rawBuffer = buffer->data(); - const uint8_t* segmentBegin = rawBuffer; - const uint8_t* end = rawBuffer + buffer->size(); - - const size_t maxPacketSize = ndn::MAX_NDN_PACKET_SIZE >> 1; - uint64_t totalSegments = buffer->size() / maxPacketSize; - - ndn::Name segmentPrefix(dataName); - segmentPrefix.appendVersion(); - - uint64_t segmentNo = 0; - do { - const uint8_t* segmentEnd = segmentBegin + maxPacketSize; - if (segmentEnd > end) { - segmentEnd = end; - } - - ndn::Name segmentName(segmentPrefix); - segmentName.appendSegment(segmentNo); - - // We get a std::exception: bad_weak_ptr from m_ims if we don't use shared_ptr for data - auto data = std::make_shared(segmentName); - data->setContent(ndn::span(segmentBegin, segmentEnd)); - data->setFreshnessPeriod(freshness); - data->setFinalBlock(ndn::name::Component::fromSegment(totalSegments)); - - m_keyChain.sign(*data, signingInfo); - - segmentBegin = segmentEnd; - - // Put on face only the segment which has a pending interest - // otherwise the segment is unsolicited - if (interestSegment == segmentNo) { - m_face.put(*data); - } - - m_ims.insert(*data, freshness); - m_scheduler.schedule(freshness, [this, segmentName] { m_ims.erase(segmentName); }); - - ++segmentNo; - } while (segmentBegin < end); + if (interestSegment < segments.size()) { + m_face.put(*segments[interestSegment]); + } } bool diff --git a/PSync/segment-publisher.hpp b/PSync/segment-publisher.hpp index 5817c24..481fa03 100644 --- a/PSync/segment-publisher.hpp +++ b/PSync/segment-publisher.hpp @@ -1,6 +1,6 @@ /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ /* - * Copyright (c) 2014-2020, The University of Memphis + * Copyright (c) 2014-2022, The University of Memphis * * This file is part of PSync. * See AUTHORS.md for complete list of PSync authors and contributors. @@ -23,55 +23,38 @@ #include "PSync/detail/access-specifiers.hpp" #include -#include #include -#include #include -#include +#include namespace psync { /** - * @brief Segment Publisher to publish segmented data + * @brief Helper class to publish segmented data. */ class SegmentPublisher { public: SegmentPublisher(ndn::Face& face, ndn::KeyChain& keyChain, + const ndn::security::SigningInfo& signingInfo = ndn::security::SigningInfo(), size_t imsLimit = 100); - /** - * @brief Put all the segments in memory. - * - * @param interestName the interest name, to determine the sequence to be answered immediately - * @param dataName the data name, has components after interest name - * @param block the content of the data - * @param freshness freshness of the segments - * @param signingInfo signing info to sign the data with - */ - void - publish(const ndn::Name& interestName, const ndn::Name& dataName, - const ndn::Block& block, ndn::time::milliseconds freshness, - const ndn::security::SigningInfo& signingInfo = ndn::security::SigningInfo()); - /** * @brief Put all the segments in memory. * * @param interestName the interest name, to determine the sequence to be answered immediately * @param dataName the data name, has components after interest name * @param buffer the content of the data - * @param freshness freshness of the segments - * @param signingInfo signing info to sign the data with + * @param freshness freshness period of the segments */ void publish(const ndn::Name& interestName, const ndn::Name& dataName, - const ndn::ConstBufferPtr& buffer, ndn::time::milliseconds freshness, - const ndn::security::SigningInfo& signingInfo = ndn::security::SigningInfo()); + ndn::span buffer, ndn::time::milliseconds freshness); /** * @brief Try to reply from memory, return false if we cannot find the segment. * - * The caller is then expected to use publish if this returns false. + * The caller is then expected to use publish() if this returns false. */ bool replyFromStore(const ndn::Name& interestName); @@ -79,7 +62,7 @@ public: private: ndn::Face& m_face; ndn::Scheduler m_scheduler; - ndn::KeyChain& m_keyChain; + ndn::util::Segmenter m_segmenter; PSYNC_PUBLIC_WITH_TESTS_ELSE_PRIVATE: ndn::InMemoryStorageFifo m_ims; diff --git a/tests/test-segment-publisher.cpp b/tests/test-segment-publisher.cpp index bccaee1..210555d 100644 --- a/tests/test-segment-publisher.cpp +++ b/tests/test-segment-publisher.cpp @@ -38,7 +38,7 @@ protected: SegmentPublisherFixture() { m_face.setInterestFilter(InterestFilter("/hello/world"), - bind(&SegmentPublisherFixture::onInterest, this, _1, _2), + bind(&SegmentPublisherFixture::onInterest, this, _2), [] (auto&&...) { BOOST_CHECK(false); }); advanceClocks(10_ms); @@ -63,7 +63,7 @@ protected: } void - onInterest(const Name& prefix, const Interest& interest) + onInterest(const Interest& interest) { if (publisher.replyFromStore(interest.getName())) { numRepliesFromStore++; @@ -82,7 +82,7 @@ protected: protected: util::DummyClientFace m_face{m_io, m_keyChain, {true, true}}; SegmentPublisher publisher{m_face, m_keyChain}; - shared_ptr fetcher; + std::shared_ptr fetcher; Name dataName; detail::State state; @@ -97,6 +97,7 @@ BOOST_FIXTURE_TEST_SUITE(TestSegmentPublisher, SegmentPublisherFixture) BOOST_AUTO_TEST_CASE(Basic) { BOOST_CHECK_EQUAL(publisher.m_ims.size(), 0); + expressInterest(Interest("/hello/world")); BOOST_CHECK_EQUAL(numComplete, 1); // First segment is answered directly in publish, @@ -104,12 +105,20 @@ BOOST_AUTO_TEST_CASE(Basic) BOOST_CHECK_EQUAL(numRepliesFromStore, 2); BOOST_CHECK_EQUAL(publisher.m_ims.size(), 3); + for (const auto& data : publisher.m_ims) { + BOOST_TEST_CONTEXT(data.getName()) { + BOOST_REQUIRE_EQUAL(data.getName().size(), 4); + BOOST_CHECK(data.getName()[-1].isSegment()); + BOOST_CHECK(data.getName()[-2].isVersion()); + } + } + numRepliesFromStore = 0; expressInterest(Interest("/hello/world")); BOOST_CHECK_EQUAL(numComplete, 2); BOOST_CHECK_EQUAL(numRepliesFromStore, 3); - advanceClocks(time::milliseconds(freshness)); + advanceClocks(freshness); BOOST_CHECK_EQUAL(publisher.m_ims.size(), 0); numRepliesFromStore = 0; @@ -118,7 +127,7 @@ BOOST_AUTO_TEST_CASE(Basic) BOOST_CHECK_EQUAL(numRepliesFromStore, 2); numRepliesFromStore = 0; - m_face.expressInterest(Interest("/hello/world/").setCanBePrefix(true), + m_face.expressInterest(Interest("/hello/world").setCanBePrefix(true), [this] (auto&&...) { this->numComplete++; }, [] (auto&&...) { BOOST_CHECK(false); }, [] (auto&&...) { BOOST_CHECK(false); }); @@ -127,10 +136,10 @@ BOOST_AUTO_TEST_CASE(Basic) BOOST_CHECK_EQUAL(numRepliesFromStore, 1); } -BOOST_AUTO_TEST_CASE(LargerDataName) +BOOST_AUTO_TEST_CASE(LongerDataName) { - BOOST_CHECK_EQUAL(publisher.m_ims.size(), 0); dataName = Name("/hello/world/IBF"); + BOOST_CHECK_EQUAL(publisher.m_ims.size(), 0); expressInterest(Interest("/hello/world")); BOOST_CHECK_EQUAL(numComplete, 1); @@ -139,7 +148,15 @@ BOOST_AUTO_TEST_CASE(LargerDataName) BOOST_CHECK_EQUAL(numRepliesFromStore, 2); BOOST_CHECK_EQUAL(publisher.m_ims.size(), 3); - advanceClocks(time::milliseconds(freshness)); + for (const auto& data : publisher.m_ims) { + BOOST_TEST_CONTEXT(data.getName()) { + BOOST_REQUIRE_EQUAL(data.getName().size(), 5); + BOOST_CHECK(data.getName()[-1].isSegment()); + BOOST_CHECK(data.getName()[-2].isVersion()); + } + } + + advanceClocks(freshness); BOOST_CHECK_EQUAL(publisher.m_ims.size(), 0); }