From 32ccfc4d1d4db648763bf96ffdf8ac87f0bc3f50 Mon Sep 17 00:00:00 2001 From: Junxiao Shi Date: Sun, 9 Jan 2022 21:26:22 +0000 Subject: [PATCH] **breaking** use Name TLV-VALUE as input to murmurHash3 Previously the library was using Name URI string, which is not guaranteed to be stable and unique. refs #4838 Change-Id: Ic5e29f11f1320f95a754fb5def03faeb4d89cc01 --- PSync/consumer.cpp | 8 +++---- PSync/detail/bloom-filter.cpp | 6 +++--- PSync/detail/bloom-filter.hpp | 6 +++--- PSync/detail/util.cpp | 9 +++++++- PSync/detail/util.hpp | 12 +++++------ PSync/full-producer.cpp | 6 +++--- PSync/partial-producer.cpp | 23 +++++++++++--------- PSync/producer-base.cpp | 4 ++-- tests/test-iblt.cpp | 40 +++++++++++++++++------------------ tests/test-producer-base.cpp | 10 ++++----- 10 files changed, 67 insertions(+), 57 deletions(-) diff --git a/PSync/consumer.cpp b/PSync/consumer.cpp index 1b100cb..6c77ed3 100644 --- a/PSync/consumer.cpp +++ b/PSync/consumer.cpp @@ -1,6 +1,6 @@ /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ /* - * Copyright (c) 2014-2020, The University of Memphis + * Copyright (c) 2014-2022, The University of Memphis * * This file is part of PSync. * See AUTHORS.md for complete list of PSync authors and contributors. @@ -60,7 +60,7 @@ Consumer::addSubscription(const ndn::Name& prefix, uint64_t seqNo, bool callSync } m_subscriptionList.emplace(prefix); - m_bloomFilter.insert(prefix.toUri()); + m_bloomFilter.insert(prefix); if (callSyncDataCb && seqNo != 0) { m_onUpdate({{prefix, seqNo, seqNo}}); @@ -130,7 +130,7 @@ Consumer::onHelloData(const ndn::ConstBufferPtr& bufferPtr) // Extract IBF from name which is the last element in hello data's name m_iblt = m_helloDataName.getSubName(m_helloDataName.size() - 1, 1); - NDN_LOG_TRACE("m_iblt: " << std::hash{}(m_iblt.toUri())); + NDN_LOG_TRACE("m_iblt: " << std::hash{}(m_iblt)); detail::State state{ndn::Block(bufferPtr)}; std::vector updates; @@ -175,7 +175,7 @@ Consumer::sendSyncInterest() ndn::Interest syncInterest(syncInterestName); NDN_LOG_DEBUG("sendSyncInterest, nonce: " << syncInterest.getNonce() << - " hash: " << std::hash{}(syncInterest.getName().toUri())); + " hash: " << std::hash{}(syncInterest.getName())); if (m_syncFetcher) { m_syncFetcher->stop(); diff --git a/PSync/detail/bloom-filter.cpp b/PSync/detail/bloom-filter.cpp index 56c70b9..041ead0 100644 --- a/PSync/detail/bloom-filter.cpp +++ b/PSync/detail/bloom-filter.cpp @@ -1,6 +1,6 @@ /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ /* - * Copyright (c) 2014-2020, The University of Memphis + * Copyright (c) 2014-2022, The University of Memphis * * This file is part of PSync. * See AUTHORS.md for complete list of PSync authors and contributors. @@ -250,7 +250,7 @@ BloomFilter::clear() } void -BloomFilter::insert(const std::string& key) +BloomFilter::insert(const ndn::Name& key) { std::size_t bit_index = 0; std::size_t bit = 0; @@ -266,7 +266,7 @@ BloomFilter::insert(const std::string& key) } bool -BloomFilter::contains(const std::string& key) const +BloomFilter::contains(const ndn::Name& key) const { std::size_t bit_index = 0; std::size_t bit = 0; diff --git a/PSync/detail/bloom-filter.hpp b/PSync/detail/bloom-filter.hpp index e01828b..9aa4e6f 100644 --- a/PSync/detail/bloom-filter.hpp +++ b/PSync/detail/bloom-filter.hpp @@ -1,6 +1,6 @@ /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ /* - * Copyright (c) 2014-2020, The University of Memphis + * Copyright (c) 2014-2022, The University of Memphis * * This file is part of PSync. * See AUTHORS.md for complete list of PSync authors and contributors. @@ -90,10 +90,10 @@ public: clear(); void - insert(const std::string& key); + insert(const ndn::Name& key); bool - contains(const std::string& key) const; + contains(const ndn::Name& key) const; private: typedef uint32_t bloom_type; diff --git a/PSync/detail/util.cpp b/PSync/detail/util.cpp index a7fd947..3e58d77 100644 --- a/PSync/detail/util.cpp +++ b/PSync/detail/util.cpp @@ -1,6 +1,6 @@ /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ /* - * Copyright (c) 2014-2020, The University of Memphis + * Copyright (c) 2014-2022, The University of Memphis * * This file is part of PSync. * See AUTHORS.md for complete list of PSync authors and contributors. @@ -116,6 +116,13 @@ murmurHash3(const void* key, size_t len, uint32_t seed) return h1; } +uint32_t +murmurHash3(uint32_t seed, const ndn::Name& name) +{ + auto wire = name.wireEncode(); + return murmurHash3(wire.value(), wire.value_size(), seed); +} + std::shared_ptr compress(CompressionScheme scheme, const uint8_t* buffer, size_t bufferSize) { diff --git a/PSync/detail/util.hpp b/PSync/detail/util.hpp index e457832..9f6676d 100644 --- a/PSync/detail/util.hpp +++ b/PSync/detail/util.hpp @@ -1,6 +1,6 @@ /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ /* - * Copyright (c) 2014-2020, The University of Memphis + * Copyright (c) 2014-2022, The University of Memphis * * This file is part of PSync. * See AUTHORS.md for complete list of PSync authors and contributors. @@ -32,11 +32,11 @@ namespace detail { uint32_t murmurHash3(const void* key, size_t len, uint32_t seed); -inline uint32_t -murmurHash3(uint32_t seed, const std::string& str) -{ - return murmurHash3(str.data(), str.size(), seed); -} +/** + * @brief Compute 32-bit MurmurHash3 of Name TLV-VALUE. + */ +uint32_t +murmurHash3(uint32_t seed, const ndn::Name& name); inline uint32_t murmurHash3(uint32_t seed, uint32_t value) diff --git a/PSync/full-producer.cpp b/PSync/full-producer.cpp index 048bba3..106dc90 100644 --- a/PSync/full-producer.cpp +++ b/PSync/full-producer.cpp @@ -1,6 +1,6 @@ /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ /* - * Copyright (c) 2014-2020, The University of Memphis + * Copyright (c) 2014-2022, The University of Memphis * * This file is part of PSync. * See AUTHORS.md for complete list of PSync authors and contributors. @@ -207,7 +207,7 @@ FullProducer::onSyncInterest(const ndn::Name& prefixName, const ndn::Interest& i ndn::Name nameWithoutSeq = nameIt->second.getPrefix(-1); // Don't sync up sequence number zero if (m_prefixes[nameWithoutSeq] != 0 && - !isFutureHash(nameWithoutSeq.toUri(), negative)) { + !isFutureHash(nameWithoutSeq, negative)) { state.addContent(nameIt->second); } } @@ -355,7 +355,7 @@ bool FullProducer::isFutureHash(const ndn::Name& prefix, const std::set& negative) { auto nextHash = detail::murmurHash3(detail::N_HASHCHECK, - ndn::Name(prefix).appendNumber(m_prefixes[prefix] + 1).toUri()); + ndn::Name(prefix).appendNumber(m_prefixes[prefix] + 1)); return negative.find(nextHash) != negative.end(); } diff --git a/PSync/partial-producer.cpp b/PSync/partial-producer.cpp index 25e456a..202935c 100644 --- a/PSync/partial-producer.cpp +++ b/PSync/partial-producer.cpp @@ -1,6 +1,6 @@ /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ /* - * Copyright (c) 2014-2020, The University of Memphis + * Copyright (c) 2014-2022, The University of Memphis * * This file is part of PSync. * See AUTHORS.md for complete list of PSync authors and contributors. @@ -29,6 +29,9 @@ namespace psync { NDN_LOG_INIT(psync.PartialProducer); +const ndn::name::Component HELLO("hello"); +const ndn::name::Component SYNC("sync"); + PartialProducer::PartialProducer(size_t expectedNumEntries, ndn::Face& face, const ndn::Name& syncPrefix, @@ -42,9 +45,9 @@ PartialProducer::PartialProducer(size_t expectedNumEntries, { m_registeredPrefix = m_face.registerPrefix(m_syncPrefix, [this] (const ndn::Name& syncPrefix) { - m_face.setInterestFilter(ndn::Name(m_syncPrefix).append("hello"), + m_face.setInterestFilter(ndn::Name(m_syncPrefix).append(HELLO), std::bind(&PartialProducer::onHelloInterest, this, _1, _2)); - m_face.setInterestFilter(ndn::Name(m_syncPrefix).append("sync"), + m_face.setInterestFilter(ndn::Name(m_syncPrefix).append(SYNC), std::bind(&PartialProducer::onSyncInterest, this, _1, _2)); }, std::bind(&PartialProducer::onRegisterFailed, this, _1, _2)); @@ -69,14 +72,14 @@ PartialProducer::publishName(const ndn::Name& prefix, ndn::optional se void PartialProducer::onHelloInterest(const ndn::Name& prefix, const ndn::Interest& interest) { - if (m_segmentPublisher.replyFromStore(interest.getName())) { + const auto& name = interest.getName(); + if (m_segmentPublisher.replyFromStore(name)) { return; } // Last component or fourth last component (in case of interest with version and segment) // needs to be hello - if (interest.getName().get(interest.getName().size()-1).toUri() != "hello" && - interest.getName().get(interest.getName().size()-4).toUri() != "hello") { + if (name.get(name.size() - 1) != HELLO && name.get(name.size() - 4) != HELLO) { return; } @@ -103,7 +106,7 @@ PartialProducer::onSyncInterest(const ndn::Name& prefix, const ndn::Interest& in } NDN_LOG_DEBUG("Sync Interest Received, nonce: " << interest.getNonce() << - " hash: " << std::hash{}(interest.getName().toUri())); + " hash: " << std::hash{}(interest.getName())); ndn::Name nameWithoutSyncPrefix = interest.getName().getSubName(prefix.size()); ndn::Name interestName; @@ -173,7 +176,7 @@ PartialProducer::onSyncInterest(const ndn::Name& prefix, const ndn::Interest& in for (const auto& hash : positive) { auto nameIt = m_biMap.left.find(hash); if (nameIt != m_biMap.left.end()) { - if (bf.contains(nameIt->second.getPrefix(-1).toUri())) { + if (bf.contains(nameIt->second.getPrefix(-1))) { // generate data state.addContent(nameIt->second); NDN_LOG_DEBUG("Content: " << nameIt->second << " " << nameIt->first); @@ -227,8 +230,8 @@ PartialProducer::satisfyPendingSyncInterests(const ndn::Name& prefix) { } detail::State state; - if (entry.bf.contains(prefix.toUri()) || positive.size() + negative.size() >= m_threshold) { - if (entry.bf.contains(prefix.toUri())) { + if (entry.bf.contains(prefix) || positive.size() + negative.size() >= m_threshold) { + if (entry.bf.contains(prefix)) { state.addContent(ndn::Name(prefix).appendNumber(m_prefixes[prefix])); NDN_LOG_DEBUG("sending sync content " << prefix << " " << std::to_string(m_prefixes[prefix])); } diff --git a/PSync/producer-base.cpp b/PSync/producer-base.cpp index bf8bc0e..ff6870d 100644 --- a/PSync/producer-base.cpp +++ b/PSync/producer-base.cpp @@ -1,6 +1,6 @@ /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ /* - * Copyright (c) 2014-2020, The University of Memphis + * Copyright (c) 2014-2022, The University of Memphis * * This file is part of PSync. * See AUTHORS.md for complete list of PSync authors and contributors. @@ -116,7 +116,7 @@ ProducerBase::updateSeqNo(const ndn::Name& prefix, uint64_t seq) // Insert the new seq no in m_prefixes, m_biMap, and m_iblt it->second = seq; ndn::Name prefixWithSeq = ndn::Name(prefix).appendNumber(seq); - auto newHash = detail::murmurHash3(detail::N_HASHCHECK, prefixWithSeq.toUri()); + auto newHash = detail::murmurHash3(detail::N_HASHCHECK, prefixWithSeq); m_biMap.insert({newHash, prefixWithSeq}); m_iblt.insert(newHash); } diff --git a/tests/test-iblt.cpp b/tests/test-iblt.cpp index c015a19..b2f4d3e 100644 --- a/tests/test-iblt.cpp +++ b/tests/test-iblt.cpp @@ -1,6 +1,6 @@ /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ /* - * Copyright (c) 2014-2020, The University of Memphis + * Copyright (c) 2014-2022, The University of Memphis * * This file is part of PSync. * See AUTHORS.md for complete list of PSync authors and contributors. @@ -37,7 +37,7 @@ BOOST_AUTO_TEST_CASE(Equal) IBLT iblt2(size, CompressionScheme::DEFAULT); BOOST_CHECK_EQUAL(iblt1, iblt2); - std::string prefix = Name("/test/memphis").appendNumber(1).toUri(); + auto prefix = Name("/test/memphis").appendNumber(1); uint32_t newHash = murmurHash3(11, prefix); iblt1.insert(newHash); iblt2.insert(newHash); @@ -54,17 +54,17 @@ static const uint8_t IBF[] = { // Header 0x08, 0xB4, // Uncompressed IBF - 0x00, 0x00, 0x00, 0x01, 0xF6, 0xA7, 0x7A, 0xBA, 0x6B, 0xA3, 0x4D, 0x63, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x5C, 0x5B, 0xF2, 0x67, + 0x42, 0x24, 0xEE, 0x6C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x5C, 0x5B, 0xF2, 0x67, + 0x42, 0x24, 0xEE, 0x6C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xF6, 0xA7, 0x7A, 0xBA, - 0x6B, 0xA3, 0x4D, 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x01, 0xF6, 0xA7, 0x7A, 0xBA, 0x6B, 0xA3, 0x4D, 0x63, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x5C, 0x5B, 0xF2, 0x67, 0x42, 0x24, 0xEE, 0x6C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; @@ -73,7 +73,7 @@ BOOST_AUTO_TEST_CASE(NameAppendAndExtract) const int size = 10; IBLT iblt(size, CompressionScheme::DEFAULT); - auto prefix = Name("/test/memphis").appendNumber(1).toUri(); + auto prefix = Name("/test/memphis").appendNumber(1); auto hash = murmurHash3(11, prefix); iblt.insert(hash); @@ -110,18 +110,18 @@ BOOST_AUTO_TEST_CASE(CopyInsertErase) IBLT iblt1(size, CompressionScheme::DEFAULT); - std::string prefix = Name("/test/memphis").appendNumber(1).toUri(); + auto prefix = Name("/test/memphis").appendNumber(1); uint32_t hash1 = murmurHash3(11, prefix); iblt1.insert(hash1); IBLT iblt2(iblt1); iblt2.erase(hash1); - prefix = Name("/test/memphis").appendNumber(2).toUri(); + prefix = Name("/test/memphis").appendNumber(2); uint32_t hash3 = murmurHash3(11, prefix); iblt2.insert(hash3); iblt1.erase(hash1); - prefix = Name("/test/memphis").appendNumber(5).toUri(); + prefix = Name("/test/memphis").appendNumber(5); uint32_t hash5 = murmurHash3(11, prefix); iblt1.insert(hash5); @@ -141,11 +141,11 @@ BOOST_AUTO_TEST_CASE(HigherSeqTest) IBLT ownIBF(size, CompressionScheme::DEFAULT); IBLT rcvdIBF(size, CompressionScheme::DEFAULT); - std::string prefix = Name("/test/memphis").appendNumber(3).toUri(); + auto prefix = Name("/test/memphis").appendNumber(3); uint32_t hash1 = murmurHash3(11, prefix); ownIBF.insert(hash1); - std::string prefix2 = Name("/test/memphis").appendNumber(4).toUri(); + auto prefix2 = Name("/test/memphis").appendNumber(4); uint32_t hash2 = murmurHash3(11, prefix2); rcvdIBF.insert(hash2); @@ -174,7 +174,7 @@ BOOST_AUTO_TEST_CASE(Difference) BOOST_CHECK_EQUAL(positive.size(), 0); BOOST_CHECK_EQUAL(negative.size(), 0); - std::string prefix = Name("/test/memphis").appendNumber(1).toUri(); + auto prefix = Name("/test/memphis").appendNumber(1); uint32_t newHash = murmurHash3(11, prefix); ownIBF.insert(newHash); @@ -183,7 +183,7 @@ BOOST_AUTO_TEST_CASE(Difference) BOOST_CHECK_EQUAL(positive.size(), 1); BOOST_CHECK_EQUAL(negative.size(), 0); - prefix = Name("/test/csu").appendNumber(1).toUri(); + prefix = Name("/test/csu").appendNumber(1); newHash = murmurHash3(11, prefix); rcvdIBF.insert(newHash); @@ -204,14 +204,14 @@ BOOST_AUTO_TEST_CASE(DifferenceBwOversizedIBFs) IBLT ownIBF(size, CompressionScheme::DEFAULT); for (int i = 0; i < 50; i++) { - std::string prefix = Name("/test/memphis" + std::to_string(i)).appendNumber(1).toUri(); + auto prefix = Name("/test/memphis" + std::to_string(i)).appendNumber(1); uint32_t newHash = murmurHash3(11, prefix); ownIBF.insert(newHash); } IBLT rcvdIBF = ownIBF; - std::string prefix = Name("/test/ucla").appendNumber(1).toUri(); + auto prefix = Name("/test/ucla").appendNumber(1); uint32_t newHash = murmurHash3(11, prefix); ownIBF.insert(newHash); diff --git a/tests/test-producer-base.cpp b/tests/test-producer-base.cpp index 8e9dd77..30cf4c3 100644 --- a/tests/test-producer-base.cpp +++ b/tests/test-producer-base.cpp @@ -1,6 +1,6 @@ /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ /* - * Copyright (c) 2014-2020, The University of Memphis + * Copyright (c) 2014-2022, The University of Memphis * * This file is part of PSync. * See AUTHORS.md for complete list of PSync authors and contributors. @@ -47,21 +47,21 @@ BOOST_AUTO_TEST_CASE(Basic) BOOST_CHECK_EQUAL(producerBase.getSeqNo(userNode).value(), 0); producerBase.updateSeqNo(userNode, 1); - BOOST_CHECK(producerBase.getSeqNo(userNode.toUri()).value() == 1); + BOOST_CHECK(producerBase.getSeqNo(userNode).value() == 1); - std::string prefixWithSeq = Name(userNode).appendNumber(1).toUri(); + auto prefixWithSeq = Name(userNode).appendNumber(1); uint32_t hash = producerBase.m_biMap.right.find(prefixWithSeq)->second; Name prefix(producerBase.m_biMap.left.find(hash)->second); BOOST_CHECK_EQUAL(prefix.getPrefix(-1), userNode); producerBase.removeUserNode(userNode); - BOOST_CHECK(producerBase.getSeqNo(userNode.toUri()) == nullopt); + BOOST_CHECK(producerBase.getSeqNo(userNode) == nullopt); BOOST_CHECK(producerBase.m_biMap.right.find(prefixWithSeq) == producerBase.m_biMap.right.end()); BOOST_CHECK(producerBase.m_biMap.left.find(hash) == producerBase.m_biMap.left.end()); Name nonExistentUserNode("/notAUser"); producerBase.updateSeqNo(nonExistentUserNode, 1); - BOOST_CHECK(producerBase.m_biMap.right.find(Name(nonExistentUserNode).appendNumber(1).toUri()) == + BOOST_CHECK(producerBase.m_biMap.right.find(Name(nonExistentUserNode).appendNumber(1)) == producerBase.m_biMap.right.end()); }