From bd08e492554040d2ca03677f3674ae2f01464a8b Mon Sep 17 00:00:00 2001 From: joaquintides Date: Mon, 6 Mar 2023 11:56:49 +0100 Subject: [PATCH 1/7] added fast table_iterator::increment variant for regular-layout groups --- include/boost/unordered/detail/foa.hpp | 89 +++++++++++++++++++++----- 1 file changed, 73 insertions(+), 16 deletions(-) diff --git a/include/boost/unordered/detail/foa.hpp b/include/boost/unordered/detail/foa.hpp index 0f9f1b79..63b43ee4 100644 --- a/include/boost/unordered/detail/foa.hpp +++ b/include/boost/unordered/detail/foa.hpp @@ -161,6 +161,7 @@ static const std::size_t default_bucket_count = 0; struct group15 { static constexpr int N=15; + static constexpr bool regular_layout=true; struct dummy_group_type { @@ -186,6 +187,11 @@ struct group15 return at(pos)==sentinel_; } + static inline bool is_sentinel(unsigned char* pc)noexcept + { + return *pc==sentinel_; + } + inline void reset(std::size_t pos) { BOOST_ASSERT(pos friend class table; table_iterator(Group* pg,std::size_t n,const table_element_type* p_): - pc{reinterpret_cast(const_cast(pg))+n}, + pc{reinterpret_cast(const_cast(pg))+n}, p{const_cast(p_)} {} - inline std::size_t rebase() noexcept - { - std::size_t off=reinterpret_cast(pc)%sizeof(Group); - pc-=off; - return off; - } - inline void increment()noexcept { - std::size_t n0=rebase(); + increment(std::integral_constant{}); + } - int mask=(reinterpret_cast(pc)->match_occupied()>>(n0+1))<<(n0+1); + inline void increment(std::true_type /* regular layout */)noexcept + { + for(;;){ + ++p; + if(reinterpret_cast(pc)%sizeof(group_type)==N-1){ + pc+=sizeof(group_type)-(N-1); + break; + } + ++pc; + if(!group_type::is_occupied(pc))continue; + if(BOOST_UNLIKELY(group_type::is_sentinel(pc)))p=nullptr; + return; + } + + for(;;){ + int mask=reinterpret_cast(pc)->match_occupied(); + if(mask!=0){ + auto n=unchecked_countr_zero(mask); + if(BOOST_UNLIKELY(reinterpret_cast(pc)->is_sentinel(n))){ + p=nullptr; + } + else{ + pc+=n; + p+=n; + } + return; + } + pc+=sizeof(group_type); + p+=N; + } + } + + inline void increment(std::false_type /* interleaved */)noexcept + { + std::size_t n0=reinterpret_cast(pc)%sizeof(group_type); + pc-=n0; + + int mask=( + reinterpret_cast(pc)->match_occupied()>>(n0+1))<<(n0+1); if(!mask){ do{ - pc+=sizeof(Group); - p+=Group::N; + pc+=sizeof(group_type); + p+=N; } - while((mask=reinterpret_cast(pc)->match_occupied())==0); + while((mask=reinterpret_cast(pc)->match_occupied())==0); } auto n=unchecked_countr_zero(mask); - if(BOOST_UNLIKELY(reinterpret_cast(pc)->is_sentinel(n))){ + if(BOOST_UNLIKELY(reinterpret_cast(pc)->is_sentinel(n))){ p=nullptr; } else{ From c50461528c857a4ef7d97d852014bd8dc1f3cac4 Mon Sep 17 00:00:00 2001 From: joaquintides Date: Mon, 6 Mar 2023 13:50:25 +0100 Subject: [PATCH 2/7] fixed UB with begin when using fast iteration --- include/boost/unordered/detail/foa.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/boost/unordered/detail/foa.hpp b/include/boost/unordered/detail/foa.hpp index 63b43ee4..c4d8b56f 100644 --- a/include/boost/unordered/detail/foa.hpp +++ b/include/boost/unordered/detail/foa.hpp @@ -1457,7 +1457,7 @@ public: iterator begin()noexcept { iterator it{arrays.groups,0,arrays.elements}; - if(!(arrays.groups[0].match_occupied()&0x1))++it; + if(arrays.elements&&!(arrays.groups[0].match_occupied()&0x1))++it; return it; } From 15c9bc40f7e72668f2859876097b595c221a7bca Mon Sep 17 00:00:00 2001 From: joaquintides Date: Mon, 6 Mar 2023 11:56:49 +0100 Subject: [PATCH 3/7] added fast table_iterator::increment variant for regular-layout groups --- include/boost/unordered/detail/foa.hpp | 89 +++++++++++++++++++++----- 1 file changed, 73 insertions(+), 16 deletions(-) diff --git a/include/boost/unordered/detail/foa.hpp b/include/boost/unordered/detail/foa.hpp index 93036768..85bbc701 100644 --- a/include/boost/unordered/detail/foa.hpp +++ b/include/boost/unordered/detail/foa.hpp @@ -161,6 +161,7 @@ static const std::size_t default_bucket_count = 0; struct group15 { static constexpr int N=15; + static constexpr bool regular_layout=true; struct dummy_group_type { @@ -186,6 +187,11 @@ struct group15 return at(pos)==sentinel_; } + static inline bool is_sentinel(unsigned char* pc)noexcept + { + return *pc==sentinel_; + } + inline void reset(std::size_t pos) { BOOST_ASSERT(pos friend class table; table_iterator(Group* pg,std::size_t n,const table_element_type* p_): - pc{reinterpret_cast(const_cast(pg))+n}, + pc{reinterpret_cast(const_cast(pg))+n}, p{const_cast(p_)} {} - inline std::size_t rebase() noexcept - { - std::size_t off=reinterpret_cast(pc)%sizeof(Group); - pc-=off; - return off; - } - inline void increment()noexcept { - std::size_t n0=rebase(); + increment(std::integral_constant{}); + } - int mask=(reinterpret_cast(pc)->match_occupied()>>(n0+1))<<(n0+1); + inline void increment(std::true_type /* regular layout */)noexcept + { + for(;;){ + ++p; + if(reinterpret_cast(pc)%sizeof(group_type)==N-1){ + pc+=sizeof(group_type)-(N-1); + break; + } + ++pc; + if(!group_type::is_occupied(pc))continue; + if(BOOST_UNLIKELY(group_type::is_sentinel(pc)))p=nullptr; + return; + } + + for(;;){ + int mask=reinterpret_cast(pc)->match_occupied(); + if(mask!=0){ + auto n=unchecked_countr_zero(mask); + if(BOOST_UNLIKELY(reinterpret_cast(pc)->is_sentinel(n))){ + p=nullptr; + } + else{ + pc+=n; + p+=n; + } + return; + } + pc+=sizeof(group_type); + p+=N; + } + } + + inline void increment(std::false_type /* interleaved */)noexcept + { + std::size_t n0=reinterpret_cast(pc)%sizeof(group_type); + pc-=n0; + + int mask=( + reinterpret_cast(pc)->match_occupied()>>(n0+1))<<(n0+1); if(!mask){ do{ - pc+=sizeof(Group); - p+=Group::N; + pc+=sizeof(group_type); + p+=N; } - while((mask=reinterpret_cast(pc)->match_occupied())==0); + while((mask=reinterpret_cast(pc)->match_occupied())==0); } auto n=unchecked_countr_zero(mask); - if(BOOST_UNLIKELY(reinterpret_cast(pc)->is_sentinel(n))){ + if(BOOST_UNLIKELY(reinterpret_cast(pc)->is_sentinel(n))){ p=nullptr; } else{ From ced8b45add865d6ef60e0ad8be047f75ec1cfe23 Mon Sep 17 00:00:00 2001 From: joaquintides Date: Mon, 6 Mar 2023 13:50:25 +0100 Subject: [PATCH 4/7] fixed UB with begin when using fast iteration --- include/boost/unordered/detail/foa.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/boost/unordered/detail/foa.hpp b/include/boost/unordered/detail/foa.hpp index 85bbc701..c5c7abe6 100644 --- a/include/boost/unordered/detail/foa.hpp +++ b/include/boost/unordered/detail/foa.hpp @@ -1471,7 +1471,7 @@ public: iterator begin()noexcept { iterator it{arrays.groups,0,arrays.elements}; - if(!(arrays.groups[0].match_occupied()&0x1))++it; + if(arrays.elements&&!(arrays.groups[0].match_occupied()&0x1))++it; return it; } From 1bf193b8fc3091e4a448e971dd838f55e40045ee Mon Sep 17 00:00:00 2001 From: joaquintides Date: Mon, 6 Mar 2023 21:46:42 +0100 Subject: [PATCH 5/7] updated release notes --- doc/unordered/changes.adoc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/unordered/changes.adoc b/doc/unordered/changes.adoc index 8f1c37b0..dfd12081 100644 --- a/doc/unordered/changes.adoc +++ b/doc/unordered/changes.adoc @@ -6,6 +6,10 @@ :github-pr-url: https://github.com/boostorg/unordered/pull :cpp: C++ +== Release 1.83.0 + +* Sped up iteration of open-addressing containers. + == Release 1.82.0 - Major update * {cpp}03 support is planned for deprecation. Boost 1.84.0 will no longer support From 4d982e133ec7f0215cfffb05cb82b0278c99c7f3 Mon Sep 17 00:00:00 2001 From: joaquintides Date: Wed, 8 Mar 2023 18:29:55 +0100 Subject: [PATCH 6/7] added is-not-end assumption on table_iterator::increment --- include/boost/unordered/detail/foa.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/boost/unordered/detail/foa.hpp b/include/boost/unordered/detail/foa.hpp index c5c7abe6..67c0a46f 100644 --- a/include/boost/unordered/detail/foa.hpp +++ b/include/boost/unordered/detail/foa.hpp @@ -892,6 +892,7 @@ private: inline void increment()noexcept { + BOOST_UNORDERED_ASSUME(p!=nullptr); increment(std::integral_constant{}); } From 1c5640cfbe84213b9d2a7eff68e2db744cfa3817 Mon Sep 17 00:00:00 2001 From: joaquintides Date: Wed, 8 Mar 2023 19:12:46 +0100 Subject: [PATCH 7/7] s/BOOST_UNORDERED_ASSUME/BOOST_ASSERT --- include/boost/unordered/detail/foa.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/boost/unordered/detail/foa.hpp b/include/boost/unordered/detail/foa.hpp index 67c0a46f..30a27272 100644 --- a/include/boost/unordered/detail/foa.hpp +++ b/include/boost/unordered/detail/foa.hpp @@ -892,7 +892,7 @@ private: inline void increment()noexcept { - BOOST_UNORDERED_ASSUME(p!=nullptr); + BOOST_ASSERT(p!=nullptr); increment(std::integral_constant{}); }