. /include/boost/corosio /detail /object_pool.hpp
94.20% Lines (65/69) 100.00% Functions (10/10)
[ ] prev/next file n p prev/next uncovered line
TLA Baseline Branch
Line Hits Code Line Hits Code
1   // 1   //
2   // Copyright (c) 2025 Steve Gerbino (steve@gerbino.co) 2   // Copyright (c) 2025 Steve Gerbino (steve@gerbino.co)
3   // 3   //
4   // Distributed under the Boost Software License, Version 1.0. (See accompanying 4   // Distributed under the Boost Software License, Version 1.0. (See accompanying
5   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) 5   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6   // 6   //
7   // Official repository: https://github.com/cppalliance/corosio 7   // Official repository: https://github.com/cppalliance/corosio
8   // 8   //
9   9  
10   #ifndef BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP 10   #ifndef BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP
11   #define BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP 11   #define BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP
12   12  
13   #include <boost/corosio/detail/config.hpp> 13   #include <boost/corosio/detail/config.hpp>
14   #include <boost/corosio/detail/intrusive.hpp> 14   #include <boost/corosio/detail/intrusive.hpp>
15   #include <boost/corosio/detail/object_ref.hpp> 15   #include <boost/corosio/detail/object_ref.hpp>
16   #include <atomic> 16   #include <atomic>
17   #include <cstddef> 17   #include <cstddef>
18   #include <initializer_list> 18   #include <initializer_list>
19   #include <limits> 19   #include <limits>
20   #include <mutex> 20   #include <mutex>
21   #include <utility> 21   #include <utility>
22   #include <vector> 22   #include <vector>
23   23  
24   namespace boost::corosio::detail { 24   namespace boost::corosio::detail {
25   25  
26   /** Per-service recycling pool for I/O implementations. 26   /** Per-service recycling pool for I/O implementations.
27   27  
28   Replaces per-impl `shared_ptr` ownership: a service news an impl 28   Replaces per-impl `shared_ptr` ownership: a service news an impl
29   once, then `recycle()` (the impl's `retire()` action) parks 29   once, then `recycle()` (the impl's `retire()` action) parks
30   it on the free list and `acquire()` hands it back out, so steady-state 30   it on the free list and `acquire()` hands it back out, so steady-state
31   construct/destroy allocates nothing. Unbounded; memory is released 31   construct/destroy allocates nothing. Unbounded; memory is released
32   only on pool destruction. Debug poisoning, where an impl does it, 32   only on pool destruction. Debug poisoning, where an impl does it,
33   happens in that impl's `retire()`, not here (the pool cannot know 33   happens in that impl's `retire()`, not here (the pool cannot know
34   which bytes are resettable). 34   which bytes are resettable).
35   35  
36   A service tears down its impls by calling `shutdown()`, which 36   A service tears down its impls by calling `shutdown()`, which
37   enters shutting-down mode and visits every live impl in one 37   enters shutting-down mode and visits every live impl in one
38   critical section; subsequent zero-ref crossings delete instead of 38   critical section; subsequent zero-ref crossings delete instead of
39   recycling. 39   recycling.
40   40  
41   `adopt()` and `remove()` are protected ownership-transfer 41   `adopt()` and `remove()` are protected ownership-transfer
42   primitives for sanctioned extensions — a derived pool exposing 42   primitives for sanctioned extensions — a derived pool exposing
43   them to a caller that tracks some of its live impls through its 43   them to a caller that tracks some of its live impls through its
44   own structure instead of this pool's `live_` list (the timer 44   own structure instead of this pool's `live_` list (the timer
45   service, via its expiry heap) — rather than part of the public 45   service, via its expiry heap) — rather than part of the public
46   surface every ordinary service uses. 46   surface every ordinary service uses.
47   47  
48   @par Thread Safety 48   @par Thread Safety
49   Distinct objects: Safe. Shared objects: Safe; all operations lock 49   Distinct objects: Safe. Shared objects: Safe; all operations lock
50   an internal mutex. 50   an internal mutex.
51   */ 51   */
52   template<class Impl> 52   template<class Impl>
53   class object_pool 53   class object_pool
54   { 54   {
55   std::mutex mutex_; 55   std::mutex mutex_;
56   intrusive_list<Impl> live_; 56   intrusive_list<Impl> live_;
57   intrusive_list<Impl> free_; 57   intrusive_list<Impl> free_;
58   bool shutting_down_ = false; 58   bool shutting_down_ = false;
59   59  
60   /** Marks `refs_` on an impl currently parked in `free_`. 60   /** Marks `refs_` on an impl currently parked in `free_`.
61   61  
62   `intrusive_list::remove()` cannot distinguish "linked in 62   `intrusive_list::remove()` cannot distinguish "linked in
63   `free_`" from "linked in `live_`" by inspecting the node 63   `free_`" from "linked in `live_`" by inspecting the node
64   alone — both lists share one `next_`/`prev_` pair per impl — 64   alone — both lists share one `next_`/`prev_` pair per impl —
65   so `recycle()` cannot safely call `live_.remove()` on an impl 65   so `recycle()` cannot safely call `live_.remove()` on an impl
66   it does not already know is live. Stamping this sentinel into 66   it does not already know is live. Stamping this sentinel into
67   `refs_` the moment an impl joins `free_`, and checking it 67   `refs_` the moment an impl joins `free_`, and checking it
68   before touching either list, is what lets `recycle()` and 68   before touching either list, is what lets `recycle()` and
69   `remove()` tell the two cases apart in every build mode, not 69   `remove()` tell the two cases apart in every build mode, not
70   just under asserts. `acquire()` always overwrites it with 1 70   just under asserts. `acquire()` always overwrites it with 1
71   before the impl is usable again. 71   before the impl is usable again.
72   */ 72   */
73   static constexpr std::size_t pooled_sentinel = pooled_refs; 73   static constexpr std::size_t pooled_sentinel = pooled_refs;
74   74  
75   /// Check whether @p impl is parked in `free_` (`refs_` reads the 75   /// Check whether @p impl is parked in `free_` (`refs_` reads the
76   /// sentinel). Callers must check this before any list operation — 76   /// sentinel). Callers must check this before any list operation —
77   /// see the class-level comment on `pooled_sentinel`. 77   /// see the class-level comment on `pooled_sentinel`.
HITCBC 78   85603 static bool is_pooled(Impl const* impl) noexcept 78   86532 static bool is_pooled(Impl const* impl) noexcept
79   { 79   {
HITCBC 80   85603 return impl->refs_.load(std::memory_order_relaxed) == 80   86532 return impl->refs_.load(std::memory_order_relaxed) ==
HITCBC 81   85603 pooled_sentinel; 81   86532 pooled_sentinel;
82   } 82   }
83   83  
84   protected: 84   protected:
85   /** Track a fresh or reset impl as live. 85   /** Track a fresh or reset impl as live.
86   86  
87   @pre @p impl is not parked in `free_` (`refs_` must not read 87   @pre @p impl is not parked in `free_` (`refs_` must not read
88   `pooled_sentinel`). The same list-corruption hazard 88   `pooled_sentinel`). The same list-corruption hazard
89   `recycle()`'s class-level comment documents applies here: a 89   `recycle()`'s class-level comment documents applies here: a
90   free-listed impl pushed onto `live_` directly, bypassing 90   free-listed impl pushed onto `live_` directly, bypassing
91   `acquire()`, would end up linked in both lists through one shared 91   `acquire()`, would end up linked in both lists through one shared
92   `next_`/`prev_` pair. 92   `next_`/`prev_` pair.
93   93  
94   @pre The caller is not racing `shutdown()`. Every caller adopts 94   @pre The caller is not racing `shutdown()`. Every caller adopts
95   while constructing an object (`acquire()`, or a derived pool's 95   while constructing an object (`acquire()`, or a derived pool's
96   cache handing a parked impl back out), and construction is 96   cache handing a parked impl back out), and construction is
97   expected to stop before the service's own `shutdown()` runs; 97   expected to stop before the service's own `shutdown()` runs;
98   this is checked only in debug builds because the failure mode 98   this is checked only in debug builds because the failure mode
99   is a silently orphaned impl (no close callback), not memory 99   is a silently orphaned impl (no close callback), not memory
100   corruption. 100   corruption.
101   */ 101   */
HITCBC 102   31686 void adopt(Impl* impl) 102   32058 void adopt(Impl* impl)
103   { 103   {
HITCBC 104   31686 std::lock_guard lock(mutex_); 104   32058 std::lock_guard lock(mutex_);
HITCBC 105   31686 if (is_pooled(impl)) 105   32058 if (is_pooled(impl))
106   { 106   {
MISUBC 107   ✗ BOOST_COROSIO_ASSERT(false); 107   ✗ BOOST_COROSIO_ASSERT(false);
108   return; 108   return;
109   } 109   }
HITCBC 110   31686 BOOST_COROSIO_ASSERT(!shutting_down_); 110   32058 BOOST_COROSIO_ASSERT(!shutting_down_);
HITCBC 111   31686 live_.push_back(impl); 111   32058 live_.push_back(impl);
HITCBC 112   31686 } 112   32058 }
113   113  
114   /** Unlink a live impl without recycling (ownership transfer). 114   /** Unlink a live impl without recycling (ownership transfer).
115   115  
116   Used by a caller that is taking an impl out of this pool's 116   Used by a caller that is taking an impl out of this pool's
117   bookkeeping entirely (a thread-local cache slot, a direct 117   bookkeeping entirely (a thread-local cache slot, a direct
118   delete during shutdown) rather than parking it on `free_`. 118   delete during shutdown) rather than parking it on `free_`.
119   119  
120   @pre @p impl is not parked in `free_` (`refs_` must not read 120   @pre @p impl is not parked in `free_` (`refs_` must not read
121   `pooled_sentinel`). The same list-corruption hazard 121   `pooled_sentinel`). The same list-corruption hazard
122   `recycle()`'s class-level comment documents applies here: a 122   `recycle()`'s class-level comment documents applies here: a
123   free-listed impl would be spliced out using `live_`'s 123   free-listed impl would be spliced out using `live_`'s
124   boundary pointers instead of `free_`'s. 124   boundary pointers instead of `free_`'s.
125   125  
126   @return True if @p impl was linked in `live_` and has now 126   @return True if @p impl was linked in `live_` and has now
127   been removed; false if it was already detached. 127   been removed; false if it was already detached.
128   */ 128   */
HITCBC 129   18434 bool remove(Impl* impl) noexcept 129   18336 bool remove(Impl* impl) noexcept
130   { 130   {
HITCBC 131   18434 std::lock_guard lock(mutex_); 131   18336 std::lock_guard lock(mutex_);
HITCBC 132   18434 if (is_pooled(impl)) 132   18336 if (is_pooled(impl))
133   { 133   {
MISUBC 134   ✗ BOOST_COROSIO_ASSERT(false); 134   ✗ BOOST_COROSIO_ASSERT(false);
135   return false; 135   return false;
136   } 136   }
HITCBC 137   18434 return live_.remove(impl); 137   18336 return live_.remove(impl);
HITCBC 138   18434 } 138   18336 }
139   139  
140   public: 140   public:
141   /** Destroy the pool, freeing every impl still parked or live. 141   /** Destroy the pool, freeing every impl still parked or live.
142   142  
143   A live impl is deleted whatever its count. Keepalives it holds 143   A live impl is deleted whatever its count. Keepalives it holds
144   on itself (an embedded op's `object_ref`) are released by its 144   on itself (an embedded op's `object_ref`) are released by its
145   own destructor; the count is pinned first so none of those 145   own destructor; the count is pinned first so none of those
146   releases reaches zero and calls `retire()` on an object that 146   releases reaches zero and calls `retire()` on an object that
147   is already being destroyed. 147   is already being destroyed.
148   148  
149   @pre `shutdown()` has already run, or no impls are live. 149   @pre `shutdown()` has already run, or no impls are live.
150   */ 150   */
HITCBC 151   5572 ~object_pool() 151   5578 ~object_pool()
152   { 152   {
HITCBC 153   5572 BOOST_COROSIO_ASSERT(shutting_down_ || live_.empty()); 153   5578 BOOST_COROSIO_ASSERT(shutting_down_ || live_.empty());
HITCBC 154   16716 for (auto* list : {&free_, &live_}) 154   16734 for (auto* list : {&free_, &live_})
155   { 155   {
HITCBC 156   18431 while (auto* impl = list->pop_front()) 156   18910 while (auto* impl = list->pop_front())
157   { 157   {
HITCBC 158   7287 impl->refs_.store(pooled_sentinel, std::memory_order_relaxed); 158   7754 impl->refs_.store(pooled_sentinel, std::memory_order_relaxed);
HITCBC 159   7287 delete impl; 159   7754 delete impl;
160   } 160   }
161   } 161   }
HITCBC 162   5572 } 162   5578 }
163   163  
HITCBC 164   5562 object_pool() = default; 164   5568 object_pool() = default;
165   object_pool(object_pool const&) = delete; 165   object_pool(object_pool const&) = delete;
166   object_pool& operator=(object_pool const&) = delete; 166   object_pool& operator=(object_pool const&) = delete;
167   167  
168   /** Pop-or-create an impl with the acquisition protocol applied. 168   /** Pop-or-create an impl with the acquisition protocol applied.
169   169  
170   Recycled impls are `reuse()`-reset and their reference count 170   Recycled impls are `reuse()`-reset and their reference count
171   restored from the pooled sentinel; fresh impls are constructed 171   restored from the pooled sentinel; fresh impls are constructed
172   in place from `args` on the cold path. Either way the impl is 172   in place from `args` on the cold path. Either way the impl is
173   adopted as live. This is the sequence every service's 173   adopted as live. This is the sequence every service's
174   `construct()` used to hand-roll. Making the pool itself the 174   `construct()` used to hand-roll. Making the pool itself the
175   only place that calls `new Impl` gives the whole library one 175   only place that calls `new Impl` gives the whole library one
176   audit point for allocation policy. 176   audit point for allocation policy.
177   177  
178   @pre Not called concurrently with, or after, `shutdown()` on 178   @pre Not called concurrently with, or after, `shutdown()` on
179   this pool. Every caller reaches this through a service's 179   this pool. Every caller reaches this through a service's
180   `construct()`, and service shutdown order guarantees no 180   `construct()`, and service shutdown order guarantees no
181   `construct()` outlives that service's own `shutdown()`; the 181   `construct()` outlives that service's own `shutdown()`; the
182   window is only asserted in debug builds (see `adopt()`) 182   window is only asserted in debug builds (see `adopt()`)
183   because the failure mode is an orphaned impl, not corruption. 183   because the failure mode is an orphaned impl, not corruption.
184   184  
185   @param args Constructor arguments for `Impl` (cold path only). 185   @param args Constructor arguments for `Impl` (cold path only).
186   186  
187   @return The acquired impl, `refs_ == 1`, tracked live. 187   @return The acquired impl, `refs_ == 1`, tracked live.
188   */ 188   */
189   template<class... Args> 189   template<class... Args>
HITCBC 190   39239 Impl* acquire(Args&&... args) 190   39898 Impl* acquire(Args&&... args)
191   { 191   {
192   Impl* impl; 192   Impl* impl;
193   { 193   {
HITCBC 194   39239 std::lock_guard lock(mutex_); 194   39898 std::lock_guard lock(mutex_);
HITCBC 195   39239 BOOST_COROSIO_ASSERT(!shutting_down_); 195   39898 BOOST_COROSIO_ASSERT(!shutting_down_);
HITCBC 196   39239 impl = free_.pop_front(); 196   39898 impl = free_.pop_front();
HITCBC 197   39239 if (impl) 197   39898 if (impl)
198   { 198   {
HITCBC 199   22236 impl->refs_.store(1, std::memory_order_relaxed); 199   22422 impl->refs_.store(1, std::memory_order_relaxed);
HITCBC 200   22236 live_.push_back(impl); 200   22422 live_.push_back(impl);
201   } 201   }
HITCBC 202   39239 } 202   39898 }
203   // reuse() runs unlocked: acquire()'s precondition excludes a 203   // reuse() runs unlocked: acquire()'s precondition excludes a
204   // concurrent shutdown() visiting the impl mid-reset. 204   // concurrent shutdown() visiting the impl mid-reset.
HITCBC 205   39239 if (impl) 205   39898 if (impl)
206   { 206   {
HITCBC 207   22236 impl->reuse(); 207   22422 impl->reuse();
HITCBC 208   22236 return impl; 208   22422 return impl;
209   } 209   }
HITCBC 210   17003 impl = new Impl(std::forward<Args>(args)...); 210   17476 impl = new Impl(std::forward<Args>(args)...);
HITCBC 211   17003 adopt(impl); 211   17476 adopt(impl);
HITCBC 212   17003 return impl; 212   17476 return impl;
213   } 213   }
214   214  
215   /** Move a zero-ref impl from live to the free list. 215   /** Move a zero-ref impl from live to the free list.
216   216  
217   Called from `retire()`, possibly on a scheduler thread. 217   Called from `retire()`, possibly on a scheduler thread.
218   Deletes instead when shutting down. The impl must not be 218   Deletes instead when shutting down. The impl must not be
219   touched after this returns. 219   touched after this returns.
220   220  
221   Two accounting bugs are possible here, neither from this 221   Two accounting bugs are possible here, neither from this
222   pool's own bookkeeping but from a caller's refcounting (e.g. 222   pool's own bookkeeping but from a caller's refcounting (e.g.
223   a duplicate `retire()` for one zero-crossing). Both are 223   a duplicate `retire()` for one zero-crossing). Both are
224   made safe in every build mode, not just under asserts, because 224   made safe in every build mode, not just under asserts, because
225   neither can be told apart from the correct case by a 225   neither can be told apart from the correct case by a
226   `BOOST_COROSIO_ASSERT` alone that still runs the list 226   `BOOST_COROSIO_ASSERT` alone that still runs the list
227   operations around it: 227   operations around it:
228   228  
229   - `impl` is already parked in `free_` — the list-corruption 229   - `impl` is already parked in `free_` — the list-corruption
230   hazard the class-level `pooled_sentinel` comment documents. 230   hazard the class-level `pooled_sentinel` comment documents.
231   The `is_pooled()` check runs first and returns before either 231   The `is_pooled()` check runs first and returns before either
232   list is touched. 232   list is touched.
233   - `impl` is fully detached from `live_` and is not the 233   - `impl` is fully detached from `live_` and is not the
234   sentinel (not shutting down): `live_.remove()` correctly 234   sentinel (not shutting down): `live_.remove()` correctly
235   returns false, but then the impl is reachable through 235   returns false, but then the impl is reachable through
236   neither `acquire()` nor `shutdown()`'s visit while still 236   neither `acquire()` nor `shutdown()`'s visit while still
237   allocated — stranded. Deleted instead of left to leak. 237   allocated — stranded. Deleted instead of left to leak.
238   238  
239   While shutting down, an impl that is no longer in `live_` is 239   While shutting down, an impl that is no longer in `live_` is
240   left alone rather than deleted: something else already took 240   left alone rather than deleted: something else already took
241   it out of the pool's bookkeeping (a derived pool's owner, or a 241   it out of the pool's bookkeeping (a derived pool's owner, or a
242   duplicate retirement of an impl already deleted), and deleting 242   duplicate retirement of an impl already deleted), and deleting
243   it again here could be a double free. 243   it again here could be a double free.
244   */ 244   */
HITCBC 245   35483 void recycle(Impl* impl) 245   36138 void recycle(Impl* impl)
246   { 246   {
HITCBC 247   35483 bool do_delete = false; 247   36138 bool do_delete = false;
248   { 248   {
HITCBC 249   35483 std::lock_guard lock(mutex_); 249   36138 std::lock_guard lock(mutex_);
HITCBC 250   35483 if (is_pooled(impl)) 250   36138 if (is_pooled(impl))
251   { 251   {
MISUBC 252   ✗ BOOST_COROSIO_ASSERT(false); 252   ✗ BOOST_COROSIO_ASSERT(false);
253   return; 253   return;
254   } 254   }
255   255  
HITCBC 256   35483 bool const was_live = live_.remove(impl); 256   36138 bool const was_live = live_.remove(impl);
HITCBC 257   35483 if (shutting_down_) 257   36138 if (shutting_down_)
258   { 258   {
HITCBC 259   5966 do_delete = was_live; 259   5969 do_delete = was_live;
260   } 260   }
HITCBC 261   29517 else if (was_live) 261   30169 else if (was_live)
262   { 262   {
HITCBC 263   29517 impl->refs_.store( 263   30169 impl->refs_.store(
264   pooled_sentinel, std::memory_order_relaxed); 264   pooled_sentinel, std::memory_order_relaxed);
265   // LIFO: the most recently retired impl is the 265   // LIFO: the most recently retired impl is the
266   // cache-warmest candidate for the next acquire(). 266   // cache-warmest candidate for the next acquire().
HITCBC 267   29517 free_.push_front(impl); 267   30169 free_.push_front(impl);
268   } 268   }
269   else 269   else
270   { 270   {
MISUBC 271   ✗ BOOST_COROSIO_ASSERT(false); 271   ✗ BOOST_COROSIO_ASSERT(false);
272   do_delete = true; 272   do_delete = true;
273   } 273   }
HITCBC 274   35483 } 274   36138 }
HITCBC 275   35483 if (do_delete) 275   36138 if (do_delete)
HITCBC 276   5966 delete impl; 276   5969 delete impl;
277   } 277   }
278   278  
279   /** Enter shutdown mode and visit every live impl. 279   /** Enter shutdown mode and visit every live impl.
280   280  
281   Sets shutting-down (subsequent zero-crossings delete instead 281   Sets shutting-down (subsequent zero-crossings delete instead
282   of recycling) and runs `f` on each live impl, all under one 282   of recycling) and runs `f` on each live impl, all under one
283   critical section. Each visited impl is pinned with a reference 283   critical section. Each visited impl is pinned with a reference
284   for the call and released after the critical section ends, so 284   for the call and released after the critical section ends, so
285   `f` may hold on to it; an impl whose count already reached zero 285   `f` may hold on to it; an impl whose count already reached zero
286   is retiring and is skipped rather than revived. 286   is retiring and is skipped rather than revived.
287   287  
288   @note The callback runs under the pool's non-recursive mutex; 288   @note The callback runs under the pool's non-recursive mutex;
289   it must not call recycle(), adopt(), or remove() on this pool. 289   it must not call recycle(), adopt(), or remove() on this pool.
290   */ 290   */
291   template<class F> 291   template<class F>
HITCBC 292   5562 void shutdown(F&& f) 292   5568 void shutdown(F&& f)
293   { 293   {
HITCBC 294   5562 std::vector<Impl*> pinned; 294   5568 std::vector<Impl*> pinned;
295   { 295   {
HITCBC 296   5562 std::lock_guard lock(mutex_); 296   5568 std::lock_guard lock(mutex_);
HITCBC 297   5562 shutting_down_ = true; 297   5568 shutting_down_ = true;
298   // Sized before any pin, so a failure here leaves none held. 298   // Sized before any pin, so a failure here leaves none held.
HITCBC 299   5562 std::size_t n = 0; 299   5568 std::size_t n = 0;
HITCBC 300   11527 live_.for_each([&](Impl*) { ++n; }); 300   11536 live_.for_each([&](Impl*) { ++n; });
HITCBC 301   5562 pinned.reserve(n); 301   5568 pinned.reserve(n);
HITCBC 302   5562 live_.for_each( 302   5568 live_.for_each(
HITCBC 303   18438 [&](Impl* impl) 303   18444 [&](Impl* impl)
304   { 304   {
HITCBC 305   9219 if (try_acquire(impl)) 305   9222 if (try_acquire(impl))
306   { 306   {
HITCBC 307   9218 pinned.push_back(impl); 307   9221 pinned.push_back(impl);
HITCBC 308   9218 f(impl); 308   9221 f(impl);
309   } 309   }
310   }); 310   });
HITCBC 311   5562 } 311   5568 }
HITCBC 312   14780 for (auto* impl : pinned) 312   14789 for (auto* impl : pinned)
HITCBC 313   9218 release(impl); 313   9221 release(impl);
HITCBC 314   5562 } 314   5568 }
315   }; 315   };
316   316  
317   } // namespace boost::corosio::detail 317   } // namespace boost::corosio::detail
318   318  
319   #endif 319   #endif