GCC Code Coverage Report

Directory: src/
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 83.5% 142 / 0 / 170
Functions: 87.5% 21 / 0 / 24
Branches: 42.3% 60 / 0 / 142
memory/allocators/r2/allocator_master.hh
Line Branch Exec Source
1 #pragma once
2
3 #include <mutex>
4 #include <unordered_map>
5 #include <unordered_set>
6 #include <vector>
7 #include <cstdint>
8 #include <cstddef>
9 #include <cstring>
10 #include <atomic>
11
12 #define JEMALLOC_NO_DEMANGLE
13 #include <jemalloc/jemalloc.h>
14 #include "allocator.hh"
15
16 namespace r2 {
17
18 #ifndef container_of
19 # define container_of(ptr, type, member) \
20 ((type*)((char*)(ptr)-offsetof(type, member)))
21 #endif
22
23 class AllocatorMaster {
24 public:
25 186 AllocatorMaster() = default;
26
27 186 ~AllocatorMaster() {
28 186 shutting_down_.store(true, std::memory_order_release);
29
30 186 auto& tls = tls_allocs();
31
2/2
✓ Branch 3 taken 154 times.
✓ Branch 4 taken 32 times.
 186 if (auto it = tls.find(this); it != tls.end()) {
32
1/2
✓ Branch 1 taken 154 times.
✗ Branch 2 not taken.
 154 delete it->second;
33 154 tls.erase(it);
34 }
35
36 186 (void)je_mallctl("thread.tcache.flush", nullptr, nullptr, nullptr, 0);
37 186 std::vector<Entry> entries;
38 {
39 186 std::lock_guard<std::mutex> g(created_mu_);
40 186 entries.swap(created_);
41 186 }
42
43
2/2
✓ Branch 5 taken 396 times.
✓ Branch 6 taken 186 times.
 582 for (auto& e : entries) {
44
2/2
✓ Branch 0 taken 230 times.
✓ Branch 1 taken 166 times.
 396 if (e.tcache != 0) {
45 230 size_t z = 0;
46 230 (void)je_mallctl("tcache.destroy", &e.tcache, &z, nullptr, 0);
47 }
48 }
49
50
2/2
✓ Branch 5 taken 396 times.
✓ Branch 6 taken 186 times.
 582 for (auto& e : entries) {
51
2/2
✓ Branch 0 taken 234 times.
✓ Branch 1 taken 162 times.
 396 if (!e.owns_arena)
52 234 continue;
53
54 char purge[64];
55 162 std::snprintf(purge, sizeof(purge), "arena.%u.purge", e.arena);
56 162 (void)je_mallctl(purge, nullptr, nullptr, nullptr, 0);
57
58 char d1[64];
59 162 std::snprintf(d1, sizeof(d1), "arena.%u.destroy", e.arena);
60 char d2[64];
61 162 std::snprintf(d2, sizeof(d2), "arenas.%u.destroy", e.arena);
62 162 int rc1 = je_mallctl(d1, nullptr, nullptr, nullptr, 0);
63
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 162 times.
 162 int rc2 = (rc1 == 0) ? 0 : je_mallctl(d2, nullptr, nullptr, nullptr, 0);
64
65
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 162 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 162 if (rc1 == 0 || rc2 == 0) {
66
1/2
✓ Branch 0 taken 162 times.
✗ Branch 1 not taken.
 162 if (e.pack) {
67 {
68 162 std::lock_guard<std::mutex> gp(packs_mu());
69 162 packs().erase(e.pack);
70 162 }
71
1/2
✓ Branch 0 taken 162 times.
✗ Branch 1 not taken.
 162 delete e.pack;
72 162 e.pack = nullptr;
73 }
74 }
75 }
76
77 186 std::lock_guard<std::mutex> guard(lock_);
78 186 start_addr_ = end_addr_ = heap_top_ = nullptr;
79 186 }
80
81 AllocatorMaster(const AllocatorMaster&) = delete;
82 AllocatorMaster& operator=(const AllocatorMaster&) = delete;
83 AllocatorMaster(AllocatorMaster&&) = delete;
84 AllocatorMaster& operator=(AllocatorMaster&&) = delete;
85
86 186 void init(char* mem, uint64_t mem_size) {
87
1/2
✓ Branch 1 taken 186 times.
✗ Branch 2 not taken.
 186 std::lock_guard<std::mutex> guard(lock_);
88
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 186 times.
 186 if (total_managed_mem() != 0)
89 ✗ return;
90 186 start_addr_ = mem;
91 186 end_addr_ = mem + mem_size;
92 186 heap_top_ = start_addr_;
93
1/2
✓ Branch 1 taken 186 times.
✗ Branch 2 not taken.
 186 }
94
95 43232 Allocator* get_thread_allocator() {
96 43232 auto& tls = tls_allocs();
97
3/4
✓ Branch 1 taken 43232 times.
✗ Branch 2 not taken.
✓ Branch 5 taken 42998 times.
✓ Branch 6 taken 234 times.
 43232 if (auto it = tls.find(this); it != tls.end())
98 42998 return it->second;
99
1/2
✓ Branch 1 taken 234 times.
✗ Branch 2 not taken.
 234 Allocator* al = get_allocator();
100
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 234 times.
 234 if (!al) {
101 // Fallback to an already-created arena so one failed arena creation
102 // does not make all allocations on this thread fail.
103 ✗ al = get_fallback_allocator();
104 }
105
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 234 times.
 234 if (!al)
106 ✗ return nullptr;
107
1/2
✓ Branch 1 taken 234 times.
✗ Branch 2 not taken.
 234 tls.emplace(this, al);
108 234 return al;
109 }
110
111 234 Allocator* get_allocator() {
112
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 234 times.
 234 if (shutting_down_.load(std::memory_order_acquire)) {
113 ✗ LOG(INFO) << "has benn shutting down";
114 ✗ return nullptr;
115 }
116 {
117
1/2
✓ Branch 1 taken 234 times.
✗ Branch 2 not taken.
 234 std::lock_guard<std::mutex> g(lock_);
118
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 234 times.
 234 if (memory_usblae() == 0) {
119 // LOG(INFO) << "R2 allocator has no memory!";
120 ✗ return nullptr;
121 }
122
1/2
✓ Branch 1 taken 234 times.
✗ Branch 2 not taken.
 234 }
123
124 234 unsigned arena_id = 0, cache_id = 0;
125 234 size_t olen = sizeof(unsigned);
126
127
2/4
✓ Branch 1 taken 234 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 234 times.
 234 if (!ensure_shared_arena(&arena_id)) {
128 ✗ return nullptr;
129 }
130
131 234 int e = je_mallctl("tcache.create", &cache_id, &olen, nullptr, 0);
132
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 234 times.
 234 if (e != 0) {
133 ✗ LOG(INFO) << "tcache.create fail!";
134 ✗ return new Allocator(MALLOCX_ARENA(arena_id));
135 }
136
137
1/2
✓ Branch 1 taken 234 times.
✗ Branch 2 not taken.
 234 record_entry(arena_id, cache_id, /*pack=*/nullptr, /*owns_arena=*/false);
138
1/2
✓ Branch 1 taken 234 times.
✗ Branch 2 not taken.
 234 return new Allocator(MALLOCX_ARENA(arena_id) | MALLOCX_TCACHE(cache_id));
139 }
140
141 188 uint64_t total_managed_mem() const {
142 188 return static_cast<uint64_t>(end_addr_ - start_addr_);
143 }
144
145 236 uint64 memory_usblae() const {
146 236 return (uint64_t)end_addr_ - (uint64_t)heap_top_;
147 }
148
149 bool within_range(ptr_t p) const {
150 char* c = static_cast<char*>(p);
151 return (c >= start_addr_) && (c < end_addr_);
152 }
153
154 private:
155 char* start_addr_ = nullptr;
156 char* end_addr_ = nullptr;
157 char* heap_top_ = nullptr;
158 mutable std::mutex lock_;
159 std::atomic<bool> shutting_down_{false};
160
161 struct HooksPack {
162 extent_hooks_t hooks{};
163 AllocatorMaster* owner{nullptr};
164 };
165
166 struct Entry {
167 unsigned arena{0};
168 unsigned tcache{0};
169 HooksPack* pack{nullptr};
170 bool owns_arena{false};
171 };
172
173 234 void record_entry(
174 unsigned arena, unsigned tcache, HooksPack* pack, bool owns_arena) {
175
1/2
✓ Branch 1 taken 234 times.
✗ Branch 2 not taken.
 234 std::lock_guard<std::mutex> g(created_mu_);
176
1/2
✓ Branch 1 taken 234 times.
✗ Branch 2 not taken.
 234 created_.push_back(Entry{arena, tcache, pack, owns_arena});
177 234 }
178
179 std::mutex created_mu_;
180 std::vector<Entry> created_;
181 bool shared_arena_created_{false};
182 unsigned shared_arena_{0};
183
184 234 bool ensure_shared_arena(unsigned* arena_id) {
185
1/2
✓ Branch 1 taken 234 times.
✗ Branch 2 not taken.
 234 std::lock_guard<std::mutex> g(created_mu_);
186
2/2
✓ Branch 0 taken 72 times.
✓ Branch 1 taken 162 times.
 234 if (shared_arena_created_) {
187 72 *arena_id = shared_arena_;
188 72 return true;
189 }
190
191
1/2
✓ Branch 1 taken 162 times.
✗ Branch 2 not taken.
 162 HooksPack* pack = new HooksPack();
192 162 pack->owner = this;
193 162 pack->hooks.alloc = &AllocatorMaster::extent_alloc_hook;
194 162 pack->hooks.dalloc = &AllocatorMaster::extent_dalloc_hook;
195 162 pack->hooks.destroy = &AllocatorMaster::extent_destroy_hook;
196 162 pack->hooks.commit = &AllocatorMaster::extent_commit_hook;
197 162 pack->hooks.decommit = &AllocatorMaster::extent_decommit_hook;
198 162 pack->hooks.purge_lazy = &AllocatorMaster::extent_purge_lazy_hook;
199 162 pack->hooks.purge_forced = &AllocatorMaster::extent_purge_forced_hook;
200 162 pack->hooks.split = &AllocatorMaster::extent_split_hook;
201 162 pack->hooks.merge = &AllocatorMaster::extent_merge_hook;
202
203 {
204
1/2
✓ Branch 2 taken 162 times.
✗ Branch 3 not taken.
 162 std::lock_guard<std::mutex> gp(packs_mu());
205
1/2
✓ Branch 2 taken 162 times.
✗ Branch 3 not taken.
 162 packs().emplace(pack);
206 162 }
207
208 162 unsigned new_arena_id = 0;
209 162 size_t olen = sizeof(unsigned);
210 162 extent_hooks_t* hooks_ptr = &pack->hooks;
211 162 int e = je_mallctl(
212 "arenas.create",
213 &new_arena_id,
214 &olen,
215 &hooks_ptr,
216 sizeof(extent_hooks_t*));
217
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 162 times.
 162 if (e != 0) {
218 ✗ LOG(INFO) << "arenas.create fail, err=" << e
219 ✗ << " memory_usable=" << memory_usblae();
220 {
221 ✗ std::lock_guard<std::mutex> gp(packs_mu());
222 ✗ packs().erase(pack);
223 ✗ }
224 ✗ delete pack;
225 ✗ return false;
226 }
227
228 162 shared_arena_created_ = true;
229 162 shared_arena_ = new_arena_id;
230 162 created_.push_back(
231
1/2
✓ Branch 1 taken 162 times.
✗ Branch 2 not taken.
 162 Entry{new_arena_id, /*tcache=*/0, pack, /*owns_arena=*/true});
232 162 *arena_id = new_arena_id;
233 162 return true;
234 234 }
235
236 ✗ Allocator* get_fallback_allocator() {
237 ✗ std::lock_guard<std::mutex> g(created_mu_);
238 ✗ if (!shared_arena_created_) {
239 ✗ return nullptr;
240 }
241 ✗ return new Allocator(MALLOCX_ARENA(shared_arena_) | MALLOCX_TCACHE_NONE);
242 ✗ }
243
244 324 static std::unordered_set<HooksPack*>& packs() {
245
3/4
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 320 times.
✓ Branch 3 taken 4 times.
✗ Branch 4 not taken.
 324 static std::unordered_set<HooksPack*> s;
246 324 return s;
247 }
248 324 static std::mutex& packs_mu() {
249 static std::mutex m;
250 324 return m;
251 }
252
253 162 static std::atomic<uint64_t>& zero_extent_requests() {
254 static std::atomic<uint64_t> c{0};
255 162 return c;
256 }
257
258 static thread_local std::unordered_map<const AllocatorMaster*, Allocator*>&
259 43418 tls_allocs() {
260 static thread_local std::unordered_map<const AllocatorMaster*, Allocator*>
261
2/2
✓ Branch 0 taken 84 times.
✓ Branch 1 taken 43334 times.
 43418 m;
262 43418 return m;
263 }
264
265 334 static AllocatorMaster* owner_from_hooks(extent_hooks_t* eh) {
266 334 HooksPack* pack = container_of(eh, HooksPack, hooks);
267 334 return pack->owner;
268 }
269
270 334 static void* extent_alloc_hook(
271 extent_hooks_t* eh,
272 void*,
273 size_t size,
274 size_t alignment,
275 bool* zero,
276 bool*,
277 unsigned) {
278 334 AllocatorMaster* self = owner_from_hooks(eh);
279
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 334 times.
 334 if (!self)
280 ✗ return nullptr;
281
282
1/2
✓ Branch 1 taken 334 times.
✗ Branch 2 not taken.
 334 std::lock_guard<std::mutex> guard(self->lock_);
283 334 uintptr_t p = reinterpret_cast<uintptr_t>(self->heap_top_);
284 334 uintptr_t ret_u = (p + (alignment - 1)) & ~(alignment - 1);
285 334 char* ret = reinterpret_cast<char*>(ret_u);
286
287
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 334 times.
 334 if (ret + size > self->end_addr_)
288 ✗ return nullptr;
289
290 334 self->heap_top_ = ret + size;
291
2/2
✓ Branch 0 taken 162 times.
✓ Branch 1 taken 172 times.
 334 if (*zero) {
292 uint64_t n =
293 162 zero_extent_requests().fetch_add(1, std::memory_order_relaxed) + 1;
294
4/8
✓ Branch 1 taken 162 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 162 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 162 times.
✗ Branch 8 not taken.
✓ Branch 10 taken 162 times.
✗ Branch 11 not taken.
 324 LOG(INFO) << "R2 extent_alloc_hook zero-filled extent, count=" << n
295
2/4
✓ Branch 1 taken 162 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 162 times.
✗ Branch 5 not taken.
 162 << ", size=" << size;
296 162 std::memset(ret, 0, size);
297 }
298 334 return ret;
299 334 }
300
301 static bool
302 334 extent_dalloc_hook(extent_hooks_t*, void*, size_t, bool, unsigned) {
303 334 return true;
304 }
305 static void
306 162 extent_destroy_hook(extent_hooks_t*, void*, size_t, bool, unsigned) {}
307 static bool
308 10944 extent_commit_hook(extent_hooks_t*, void*, size_t, size_t, size_t, unsigned) {
309 10944 return false;
310 }
311 334 static bool extent_decommit_hook(
312 extent_hooks_t*, void*, size_t, size_t, size_t, unsigned) {
313 334 return false;
314 }
315 ✗ static bool extent_purge_lazy_hook(
316 extent_hooks_t*, void*, size_t, size_t, size_t, unsigned) {
317 ✗ return true;
318 }
319 ✗ static bool extent_purge_forced_hook(
320 extent_hooks_t*, void*, size_t, size_t, size_t, unsigned) {
321 ✗ return true;
322 }
323 10934 static bool extent_split_hook(
324 extent_hooks_t*, void*, size_t, size_t, size_t, bool, unsigned) {
325 10934 return false;
326 }
327 10944 static bool extent_merge_hook(
328 extent_hooks_t*, void*, size_t, void*, size_t, bool, unsigned) {
329 10944 return false;
330 }
331 };
332
333 } // namespace r2
334