GCC Code Coverage Report

src/
Coverage:
low: ≥ 0%
medium: ≥ 75.0%
high: ≥ 90.0%
Lines:
128 / 183
69.9%
Branches:
80 / 176
45.5%
Functions:
17 / 22
77.3%
storage/value_store/dram_value_store.h
Line Branch Exec Source
1 #pragma once
2
3 #include <cstring>
4 #include <memory>
5 #include <stdexcept>
6 #include <string>
7
8 #include <glog/logging.h>
9
10 #include "base/factory.h"
11 #include "memory/malloc.h"
12 #include "memory/memory_factory.h"
13 #include "storage/value_store/value_store.h"
14
15 class DramValueStore : public ValueStore {
16 public:
17 806 explicit DramValueStore(const BaseKVConfig& config) {
18 806 const auto& j = config.json_config_;
19
6/12
✓ Branch 1 taken 806 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 806 times.
✗ Branch 4 not taken.
✓ Branch 6 taken 806 times.
✗ Branch 7 not taken.
✓ Branch 9 taken 806 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✓ Branch 12 taken 806 times.
✗ Branch 13 not taken.
✓ Branch 14 taken 806 times.
 806 if (!j.contains("value") || !j.at("value").contains("dram_allocator")) {
20
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
 ✗ throw std::invalid_argument(
21 ✗ "DramValueStore requires value.dram_allocator");
22 }
23
1/2
✓ Branch 1 taken 806 times.
✗ Branch 2 not taken.
 806 const auto& value = j.at("value");
24
4/6
✓ Branch 1 taken 806 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 806 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 2 times.
✓ Branch 6 taken 804 times.
 1612 if (!value.contains("path") ||
25
5/10
✓ Branch 1 taken 806 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 806 times.
✗ Branch 5 not taken.
✓ Branch 7 taken 806 times.
✗ Branch 8 not taken.
✓ Branch 10 taken 2 times.
✓ Branch 11 taken 804 times.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
 1612 value.at("path").get<std::string>().empty()) {
26
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 throw std::invalid_argument(
27 4 "DramValueStore requires non-empty value.path");
28 }
29
2/4
✓ Branch 1 taken 804 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 804 times.
✗ Branch 5 not taken.
 804 const std::string path = value.at("path").get<std::string>();
30
1/2
✓ Branch 1 taken 804 times.
✗ Branch 2 not taken.
 804 const auto& dram = value.at("dram_allocator");
31
1/2
✓ Branch 1 taken 804 times.
✗ Branch 2 not taken.
 804 const std::string allocator_type = dram.value("type", "R2_SLAB");
32
2/4
✓ Branch 1 taken 804 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 804 times.
✗ Branch 5 not taken.
 804 const uint64_t capacity_bytes = dram.at("capacity_bytes").get<uint64_t>();
33 804 std::vector<int> size_classes;
34
2/4
✓ Branch 1 taken 804 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 804 times.
 804 if (dram.contains("size_classes")) {
35
0/4
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 ✗ size_classes = dram.at("size_classes").get<std::vector<int>>();
36
0/2
✗ Branch 5 not taken.
✗ Branch 6 not taken.
 ✗ for (int slab_size : size_classes) {
37
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (slab_size <= 0) {
38
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
 ✗ throw std::invalid_argument(
39 ✗ "DramValueStore requires positive dram_allocator.size_classes");
40 }
41 }
42 }
43
44 using MF = base::
45 Factory<base::MallocApi, const std::string&, int64, const std::string&>;
46 using MFS =
47 base::Factory<base::MallocApi,
48 const std::string&,
49 int64,
50 const std::string&,
51 const std::vector<int>&>;
52
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 804 times.
 804 if (!size_classes.empty()) {
53 // Only allocators that registered the sized factory accept an explicit
54 // size-class list; everything else keeps the fixed-class constructor.
55
0/4
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 ✗ if (MFS::creators().count(allocator_type) == 0) {
56
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
 ✗ throw std::invalid_argument(
57
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 ✗ "allocator does not support size_classes: " + allocator_type);
58 }
59
0/4
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
 ✗ allocator_.reset(MFS::NewInstance(
60 allocator_type,
61 path,
62 static_cast<int64>(capacity_bytes),
63 "DRAM",
64 size_classes));
65 } else {
66
2/4
✓ Branch 2 taken 804 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 804 times.
✗ Branch 6 not taken.
 804 allocator_.reset(MF::NewInstance(
67 allocator_type, path, static_cast<int64>(capacity_bytes), "DRAM"));
68 }
69
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 804 times.
 804 if (!allocator_) {
70
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 ✗ throw std::runtime_error("failed to create DramValueStore allocator");
71 }
72
1/2
✓ Branch 2 taken 804 times.
✗ Branch 3 not taken.
 804 allocator_->Initialize();
73 804 recycler_ = std::make_unique<base::ThreadSafeDelayedRecycle>(
74
1/2
✓ Branch 2 taken 804 times.
✗ Branch 3 not taken.
 804 allocator_.get(), kRecycleDelayUs);
75 810 }
76
77 309870 uint64_t Alloc(size_t size) override {
78 309870 char* data = allocator_->New(static_cast<int>(size));
79
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 309870 times.
 309870 if (data == nullptr) {
80 ✗ return kValueHandleNone;
81 }
82 309870 return EncodeOffset(allocator_->GetMallocOffset(data));
83 }
84
85 309870 void Write(uint64_t handle, const void* data, size_t size) override {
86 309870 char* dst = Ptr(handle);
87
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 309870 times.
 309870 if (dst == nullptr) {
88 ✗ return;
89 }
90 309870 std::memcpy(dst, data, size);
91 }
92
93 211278 uint64_t AllocAndWrite(const void* data, size_t size) override {
94 211278 const uint64_t handle = Alloc(size);
95
1/2
✓ Branch 0 taken 211278 times.
✗ Branch 1 not taken.
 211278 if (handle != kValueHandleNone) {
96 211278 Write(handle, data, size);
97 }
98 211278 return handle;
99 }
100
101 ✗ size_t Read(uint64_t handle, void* out_buf, size_t buf_size) override {
102 ✗ char* src = Ptr(handle);
103
0/4
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 ✗ if (src == nullptr || out_buf == nullptr) {
104 ✗ return 0;
105 }
106 ✗ const size_t n = std::min(
107 buf_size,
108 ✗ static_cast<size_t>(allocator_->GetMallocSize(DecodeOffset(handle))));
109 ✗ std::memcpy(out_buf, src, n);
110 ✗ return n;
111 }
112
113 41756 void Free(uint64_t handle) override {
114 41756 char* data = Ptr(handle);
115
1/2
✓ Branch 0 taken 41756 times.
✗ Branch 1 not taken.
 41756 if (data != nullptr) {
116 41756 allocator_->Free(data);
117 }
118 41756 }
119
120 124378 void Retire(uint64_t handle) override {
121
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 124378 times.
 124378 if (handle == kValueHandleNone) {
122 ✗ return;
123 }
124 124378 recycler_->Recycle(allocator_->GetMallocData(DecodeOffset(handle)));
125 }
126
127 190342 const char* DirectPtr(uint64_t handle) const override {
128 190342 return allocator_->GetMallocData(DecodeOffset(handle));
129 }
130
131 ✗ char* RDMABackingData() const override { return allocator_->BackingData(); }
132
133 ✗ size_t RDMABackingSize() const override {
134 ✗ return static_cast<size_t>(allocator_->BackingSize());
135 }
136
137 330718 size_t SlotCapacity(uint64_t handle) const override {
138
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 330718 times.
 330718 if (handle == kValueHandleNone) {
139 ✗ return 0;
140 }
141 330718 return static_cast<size_t>(allocator_->GetMallocSize(DecodeOffset(handle)));
142 }
143
144 48 bool ReadFlatFixedRows(const uint64_t* handles,
145 size_t num_rows,
146 void* out_buf,
147 size_t row_bytes,
148 uint64_t* missing_rows) const override {
149
3/6
✓ Branch 0 taken 48 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 48 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 48 times.
 48 if (handles == nullptr || out_buf == nullptr || row_bytes == 0) {
150 ✗ return false;
151 }
152
2/2
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 46 times.
 48 if (!IsPowerOfTwo(row_bytes)) {
153 2 return ReadFlatFixedRowsGeneric(
154 2 handles, num_rows, out_buf, row_bytes, missing_rows);
155 }
156 46 uint64_t local_missing = 0;
157 46 char* dst = static_cast<char*>(out_buf);
158 46 const unsigned row_shift = Log2PowerOfTwo(row_bytes);
159
2/2
✓ Branch 0 taken 246 times.
✓ Branch 1 taken 46 times.
 292 for (size_t row = 0; row < num_rows; ++row) {
160 246 char* row_dst = dst + (row << row_shift);
161
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 244 times.
 246 if (handles[row] == kValueHandleNone) {
162 2 std::memset(row_dst, 0, row_bytes);
163 2 ++local_missing;
164 2 continue;
165 }
166 244 const char* src = Ptr(handles[row]);
167
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 244 times.
 244 if (src == nullptr) {
168 ✗ return false;
169 }
170 244 std::memcpy(row_dst, src, row_bytes);
171 }
172
1/2
✓ Branch 0 taken 46 times.
✗ Branch 1 not taken.
 46 if (missing_rows != nullptr) {
173 46 *missing_rows = local_missing;
174 }
175 46 return true;
176 }
177
178 2 bool ReadFlatFixedRowSlices(
179 const uint64_t* handles,
180 size_t num_rows,
181 void* out_buf,
182 size_t stored_row_bytes,
183 size_t source_offset_bytes,
184 size_t output_row_bytes,
185 uint64_t* missing_rows) const override {
186
4/8
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 2 times.
✗ Branch 7 not taken.
 2 if (handles == nullptr || out_buf == nullptr || output_row_bytes == 0 ||
187 2 source_offset_bytes > stored_row_bytes ||
188
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
 2 output_row_bytes > stored_row_bytes - source_offset_bytes) {
189 ✗ return false;
190 }
191
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
 2 if (!IsPowerOfTwo(output_row_bytes)) {
192 ✗ return ReadFlatFixedRowSlicesGeneric(
193 handles,
194 num_rows,
195 out_buf,
196 source_offset_bytes,
197 output_row_bytes,
198 ✗ missing_rows);
199 }
200 2 uint64_t local_missing = 0;
201 2 char* dst = static_cast<char*>(out_buf);
202 2 const unsigned row_shift = Log2PowerOfTwo(output_row_bytes);
203
2/2
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 2 times.
 8 for (size_t row = 0; row < num_rows; ++row) {
204 6 char* row_dst = dst + (row << row_shift);
205
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 4 times.
 6 if (handles[row] == kValueHandleNone) {
206 2 std::memset(row_dst, 0, output_row_bytes);
207 2 ++local_missing;
208 2 continue;
209 }
210 4 const char* src = Ptr(handles[row]);
211
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
 4 if (src == nullptr) {
212 ✗ return false;
213 }
214 4 std::memcpy(row_dst, src + source_offset_bytes, output_row_bytes);
215 }
216
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 if (missing_rows != nullptr) {
217 2 *missing_rows = local_missing;
218 }
219 2 return true;
220 }
221
222 26 bool GetDirectFixedRows(const uint64_t* handles,
223 size_t num_rows,
224 size_t row_bytes,
225 DirectFixedRow* rows,
226 uint64_t* missing_rows) const override {
227
3/6
✓ Branch 0 taken 26 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 26 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 26 times.
 26 if (handles == nullptr || rows == nullptr || row_bytes == 0) {
228 ✗ return false;
229 }
230 26 uint64_t local_missing = 0;
231
2/2
✓ Branch 0 taken 48 times.
✓ Branch 1 taken 26 times.
 74 for (size_t row = 0; row < num_rows; ++row) {
232
2/2
✓ Branch 0 taken 20 times.
✓ Branch 1 taken 28 times.
 48 if (handles[row] == kValueHandleNone) {
233 20 rows[row] = DirectFixedRow{nullptr, row_bytes, true};
234 20 ++local_missing;
235 20 continue;
236 }
237 28 const char* src = Ptr(handles[row]);
238
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 28 times.
 28 if (src == nullptr) {
239 ✗ return false;
240 }
241
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 28 times.
 28 if (SlotCapacity(handles[row]) < row_bytes) {
242 ✗ return false;
243 }
244 28 rows[row] = DirectFixedRow{src, row_bytes, false};
245 }
246
1/2
✓ Branch 0 taken 26 times.
✗ Branch 1 not taken.
 26 if (missing_rows != nullptr) {
247 26 *missing_rows = local_missing;
248 }
249 26 return true;
250 }
251
252 ✗ std::string GetInfo() const override { return allocator_->GetInfo(); }
253 uint64_t TotalAllocCount() const { return allocator_->total_malloc(); }
254
255 private:
256 2 [[gnu::noinline]] bool ReadFlatFixedRowsGeneric(
257 const uint64_t* handles,
258 size_t num_rows,
259 void* out_buf,
260 size_t row_bytes,
261 uint64_t* missing_rows) const {
262 2 uint64_t local_missing = 0;
263 2 char* dst = static_cast<char*>(out_buf);
264
2/2
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 2 times.
 8 for (size_t row = 0; row < num_rows; ++row) {
265 6 char* row_dst = dst + row * row_bytes;
266
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 4 times.
 6 if (handles[row] == kValueHandleNone) {
267 2 std::memset(row_dst, 0, row_bytes);
268 2 ++local_missing;
269 2 continue;
270 }
271 4 const char* src = Ptr(handles[row]);
272
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
 4 if (src == nullptr) {
273 ✗ return false;
274 }
275 4 std::memcpy(row_dst, src, row_bytes);
276 }
277
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 if (missing_rows != nullptr) {
278 2 *missing_rows = local_missing;
279 }
280 2 return true;
281 }
282
283 ✗ [[gnu::noinline]] bool ReadFlatFixedRowSlicesGeneric(
284 const uint64_t* handles,
285 size_t num_rows,
286 void* out_buf,
287 size_t source_offset_bytes,
288 size_t output_row_bytes,
289 uint64_t* missing_rows) const {
290 ✗ uint64_t local_missing = 0;
291 ✗ char* dst = static_cast<char*>(out_buf);
292
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ for (size_t row = 0; row < num_rows; ++row) {
293 ✗ char* row_dst = dst + row * output_row_bytes;
294
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (handles[row] == kValueHandleNone) {
295 ✗ std::memset(row_dst, 0, output_row_bytes);
296 ✗ ++local_missing;
297 ✗ continue;
298 }
299 ✗ const char* src = Ptr(handles[row]);
300
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (src == nullptr) {
301 ✗ return false;
302 }
303 ✗ std::memcpy(row_dst, src + source_offset_bytes, output_row_bytes);
304 }
305
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (missing_rows != nullptr) {
306 ✗ *missing_rows = local_missing;
307 }
308 ✗ return true;
309 }
310
311 309870 static uint64_t EncodeOffset(int64 offset) {
312 309870 return static_cast<uint64_t>(offset) + 1;
313 }
314
315 997344 static int64 DecodeOffset(uint64_t handle) {
316 997344 return static_cast<int64>(handle - 1);
317 }
318
319 351906 char* Ptr(uint64_t handle) const {
320
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 351906 times.
 351906 if (handle == kValueHandleNone) {
321 ✗ return nullptr;
322 }
323 351906 return allocator_->GetMallocData(DecodeOffset(handle));
324 }
325
326 50 static bool IsPowerOfTwo(size_t value) {
327
3/4
✓ Branch 0 taken 50 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 48 times.
✓ Branch 3 taken 2 times.
 50 return value != 0 && (value & (value - 1)) == 0;
328 }
329
330 48 static unsigned Log2PowerOfTwo(size_t value) {
331 return static_cast<unsigned>(
332 48 __builtin_ctzll(static_cast<unsigned long long>(value)));
333 }
334
335 std::unique_ptr<base::MallocApi> allocator_;
336 std::unique_ptr<base::ThreadSafeDelayedRecycle> recycler_;
337 static constexpr int64 kRecycleDelayUs = 1000;
338 };
339
340 FACTORY_REGISTER(
341 ValueStore, DRAM_VALUE_STORE, DramValueStore, const BaseKVConfig&);
342