| Line |
Branch |
Exec |
Source |
| 1 |
|
|
#include "pet_kv.h" |
| 2 |
|
|
|
| 3 |
|
|
#include "base/bind_core.h" |
| 4 |
|
|
|
| 5 |
|
|
#include <algorithm> |
| 6 |
|
|
#include <atomic> |
| 7 |
|
|
#include <cstdio> |
| 8 |
|
|
#include <fstream> |
| 9 |
|
|
#include <iostream> |
| 10 |
|
|
#include <utility> |
| 11 |
|
|
|
| 12 |
|
|
#define FREE_SHM(ptr) shm_recycle_->Recycle(ptr); |
| 13 |
|
|
|
| 14 |
|
|
namespace base { |
| 15 |
|
|
|
| 16 |
|
|
constexpr bool IGNORE_LOAD_FACTOR = true; |
| 17 |
|
|
|
| 18 |
|
|
PetKVData::Config PetKVData::kConfig = PetKVData::Config(); |
| 19 |
|
|
|
| 20 |
|
✗ |
PetKV::PetKV(const std::string& shm_dir, |
| 21 |
|
|
int64 memory_size, |
| 22 |
|
|
int capacity, |
| 23 |
|
✗ |
int pre_known_value_size) |
| 24 |
|
✗ |
: start_ts_(base::GetTimestamp()), |
| 25 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
shm_dir_(shm_dir), |
| 26 |
|
✗ |
pre_known_value_size_(pre_known_value_size) { |
| 27 |
|
✗ |
ts_getter_ = &AsyncTimeHelper::GetTimestamp; |
| 28 |
|
|
#if 1 |
| 29 |
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
|
✗ |
if (pre_known_value_size == 0) |
| 30 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
shm_malloc_ = new PersistMemoryPool<false>( |
| 31 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
shm_dir + "/value", |
| 32 |
|
|
memory_size, |
| 33 |
0/4
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
✗ |
{8 + 32, 8 + 64, 8 + 128, 8 + 512, 8 + 1024}); |
| 34 |
|
|
else |
| 35 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
shm_malloc_ = new PersistMemoryPool<true>( |
| 36 |
0/6
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
|
✗ |
shm_dir + "/value", memory_size, {pre_known_value_size}); |
| 37 |
|
|
#else |
| 38 |
|
|
shm_malloc_ = new PersistLoopShmMalloc(shm_dir + "/value", memory_size); |
| 39 |
|
|
#endif |
| 40 |
|
|
|
| 41 |
|
|
// shm_recycle_ = new DirectRecycle (shm_malloc_); |
| 42 |
|
|
// shm_recycle_ = new DelayedRecycle(shm_malloc_); |
| 43 |
0/4
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
✗ |
shm_recycle_ = new ShmEpochRecycle(shm_malloc_); |
| 44 |
|
|
|
| 45 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
auto begin_ts = base::GetTimestamp() / 1000; |
| 46 |
0/4
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
✗ |
LOG(INFO) << "PetKV " << shm_dir |
| 47 |
0/8
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
|
✗ |
<< " start initialize, pre_ms: " << (begin_ts - start_ts_ / 1000); |
| 48 |
|
|
|
| 49 |
|
✗ |
uint64_t dict_size = capacity; |
| 50 |
|
|
auto dict_memory_size = |
| 51 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
ShmKDoubleDict::MemorySize(dict_size, IGNORE_LOAD_FACTOR); |
| 52 |
0/4
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
✗ |
LOG(INFO) << fmt::format( |
| 53 |
|
|
"PetKV allocate {:.2f} GB; capacity={} for dict", |
| 54 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
(double)dict_memory_size / 1024 / 1024 / 1024, |
| 55 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
dict_size); |
| 56 |
0/6
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
|
✗ |
dict_shm_file_ = base::ShmFile::New(base::ShmFile::ConfigForMedium( |
| 57 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
"DRAM", shm_dir + "/dict", dict_memory_size)); |
| 58 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
if (!dict_shm_file_) { |
| 59 |
0/4
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
✗ |
base::file_util::Delete(shm_dir + "/dict", false); |
| 60 |
0/6
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
|
✗ |
dict_shm_file_ = base::ShmFile::New(base::ShmFile::ConfigForMedium( |
| 61 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
"DRAM", shm_dir + "/dict", dict_memory_size)); |
| 62 |
0/12
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✗ Branch 17 not taken.
✗ Branch 18 not taken.
|
✗ |
CHECK(dict_shm_file_); |
| 63 |
0/8
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
|
✗ |
LOG(INFO) << "Reinitialize shm dict size: " << dict_memory_size; |
| 64 |
|
|
} |
| 65 |
|
|
|
| 66 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
auto dict_file_init_ts = base::GetTimestamp() / 1000; |
| 67 |
|
|
|
| 68 |
0/6
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
|
✗ |
valid_shm_file_ = base::ShmFile::New(base::ShmFile::ConfigForMedium( |
| 69 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
"DRAM", shm_dir + "/valid", valid_file_size)); |
| 70 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
if (!valid_shm_file_) { |
| 71 |
0/4
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
✗ |
base::file_util::Delete(shm_dir + "/valid", false); |
| 72 |
0/6
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
|
✗ |
valid_shm_file_ = base::ShmFile::New(base::ShmFile::ConfigForMedium( |
| 73 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
"DRAM", shm_dir + "/valid", valid_file_size)); |
| 74 |
0/12
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✗ Branch 17 not taken.
✗ Branch 18 not taken.
|
✗ |
CHECK(valid_shm_file_); |
| 75 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
shm_malloc_->Initialize(); |
| 76 |
|
|
} |
| 77 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
auto valid_file_init_ts = base::GetTimestamp() / 1000; |
| 78 |
|
|
|
| 79 |
|
✗ |
dict_ = reinterpret_cast<ShmKDoubleDict*>(dict_shm_file_->Data()); |
| 80 |
0/4
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
|
✗ |
if (!dict_->Valid(dict_shm_file_->Size())) { |
| 81 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
dict_->Initialize(dict_size, IGNORE_LOAD_FACTOR); |
| 82 |
|
|
} else { |
| 83 |
0/10
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
|
✗ |
LOG(INFO) << "Before recovery: [shm_malloc] " << shm_malloc_->GetInfo(); |
| 84 |
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
✗ |
dict_->Reload( |
| 85 |
|
✗ |
dict_size, IGNORE_LOAD_FACTOR, [&](uint64 key, PetKVData value) { |
| 86 |
|
✗ |
shm_malloc_->AddMallocs4Recovery(value.shm_malloc_offset()); |
| 87 |
|
✗ |
}); |
| 88 |
0/10
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
|
✗ |
LOG(INFO) << "After recovery: [Dict] find " << dict_->Size() << " kvs"; |
| 89 |
0/10
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
|
✗ |
LOG(INFO) << "After recovery: [shm_malloc] " << shm_malloc_->GetInfo(); |
| 90 |
|
|
} |
| 91 |
|
|
|
| 92 |
0/10
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 16 not taken.
✗ Branch 17 not taken.
|
✗ |
CHECK_EQ(dict_->Size(), shm_malloc_->total_malloc()); |
| 93 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
auto all_valid_ts = base::GetTimestamp() / 1000; |
| 94 |
0/10
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
|
✗ |
LOG(INFO) << "PetKV " << shm_dir << " initialize succeed, dict_file_init_ms: " |
| 95 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
<< (dict_file_init_ts - begin_ts) |
| 96 |
0/4
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
✗ |
<< ", dict_init_ms: " << (valid_file_init_ts - dict_file_init_ts) |
| 97 |
0/4
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
✗ |
<< ", all_valid_ms: " << (all_valid_ts - valid_file_init_ts); |
| 98 |
|
✗ |
} |
| 99 |
|
|
|
| 100 |
|
✗ |
PetKV::~PetKV() { |
| 101 |
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
|
✗ |
delete shm_recycle_; |
| 102 |
|
✗ |
LOG(INFO) << "shm kv safe quit, remain dict size = " << dict_->Size(); |
| 103 |
|
✗ |
LOG(INFO) << shm_malloc_->GetInfo(); |
| 104 |
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
|
✗ |
delete shm_malloc_; |
| 105 |
|
✗ |
} |
| 106 |
|
|
|
| 107 |
|
✗ |
bool PetKV::Valid() { |
| 108 |
|
✗ |
const int kCheckThreadNum = 3; |
| 109 |
|
✗ |
auto begin_ts = base::GetTimestamp() / 1000; |
| 110 |
0/2
✗ Branch 3 not taken.
✗ Branch 4 not taken.
|
✗ |
if (!dict_->Valid(dict_shm_file_->Size())) { |
| 111 |
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
✗ |
LOG(ERROR) << "dict load error: " << dict_shm_file_->filename() |
| 112 |
0/8
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
|
✗ |
<< ", size: " << dict_shm_file_->Size(); |
| 113 |
|
✗ |
return false; |
| 114 |
|
|
} |
| 115 |
|
✗ |
auto dict_check_ts = base::GetTimestamp() / 1000; |
| 116 |
|
✗ |
auto check_ts = base::GetTimestamp() / 1000; |
| 117 |
|
✗ |
auto shm_free_ts = base::GetTimestamp() / 1000; |
| 118 |
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
✗ |
LOG(INFO) << "shm kv " << shm_dir_ |
| 119 |
0/8
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
|
✗ |
<< " check valid, dict_check_ms: " << (dict_check_ts - begin_ts) |
| 120 |
0/4
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
✗ |
<< ", check_ms: " << (check_ts - dict_check_ts) |
| 121 |
0/4
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
✗ |
<< ", shm_free_ms: " << (shm_free_ts - check_ts); |
| 122 |
|
|
|
| 123 |
|
✗ |
return true; |
| 124 |
|
|
} |
| 125 |
|
|
|
| 126 |
|
✗ |
bool PetKV::Update(uint64 key, const char* log, int log_size) { |
| 127 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
base::AutoLock lock(modify_lock_); |
| 128 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
auto [p_cache, exists] = dict_->GetReturnPtr(key); |
| 129 |
|
|
|
| 130 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
char* value = shm_malloc_->New(log_size); |
| 131 |
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
|
✗ |
if (nullptr == value) |
| 132 |
|
✗ |
return false; |
| 133 |
|
✗ |
memcpy(value, log, log_size); |
| 134 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
base::clflushopt_range(value, log_size); |
| 135 |
0/4
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
✗ |
PetKVData new_cache(shm_malloc_->GetMallocOffset(value)); |
| 136 |
|
|
|
| 137 |
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
|
✗ |
if (!exists) { |
| 138 |
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
✗ |
p_cache = dict_->Insert(key, new_cache, nullptr, true); |
| 139 |
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
|
✗ |
if (p_cache == nullptr) { |
| 140 |
0/16
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 15 not taken.
✗ Branch 16 not taken.
✗ Branch 18 not taken.
✗ Branch 19 not taken.
✗ Branch 20 not taken.
✗ Branch 21 not taken.
|
✗ |
RECSTORE_LOG_EVERY_MS(WARNING, 2000) << "Update fail: " << key; |
| 141 |
0/4
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
|
✗ |
FREE_SHM(shm_malloc_->GetMallocData(new_cache.shm_malloc_offset())); |
| 142 |
|
✗ |
return false; |
| 143 |
|
|
} |
| 144 |
|
✗ |
p_cache->SetShmMallocOffset(new_cache.shm_malloc_offset()); |
| 145 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
p_cache->DoFlush(); |
| 146 |
|
|
} else { |
| 147 |
0/4
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
|
✗ |
FREE_SHM(shm_malloc_->GetMallocData(p_cache->shm_malloc_offset())); |
| 148 |
|
✗ |
p_cache->SetShmMallocOffset(new_cache.shm_malloc_offset()); |
| 149 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
p_cache->DoFlush(); |
| 150 |
|
|
} |
| 151 |
|
|
|
| 152 |
|
|
// re-read the KV pair |
| 153 |
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
✗ |
if (p_cache->shm_malloc_offset() != new_cache.shm_malloc_offset()) |
| 154 |
0/4
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
|
✗ |
FREE_SHM(shm_malloc_->GetMallocData(new_cache.shm_malloc_offset())); |
| 155 |
|
✗ |
return true; |
| 156 |
|
✗ |
} |
| 157 |
|
|
|
| 158 |
|
✗ |
std::string PetKV::GetInfo() { |
| 159 |
|
✗ |
std::string info; |
| 160 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
info.append( |
| 161 |
0/4
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
|
✗ |
base::SFormat("cache: {}/{}\n", dict_->Size(), dict_->Capacity())); |
| 162 |
0/4
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
✗ |
info.append(shm_malloc_->GetInfo()); |
| 163 |
|
|
|
| 164 |
0/6
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
|
✗ |
LOG(INFO) << base::SFormat( |
| 165 |
|
|
"LoadFactor : {}/{}={}", |
| 166 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
dict_->Size(), |
| 167 |
|
✗ |
dict_->Capacity(), |
| 168 |
0/2
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
✗ |
dict_->Size() / (float)dict_->Capacity()); |
| 169 |
0/6
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
|
✗ |
LOG(INFO) << "MemoryUtil: " |
| 170 |
|
✗ |
<< dict_->Size() * 16 / |
| 171 |
|
✗ |
(float)ShmKDoubleDict::MemorySize( |
| 172 |
0/4
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
|
✗ |
(float)dict_->Capacity(), IGNORE_LOAD_FACTOR); |
| 173 |
|
|
|
| 174 |
|
✗ |
return info; |
| 175 |
|
✗ |
} |
| 176 |
|
|
|
| 177 |
|
✗ |
PetMultiKV::PetMultiKV(const std::vector<std::string>& shm_dir, |
| 178 |
|
|
int shard_num, |
| 179 |
|
|
int64 shard_memory, |
| 180 |
|
|
int shard_cache_capacity, |
| 181 |
|
✗ |
int pre_known_value_size) |
| 182 |
|
✗ |
: shm_dir_(shm_dir), |
| 183 |
|
✗ |
shard_num_(shard_num), |
| 184 |
|
✗ |
shard_memory_(shard_memory), |
| 185 |
|
✗ |
shard_cache_capacity_(shard_cache_capacity), |
| 186 |
|
✗ |
pre_known_value_size_(pre_known_value_size) { |
| 187 |
0/12
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✗ Branch 17 not taken.
✗ Branch 18 not taken.
|
✗ |
CHECK(!shm_dir_.empty()); |
| 188 |
0/2
✗ Branch 5 not taken.
✗ Branch 6 not taken.
|
✗ |
for (const auto& dir : shm_dir) { |
| 189 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
base::file_util::CreateDirectory(dir); |
| 190 |
|
|
} |
| 191 |
|
|
|
| 192 |
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
|
✗ |
if (shard_memory_ >= (1LL << 35)) { |
| 193 |
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
|
✗ |
for (int i = 36; i <= 64; ++i) { |
| 194 |
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
|
✗ |
if ((1LL << i) > shard_memory_) { |
| 195 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
PetKVData::kConfig = PetKVData::Config(i - 3); |
| 196 |
0/6
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
|
✗ |
LOG(INFO) << "shard_memory over 32G, change expire_time bit: " |
| 197 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
<< PetKVData::kConfig.kExpireBit |
| 198 |
0/4
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
✗ |
<< ", shm_offset bit: " << (i - 3); |
| 199 |
|
✗ |
break; |
| 200 |
|
|
} |
| 201 |
|
|
} |
| 202 |
|
|
} |
| 203 |
|
|
|
| 204 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
shm_kv_.resize(shard_num); |
| 205 |
|
✗ |
std::vector<std::thread> thread_pool; |
| 206 |
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
|
✗ |
for (int i = 0; i < shard_num; ++i) { |
| 207 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
thread_pool.emplace_back([this, i]() { |
| 208 |
|
✗ |
base::auto_bind_core(); |
| 209 |
|
✗ |
LoadShard(i); |
| 210 |
|
✗ |
}); |
| 211 |
|
|
} |
| 212 |
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
|
✗ |
for (int i = 0; i < shard_num; ++i) { |
| 213 |
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
✗ |
thread_pool[i].join(); |
| 214 |
|
|
} |
| 215 |
|
✗ |
} |
| 216 |
|
|
|
| 217 |
|
✗ |
void PetMultiKV::LoadShard(int shard) { |
| 218 |
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
✗ |
base::file_util::CreateDirectory(shm_dir(shard)); |
| 219 |
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
✗ |
LOG(INFO) << "PetMultiKV LoadShard shm_file:" << shm_dir(shard) |
| 220 |
0/10
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
|
✗ |
<< ", shard memory_size:" << shard_memory_ |
| 221 |
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
|
✗ |
<< ", shard_cache_capacity:" << shard_cache_capacity_ |
| 222 |
0/6
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
|
✗ |
<< ", pre_known_value_size:" << pre_known_value_size_; |
| 223 |
|
|
|
| 224 |
|
✗ |
shm_kv_[shard] = new PetKV( |
| 225 |
|
✗ |
shm_dir(shard), |
| 226 |
|
|
shard_memory_, |
| 227 |
|
|
shard_cache_capacity_, |
| 228 |
0/4
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
✗ |
pre_known_value_size_); |
| 229 |
|
✗ |
} |
| 230 |
|
✗ |
std::string PetMultiKV::GetInfo() { |
| 231 |
|
✗ |
std::string info; |
| 232 |
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
|
✗ |
for (int shard = 0; shard < shard_num_; ++shard) { |
| 233 |
0/8
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
|
✗ |
info.append("shard " + base::IntToString(shard) + "\n"); |
| 234 |
0/4
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
|
✗ |
info.append(shm_kv_[shard]->GetInfo()); |
| 235 |
|
|
} |
| 236 |
|
✗ |
return info; |
| 237 |
|
✗ |
} |
| 238 |
|
|
|
| 239 |
|
✗ |
std::string PetMultiKV::shm_dir(int shard_id) { |
| 240 |
|
✗ |
int idx = shard_id % shm_dir_.size(); |
| 241 |
0/4
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
|
✗ |
return shm_dir_[idx] + "/" + base::IntToString(shard_id); |
| 242 |
|
|
} |
| 243 |
|
|
|
| 244 |
|
|
} // namespace base |
| 245 |
|
|
|