memory/shm_file.h
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | #pragma once | ||
| 2 | |||
| 3 | #include <fcntl.h> | ||
| 4 | #include <sys/mman.h> | ||
| 5 | #include <atomic> | ||
| 6 | #include <deque> | ||
| 7 | #include <fstream> | ||
| 8 | #include <memory> | ||
| 9 | #include <queue> | ||
| 10 | #include <string> | ||
| 11 | #include <utility> | ||
| 12 | #include <vector> | ||
| 13 | |||
| 14 | #include "base/base.h" | ||
| 15 | #include "base/factory.h" | ||
| 16 | #include "base/json.h" | ||
| 17 | #include "base/string.h" | ||
| 18 | #include "storage/nvm/pet_kv/persistence.h" | ||
| 19 | |||
| 20 | namespace base { | ||
| 21 | |||
| 22 | class PMMmapRegisterCenter { | ||
| 23 | struct MetaBlock { | ||
| 24 | static const int MM_META_NR = 1000; | ||
| 25 | |||
| 26 | ✗ | MetaBlock() { | |
| 27 | ✗ | mmMetaNr_ = 0; | |
| 28 | ✗ | magic_ = 0xdeadbeef; | |
| 29 | ✗ | for (int i = 0; i < MetaBlock::MM_META_NR; i++) | |
| 30 | ✗ | mmMeta_[i].magic_ = 0x66666666; | |
| 31 | ✗ | } | |
| 32 | struct MmMeta { | ||
| 33 | MmMeta() {} | ||
| 34 | char file_name_[64]; | ||
| 35 | uint64_t offset_; | ||
| 36 | uint64_t size_; | ||
| 37 | int magic_ = 0x66666666; | ||
| 38 | }; | ||
| 39 | union { | ||
| 40 | struct { | ||
| 41 | int magic_ = 0xdeadbeef; | ||
| 42 | int mmMetaNr_ = 0; | ||
| 43 | MmMeta mmMeta_[MM_META_NR]; | ||
| 44 | }; | ||
| 45 | char _[2 * 4096 * 4096LL]; | ||
| 46 | }; | ||
| 47 | }; | ||
| 48 | |||
| 49 | public: | ||
| 50 | ✗ | static PMMmapRegisterCenter* GetInstance() { | |
| 51 | ✗ | static PMMmapRegisterCenter instance; | |
| 52 | ✗ | return &instance; | |
| 53 | } | ||
| 54 | static constexpr int64_t align = 2 * 1024 * 1024LL; | ||
| 55 | // static constexpr int64_t align = 4 * 1024LL; | ||
| 56 | |||
| 57 | ✗ | void* Register(const std::string& name, uint64_t size) { | |
| 58 | ✗ | std::lock_guard<std::mutex> _(mutex_); | |
| 59 | ✗ | auto* item = FindIfExists(name); | |
| 60 | ✗ | if (item == nullptr) { | |
| 61 | ✗ | return append_register(name, size); | |
| 62 | ✗ | } else if (item->size_ != size) { | |
| 63 | // reinit MMAP | ||
| 64 | ✗ | LOG(INFO) << base::SFormat( | |
| 65 | "PM mmap {} registered size{} != existing{}", | ||
| 66 | name, | ||
| 67 | size, | ||
| 68 | ✗ | item->size_); | |
| 69 | ✗ | ReInitialize(); | |
| 70 | ✗ | return append_register(name, size); | |
| 71 | } else { | ||
| 72 | ✗ | LOG(INFO) << base::SFormat("PM mmap {} use existing map", name); | |
| 73 | |||
| 74 | ✗ | shmfile_mmap_addr_ = | |
| 75 | ✗ | std::max(shmfile_mmap_addr_.load(), | |
| 76 | ✗ | shmfile_mmap_data_addr_start_ + item->offset_ + item->size_); | |
| 77 | ✗ | return (void*)(shmfile_mmap_data_addr_start_ + item->offset_); | |
| 78 | } | ||
| 79 | ✗ | } | |
| 80 | |||
| 81 | std::pair<uint64_t, uint64_t> ForRDMAMemoryRegion() const { | ||
| 82 | return std::make_pair( | ||
| 83 | shmfile_mmap_data_addr_start_, | ||
| 84 | shmfile_mmap_addr_.load() - shmfile_mmap_data_addr_start_); | ||
| 85 | } | ||
| 86 | // this function is non-reentrant | ||
| 87 | ✗ | void ReInitialize() { | |
| 88 | ✗ | LOG(WARNING) << "reinitialize the whole PM space"; | |
| 89 | static std::atomic_int flag{false}; | ||
| 90 | ✗ | if (!flag.fetch_or(true)) { | |
| 91 | ✗ | LOG(WARNING) << "memset whole meta block"; | |
| 92 | ✗ | memset((void*)metaBlock_, 0, sizeof(MetaBlock)); | |
| 93 | ✗ | new (metaBlock_) MetaBlock(); | |
| 94 | } else { | ||
| 95 | ✗ | LOG(FATAL) << "ReInitialize is non-reentrant"; | |
| 96 | } | ||
| 97 | ✗ | CHECK(Valid()); | |
| 98 | ✗ | } | |
| 99 | |||
| 100 | enum class Backend { | ||
| 101 | kAnonymousDram, | ||
| 102 | kDevDax, | ||
| 103 | }; | ||
| 104 | |||
| 105 | struct Config { | ||
| 106 | Backend backend = Backend::kAnonymousDram; | ||
| 107 | int numa_id = 0; | ||
| 108 | }; | ||
| 109 | |||
| 110 | 2164 | static Config& GetConfig() { | |
| 111 | static Config config; | ||
| 112 | 2164 | return config; | |
| 113 | } | ||
| 114 | |||
| 115 | ✗ | static Backend BackendFromUseDram(bool use_dram) { | |
| 116 | ✗ | return use_dram ? Backend::kAnonymousDram : Backend::kDevDax; | |
| 117 | } | ||
| 118 | |||
| 119 | private: | ||
| 120 | ✗ | bool Valid() const { | |
| 121 | ✗ | if (metaBlock_->magic_ != 0xdeadbeef) | |
| 122 | ✗ | return false; | |
| 123 | ✗ | for (int i = 0; i < MetaBlock::MM_META_NR; i++) | |
| 124 | ✗ | if (metaBlock_->mmMeta_[i].magic_ != 0x66666666) | |
| 125 | ✗ | return false; | |
| 126 | ✗ | return true; | |
| 127 | } | ||
| 128 | |||
| 129 | ✗ | void* append_register(const std::string& name, uint64_t size) { | |
| 130 | ✗ | CHECK_NE(size, 0); | |
| 131 | // LOG(INFO) << fmt::format("PM mmap register {}, size={}", name, size); | ||
| 132 | ✗ | auto new_size = (size + align - 1) / align * align; | |
| 133 | ✗ | auto ret = shmfile_mmap_addr_.fetch_add(new_size); | |
| 134 | ✗ | LOG(ERROR) << "xmh " | |
| 135 | ✗ | << (shmfile_mmap_addr_ - shmfile_mmap_addr_start_) / | |
| 136 | ✗ | (1024 * 1024 * 1024LL); | |
| 137 | ✗ | CHECK_GE((uint64_t)ret, shmfile_mmap_data_addr_start_); | |
| 138 | ✗ | CHECK_LT((uint64_t)ret + size, | |
| 139 | shmfile_mmap_data_addr_start_ + dax_mm_size_); | ||
| 140 | ✗ | memset((void*)ret, 0, size); | |
| 141 | |||
| 142 | // persist meta data | ||
| 143 | ✗ | auto& item = metaBlock_->mmMeta_[metaBlock_->mmMetaNr_]; | |
| 144 | ✗ | std::strncpy(item.file_name_, name.c_str(), 50); | |
| 145 | ✗ | item.offset_ = ret - shmfile_mmap_data_addr_start_; | |
| 146 | ✗ | item.size_ = size; | |
| 147 | ✗ | metaBlock_->mmMetaNr_++; | |
| 148 | ✗ | CHECK_LE(metaBlock_->mmMetaNr_, MetaBlock::MM_META_NR); | |
| 149 | ✗ | base::clflushopt_range(metaBlock_, sizeof(MetaBlock)); | |
| 150 | ✗ | return (char*)ret; | |
| 151 | } | ||
| 152 | |||
| 153 | ✗ | MetaBlock::MmMeta* FindIfExists(const std::string& name) { | |
| 154 | ✗ | for (int i = 0; i < metaBlock_->mmMetaNr_; i++) { | |
| 155 | ✗ | auto& item = metaBlock_->mmMeta_[i]; | |
| 156 | ✗ | if (std::strncmp(item.file_name_, name.c_str(), 64) == 0) { | |
| 157 | ✗ | CHECK_EQ(item.magic_, 0x66666666); | |
| 158 | ✗ | return &item; | |
| 159 | } | ||
| 160 | } | ||
| 161 | ✗ | return nullptr; | |
| 162 | } | ||
| 163 | |||
| 164 | ✗ | PMMmapRegisterCenter() { | |
| 165 | ✗ | CHECK_NE(GetConfig().numa_id, -1); | |
| 166 | |||
| 167 | char* data; | ||
| 168 | ✗ | if (GetConfig().backend == Backend::kAnonymousDram) { | |
| 169 | // filename_ = folly::sformat("/dev/shm/big_file{}", GetConfig().numa_id); | ||
| 170 | ✗ | fd_ = 0; | |
| 171 | ✗ | LOG(WARNING) << "use dram mmap for PMMmapRegisterCenter"; | |
| 172 | ✗ | data = reinterpret_cast<char*>(mmap( | |
| 173 | nullptr, | ||
| 174 | ✗ | dax_mm_size_, | |
| 175 | PROT_READ | PROT_WRITE, | ||
| 176 | MAP_PRIVATE | MAP_POPULATE | MAP_ANONYMOUS, | ||
| 177 | -1, | ||
| 178 | 0)); | ||
| 179 | ✗ | metaBlock_ = (MetaBlock*)data; | |
| 180 | ✗ | CHECK_NE(data, MAP_FAILED) << "map failed"; | |
| 181 | ✗ | mlock(data, dax_mm_size_); | |
| 182 | ✗ | ReInitialize(); | |
| 183 | } else { | ||
| 184 | ✗ | filename_ = base::SFormat("/dev/dax{}.0", GetConfig().numa_id); | |
| 185 | ✗ | fd_ = open(filename_.c_str(), 0666); | |
| 186 | ✗ | if (fd_ < 0) { | |
| 187 | ✗ | LOG(FATAL) << "Failed to open file " << filename_ << ": " | |
| 188 | ✗ | << strerror(errno); | |
| 189 | } | ||
| 190 | ✗ | data = reinterpret_cast<char*>(mmap( | |
| 191 | nullptr, | ||
| 192 | ✗ | dax_mm_size_, | |
| 193 | PROT_READ | PROT_WRITE, | ||
| 194 | MAP_SHARED | MAP_POPULATE, | ||
| 195 | fd_, | ||
| 196 | 0)); | ||
| 197 | } | ||
| 198 | |||
| 199 | ✗ | LOG(INFO) << "mmap " << filename_ << " ; size = " << dax_mm_size_; | |
| 200 | ✗ | CHECK_NE(data, MAP_FAILED) << "map failed"; | |
| 201 | // read meta file | ||
| 202 | ✗ | metaBlock_ = (MetaBlock*)data; | |
| 203 | ✗ | if (!Valid()) { | |
| 204 | // first use this device | ||
| 205 | ✗ | LOG(ERROR) | |
| 206 | << "metadata of dax is not valid, maybe you first use this device;" | ||
| 207 | ✗ | << " Please use pm_command to reinit"; | |
| 208 | // ReInitialize(); | ||
| 209 | } | ||
| 210 | |||
| 211 | ✗ | *(uint64_t*)&shmfile_mmap_addr_start_ = (uint64_t)data; | |
| 212 | ✗ | *(uint64_t*)&shmfile_mmap_data_addr_start_ = | |
| 213 | ✗ | (uint64_t)data + sizeof(MetaBlock); | |
| 214 | ✗ | shmfile_mmap_addr_.store(shmfile_mmap_data_addr_start_); | |
| 215 | ✗ | } | |
| 216 | ✗ | ~PMMmapRegisterCenter() { | |
| 217 | ✗ | if (fd_ >= 0) { | |
| 218 | ✗ | LOG(INFO) << "ummap shm file: " << filename_ << ", size: " << dax_mm_size_ | |
| 219 | ✗ | << ", fd: " << fd_; | |
| 220 | ✗ | munmap((void*)shmfile_mmap_addr_start_, dax_mm_size_); | |
| 221 | ✗ | close(fd_); | |
| 222 | ✗ | fd_ = -1; | |
| 223 | } | ||
| 224 | ✗ | } | |
| 225 | |||
| 226 | std::string filename_; | ||
| 227 | // const int64_t dax_mm_size_ = 300 * 1024 * 1024 * 1024LL; | ||
| 228 | const int64_t dax_mm_size_ = 140 * 1024 * 1024 * 1024LL; | ||
| 229 | // const int64_t dax_mm_size_ = 500 * 1024 * 1024 * 1024LL; | ||
| 230 | // const int64_t dax_mm_size_ = 75 * 1024 * 1024 * 1024LL; | ||
| 231 | int fd_; | ||
| 232 | const uint64_t shmfile_mmap_addr_start_ = 0; | ||
| 233 | const uint64_t shmfile_mmap_data_addr_start_ = 0; | ||
| 234 | // const uint64_t shmfile_mmap_addr_start_ = 0xeff4200000ull; | ||
| 235 | std::atomic<uint64_t> shmfile_mmap_addr_; | ||
| 236 | |||
| 237 | std::mutex mutex_; | ||
| 238 | |||
| 239 | MetaBlock* metaBlock_ = nullptr; | ||
| 240 | }; | ||
| 241 | |||
| 242 | class ShmFile { | ||
| 243 | public: | ||
| 244 | 1248 | ShmFile() : data_(NULL), size_(0) {} | |
| 245 | 1248 | virtual ~ShmFile() = default; | |
| 246 | |||
| 247 | static std::unique_ptr<ShmFile> New(const json& config); | ||
| 248 | static json ConfigForMedium( | ||
| 249 | const std::string& medium, const std::string& filename, int64 size); | ||
| 250 | |||
| 251 | virtual bool Initialize(const json& config) = 0; | ||
| 252 | virtual void Clear() = 0; | ||
| 253 | 691474 | char* Data() const { return data_; } | |
| 254 | 4 | int64 Size() const { return size_; } | |
| 255 | 2 | const std::string& filename() const { return filename_; } | |
| 256 | |||
| 257 | protected: | ||
| 258 | static std::string ConfigFilename(const json& config); | ||
| 259 | static int64 ConfigSize(const json& config); | ||
| 260 | |||
| 261 | std::string filename_; | ||
| 262 | char* data_; | ||
| 263 | int64 size_; | ||
| 264 | |||
| 265 | private: | ||
| 266 | DISALLOW_COPY_AND_ASSIGN(ShmFile); | ||
| 267 | }; | ||
| 268 | |||
| 269 | class FsDaxShmFile : public ShmFile { | ||
| 270 | public: | ||
| 271 | 6 | explicit FsDaxShmFile(const json& config) : fd_(-1) {} | |
| 272 | 12 | ~FsDaxShmFile() override { Clear(); } | |
| 273 | bool Initialize(const json& config) override; | ||
| 274 | void Clear() override; | ||
| 275 | |||
| 276 | private: | ||
| 277 | int fd_; | ||
| 278 | DISALLOW_COPY_AND_ASSIGN(FsDaxShmFile); | ||
| 279 | }; | ||
| 280 | |||
| 281 | class AnonyDramShmFile : public ShmFile { | ||
| 282 | public: | ||
| 283 | 1242 | explicit AnonyDramShmFile(const json& config) {} | |
| 284 | 2484 | ~AnonyDramShmFile() override { Clear(); } | |
| 285 | bool Initialize(const json& config) override; | ||
| 286 | void Clear() override; | ||
| 287 | |||
| 288 | private: | ||
| 289 | DISALLOW_COPY_AND_ASSIGN(AnonyDramShmFile); | ||
| 290 | }; | ||
| 291 | |||
| 292 | class DevDaxShmFile : public ShmFile { | ||
| 293 | public: | ||
| 294 | ✗ | explicit DevDaxShmFile(const json& config) {} | |
| 295 | ✗ | ~DevDaxShmFile() override { Clear(); } | |
| 296 | bool Initialize(const json& config) override; | ||
| 297 | void Clear() override; | ||
| 298 | |||
| 299 | private: | ||
| 300 | DISALLOW_COPY_AND_ASSIGN(DevDaxShmFile); | ||
| 301 | }; | ||
| 302 | |||
| 303 | } // namespace base | ||
| 304 |