GCC Code Coverage Report

Directory: src/
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 0.0% 0 / 0 / 788
Functions: 0.0% 0 / 0 / 39
Branches: 0.0% 0 / 0 / 1306
ps/rdma/petps_server.cc
Line Branch Exec Source
1 #include <folly/init/Init.h>
2
3 #include <boost/coroutine2/all.hpp>
4
5 #include <atomic>
6 #include <array>
7 #include <algorithm>
8 #include <chrono>
9 #include <condition_variable>
10 #include <deque>
11 #include <cstdint>
12 #include <cstdlib>
13 #include <cstring>
14 #include <fstream>
15 #include <iostream>
16 #include <limits>
17 #include <memory>
18 #include <mutex>
19 #include <stdexcept>
20 #include <string>
21 #include <thread>
22 #include <vector>
23
24 #include "base/bind_core.h"
25 #include "base/config.h"
26 #include "base/log.h"
27 #include "base/timer.h"
28 #include "memory/shm_file.h"
29 #include "ps/rdma/rdma_common.h"
30 #include "ps/base/cache_ps_impl.h"
31 #include "ps/rdma/control_plane.h"
32 #include "ps/rdma/rc_options.h"
33 #include "ps/rdma/rc_transport.h"
34 #include "ps/rdma/rdma_protocol.h"
35 #include "ps/rdma/rdma_status.h"
36
37 DEFINE_string(config_path, "", "config file path");
38 DEFINE_int32(thread_num, 1, "RC write poll thread count");
39 DECLARE_int32(global_id);
40 DECLARE_int32(num_server_processes);
41 DECLARE_int32(num_client_processes);
42 DEFINE_int32(value_size, 128, "embedding row bytes");
43 DEFINE_int32(max_kv_num_per_request, 500, "max keys per request");
44 DEFINE_bool(use_dram, false, "unused compatibility flag");
45 DEFINE_int32(numa_id, 0, "NUMA node id for mmap and core binding");
46
47 namespace {
48
49 using petps::Exchange;
50 using petps::NamespaceToken;
51 using petps::NowNs;
52
53 constexpr std::size_t kMaxDirectSgesPerWr = 32;
54
55 ✗ bool ShouldTraceRdmaGet() {
56 ✗ static const bool enabled = [] {
57 ✗ const char* env = std::getenv("RECSTORE_RDMA_GET_TRACE");
58 ✗ return env != nullptr && std::string(env) != "0";
59 ✗ }();
60 ✗ return enabled;
61 }
62
63 ✗ std::uint64_t RdmaGetTraceInterval() {
64 ✗ static const std::uint64_t interval = [] {
65 ✗ const char* env = std::getenv("RECSTORE_RDMA_GET_TRACE_INTERVAL");
66 ✗ if (env == nullptr) {
67 ✗ return std::uint64_t{5000};
68 }
69 const auto parsed =
70 ✗ static_cast<std::uint64_t>(std::strtoull(env, nullptr, 10));
71 ✗ return parsed == 0 ? std::uint64_t{5000} : parsed;
72 ✗ }();
73 ✗ return interval;
74 }
75
76 ✗ std::string TimestampNow() {
77 ✗ const auto now = std::chrono::system_clock::now().time_since_epoch();
78 return std::to_string(
79 ✗ std::chrono::duration_cast<std::chrono::microseconds>(now).count());
80 }
81
82 ✗ int ResolveShardId(const nlohmann::json& config) {
83 ✗ const int default_shard = FLAGS_global_id;
84 ✗ if (!config.contains("cache_ps") || !config["cache_ps"].is_object()) {
85 ✗ return default_shard;
86 }
87 ✗ const auto& cache_ps = config["cache_ps"];
88 ✗ if (cache_ps.contains("servers") && cache_ps["servers"].is_array()) {
89 ✗ for (const auto& server : cache_ps["servers"]) {
90 ✗ if (server.value("shard", -1) == FLAGS_global_id) {
91 ✗ return server.value("shard", default_shard);
92 }
93 }
94 }
95 ✗ return default_shard;
96 }
97
98 ✗ void NormalizeDramValuePath(nlohmann::json* base_kv_config) {
99 ✗ if (base_kv_config == nullptr || !base_kv_config->is_object()) {
100 ✗ return;
101 }
102 ✗ if (!base_kv_config->contains("value") ||
103 ✗ !(*base_kv_config)["value"].is_object()) {
104 ✗ return;
105 }
106 ✗ auto& value_cfg = (*base_kv_config)["value"];
107 const std::string value_type =
108 ✗ value_cfg.value("type", std::string("DRAM_VALUE_STORE"));
109 ✗ if (value_type != "DRAM_VALUE_STORE") {
110 ✗ return;
111 }
112 ✗ const std::string path = value_cfg.value("path", std::string());
113 ✗ if (path.empty() || path.rfind("/dev/shm", 0) == 0) {
114 ✗ return;
115 }
116 ✗ value_cfg["path"] = "/dev/shm/recstore_rdma_rc_" + TimestampNow() + "/value";
117 ✗ }
118
119 class PetPSServer {
120 public:
121 ✗ PetPSServer(CachePS* cache_ps,
122 int thread_count,
123 int shard_id,
124 const std::string& namespace_token)
125 ✗ : cache_ps_(cache_ps),
126 ✗ thread_count_(thread_count),
127 ✗ shard_id_(shard_id),
128 ✗ control_plane_client_(petps::RdmaControlPlaneEndpoint{
129 FLAGS_rdma_control_plane_host,
130 FLAGS_rdma_control_plane_port,
131 FLAGS_rdma_control_plane_timeout_ms,
132 ✗ }) {
133 ✗ petps::RcTransportConfig config;
134 ✗ config.shard_id = shard_id_;
135 ✗ config.num_clients =
136 ✗ FLAGS_rdma_rc_num_logical_clients >= 0
137 ✗ ? FLAGS_rdma_rc_num_logical_clients
138 : FLAGS_num_client_processes;
139 ✗ config.qps_per_client_per_shard = FLAGS_rdma_rc_qps_per_client_per_shard;
140 ✗ config.slots_per_qp = FLAGS_rdma_rc_slots_per_qp;
141 ✗ config.request_slot_bytes =
142 ✗ static_cast<std::size_t>(FLAGS_rdma_rc_request_slot_bytes);
143 ✗ config.response_slot_bytes =
144 ✗ static_cast<std::size_t>(FLAGS_rdma_rc_response_slot_bytes);
145 ✗ config.control_plane_host = FLAGS_rdma_control_plane_host;
146 ✗ config.control_plane_port = FLAGS_rdma_control_plane_port;
147 ✗ config.control_plane_timeout_ms = FLAGS_rdma_control_plane_timeout_ms;
148 ✗ config.namespace_token = namespace_token;
149 ✗ transport_ = std::make_unique<petps::RcShardServerTransport>(config);
150 ✗ const auto backing = cache_ps_->GetRDMABackingRegion();
151 ✗ if (backing.data != nullptr && backing.size > 0) {
152 ✗ transport_->RegisterLocalMemoryRegion(backing.data, backing.size);
153 ✗ LOG(INFO) << "component=rdma_rc_server event=value_region_registered"
154 ✗ << " bytes=" << backing.size;
155 } else {
156 ✗ LOG(INFO) << "component=rdma_rc_server event=value_region_unavailable";
157 }
158 ✗ last_seq_.assign(
159 ✗ static_cast<std::size_t>(transport_->TotalSlots()), std::uint64_t{0});
160 ✗ inflight_seq_.assign(
161 ✗ static_cast<std::size_t>(transport_->TotalSlots()), std::uint64_t{0});
162 ✗ get_payload_worker_count_ = FLAGS_rdma_rc_server_get_workers;
163 ✗ if (get_payload_worker_count_ < 0) {
164 ✗ LOG(FATAL) << "--rdma_rc_server_get_workers must be non-negative";
165 }
166 ✗ poller_profiles_.reserve(
167 ✗ static_cast<std::size_t>(std::max(1, thread_count_)));
168 ✗ for (int i = 0; i < std::max(1, thread_count_); ++i) {
169 ✗ poller_profiles_.emplace_back(std::make_unique<PollerProfile>());
170 }
171 ✗ get_payload_completions_.resize(
172 ✗ static_cast<std::size_t>(std::max(1, thread_count_)));
173 ✗ }
174
175 ✗ void Run() {
176 ✗ StartGetPayloadWorkers();
177 ✗ for (int i = 0; i < thread_count_; ++i) {
178 ✗ threads_.emplace_back(&PetPSServer::PollingThread, this, i);
179 }
180 ✗ }
181
182 private:
183 struct GetPayloadTask {
184 int slot = -1;
185 int client_id = -1;
186 int qp_index = -1;
187 int slot_in_qp = -1;
188 int poll_thread_id = -1;
189 std::uint64_t seq = 0;
190 petps::RequestDescriptor descriptor{};
191 const char* payload = nullptr;
192 petps::RcShardServerTransport::ResponseView response{};
193 };
194
195 struct GetPayloadCompletion {
196 int slot = -1;
197 int client_id = -1;
198 int qp_index = -1;
199 int slot_in_qp = -1;
200 int poll_thread_id = -1;
201 std::uint64_t seq = 0;
202 petps::RcShardServerTransport::ResponseView response{};
203 bool payload_written_direct = false;
204 };
205
206 struct ProfileCounters {
207 std::atomic<std::uint64_t> scan_rounds{0};
208 std::atomic<std::uint64_t> scanned_slots{0};
209 std::atomic<std::uint64_t> ready_slots{0};
210 std::atomic<std::uint64_t> not_ready_slots{0};
211 std::atomic<std::uint64_t> zero_seq_ready{0};
212 std::atomic<std::uint64_t> duplicate_seq_ready{0};
213 std::atomic<std::uint64_t> inflight_seq_ready{0};
214 std::atomic<std::uint64_t> empty_scan_rounds{0};
215 std::atomic<std::uint64_t> max_ready_per_round{0};
216 std::atomic<std::uint64_t> handled_get{0};
217 std::atomic<std::uint64_t> handled_put{0};
218 std::atomic<std::uint64_t> handled_update{0};
219 std::atomic<std::uint64_t> handled_init{0};
220 std::atomic<std::uint64_t> invalid_descriptor{0};
221 std::atomic<std::uint64_t> wrong_shard{0};
222 std::atomic<std::uint64_t> handle_get_ns{0};
223 std::atomic<std::uint64_t> get_batch_get_ns{0};
224 std::atomic<std::uint64_t> get_index_lookup_ns{0};
225 std::atomic<std::uint64_t> get_zero_fill_ns{0};
226 std::atomic<std::uint64_t> get_row_copy_ns{0};
227 std::atomic<std::uint64_t> get_rows{0};
228 std::atomic<std::uint64_t> get_value_bytes{0};
229 std::atomic<std::uint64_t> get_missing_rows{0};
230 std::atomic<std::uint64_t> get_direct_sg{0};
231 std::atomic<std::uint64_t> get_direct_sg_fallback{0};
232 std::atomic<std::uint64_t> get_direct_sg_ns{0};
233 std::atomic<std::uint64_t> get_direct_sg_wr{0};
234 std::atomic<std::uint64_t> handle_put_ns{0};
235 std::atomic<std::uint64_t> handle_update_ns{0};
236 std::atomic<std::uint64_t> handle_init_ns{0};
237 std::atomic<std::uint64_t> complete_response_ns{0};
238 std::atomic<std::uint64_t> poll_loop_ns{0};
239 std::atomic<std::uint64_t> next_report_ns{0};
240 };
241
242 struct PollerProfile {
243 std::atomic<std::uint64_t> scan_rounds{0};
244 std::atomic<std::uint64_t> scanned_slots{0};
245 std::atomic<std::uint64_t> ready_slots{0};
246 std::atomic<std::uint64_t> not_ready_slots{0};
247 std::atomic<std::uint64_t> duplicate_seq_ready{0};
248 std::atomic<std::uint64_t> inflight_seq_ready{0};
249 std::atomic<std::uint64_t> handled_get{0};
250 std::atomic<std::uint64_t> poll_loop_ns{0};
251 };
252
253 static void
254 ✗ UpdateMax(std::atomic<std::uint64_t>* value, std::uint64_t candidate) {
255 ✗ std::uint64_t current = value->load(std::memory_order_relaxed);
256 ✗ while (candidate > current &&
257 ✗ !value->compare_exchange_weak(
258 current, candidate, std::memory_order_relaxed)) {
259 }
260 ✗ }
261
262 ✗ void MaybeReportProfile(int thread_id) {
263 ✗ if (FLAGS_rdma_rc_profile_interval_ms <= 0 || thread_id != 0) {
264 ✗ return;
265 }
266 ✗ const std::uint64_t now = NowNs();
267 ✗ const std::uint64_t interval =
268 ✗ static_cast<std::uint64_t>(FLAGS_rdma_rc_profile_interval_ms) * 1000000;
269 std::uint64_t expected =
270 ✗ profile_.next_report_ns.load(std::memory_order_relaxed);
271 ✗ if (expected == 0) {
272 ✗ profile_.next_report_ns.compare_exchange_strong(
273 expected, now + interval, std::memory_order_relaxed);
274 ✗ return;
275 }
276 ✗ if (now < expected ||
277 ✗ !profile_.next_report_ns.compare_exchange_strong(
278 expected, now + interval, std::memory_order_relaxed)) {
279 ✗ return;
280 }
281
282 ✗ const std::uint64_t scan_rounds = Exchange(&profile_.scan_rounds);
283 ✗ const std::uint64_t scanned_slots = Exchange(&profile_.scanned_slots);
284 ✗ const std::uint64_t ready_slots = Exchange(&profile_.ready_slots);
285 ✗ const std::uint64_t not_ready_slots = Exchange(&profile_.not_ready_slots);
286 ✗ const std::uint64_t zero_seq_ready = Exchange(&profile_.zero_seq_ready);
287 const std::uint64_t duplicate_seq_ready =
288 ✗ Exchange(&profile_.duplicate_seq_ready);
289 const std::uint64_t inflight_seq_ready =
290 ✗ Exchange(&profile_.inflight_seq_ready);
291 const std::uint64_t empty_scan_rounds =
292 ✗ Exchange(&profile_.empty_scan_rounds);
293 const std::uint64_t max_ready_per_round =
294 ✗ Exchange(&profile_.max_ready_per_round);
295 ✗ const std::uint64_t handled_get = Exchange(&profile_.handled_get);
296 ✗ const std::uint64_t handled_put = Exchange(&profile_.handled_put);
297 ✗ const std::uint64_t handled_update = Exchange(&profile_.handled_update);
298 ✗ const std::uint64_t handled_init = Exchange(&profile_.handled_init);
299 ✗ const std::uint64_t complete_count =
300 ✗ handled_get + handled_put + handled_update + handled_init;
301 ✗ const std::uint64_t handle_get_ns = Exchange(&profile_.handle_get_ns);
302 ✗ const std::uint64_t get_batch_get_ns = Exchange(&profile_.get_batch_get_ns);
303 const std::uint64_t get_index_lookup_ns =
304 ✗ Exchange(&profile_.get_index_lookup_ns);
305 ✗ const std::uint64_t get_zero_fill_ns = Exchange(&profile_.get_zero_fill_ns);
306 ✗ const std::uint64_t get_row_copy_ns = Exchange(&profile_.get_row_copy_ns);
307 ✗ const std::uint64_t get_rows = Exchange(&profile_.get_rows);
308 ✗ const std::uint64_t get_value_bytes = Exchange(&profile_.get_value_bytes);
309 ✗ const std::uint64_t get_missing_rows = Exchange(&profile_.get_missing_rows);
310 ✗ const std::uint64_t get_direct_sg = Exchange(&profile_.get_direct_sg);
311 ✗ const std::uint64_t get_direct_sg_ns = Exchange(&profile_.get_direct_sg_ns);
312 ✗ const std::uint64_t handle_put_ns = Exchange(&profile_.handle_put_ns);
313 ✗ const std::uint64_t handle_update_ns = Exchange(&profile_.handle_update_ns);
314 ✗ const std::uint64_t handle_init_ns = Exchange(&profile_.handle_init_ns);
315 const std::uint64_t complete_response_ns =
316 ✗ Exchange(&profile_.complete_response_ns);
317 ✗ const std::uint64_t poll_loop_ns = Exchange(&profile_.poll_loop_ns);
318 ✗ std::uint64_t poller_min_get = std::numeric_limits<std::uint64_t>::max();
319 ✗ std::uint64_t poller_max_get = 0;
320 ✗ int poller_min_get_thread = -1;
321 ✗ int poller_max_get_thread = -1;
322 ✗ std::uint64_t poller_total_get = 0;
323 ✗ std::uint64_t poller_active = 0;
324 ✗ for (std::size_t i = 0; i < poller_profiles_.size(); ++i) {
325 ✗ auto& poller = *poller_profiles_[i];
326 ✗ const std::uint64_t poller_get = Exchange(&poller.handled_get);
327 ✗ const std::uint64_t poller_scan_rounds = Exchange(&poller.scan_rounds);
328 const std::uint64_t poller_scanned_slots =
329 ✗ Exchange(&poller.scanned_slots);
330 ✗ const std::uint64_t poller_ready_slots = Exchange(&poller.ready_slots);
331 const std::uint64_t poller_not_ready_slots =
332 ✗ Exchange(&poller.not_ready_slots);
333 const std::uint64_t poller_duplicate_seq_ready =
334 ✗ Exchange(&poller.duplicate_seq_ready);
335 const std::uint64_t poller_inflight_seq_ready =
336 ✗ Exchange(&poller.inflight_seq_ready);
337 ✗ const std::uint64_t poller_poll_loop_ns = Exchange(&poller.poll_loop_ns);
338 ✗ if (poller_get > 0) {
339 ✗ ++poller_active;
340 }
341 ✗ poller_total_get += poller_get;
342 ✗ if (poller_get < poller_min_get) {
343 ✗ poller_min_get = poller_get;
344 ✗ poller_min_get_thread = static_cast<int>(i);
345 }
346 ✗ if (poller_get > poller_max_get) {
347 ✗ poller_max_get = poller_get;
348 ✗ poller_max_get_thread = static_cast<int>(i);
349 }
350 std::cout
351 << "component=rdma_rc_server_poller_profile"
352 ✗ << " shard=" << shard_id_ << " thread_id=" << i << " scan_rounds="
353 ✗ << poller_scan_rounds << " scanned_slots=" << poller_scanned_slots
354 ✗ << " ready_slots=" << poller_ready_slots << " scan_hit_pct="
355 << (poller_scanned_slots == 0
356 ? 0.0
357 ✗ : 100.0 * static_cast<double>(poller_ready_slots) /
358 ✗ static_cast<double>(poller_scanned_slots))
359 ✗ << " not_ready_slots=" << poller_not_ready_slots
360 ✗ << " duplicate_seq_ready=" << poller_duplicate_seq_ready
361 ✗ << " inflight_seq_ready=" << poller_inflight_seq_ready
362 ✗ << " handled_get=" << poller_get << " poll_loop_avg_ns="
363 << (poller_scan_rounds == 0
364 ? 0
365 ✗ : poller_poll_loop_ns / poller_scan_rounds)
366 ✗ << std::endl;
367 }
368 ✗ if (poller_min_get == std::numeric_limits<std::uint64_t>::max()) {
369 ✗ poller_min_get = 0;
370 }
371 std::cout
372 << "component=rdma_rc_server_profile"
373 ✗ << " shard=" << shard_id_ << " threads=" << thread_count_
374 ✗ << " scan_rounds=" << scan_rounds << " scanned_slots=" << scanned_slots
375 ✗ << " ready_slots=" << ready_slots << " not_ready_slots="
376 ✗ << not_ready_slots << " zero_seq_ready=" << zero_seq_ready
377 ✗ << " duplicate_seq_ready=" << duplicate_seq_ready
378 ✗ << " inflight_seq_ready=" << inflight_seq_ready
379 ✗ << " empty_scan_rounds=" << empty_scan_rounds << " scan_hit_pct="
380 << (scanned_slots == 0 ? 0.0
381 ✗ : 100.0 * static_cast<double>(ready_slots) /
382 ✗ static_cast<double>(scanned_slots))
383 ✗ << " ready_round_pct="
384 << (scan_rounds == 0
385 ? 0.0
386 ✗ : 100.0 * static_cast<double>(scan_rounds - empty_scan_rounds) /
387 ✗ static_cast<double>(scan_rounds))
388 ✗ << " avg_ready_per_round="
389 << (scan_rounds == 0 ? 0.0
390 ✗ : static_cast<double>(ready_slots) /
391 ✗ static_cast<double>(scan_rounds))
392 ✗ << " max_ready_per_round=" << max_ready_per_round
393 ✗ << " handled_get=" << handled_get << " handled_put=" << handled_put
394 ✗ << " handled_update=" << handled_update
395 ✗ << " handled_init=" << handled_init
396 ✗ << " invalid_descriptor=" << Exchange(&profile_.invalid_descriptor)
397 ✗ << " wrong_shard=" << Exchange(&profile_.wrong_shard)
398 ✗ << " handle_get_avg_ns="
399 ✗ << (handled_get == 0 ? 0 : handle_get_ns / handled_get)
400 ✗ << " get_batch_get_avg_ns="
401 ✗ << (handled_get == 0 ? 0 : get_batch_get_ns / handled_get)
402 ✗ << " get_index_lookup_avg_ns="
403 ✗ << (handled_get == 0 ? 0 : get_index_lookup_ns / handled_get)
404 ✗ << " get_zero_fill_avg_ns="
405 ✗ << (handled_get == 0 ? 0 : get_zero_fill_ns / handled_get)
406 ✗ << " get_row_copy_avg_ns="
407 ✗ << (handled_get == 0 ? 0 : get_row_copy_ns / handled_get)
408 ✗ << " get_rows=" << get_rows << " get_value_bytes=" << get_value_bytes
409 ✗ << " get_missing_rows=" << get_missing_rows
410 ✗ << " get_direct_sg=" << get_direct_sg << " get_direct_sg_fallback="
411 ✗ << Exchange(&profile_.get_direct_sg_fallback)
412 ✗ << " get_direct_sg_avg_ns="
413 ✗ << (get_direct_sg == 0 ? 0 : get_direct_sg_ns / get_direct_sg)
414 ✗ << " get_direct_sg_wr=" << Exchange(&profile_.get_direct_sg_wr)
415 ✗ << " handle_put_avg_ns="
416 ✗ << (handled_put == 0 ? 0 : handle_put_ns / handled_put)
417 ✗ << " handle_update_avg_ns="
418 ✗ << (handled_update == 0 ? 0 : handle_update_ns / handled_update)
419 ✗ << " handle_init_avg_ns="
420 ✗ << (handled_init == 0 ? 0 : handle_init_ns / handled_init)
421 ✗ << " complete_response_avg_ns="
422 ✗ << (complete_count == 0 ? 0 : complete_response_ns / complete_count)
423 ✗ << " poll_loop_avg_ns="
424 ✗ << (scan_rounds == 0 ? 0 : poll_loop_ns / scan_rounds)
425 ✗ << " poller_active=" << poller_active << " poller_total_get="
426 ✗ << poller_total_get << " poller_min_get=" << poller_min_get
427 ✗ << " poller_min_get_thread=" << poller_min_get_thread
428 ✗ << " poller_max_get=" << poller_max_get
429 ✗ << " poller_max_get_thread=" << poller_max_get_thread << std::endl;
430 }
431
432 ✗ bool GetPayloadOffloadEnabled() const {
433 ✗ return get_payload_worker_count_ > 0;
434 }
435
436 ✗ std::size_t MaxGetPayloadQueueDepth() const {
437 ✗ return static_cast<std::size_t>(std::max(1, transport_->TotalSlots()));
438 }
439
440 ✗ void StartGetPayloadWorkers() {
441 ✗ if (!GetPayloadOffloadEnabled()) {
442 ✗ return;
443 }
444 ✗ for (int worker_id = 0; worker_id < get_payload_worker_count_;
445 ++worker_id) {
446 ✗ get_payload_workers_.emplace_back(
447 ✗ &PetPSServer::GetPayloadWorkerLoop, this, worker_id);
448 }
449 ✗ LOG(INFO) << "component=rdma_rc_server event=get_payload_workers_started"
450 ✗ << " count=" << get_payload_worker_count_;
451 }
452
453 ✗ void BindServerCore(int core_index) {
454 ✗ base::bind_core_with_env_offset(core_index);
455 ✗ }
456
457 ✗ bool EnqueueGetPayloadTask(const GetPayloadTask& task) {
458 ✗ std::lock_guard<std::mutex> guard(get_payload_mu_);
459 ✗ if (get_payload_tasks_.size() >= MaxGetPayloadQueueDepth()) {
460 ✗ return false;
461 }
462 ✗ get_payload_tasks_.push_back(task);
463 ✗ get_payload_cv_.notify_one();
464 ✗ return true;
465 ✗ }
466
467 ✗ std::size_t PollThreadIndex(int poll_thread_id) const {
468 ✗ return static_cast<std::size_t>(poll_thread_id);
469 }
470
471 ✗ bool TryPopGetPayloadCompletion(int poll_thread_id,
472 GetPayloadCompletion* completion) {
473 ✗ std::lock_guard<std::mutex> guard(get_payload_mu_);
474 auto& completions =
475 ✗ get_payload_completions_.at(PollThreadIndex(poll_thread_id));
476 ✗ if (completions.empty()) {
477 ✗ return false;
478 }
479 ✗ *completion = completions.front();
480 ✗ completions.pop_front();
481 ✗ return true;
482 ✗ }
483
484 ✗ void PushGetPayloadCompletion(const GetPayloadCompletion& completion) {
485 ✗ std::lock_guard<std::mutex> guard(get_payload_mu_);
486 ✗ get_payload_completions_.at(PollThreadIndex(completion.poll_thread_id))
487 ✗ .push_back(completion);
488 ✗ }
489
490 ✗ void AccumulateFlatGetProfile(const CachePS::FlatGetProfile& get_profile) {
491 ✗ profile_.get_batch_get_ns.fetch_add(
492 ✗ get_profile.batch_get_ns, std::memory_order_relaxed);
493 ✗ profile_.get_index_lookup_ns.fetch_add(
494 ✗ get_profile.index_lookup_ns, std::memory_order_relaxed);
495 ✗ profile_.get_zero_fill_ns.fetch_add(
496 ✗ get_profile.zero_fill_ns, std::memory_order_relaxed);
497 ✗ profile_.get_row_copy_ns.fetch_add(
498 ✗ get_profile.row_copy_ns, std::memory_order_relaxed);
499 ✗ profile_.get_rows.fetch_add(get_profile.rows, std::memory_order_relaxed);
500 ✗ profile_.get_value_bytes.fetch_add(
501 ✗ get_profile.value_bytes, std::memory_order_relaxed);
502 ✗ profile_.get_missing_rows.fetch_add(
503 ✗ get_profile.missing_rows, std::memory_order_relaxed);
504 ✗ }
505
506 ✗ void GetPayloadWorkerLoop(int worker_id) {
507 ✗ BindServerCore(thread_count_ + worker_id);
508 ✗ LOG(INFO) << "component=rdma_rc_server event=get_payload_worker_ready"
509 ✗ << " worker_id=" << worker_id;
510 while (true) {
511 ✗ GetPayloadTask task;
512 {
513 ✗ std::unique_lock<std::mutex> lock(get_payload_mu_);
514 ✗ get_payload_cv_.wait(lock, [this] {
515 ✗ return !get_payload_tasks_.empty();
516 });
517 ✗ task = get_payload_tasks_.front();
518 ✗ get_payload_tasks_.pop_front();
519 ✗ }
520
521 ✗ const bool profile_enabled = FLAGS_rdma_rc_profile_interval_ms > 0;
522 ✗ const std::uint64_t handle_start_ns = profile_enabled ? NowNs() : 0;
523 ✗ const bool payload_written_direct = HandleGet(
524 task.descriptor,
525 task.payload,
526 &task.response,
527 worker_id,
528 task.slot_in_qp);
529 ✗ if (profile_enabled) {
530 ✗ profile_.handled_get.fetch_add(1, std::memory_order_relaxed);
531 ✗ profile_.handle_get_ns.fetch_add(
532 ✗ NowNs() - handle_start_ns, std::memory_order_relaxed);
533 }
534 const GetPayloadCompletion completion{
535 ✗ task.slot,
536 ✗ task.client_id,
537 ✗ task.qp_index,
538 ✗ task.slot_in_qp,
539 ✗ task.poll_thread_id,
540 ✗ task.seq,
541 task.response,
542 payload_written_direct,
543 ✗ };
544 ✗ PushGetPayloadCompletion(completion);
545 ✗ }
546 }
547
548 ✗ void CompleteResponseForSlot(
549 int slot,
550 int client_id,
551 int qp_index,
552 int slot_in_qp,
553 const petps::RcShardServerTransport::ResponseView& response,
554 std::uint64_t seq,
555 bool profile_enabled) {
556 std::atomic_thread_fence(std::memory_order_release);
557 ✗ const std::uint64_t complete_start_ns = profile_enabled ? NowNs() : 0;
558 ✗ transport_->CompleteResponse(
559 client_id, qp_index, slot_in_qp, response, seq);
560 ✗ if (profile_enabled) {
561 ✗ profile_.complete_response_ns.fetch_add(
562 ✗ NowNs() - complete_start_ns, std::memory_order_relaxed);
563 }
564 ✗ VLOG(1) << "component=rdma_rc_server event=complete shard=" << shard_id_
565 ✗ << " slot=" << slot << " client_id=" << client_id
566 ✗ << " qp=" << qp_index << " seq=" << seq
567 ✗ << " status=" << response.status->status
568 ✗ << " response_bytes=" << response.status->response_bytes;
569 ✗ last_seq_[static_cast<std::size_t>(slot)] = seq;
570 ✗ if (GetPayloadOffloadEnabled()) {
571 ✗ inflight_seq_[static_cast<std::size_t>(slot)] = 0;
572 }
573 ✗ }
574
575 ✗ void CompleteResponseStatusOnlyForSlot(
576 int slot,
577 int client_id,
578 int qp_index,
579 int slot_in_qp,
580 const petps::RcShardServerTransport::ResponseView& response,
581 std::uint64_t seq,
582 bool profile_enabled) {
583 std::atomic_thread_fence(std::memory_order_release);
584 ✗ const std::uint64_t complete_start_ns = profile_enabled ? NowNs() : 0;
585 ✗ transport_->CompleteResponseStatusOnly(
586 client_id, qp_index, slot_in_qp, response, seq);
587 ✗ if (profile_enabled) {
588 ✗ profile_.complete_response_ns.fetch_add(
589 ✗ NowNs() - complete_start_ns, std::memory_order_relaxed);
590 }
591 ✗ VLOG(1) << "component=rdma_rc_server event=complete_direct shard="
592 ✗ << shard_id_ << " slot=" << slot << " client_id=" << client_id
593 ✗ << " qp=" << qp_index << " seq=" << seq
594 ✗ << " status=" << response.status->status
595 ✗ << " response_bytes=" << response.status->response_bytes;
596 ✗ last_seq_[static_cast<std::size_t>(slot)] = seq;
597 ✗ if (GetPayloadOffloadEnabled()) {
598 ✗ inflight_seq_[static_cast<std::size_t>(slot)] = 0;
599 }
600 ✗ }
601
602 ✗ void DrainGetPayloadCompletions(int poll_thread_id, bool profile_enabled) {
603 ✗ GetPayloadCompletion completion;
604 ✗ while (TryPopGetPayloadCompletion(poll_thread_id, &completion)) {
605 ✗ if (completion.payload_written_direct) {
606 ✗ CompleteResponseStatusOnlyForSlot(
607 completion.slot,
608 completion.client_id,
609 completion.qp_index,
610 completion.slot_in_qp,
611 completion.response,
612 completion.seq,
613 profile_enabled);
614 } else {
615 ✗ CompleteResponseForSlot(
616 completion.slot,
617 completion.client_id,
618 completion.qp_index,
619 completion.slot_in_qp,
620 completion.response,
621 completion.seq,
622 profile_enabled);
623 }
624 }
625 ✗ }
626
627 ✗ bool HandleGetDirectSg(
628 const petps::RequestDescriptor& descriptor,
629 base::ConstArray<std::uint64_t> keys,
630 petps::RcShardServerTransport::ResponseView* response,
631 int thread_id,
632 int slot_in_qp,
633 CachePS::FlatGetProfile* get_profile) {
634 ✗ if (descriptor.response_bytes == 0 || descriptor.embedding_dim == 0) {
635 ✗ return false;
636 }
637 ✗ const std::size_t row_bytes =
638 ✗ static_cast<std::size_t>(descriptor.embedding_dim) * sizeof(float);
639 ✗ if (row_bytes == 0 ||
640 ✗ descriptor.response_bytes !=
641 ✗ descriptor.key_count * static_cast<std::uint32_t>(row_bytes)) {
642 ✗ return false;
643 }
644
645 ✗ thread_local std::vector<CachePS::DirectFixedRow> rows;
646 ✗ rows.clear();
647 const std::uint64_t direct_start_ns =
648 ✗ FLAGS_rdma_rc_profile_interval_ms > 0 ? NowNs() : 0;
649 ✗ const bool ok = cache_ps_->GetParameterDirectFixedRows(
650 keys,
651 ✗ descriptor.key_count,
652 ✗ descriptor.embedding_dim,
653 thread_id,
654 &rows,
655 get_profile);
656 ✗ if (!ok || rows.size() != descriptor.key_count) {
657 ✗ return false;
658 }
659 ✗ std::uint64_t response_offset = 0;
660 ✗ std::uint64_t wr_count = 0;
661 ✗ for (std::size_t row = 0; row < rows.size();) {
662 ✗ std::array<petps::RawVerbsSge, kMaxDirectSgesPerWr> sges{};
663 ✗ std::size_t sge_count = 0;
664 ✗ std::size_t row_count = 0;
665 ✗ for (; row < rows.size(); ++row) {
666 ✗ const auto& ref = rows[row];
667 ✗ if (ref.missing || ref.data == nullptr || ref.size != row_bytes) {
668 ✗ return false;
669 }
670 ✗ if (sge_count > 0) {
671 ✗ auto& last = sges[sge_count - 1];
672 ✗ const char* last_end =
673 ✗ static_cast<const char*>(last.data) + last.bytes;
674 ✗ if (last_end == ref.data) {
675 ✗ last.bytes += row_bytes;
676 ✗ ++row_count;
677 ✗ continue;
678 }
679 }
680 ✗ if (sge_count == kMaxDirectSgesPerWr) {
681 ✗ break;
682 }
683 ✗ sges[sge_count++] = petps::RawVerbsSge{ref.data, row_bytes};
684 ✗ ++row_count;
685 }
686 ✗ const std::uint64_t bytes =
687 static_cast<std::uint64_t>(row_count * row_bytes);
688 ✗ transport_->WriteResponsePayloadSg(
689 ✗ descriptor.client_id,
690 ✗ descriptor.qp_index,
691 slot_in_qp,
692 base::ConstArray<petps::RawVerbsSge>(
693 ✗ sges.data(), static_cast<int>(sge_count)),
694 response_offset,
695 bytes);
696 ✗ response_offset += bytes;
697 ✗ ++wr_count;
698 }
699 ✗ response->status->status = static_cast<std::int32_t>(petps::RpcStatus::kOk);
700 ✗ response->status->response_bytes =
701 ✗ static_cast<std::uint32_t>(descriptor.response_bytes);
702 ✗ if (FLAGS_rdma_rc_profile_interval_ms > 0) {
703 ✗ profile_.get_direct_sg.fetch_add(1, std::memory_order_relaxed);
704 ✗ profile_.get_direct_sg_ns.fetch_add(
705 ✗ NowNs() - direct_start_ns, std::memory_order_relaxed);
706 ✗ profile_.get_direct_sg_wr.fetch_add(wr_count, std::memory_order_relaxed);
707 ✗ if (get_profile != nullptr) {
708 ✗ AccumulateFlatGetProfile(*get_profile);
709 }
710 }
711 ✗ return true;
712 }
713
714 ✗ bool HandleGet(const petps::RequestDescriptor& descriptor,
715 const char* payload,
716 petps::RcShardServerTransport::ResponseView* response,
717 int thread_id,
718 int slot_in_qp) {
719 ✗ if (FLAGS_rdma_rc_fake_get_mode == "status_only") {
720 ✗ response->status->status =
721 static_cast<std::int32_t>(petps::RpcStatus::kOk);
722 ✗ response->status->response_bytes = 0;
723 ✗ return false;
724 }
725 ✗ if (FLAGS_rdma_rc_fake_get_mode == "payload_memset") {
726 ✗ std::memset(response->payload, 0, descriptor.response_bytes);
727 ✗ response->status->status =
728 static_cast<std::int32_t>(petps::RpcStatus::kOk);
729 ✗ response->status->response_bytes =
730 ✗ static_cast<std::uint32_t>(descriptor.response_bytes);
731 ✗ return false;
732 }
733 ✗ if (FLAGS_rdma_rc_fake_get_mode == "index_only") {
734 base::ConstArray<std::uint64_t> keys(
735 reinterpret_cast<const std::uint64_t*>(payload),
736 ✗ descriptor.key_count);
737 ✗ CachePS::FlatGetProfile get_profile;
738 ✗ CachePS::FlatGetProfile* get_profile_ptr =
739 ✗ FLAGS_rdma_rc_profile_interval_ms > 0 ? &get_profile : nullptr;
740 const bool ok =
741 ✗ cache_ps_->ProbeParameterIndex(keys, thread_id, get_profile_ptr);
742 ✗ if (get_profile_ptr != nullptr) {
743 ✗ AccumulateFlatGetProfile(get_profile);
744 }
745 ✗ response->status->status = static_cast<std::int32_t>(
746 ok ? petps::RpcStatus::kOk : petps::RpcStatus::kValueSizeMismatch);
747 ✗ response->status->response_bytes = 0;
748 ✗ return false;
749 }
750 ✗ if (FLAGS_rdma_rc_fake_get_mode != "none" &&
751 ✗ !FLAGS_rdma_rc_fake_get_mode.empty()) {
752 ✗ response->status->status =
753 static_cast<std::int32_t>(petps::RpcStatus::kInvalidPayload);
754 ✗ response->status->response_bytes = 0;
755 ✗ return false;
756 }
757
758 base::ConstArray<std::uint64_t> keys(
759 ✗ reinterpret_cast<const std::uint64_t*>(payload), descriptor.key_count);
760 ✗ CachePS::FlatGetProfile get_profile;
761 ✗ CachePS::FlatGetProfile* get_profile_ptr =
762 ✗ FLAGS_rdma_rc_profile_interval_ms > 0 ? &get_profile : nullptr;
763 ✗ if ((descriptor.flags & petps::kRcFlagGetDirectSg) != 0) {
764 ✗ const bool direct_ok = HandleGetDirectSg(
765 descriptor, keys, response, thread_id, slot_in_qp, get_profile_ptr);
766 ✗ if (direct_ok) {
767 ✗ return true;
768 }
769 ✗ if (FLAGS_rdma_rc_profile_interval_ms > 0) {
770 ✗ profile_.get_direct_sg_fallback.fetch_add(1, std::memory_order_relaxed);
771 }
772 ✗ if ((descriptor.flags & petps::kRcFlagGetAllowFallbackCopy) == 0) {
773 ✗ response->status->status =
774 static_cast<std::int32_t>(petps::RpcStatus::kInvalidPayload);
775 ✗ response->status->response_bytes = 0;
776 ✗ return false;
777 }
778 }
779 ✗ const bool ok = cache_ps_->GetParameterFlat(
780 keys,
781 ✗ reinterpret_cast<float*>(response->payload),
782 ✗ descriptor.key_count,
783 ✗ descriptor.embedding_dim,
784 thread_id,
785 get_profile_ptr);
786 ✗ if (get_profile_ptr != nullptr) {
787 ✗ AccumulateFlatGetProfile(get_profile);
788 }
789 ✗ response->status->status = static_cast<std::int32_t>(
790 ok ? petps::RpcStatus::kOk : petps::RpcStatus::kValueSizeMismatch);
791 ✗ response->status->response_bytes =
792 ✗ static_cast<std::uint32_t>(descriptor.response_bytes);
793 ✗ return false;
794 }
795
796 ✗ void HandlePut(const petps::RequestDescriptor& descriptor,
797 const char* payload,
798 petps::RcShardServerTransport::ResponseView* response,
799 int thread_id) {
800 ✗ const auto* reader =
801 reinterpret_cast<const ParameterCompressReader*>(payload);
802 ✗ if (!reader->Valid(static_cast<int>(descriptor.payload_bytes))) {
803 ✗ response->status->status =
804 static_cast<std::int32_t>(petps::RpcStatus::kInvalidPayload);
805 ✗ response->status->response_bytes = 0;
806 ✗ return;
807 }
808 ✗ for (int i = 0; i < reader->item_size(); ++i) {
809 ✗ cache_ps_->PutSingleParameter(reader->item(i), thread_id);
810 }
811 ✗ response->status->status = static_cast<std::int32_t>(petps::RpcStatus::kOk);
812 ✗ response->status->response_bytes = 0;
813 }
814
815 ✗ void HandleUpdate(const petps::RequestDescriptor& descriptor,
816 const char* payload,
817 petps::RcShardServerTransport::ResponseView* response,
818 int thread_id) {
819 ✗ const std::string_view table_name = petps::DescriptorTableName(descriptor);
820 ✗ if (table_name.empty()) {
821 ✗ response->status->status =
822 static_cast<std::int32_t>(petps::RpcStatus::kInvalidPayload);
823 ✗ response->status->response_bytes = 0;
824 ✗ return;
825 }
826
827 ✗ const auto* reader =
828 reinterpret_cast<const ParameterCompressReader*>(payload);
829 ✗ if (!reader->Valid(static_cast<int>(descriptor.payload_bytes))) {
830 ✗ response->status->status =
831 static_cast<std::int32_t>(petps::RpcStatus::kInvalidPayload);
832 ✗ response->status->response_bytes = 0;
833 ✗ return;
834 }
835
836 ✗ const bool ok = cache_ps_->UpdateParameter(
837 ✗ std::string(table_name), reader, static_cast<unsigned>(thread_id));
838 ✗ response->status->status = static_cast<std::int32_t>(
839 ok ? petps::RpcStatus::kOk : petps::RpcStatus::kInvalidPayload);
840 ✗ response->status->response_bytes = 0;
841 }
842
843 ✗ void HandleUpdateFlat(const petps::RequestDescriptor& descriptor,
844 const char* payload,
845 petps::RcShardServerTransport::ResponseView* response,
846 int thread_id) {
847 ✗ const std::string_view table_name = petps::DescriptorTableName(descriptor);
848 ✗ const std::size_t expected_bytes = petps::FlatUpdatePayloadBytes(
849 ✗ descriptor.key_count, descriptor.embedding_dim);
850 ✗ if (table_name.empty() || descriptor.key_count == 0 || expected_bytes == 0 ||
851 ✗ descriptor.payload_bytes != expected_bytes) {
852 ✗ response->status->status =
853 static_cast<std::int32_t>(petps::RpcStatus::kInvalidPayload);
854 ✗ response->status->response_bytes = 0;
855 ✗ return;
856 }
857
858 ✗ const std::size_t key_bytes =
859 ✗ static_cast<std::size_t>(descriptor.key_count) * sizeof(std::uint64_t);
860 ✗ const auto* keys = reinterpret_cast<const std::uint64_t*>(payload);
861 ✗ const auto* grads = reinterpret_cast<const float*>(payload + key_bytes);
862 ✗ const bool ok = cache_ps_->UpdateParameterFlat(
863 ✗ std::string(table_name),
864 ✗ base::ConstArray<std::uint64_t>(keys, descriptor.key_count),
865 grads,
866 ✗ descriptor.key_count,
867 ✗ descriptor.embedding_dim,
868 static_cast<unsigned>(thread_id));
869 ✗ response->status->status = static_cast<std::int32_t>(
870 ok ? petps::RpcStatus::kOk : petps::RpcStatus::kInvalidPayload);
871 ✗ response->status->response_bytes = 0;
872 }
873
874 ✗ void HandleInitTable(const petps::RequestDescriptor& descriptor,
875 const char* payload,
876 petps::RcShardServerTransport::ResponseView* response) {
877 ✗ const std::string_view table_name = petps::DescriptorTableName(descriptor);
878 ✗ if (table_name.empty() ||
879 ✗ descriptor.payload_bytes != petps::InitTablePayloadBytes()) {
880 ✗ response->status->status =
881 static_cast<std::int32_t>(petps::RpcStatus::kInvalidPayload);
882 ✗ response->status->response_bytes = 0;
883 ✗ return;
884 }
885
886 ✗ std::uint64_t num_embeddings = 0;
887 ✗ std::uint64_t embedding_dim = 0;
888 ✗ std::memcpy(&num_embeddings, payload, sizeof(num_embeddings));
889 ✗ std::memcpy(&embedding_dim,
890 ✗ payload + sizeof(num_embeddings),
891 sizeof(embedding_dim));
892 ✗ const bool ok = cache_ps_->InitTable(
893 ✗ std::string(table_name), num_embeddings, embedding_dim);
894 ✗ response->status->status = static_cast<std::int32_t>(
895 ok ? petps::RpcStatus::kOk : petps::RpcStatus::kInvalidPayload);
896 ✗ response->status->response_bytes = 0;
897 }
898
899 ✗ void MaybePublishServerReady() {
900 const int started =
901 ✗ started_threads_.fetch_add(1, std::memory_order_relaxed) + 1;
902 ✗ if (started != thread_count_ ||
903 ✗ ready_published_.exchange(true, std::memory_order_acq_rel)) {
904 ✗ return;
905 }
906 ✗ control_plane_client_.PublishServerReady(FLAGS_global_id);
907 ✗ LOG(INFO) << "component=rdma_control_plane event=server_ready_published"
908 ✗ << " server_id=" << FLAGS_global_id
909 << " host=" << FLAGS_rdma_control_plane_host
910 ✗ << " port=" << FLAGS_rdma_control_plane_port;
911 }
912
913 ✗ void PollingThread(int thread_id) {
914 ✗ BindServerCore(thread_id);
915 ✗ LOG(INFO) << "component=rdma_server event=polling_thread_ready thread_id="
916 ✗ << thread_id;
917 ✗ MaybePublishServerReady();
918 const int coroutines_per_thread =
919 ✗ std::max(1, FLAGS_rdma_rc_server_coroutines_per_thread);
920 ✗ LOG(INFO) << "component=rdma_rc_server event=polling_thread_mode"
921 ✗ << " thread_id=" << thread_id
922 ✗ << " coroutines_per_thread=" << coroutines_per_thread;
923 ✗ if (coroutines_per_thread > 1) {
924 ✗ RunCoroutinePollingThread(thread_id, coroutines_per_thread);
925 ✗ return;
926 }
927 while (true) {
928 ✗ const bool profile_enabled = FLAGS_rdma_rc_profile_interval_ms > 0;
929 ✗ const std::uint64_t poll_start_ns = profile_enabled ? NowNs() : 0;
930 ✗ std::uint64_t scanned_slots = 0;
931 ✗ std::uint64_t ready_slots = 0;
932 ✗ DrainGetPayloadCompletions(thread_id, profile_enabled);
933 ✗ ScanAssignedSlots(
934 thread_id,
935 /*worker_id=*/0,
936 /*worker_count=*/1,
937 profile_enabled,
938 &scanned_slots,
939 &ready_slots);
940 ✗ DrainGetPayloadCompletions(thread_id, profile_enabled);
941 ✗ if (profile_enabled) {
942 ✗ profile_.scan_rounds.fetch_add(1, std::memory_order_relaxed);
943 ✗ profile_.scanned_slots.fetch_add(
944 scanned_slots, std::memory_order_relaxed);
945 ✗ if (ready_slots == 0) {
946 ✗ profile_.empty_scan_rounds.fetch_add(1, std::memory_order_relaxed);
947 }
948 ✗ UpdateMax(&profile_.max_ready_per_round, ready_slots);
949 ✗ const std::uint64_t poll_loop_ns = NowNs() - poll_start_ns;
950 ✗ profile_.poll_loop_ns.fetch_add(
951 poll_loop_ns, std::memory_order_relaxed);
952 auto& poller =
953 ✗ *poller_profiles_.at(static_cast<std::size_t>(thread_id));
954 ✗ poller.scan_rounds.fetch_add(1, std::memory_order_relaxed);
955 ✗ poller.scanned_slots.fetch_add(
956 scanned_slots, std::memory_order_relaxed);
957 ✗ poller.ready_slots.fetch_add(ready_slots, std::memory_order_relaxed);
958 ✗ poller.poll_loop_ns.fetch_add(poll_loop_ns, std::memory_order_relaxed);
959 ✗ MaybeReportProfile(thread_id);
960 }
961 ✗ std::this_thread::yield();
962 ✗ }
963 }
964
965 ✗ bool ProcessSlot(int slot, int thread_id, bool profile_enabled) {
966 ✗ int client_id = -1;
967 ✗ int qp_index = -1;
968 ✗ int slot_in_qp = -1;
969 ✗ transport_->DecodeSlotIndex(slot, &client_id, &qp_index, &slot_in_qp);
970 ✗ auto* commit = transport_->RequestCommitAt(slot);
971 ✗ if (commit->state.load(std::memory_order_acquire) != petps::kRcSlotReady) {
972 ✗ if (profile_enabled) {
973 ✗ profile_.not_ready_slots.fetch_add(1, std::memory_order_relaxed);
974 ✗ poller_profiles_.at(static_cast<std::size_t>(thread_id))
975 ✗ ->not_ready_slots.fetch_add(1, std::memory_order_relaxed);
976 }
977 ✗ return false;
978 }
979 ✗ const std::uint64_t seq = commit->seq.load(std::memory_order_acquire);
980 ✗ if (seq == 0) {
981 ✗ if (profile_enabled) {
982 ✗ profile_.zero_seq_ready.fetch_add(1, std::memory_order_relaxed);
983 }
984 ✗ return false;
985 }
986 ✗ if (seq == last_seq_[static_cast<std::size_t>(slot)]) {
987 ✗ if (profile_enabled) {
988 ✗ profile_.duplicate_seq_ready.fetch_add(1, std::memory_order_relaxed);
989 ✗ poller_profiles_.at(static_cast<std::size_t>(thread_id))
990 ✗ ->duplicate_seq_ready.fetch_add(1, std::memory_order_relaxed);
991 }
992 ✗ return false;
993 }
994 ✗ if (GetPayloadOffloadEnabled() &&
995 ✗ seq == inflight_seq_[static_cast<std::size_t>(slot)]) {
996 ✗ if (profile_enabled) {
997 ✗ profile_.inflight_seq_ready.fetch_add(1, std::memory_order_relaxed);
998 ✗ poller_profiles_.at(static_cast<std::size_t>(thread_id))
999 ✗ ->inflight_seq_ready.fetch_add(1, std::memory_order_relaxed);
1000 }
1001 ✗ return false;
1002 }
1003 ✗ if (profile_enabled) {
1004 ✗ profile_.ready_slots.fetch_add(1, std::memory_order_relaxed);
1005 }
1006
1007 ✗ auto* descriptor = transport_->RequestDescriptorAt(slot);
1008 ✗ std::string error;
1009 ✗ if (!petps::ValidateRequestDescriptor(
1010 *descriptor,
1011 ✗ transport_->config().request_slot_bytes,
1012 ✗ transport_->config().response_slot_bytes,
1013 &error)) {
1014 ✗ LOG(ERROR) << "component=rdma_rc_server event=invalid_descriptor"
1015 ✗ << " shard=" << shard_id_ << " slot=" << slot
1016 ✗ << " thread_id=" << thread_id << " seq=" << seq
1017 ✗ << " descriptor_seq=" << descriptor->seq
1018 ✗ << " client_id=" << descriptor->client_id
1019 ✗ << " qp=" << descriptor->qp_index << " op=" << descriptor->op
1020 ✗ << " key_count=" << descriptor->key_count
1021 ✗ << " payload_bytes=" << descriptor->payload_bytes
1022 ✗ << " response_bytes=" << descriptor->response_bytes
1023 ✗ << " error=\"" << error << "\"";
1024 ✗ if (profile_enabled) {
1025 ✗ profile_.invalid_descriptor.fetch_add(1, std::memory_order_relaxed);
1026 }
1027 ✗ last_seq_[static_cast<std::size_t>(slot)] = seq;
1028 ✗ commit->state.store(0, std::memory_order_release);
1029 ✗ return true;
1030 }
1031 ✗ if (descriptor->client_id != static_cast<std::uint32_t>(client_id) ||
1032 ✗ descriptor->qp_index != static_cast<std::uint32_t>(qp_index)) {
1033 ✗ LOG(ERROR) << "component=rdma_rc_server event=slot_descriptor_mismatch"
1034 ✗ << " shard=" << shard_id_ << " slot=" << slot
1035 ✗ << " thread_id=" << thread_id
1036 ✗ << " slot_client_id=" << client_id << " slot_qp=" << qp_index
1037 ✗ << " descriptor_client_id=" << descriptor->client_id
1038 ✗ << " descriptor_qp=" << descriptor->qp_index << " seq=" << seq;
1039 ✗ if (profile_enabled) {
1040 ✗ profile_.invalid_descriptor.fetch_add(1, std::memory_order_relaxed);
1041 }
1042 ✗ last_seq_[static_cast<std::size_t>(slot)] = seq;
1043 ✗ commit->state.store(0, std::memory_order_release);
1044 ✗ return true;
1045 }
1046
1047 auto response =
1048 ✗ transport_->OpenClientResponse(client_id, qp_index, slot_in_qp);
1049 ✗ const char* payload = transport_->RequestPayloadAt(slot);
1050 ✗ VLOG(1) << "component=rdma_rc_server event=consume shard=" << shard_id_
1051 ✗ << " slot=" << slot << " client_id=" << descriptor->client_id
1052 ✗ << " qp=" << descriptor->qp_index << " seq=" << seq << " op="
1053 ✗ << descriptor->op << " key_count=" << descriptor->key_count
1054 ✗ << " payload_bytes=" << descriptor->payload_bytes
1055 ✗ << " response_bytes=" << descriptor->response_bytes;
1056 ✗ response.status->status =
1057 static_cast<std::int32_t>(petps::RpcStatus::kInvalidPayload);
1058 ✗ response.status->response_bytes = 0;
1059
1060 ✗ if (descriptor->shard_id != static_cast<std::uint32_t>(shard_id_)) {
1061 ✗ LOG(ERROR) << "component=rdma_rc_server event=wrong_shard"
1062 ✗ << " expected_shard=" << shard_id_
1063 ✗ << " actual_shard=" << descriptor->shard_id << " slot=" << slot
1064 ✗ << " client_id=" << descriptor->client_id
1065 ✗ << " qp=" << descriptor->qp_index << " seq=" << seq << " op="
1066 ✗ << descriptor->op << " key_count=" << descriptor->key_count;
1067 ✗ if (profile_enabled) {
1068 ✗ profile_.wrong_shard.fetch_add(1, std::memory_order_relaxed);
1069 }
1070 ✗ response.status->status =
1071 static_cast<std::int32_t>(petps::RpcStatus::kWrongShard);
1072 ✗ } else if (descriptor->op ==
1073 static_cast<std::uint16_t>(petps::RcOp::kGet)) {
1074 ✗ if (GetPayloadOffloadEnabled()) {
1075 const GetPayloadTask task{
1076 slot,
1077 client_id,
1078 qp_index,
1079 slot_in_qp,
1080 thread_id,
1081 seq,
1082 *descriptor,
1083 payload,
1084 response,
1085 ✗ };
1086 ✗ if (!EnqueueGetPayloadTask(task)) {
1087 ✗ return false;
1088 }
1089 ✗ inflight_seq_[static_cast<std::size_t>(slot)] = seq;
1090 ✗ return true;
1091 } else {
1092 ✗ const std::uint64_t handle_start_ns = profile_enabled ? NowNs() : 0;
1093 const bool payload_written_direct =
1094 ✗ HandleGet(*descriptor, payload, &response, thread_id, slot_in_qp);
1095 ✗ if (profile_enabled) {
1096 ✗ profile_.handled_get.fetch_add(1, std::memory_order_relaxed);
1097 ✗ profile_.handle_get_ns.fetch_add(
1098 ✗ NowNs() - handle_start_ns, std::memory_order_relaxed);
1099 ✗ poller_profiles_.at(static_cast<std::size_t>(thread_id))
1100 ✗ ->handled_get.fetch_add(1, std::memory_order_relaxed);
1101 }
1102 ✗ if (payload_written_direct) {
1103 ✗ CompleteResponseStatusOnlyForSlot(
1104 slot,
1105 client_id,
1106 qp_index,
1107 slot_in_qp,
1108 response,
1109 seq,
1110 profile_enabled);
1111 ✗ return true;
1112 }
1113 }
1114 ✗ } else if (descriptor->op ==
1115 static_cast<std::uint16_t>(petps::RcOp::kPut)) {
1116 ✗ const std::uint64_t handle_start_ns = profile_enabled ? NowNs() : 0;
1117 ✗ HandlePut(*descriptor, payload, &response, thread_id);
1118 ✗ if (profile_enabled) {
1119 ✗ profile_.handled_put.fetch_add(1, std::memory_order_relaxed);
1120 ✗ profile_.handle_put_ns.fetch_add(
1121 ✗ NowNs() - handle_start_ns, std::memory_order_relaxed);
1122 }
1123 ✗ } else if (descriptor->op ==
1124 static_cast<std::uint16_t>(petps::RcOp::kUpdate)) {
1125 ✗ const std::uint64_t handle_start_ns = profile_enabled ? NowNs() : 0;
1126 ✗ HandleUpdate(*descriptor, payload, &response, thread_id);
1127 ✗ if (profile_enabled) {
1128 ✗ profile_.handled_update.fetch_add(1, std::memory_order_relaxed);
1129 ✗ profile_.handle_update_ns.fetch_add(
1130 ✗ NowNs() - handle_start_ns, std::memory_order_relaxed);
1131 }
1132 ✗ } else if (descriptor->op ==
1133 static_cast<std::uint16_t>(petps::RcOp::kUpdateFlat)) {
1134 ✗ const std::uint64_t handle_start_ns = profile_enabled ? NowNs() : 0;
1135 ✗ HandleUpdateFlat(*descriptor, payload, &response, thread_id);
1136 ✗ if (profile_enabled) {
1137 ✗ profile_.handled_update.fetch_add(1, std::memory_order_relaxed);
1138 ✗ profile_.handle_update_ns.fetch_add(
1139 ✗ NowNs() - handle_start_ns, std::memory_order_relaxed);
1140 }
1141 ✗ } else if (descriptor->op ==
1142 static_cast<std::uint16_t>(petps::RcOp::kInitTable)) {
1143 ✗ const std::uint64_t handle_start_ns = profile_enabled ? NowNs() : 0;
1144 ✗ HandleInitTable(*descriptor, payload, &response);
1145 ✗ if (profile_enabled) {
1146 ✗ profile_.handled_init.fetch_add(1, std::memory_order_relaxed);
1147 ✗ profile_.handle_init_ns.fetch_add(
1148 ✗ NowNs() - handle_start_ns, std::memory_order_relaxed);
1149 }
1150 }
1151
1152 ✗ CompleteResponseForSlot(
1153 slot, client_id, qp_index, slot_in_qp, response, seq, profile_enabled);
1154 ✗ return true;
1155 ✗ }
1156
1157 ✗ void ScanAssignedSlots(
1158 int thread_id,
1159 int worker_id,
1160 int worker_count,
1161 bool profile_enabled,
1162 std::uint64_t* scanned_slots,
1163 std::uint64_t* ready_slots) {
1164 ✗ const int qp_count = transport_->config().qps_per_client_per_shard;
1165 ✗ const int slots_per_qp = transport_->config().slots_per_qp;
1166 ✗ const int num_clients = transport_->config().num_clients;
1167 ✗ const int lane_slots = num_clients * slots_per_qp;
1168 ✗ for (int qp_index = thread_id; qp_index < qp_count;
1169 ✗ qp_index += thread_count_) {
1170 ✗ for (int lane_slot = worker_id; lane_slot < lane_slots;
1171 ✗ lane_slot += worker_count) {
1172 ✗ const int client_id = lane_slot / slots_per_qp;
1173 ✗ const int slot_in_qp = lane_slot % slots_per_qp;
1174 const int slot_index =
1175 ✗ transport_->SlotIndex(client_id, qp_index, slot_in_qp);
1176 ✗ ++(*scanned_slots);
1177 ✗ if (ProcessSlot(slot_index, thread_id, profile_enabled)) {
1178 ✗ ++(*ready_slots);
1179 }
1180 }
1181 }
1182 ✗ }
1183
1184 ✗ void CoroutineSlotScanner(
1185 boost::coroutines2::coroutine<void>::push_type& sink,
1186 int thread_id,
1187 int worker_id,
1188 int worker_count) {
1189 while (true) {
1190 ✗ const bool profile_enabled = FLAGS_rdma_rc_profile_interval_ms > 0;
1191 ✗ const std::uint64_t poll_start_ns = profile_enabled ? NowNs() : 0;
1192 ✗ std::uint64_t scanned_slots = 0;
1193 ✗ std::uint64_t ready_slots = 0;
1194 ✗ DrainGetPayloadCompletions(thread_id, profile_enabled);
1195 ✗ ScanAssignedSlots(
1196 thread_id,
1197 worker_id,
1198 worker_count,
1199 profile_enabled,
1200 &scanned_slots,
1201 &ready_slots);
1202 ✗ DrainGetPayloadCompletions(thread_id, profile_enabled);
1203 ✗ if (profile_enabled) {
1204 ✗ profile_.scan_rounds.fetch_add(1, std::memory_order_relaxed);
1205 ✗ profile_.scanned_slots.fetch_add(
1206 scanned_slots, std::memory_order_relaxed);
1207 ✗ if (ready_slots == 0) {
1208 ✗ profile_.empty_scan_rounds.fetch_add(1, std::memory_order_relaxed);
1209 }
1210 ✗ UpdateMax(&profile_.max_ready_per_round, ready_slots);
1211 ✗ const std::uint64_t poll_loop_ns = NowNs() - poll_start_ns;
1212 ✗ profile_.poll_loop_ns.fetch_add(
1213 poll_loop_ns, std::memory_order_relaxed);
1214 auto& poller =
1215 ✗ *poller_profiles_.at(static_cast<std::size_t>(thread_id));
1216 ✗ poller.scan_rounds.fetch_add(1, std::memory_order_relaxed);
1217 ✗ poller.scanned_slots.fetch_add(
1218 scanned_slots, std::memory_order_relaxed);
1219 ✗ poller.ready_slots.fetch_add(ready_slots, std::memory_order_relaxed);
1220 ✗ poller.poll_loop_ns.fetch_add(poll_loop_ns, std::memory_order_relaxed);
1221 }
1222 ✗ sink();
1223 ✗ }
1224 }
1225
1226 ✗ void RunCoroutinePollingThread(int thread_id, int coroutines_per_thread) {
1227 using Coroutine = boost::coroutines2::coroutine<void>;
1228 ✗ std::vector<std::unique_ptr<Coroutine::pull_type>> coroutines;
1229 ✗ coroutines.reserve(static_cast<std::size_t>(coroutines_per_thread));
1230 ✗ for (int coroutine_id = 0; coroutine_id < coroutines_per_thread;
1231 ++coroutine_id) {
1232 ✗ coroutines.emplace_back(std::make_unique<Coroutine::pull_type>(
1233 ✗ [this, thread_id, coroutine_id, coroutines_per_thread](
1234 ✗ Coroutine::push_type& sink) {
1235 ✗ CoroutineSlotScanner(
1236 sink, thread_id, coroutine_id, coroutines_per_thread);
1237 ✗ }));
1238 }
1239 while (true) {
1240 ✗ for (auto& coroutine : coroutines) {
1241 ✗ (*coroutine)();
1242 }
1243 ✗ MaybeReportProfile(thread_id);
1244 ✗ std::this_thread::yield();
1245 ✗ }
1246 ✗ }
1247
1248 CachePS* cache_ps_ = nullptr;
1249 int thread_count_ = 1;
1250 int shard_id_ = 0;
1251 std::unique_ptr<petps::RcShardServerTransport> transport_;
1252 petps::RdmaControlPlaneClient control_plane_client_;
1253 std::vector<std::thread> threads_;
1254 std::vector<std::uint64_t> last_seq_;
1255 std::vector<std::uint64_t> inflight_seq_;
1256 std::vector<std::unique_ptr<PollerProfile>> poller_profiles_;
1257 int get_payload_worker_count_ = 0;
1258 std::vector<std::thread> get_payload_workers_;
1259 std::mutex get_payload_mu_;
1260 std::condition_variable get_payload_cv_;
1261 std::deque<GetPayloadTask> get_payload_tasks_;
1262 std::vector<std::deque<GetPayloadCompletion>> get_payload_completions_;
1263 std::atomic<int> started_threads_{0};
1264 std::atomic<bool> ready_published_{false};
1265 ProfileCounters profile_;
1266 };
1267
1268 } // namespace
1269
1270 ✗ int main(int argc, char* argv[]) {
1271 ✗ folly::init(&argc, &argv);
1272 ✗ if (ShouldTraceRdmaGet()) {
1273 ✗ std::cerr << "component=rdma_get_trace side=server event=enabled interval="
1274 ✗ << RdmaGetTraceInterval() << std::endl;
1275 }
1276 ✗ xmh::Reporter::StartReportThread();
1277
1278 ✗ base::PMMmapRegisterCenter::GetConfig().backend =
1279 ✗ base::PMMmapRegisterCenter::BackendFromUseDram(FLAGS_use_dram);
1280 ✗ base::PMMmapRegisterCenter::GetConfig().numa_id = FLAGS_numa_id;
1281
1282 ✗ base::global_socket_id = FLAGS_numa_id;
1283 ✗ LOG(INFO) << "set NUMA ID = " << FLAGS_numa_id;
1284
1285 const std::string config_path =
1286 ✗ FLAGS_config_path.empty()
1287 ✗ ? base::ResolveRecStoreConfigPath().string()
1288 ✗ : FLAGS_config_path;
1289 ✗ std::ifstream config_file(config_path);
1290 ✗ if (!config_file.is_open()) {
1291 ✗ LOG(FATAL) << "Cannot open config file: " << config_path;
1292 }
1293
1294 ✗ nlohmann::json config;
1295 ✗ config_file >> config;
1296 ✗ if (config.contains("cache_ps") && config["cache_ps"].is_object() &&
1297 ✗ config["cache_ps"].contains("base_kv_config")) {
1298 ✗ NormalizeDramValuePath(&config["cache_ps"]["base_kv_config"]);
1299 }
1300 ✗ if (config.contains("distributed_client") &&
1301 ✗ config["distributed_client"].is_object() &&
1302 ✗ config["distributed_client"].contains("base_kv_config")) {
1303 ✗ NormalizeDramValuePath(&config["distributed_client"]["base_kv_config"]);
1304 }
1305 ✗ std::unique_ptr<petps::RdmaControlPlaneServer> control_plane_server;
1306 ✗ if (FLAGS_global_id == 0) {
1307 ✗ control_plane_server = std::make_unique<petps::RdmaControlPlaneServer>(
1308 ✗ petps::RdmaControlPlaneEndpoint{
1309 FLAGS_rdma_control_plane_host,
1310 FLAGS_rdma_control_plane_port,
1311 FLAGS_rdma_control_plane_timeout_ms,
1312 ✗ });
1313 ✗ control_plane_server->Start();
1314 ✗ LOG(INFO) << "component=rdma_control_plane event=listening"
1315 << " server_id=0"
1316 << " host=" << FLAGS_rdma_control_plane_host
1317 ✗ << " port=" << FLAGS_rdma_control_plane_port;
1318 }
1319 ✗ auto cache_ps = std::make_unique<CachePS>(config["cache_ps"]);
1320 ✗ const int shard_id = ResolveShardId(config);
1321 auto ps = std::make_unique<PetPSServer>(
1322 ✗ cache_ps.get(), FLAGS_thread_num, shard_id, NamespaceToken());
1323 ✗ ps->Run();
1324 while (true) {
1325 ✗ std::this_thread::sleep_for(std::chrono::seconds(1));
1326 }
1327 return 0;
1328 ✗ }
1329