ps/rdma/shard_routing.h
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | #pragma once | ||
| 2 | |||
| 3 | // Shared shard-routing primitives for RDMA parameter clients. | ||
| 4 | // | ||
| 5 | // RDMAPSClientAdapter (rdma_ps_client_adapter.h) fans a batch of keys out to | ||
| 6 | // per-shard RPCs and reassembles the responses into a single caller buffer. | ||
| 7 | // These structs and the partition/finalize logic are shared by the adapter's | ||
| 8 | // sync and raw-async paths, so they live here instead of being copy-pasted. | ||
| 9 | |||
| 10 | #include <algorithm> | ||
| 11 | #include <cstddef> | ||
| 12 | #include <cstdint> | ||
| 13 | #include <cstring> | ||
| 14 | #include <stdexcept> | ||
| 15 | #include <string> | ||
| 16 | #include <unordered_map> | ||
| 17 | #include <vector> | ||
| 18 | |||
| 19 | #include "base/array.h" | ||
| 20 | #include "base/hash.h" | ||
| 21 | #include "base/log.h" | ||
| 22 | #include "ps/rdma/rdma_common.h" | ||
| 23 | #include "ps/rdma/rdma_status.h" | ||
| 24 | |||
| 25 | namespace recstore { | ||
| 26 | namespace shard_routing { | ||
| 27 | |||
| 28 | // One in-flight per-shard GET RPC and where its rows land in the caller batch. | ||
| 29 | struct PendingShardRpc { | ||
| 30 | int shard_id = 0; // Logical shard this RPC belongs to. | ||
| 31 | int client_index = 0; // Underlying client selected for this shard. | ||
| 32 | int rpc_id = -1; // RPC id returned by the shard-local client. | ||
| 33 | std::vector<std::size_t> original_positions; // Positions in caller's batch. | ||
| 34 | void* recv_buffer = nullptr; // Caller-visible response buffer. | ||
| 35 | std::size_t key_count = 0; // Keys in this shard chunk. | ||
| 36 | bool released = false; // True once the shard RPC slot was returned. | ||
| 37 | }; | ||
| 38 | |||
| 39 | // A caller batch spread across one or more PendingShardRpc. | ||
| 40 | struct BatchRequest { | ||
| 41 | float* user_buffer = nullptr; // Final output buffer owned by caller. | ||
| 42 | bool assembled = false; // True once shard RPCs are merged. | ||
| 43 | // The trailing status word lives just past the last row, which callers that | ||
| 44 | // hand over their own exactly-sized buffer cannot host. Those callers read | ||
| 45 | // the status from WaitBatch instead and turn this off. | ||
| 46 | bool write_status_word = true; | ||
| 47 | std::size_t total_key_count = 0; // Total keys across all shards. | ||
| 48 | int value_size = 0; // Row bytes for this batch. | ||
| 49 | std::int32_t status_code = | ||
| 50 | static_cast<std::int32_t>(petps::RpcStatus::kPending); | ||
| 51 | std::vector<std::vector<char>> receive_buffers; | ||
| 52 | std::vector<PendingShardRpc> shard_rpcs; // One pending RPC per shard chunk. | ||
| 53 | }; | ||
| 54 | |||
| 55 | // A contiguous group of keys routed to a single shard/client. | ||
| 56 | struct ShardChunk { | ||
| 57 | int shard_id = 0; // Routed shard id. | ||
| 58 | int client_index = 0; // Client that serves this shard. | ||
| 59 | std::vector<uint64_t> keys; // Keys assigned to this shard chunk. | ||
| 60 | std::vector<std::size_t> positions; // Original positions in caller input. | ||
| 61 | }; | ||
| 62 | |||
| 63 | // Maps a key to a logical shard using the configured hash method. | ||
| 64 | inline int | ||
| 65 | ✗ | PartitionKey(uint64_t key, int num_shards, const std::string& hash_method) { | |
| 66 | ✗ | if (num_shards <= 0) { | |
| 67 | ✗ | throw std::invalid_argument("num_shards must be positive"); | |
| 68 | } | ||
| 69 | ✗ | if (hash_method == "city_hash") { | |
| 70 | ✗ | return static_cast<int>(GetHash(key) % static_cast<uint64_t>(num_shards)); | |
| 71 | } | ||
| 72 | ✗ | if (hash_method == "simple_mod") { | |
| 73 | ✗ | return static_cast<int>(key % static_cast<uint64_t>(num_shards)); | |
| 74 | } | ||
| 75 | ✗ | throw std::runtime_error("unsupported shard hash method: " + hash_method); | |
| 76 | } | ||
| 77 | |||
| 78 | // Splits keys into per-shard chunks, each no larger than max_keys_per_rpc. | ||
| 79 | inline std::vector<ShardChunk> | ||
| 80 | ✗ | BuildChunks(base::ConstArray<uint64_t> keys, | |
| 81 | int num_shards, | ||
| 82 | const std::string& hash_method, | ||
| 83 | const std::unordered_map<int, int>& shard_to_client_index, | ||
| 84 | std::size_t max_keys_per_rpc) { | ||
| 85 | ✗ | if (max_keys_per_rpc == 0) { | |
| 86 | ✗ | throw std::invalid_argument("max_keys_per_rpc must be positive"); | |
| 87 | } | ||
| 88 | ✗ | if (hash_method != "city_hash" && hash_method != "simple_mod") { | |
| 89 | ✗ | throw std::invalid_argument( | |
| 90 | ✗ | "unsupported shard hash method: " + hash_method); | |
| 91 | } | ||
| 92 | ✗ | std::vector<std::vector<uint64_t>> shard_keys(num_shards); | |
| 93 | ✗ | std::vector<std::vector<std::size_t>> shard_positions(num_shards); | |
| 94 | |||
| 95 | ✗ | for (std::size_t i = 0; i < keys.Size(); ++i) { | |
| 96 | ✗ | const int shard = PartitionKey(keys[i], num_shards, hash_method); | |
| 97 | ✗ | shard_keys[static_cast<std::size_t>(shard)].push_back(keys[i]); | |
| 98 | ✗ | shard_positions[static_cast<std::size_t>(shard)].push_back(i); | |
| 99 | } | ||
| 100 | |||
| 101 | ✗ | std::vector<ShardChunk> chunks; | |
| 102 | ✗ | for (int shard = 0; shard < num_shards; ++shard) { | |
| 103 | ✗ | const int client_index = shard_to_client_index.at(shard); | |
| 104 | ✗ | for (std::size_t offset = 0; | |
| 105 | ✗ | offset < shard_keys[static_cast<std::size_t>(shard)].size(); | |
| 106 | ✗ | offset += max_keys_per_rpc) { | |
| 107 | const std::size_t end = | ||
| 108 | ✗ | std::min(offset + max_keys_per_rpc, | |
| 109 | ✗ | shard_keys[static_cast<std::size_t>(shard)].size()); | |
| 110 | ✗ | ShardChunk chunk; | |
| 111 | ✗ | chunk.shard_id = shard; | |
| 112 | ✗ | chunk.client_index = client_index; | |
| 113 | ✗ | chunk.keys.assign( | |
| 114 | ✗ | shard_keys[static_cast<std::size_t>(shard)].begin() + offset, | |
| 115 | ✗ | shard_keys[static_cast<std::size_t>(shard)].begin() + end); | |
| 116 | ✗ | chunk.positions.assign( | |
| 117 | ✗ | shard_positions[static_cast<std::size_t>(shard)].begin() + offset, | |
| 118 | ✗ | shard_positions[static_cast<std::size_t>(shard)].begin() + end); | |
| 119 | ✗ | chunks.push_back(std::move(chunk)); | |
| 120 | ✗ | } | |
| 121 | } | ||
| 122 | ✗ | return chunks; | |
| 123 | ✗ | } | |
| 124 | |||
| 125 | inline std::int32_t | ||
| 126 | ✗ | DecodeBatchStatus(const BatchRequest& batch, int value_size) { | |
| 127 | ✗ | for (const auto& pending : batch.shard_rpcs) { | |
| 128 | ✗ | const auto* status_word = petps::FixedSlotStatusWord( | |
| 129 | ✗ | pending.recv_buffer, pending.key_count, value_size); | |
| 130 | ✗ | if (*status_word != static_cast<std::int32_t>(petps::RpcStatus::kOk)) { | |
| 131 | ✗ | return *status_word; | |
| 132 | } | ||
| 133 | } | ||
| 134 | ✗ | return static_cast<std::int32_t>(petps::RpcStatus::kOk); | |
| 135 | } | ||
| 136 | |||
| 137 | ✗ | inline void MergeBatchRows(BatchRequest* batch, int value_size) { | |
| 138 | ✗ | const int embedding_dim = value_size / sizeof(float); | |
| 139 | ✗ | for (const auto& pending : batch->shard_rpcs) { | |
| 140 | ✗ | const float* shard_values = static_cast<const float*>(pending.recv_buffer); | |
| 141 | ✗ | for (std::size_t i = 0; i < pending.original_positions.size(); ++i) { | |
| 142 | ✗ | std::memcpy( | |
| 143 | ✗ | batch->user_buffer + pending.original_positions[i] * embedding_dim, | |
| 144 | ✗ | shard_values + i * embedding_dim, | |
| 145 | value_size); | ||
| 146 | } | ||
| 147 | } | ||
| 148 | ✗ | } | |
| 149 | |||
| 150 | ✗ | inline void WriteBatchStatus(BatchRequest* batch, int value_size) { | |
| 151 | ✗ | auto* batch_status_word = reinterpret_cast<std::int32_t*>( | |
| 152 | ✗ | reinterpret_cast<char*>(batch->user_buffer) + | |
| 153 | ✗ | batch->total_key_count * static_cast<std::size_t>(value_size)); | |
| 154 | ✗ | *batch_status_word = batch->status_code; | |
| 155 | ✗ | } | |
| 156 | |||
| 157 | } // namespace shard_routing | ||
| 158 | } // namespace recstore | ||
| 159 |