ps/base/dist_sharded_client.h - GCC Code Coverage Report

ps/base/dist_sharded_client.h

Line	Branch	Exec	Source
1			#pragma once
2
3			// Shared implementation for distributed, sharded parameter-server clients.
4			//
5			// DistributedGRPCParameterClient (grpc/dist_grpc_ps_client.*) and
6			// DistributedBRPCParameterClient (brpc/dist_brpc_ps_client.*) are identical
7			// apart from the underlying single-shard client type. The routing, chunking,
8			// async fan-out, result merging and prefetch bookkeeping all live here as a
9			// CRTP-free class template parameterised on that client type.
10
11			#include <algorithm>
12			#include <cstring>
13			#include <future>
14			#include <memory>
15			#include <mutex>
16			#include <string>
17			#include <unordered_map>
18			#include <vector>
19
20			#include "base/array.h"
21			#include "base/hash.h"
22			#include "base/json.h"
23			#include "base/log.h"
24			#include "base/timer.h"
25			#include "ps/base/base_client.h"
26			#include "ps/base/shard_client.h"
27
28			#ifdef ENABLE_PERF_REPORT
29			# include <chrono>
30			# include "base/report/report_client.h"
31			#endif
32
33			using json = nlohmann::json;
34
35			namespace recstore {
36
37			// ClientT is the single-shard client type (GRPCParameterClient /
38			// BRPCParameterClient). It must expose GetParameter/PutParameter/
39			// UpdateParameter/Command/ClearPS/LoadFakeData/DumpFakeData/LoadCkpt/
40			// InitEmbeddingTable and the Prefetch* family.
41			template <typename ClientT>
42			class DistributedShardedClient : public BasePSClient {
43			static_assert(std::is_base_of<ShardClient, ClientT>::value,
44			"ClientT must derive from ShardClient");
45
46			public:
47		28	explicit DistributedShardedClient(json config, const char* transport_name)
48	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 2 taken 12 times. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 2 taken 16 times. ✗ Branch 3 not taken.	28	: BasePSClient(config), transport_name_(transport_name) {
49		28	json client_config;
50
51	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 12 times. ✗ Branch 2 not taken. ✓ Branch 3 taken 12 times. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 16 times. ✗ Branch 2 not taken. ✓ Branch 3 taken 16 times. ✗ Branch 4 not taken.	28	if (config.contains("distributed_client")) {
52	6/12 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 12 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 12 times. ✗ Branch 5 not taken. ✓ Branch 7 taken 12 times. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 16 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 16 times. ✗ Branch 5 not taken. ✓ Branch 7 taken 16 times. ✗ Branch 8 not taken.	28	LOG(INFO) << "Detected recstore config format, extracting "
53			"distributed_client section";
54	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 12 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 12 times. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 16 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 16 times. ✗ Branch 5 not taken.	28	client_config = config["distributed_client"];
55			} else {
56	0/8 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken.	✗	LOG(FATAL)
57			<< "Invalid config format. Expected either recstore config with "
58			"'distributed_client' section "
59	0/8 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken.	✗	<< "or direct client config with 'servers' and 'num_shards' fields";
60			}
61
62	6/12 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 12 times. ✗ Branch 2 not taken. ✓ Branch 3 taken 12 times. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✓ Branch 6 taken 12 times. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 16 times. ✗ Branch 2 not taken. ✓ Branch 3 taken 16 times. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✓ Branch 6 taken 16 times.	56	if (!client_config.contains("servers") \|\|
63	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 12 times. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✓ Branch 5 taken 12 times. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 16 times. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✓ Branch 5 taken 16 times.	28	!client_config["servers"].is_array()) {
64	0/8 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken.	✗	LOG(FATAL)
65	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken.	✗	<< "Missing or invalid 'servers' field in distributed client config";
66			}
67
68	6/12 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 12 times. ✗ Branch 2 not taken. ✓ Branch 3 taken 12 times. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✓ Branch 6 taken 12 times. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 16 times. ✗ Branch 2 not taken. ✓ Branch 3 taken 16 times. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✓ Branch 6 taken 16 times.	56	if (!client_config.contains("num_shards") \|\|
69	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 12 times. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✓ Branch 5 taken 12 times. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 16 times. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✓ Branch 5 taken 16 times.	28	!client_config["num_shards"].is_number_integer()) {
70	0/12 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken.	✗	LOG(FATAL) << "Missing or invalid 'num_shards' field in distributed "
71			"client config";
72			}
73
74	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 12 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 12 times. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 16 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 16 times. ✗ Branch 5 not taken.	28	num_shards_ = client_config["num_shards"].get<int>();
75	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 12 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 16 times. ✗ Branch 2 not taken.	28	max_keys_per_request_ = client_config.value("max_keys_per_request", 500);
76	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 12 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 16 times. ✗ Branch 2 not taken.	28	hash_method_ = client_config.value("hash_method", "city_hash");
77	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 0 not taken. ✓ Branch 1 taken 12 times. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 0 not taken. ✓ Branch 1 taken 16 times.	28	if (max_keys_per_request_ <= 0) {
78	0/12 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken.	✗	LOG(FATAL) << "Invalid max_keys_per_request: " << max_keys_per_request_
79	0/8 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken.	✗	<< ", must be > 0";
80			}
81
82	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 12 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 12 times. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 16 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 16 times. ✗ Branch 5 not taken.	28	auto servers = client_config["servers"];
83	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 2 taken 12 times. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 2 taken 16 times. ✗ Branch 3 not taken.	28	server_configs_.reserve(servers.size());
84
85	4/4 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 22 times. ✓ Branch 2 taken 12 times. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 32 times. ✓ Branch 2 taken 16 times.	82	for (size_t i = 0; i < servers.size(); ++i) {
86	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 22 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 32 times. ✗ Branch 2 not taken.	54	const auto& server = servers[i];
87	10/20 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 22 times. ✗ Branch 2 not taken. ✓ Branch 3 taken 22 times. ✗ Branch 4 not taken. ✓ Branch 6 taken 22 times. ✗ Branch 7 not taken. ✓ Branch 8 taken 22 times. ✗ Branch 9 not taken. ✗ Branch 10 not taken. ✓ Branch 11 taken 22 times. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 32 times. ✗ Branch 2 not taken. ✓ Branch 3 taken 32 times. ✗ Branch 4 not taken. ✓ Branch 6 taken 32 times. ✗ Branch 7 not taken. ✓ Branch 8 taken 32 times. ✗ Branch 9 not taken. ✗ Branch 10 not taken. ✓ Branch 11 taken 32 times.	108	if (!server.contains("host") \|\| !server.contains("port") \|\|
88	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 22 times. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✓ Branch 4 taken 22 times. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 32 times. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✓ Branch 4 taken 32 times.	54	!server.contains("shard")) {
89	0/16 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken.	✗	LOG(FATAL) << "Server config " << i
90	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken.	✗	<< " missing required fields (host, port, shard)";
91			}
92
93		54	ServerConfig cfg;
94	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 22 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 22 times. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 32 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 32 times. ✗ Branch 5 not taken.	54	cfg.host = server["host"].get<std::string>();
95	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 22 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 22 times. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 32 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 32 times. ✗ Branch 5 not taken.	54	cfg.port = server["port"].get<int>();
96	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 22 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 22 times. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 32 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 32 times. ✗ Branch 5 not taken.	54	cfg.shard = server["shard"].get<int>();
97
98	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 22 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 32 times. ✗ Branch 2 not taken.	54	server_configs_.push_back(cfg);
99	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 22 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 32 times. ✗ Branch 2 not taken.	54	shard_to_client_index_[cfg.shard] = i;
100			}
101
102	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✓ Branch 2 taken 12 times. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✓ Branch 2 taken 16 times.	28	if (server_configs_.size() != static_cast<size_t>(num_shards_)) {
103	0/16 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 11 not taken. ✗ Branch 12 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 11 not taken. ✗ Branch 12 not taken.	✗	LOG(WARNING) << "Number of servers (" << server_configs_.size()
104	0/12 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken.	✗	<< ") doesn't match num_shards (" << num_shards_ << ")";
105			}
106
107	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 12 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 16 times. ✗ Branch 2 not taken.	28	partitioned_key_buffer_.resize(num_shards_);
108	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 12 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 16 times. ✗ Branch 2 not taken.	28	key_index_mapping_.resize(num_shards_);
109
110	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 12 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 16 times. ✗ Branch 2 not taken.	28	InitializeClients();
111
112	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 12 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 12 times. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 16 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 16 times. ✗ Branch 5 not taken.	56	LOG(INFO) << "Initialized Distributed" << transport_name_
113	6/12 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 12 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 12 times. ✗ Branch 5 not taken. ✓ Branch 7 taken 12 times. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 16 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 16 times. ✗ Branch 5 not taken. ✓ Branch 7 taken 16 times. ✗ Branch 8 not taken.	28	<< "ParameterClient with " << num_shards_
114	6/12 recstore::DistributedShardedClient<BRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 12 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 12 times. ✗ Branch 5 not taken. ✓ Branch 7 taken 12 times. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::DistributedShardedClient(nlohmann::json_abi_v3_11_2::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::json_abi_v3_11_2::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> >, void>, char const): ✓ Branch 1 taken 16 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 16 times. ✗ Branch 5 not taken. ✓ Branch 7 taken 16 times. ✗ Branch 8 not taken.	28	<< " shards, hash method: " << hash_method_;
115		28	}
116
117		28	~DistributedShardedClient() override = default;
118
119		8	int shard_count() const { return num_shards_; }
120
121			// ---- Extended (non-virtual) API ----------------------------------------
122
123		20	bool ClearPS() {
124		20	std::vector<std::future<bool>> futures;
125	4/4 recstore::DistributedShardedClient<BRPCParameterClient>::ClearPS(): ✓ Branch 4 taken 20 times. ✓ Branch 5 taken 10 times. recstore::DistributedShardedClient<GRPCParameterClient>::ClearPS(): ✓ Branch 4 taken 20 times. ✓ Branch 5 taken 10 times.	60	for (auto& client : clients_) {
126	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::ClearPS()::{lambda()#1}::operator()() const: ✓ Branch 1 taken 20 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 20 times. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::ClearPS()::{lambda()#1}::operator()() const: ✓ Branch 1 taken 20 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 20 times. ✗ Branch 5 not taken.	40	futures.push_back(std::async(std::launch::async, [&client]() {
127		40	return client->ClearPS();
128			}));
129			}
130		20	bool all_success = true;
131	4/4 recstore::DistributedShardedClient<BRPCParameterClient>::ClearPS()::{lambda()#1}::operator()() const: ✓ Branch 5 taken 20 times. ✓ Branch 6 taken 10 times. recstore::DistributedShardedClient<GRPCParameterClient>::ClearPS()::{lambda()#1}::operator()() const: ✓ Branch 5 taken 20 times. ✓ Branch 6 taken 10 times.	60	for (auto& future : futures) {
132	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::ClearPS()::{lambda()#1}::operator()() const: ✓ Branch 1 taken 20 times. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✓ Branch 4 taken 20 times. recstore::DistributedShardedClient<GRPCParameterClient>::ClearPS()::{lambda()#1}::operator()() const: ✓ Branch 1 taken 20 times. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✓ Branch 4 taken 20 times.	40	if (!future.get()) {
133		✗	all_success = false;
134			}
135			}
136		20	return all_success;
137		20	}
138
139		2	bool LoadFakeData(int64_t n) {
140		2	std::vector<std::future<bool>> futures;
141	2/2 ✓ Branch 4 taken 4 times. ✓ Branch 5 taken 2 times.	6	for (auto& client : clients_) {
142		4	ClientT* raw = client.get();
143	2/4 ✓ Branch 1 taken 4 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 4 times. ✗ Branch 5 not taken.	4	futures.push_back(std::async(std::launch::async, [raw, n]() {
144		4	return raw->LoadFakeData(n);
145			}));
146			}
147		2	bool all_success = true;
148	2/2 ✓ Branch 5 taken 4 times. ✓ Branch 6 taken 2 times.	6	for (auto& future : futures) {
149	2/4 ✓ Branch 1 taken 4 times. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✓ Branch 4 taken 4 times.	4	if (!future.get()) {
150		✗	all_success = false;
151			}
152			}
153		2	return all_success;
154		2	}
155
156		2	bool DumpFakeData(int64_t n) {
157		2	std::vector<std::future<bool>> futures;
158	2/2 ✓ Branch 4 taken 4 times. ✓ Branch 5 taken 2 times.	6	for (auto& client : clients_) {
159		4	ClientT* raw = client.get();
160	2/4 ✓ Branch 1 taken 4 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 4 times. ✗ Branch 5 not taken.	4	futures.push_back(std::async(std::launch::async, [raw, n]() {
161		4	return raw->DumpFakeData(n);
162			}));
163			}
164		2	bool all_success = true;
165	2/2 ✓ Branch 5 taken 4 times. ✓ Branch 6 taken 2 times.	6	for (auto& future : futures) {
166	2/4 ✓ Branch 1 taken 4 times. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✓ Branch 4 taken 4 times.	4	if (!future.get()) {
167		✗	all_success = false;
168			}
169			}
170		2	return all_success;
171		2	}
172
173			bool LoadCkpt(const std::vector<std::string>& model_config_path,
174			const std::vector<std::string>& emb_file_path) {
175			std::vector<std::future<bool>> futures;
176			for (auto& client : clients_) {
177			futures.push_back(std::async(
178			std::launch::async, [&client, &model_config_path, &emb_file_path]() {
179			return client->LoadCkpt(model_config_path, emb_file_path);
180			}));
181			}
182			bool all_success = true;
183			for (auto& future : futures) {
184			if (!future.get()) {
185			all_success = false;
186			}
187			}
188			return all_success;
189			}
190
191		✗	bool SaveCheckpoint(const std::string& path,
192			const std::string& metadata) override {
193		✗	return FanOutCheckpoint(path, metadata, true);
194			}
195
196		✗	bool LoadCheckpoint(const std::string& path,
197			const std::string& metadata) override {
198		✗	return FanOutCheckpoint(path, metadata, false);
199			}
200
201			// ---- BasePSClient virtual overrides ------------------------------------
202
203		54	int GetParameter(const base::ConstArray<uint64_t>& keys,
204			base::RecTensor& values) override {
205			#ifdef ENABLE_PERF_REPORT
206			auto start_time = std::chrono::high_resolution_clock::now();
207			#endif
208	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 2 taken 14 times. ✗ Branch 3 not taken. ✗ Branch 4 not taken. ✓ Branch 5 taken 14 times. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 2 taken 40 times. ✗ Branch 3 not taken. ✗ Branch 4 not taken. ✓ Branch 5 taken 40 times.	54	if (!IsFloatEmbeddingValues(values, static_cast<int64_t>(keys.Size()))) {
209		✗	return -1;
210			}
211	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✗ Branch 1 not taken. ✓ Branch 2 taken 14 times. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✗ Branch 1 not taken. ✓ Branch 2 taken 40 times.	54	if (keys.Size() == 0) {
212		✗	return 0;
213			}
214
215	8/16 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 2 taken 14 times. ✗ Branch 3 not taken. ✓ Branch 5 taken 14 times. ✗ Branch 6 not taken. ✓ Branch 8 taken 14 times. ✗ Branch 9 not taken. ✓ Branch 11 taken 14 times. ✗ Branch 12 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 2 taken 40 times. ✗ Branch 3 not taken. ✓ Branch 5 taken 40 times. ✗ Branch 6 not taken. ✓ Branch 8 taken 40 times. ✗ Branch 9 not taken. ✓ Branch 11 taken 40 times. ✗ Branch 12 not taken.	108	xmh::Timer timer(std::string("Distributed") + transport_name_ +
216			"ParameterClient::GetParameter");
217
218		54	std::vector<std::vector<uint64_t>> partitioned_keys;
219	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 1 taken 14 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 1 taken 40 times. ✗ Branch 2 not taken.	54	PartitionKeys(keys, partitioned_keys);
220	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 1 taken 14 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 1 taken 40 times. ✗ Branch 2 not taken.	54	const int64_t D = values.shape(1);
221
222		54	std::vector<std::future<int>> futures;
223		54	std::vector<base::RecTensor> partitioned_results;
224	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 1 taken 14 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 1 taken 40 times. ✗ Branch 2 not taken.	54	partitioned_results.reserve(static_cast<size_t>(num_shards_));
225	4/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 0 taken 28 times. ✓ Branch 1 taken 14 times. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 0 taken 80 times. ✓ Branch 1 taken 40 times.	162	for (int shard_id = 0; shard_id < num_shards_; ++shard_id) {
226	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 1 taken 28 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 1 taken 80 times. ✗ Branch 2 not taken.	108	partitioned_results.emplace_back(
227	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 1 taken 28 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 1 taken 80 times. ✗ Branch 2 not taken.	216	std::vector<int64_t>{
228		108	static_cast<int64_t>(partitioned_keys[shard_id].size()), D},
229		216	base::DataType::FLOAT32);
230			}
231
232	4/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 0 taken 28 times. ✓ Branch 1 taken 14 times. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 0 taken 80 times. ✓ Branch 1 taken 40 times.	162	for (int shard_id = 0; shard_id < num_shards_; ++shard_id) {
233	3/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✗ Branch 2 not taken. ✓ Branch 3 taken 28 times. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 2 taken 16 times. ✓ Branch 3 taken 64 times.	108	if (partitioned_keys[shard_id].empty()) {
234		16	continue;
235			}
236
237	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 1 taken 28 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 1 taken 64 times. ✗ Branch 2 not taken.	92	auto it = shard_to_client_index_.find(shard_id);
238	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✗ Branch 2 not taken. ✓ Branch 3 taken 28 times. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✗ Branch 2 not taken. ✓ Branch 3 taken 64 times.	92	if (it == shard_to_client_index_.end()) {
239	0/16 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken.	✗	LOG(ERROR) << "No client found for shard " << shard_id;
240		✗	return -1;
241			}
242
243		92	int client_index = it->second;
244		92	auto* client = clients_[client_index].get();
245
246	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 1 taken 28 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 28 times. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 1 taken 64 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 64 times. ✗ Branch 5 not taken.	92	futures.push_back(std::async(
247		338	std::launch::async, [=, &partitioned_keys, &partitioned_results]() {
248		44	const auto& shard_keys_vec = partitioned_keys[shard_id];
249		44	float* shard_dst = partitioned_results[shard_id].data_as<float>();
250	6/8 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&)::{lambda()#1}::operator()() const: ✓ Branch 1 taken 30 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 30 times. ✓ Branch 5 taken 28 times. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&)::{lambda()#1}::operator()() const: ✓ Branch 1 taken 20 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 20 times. ✓ Branch 5 taken 16 times.	144	for (size_t start = 0; start < shard_keys_vec.size();
251		50	start += static_cast<size_t>(max_keys_per_request_)) {
252		50	size_t end =
253		100	std::min(start + static_cast<size_t>(max_keys_per_request_),
254		50	shard_keys_vec.size());
255		100	base::ConstArray<uint64_t> shard_chunk(
256		50	shard_keys_vec.data() + start, static_cast<int>(end - start));
257	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&)::{lambda()#1}::operator()() const: ✓ Branch 1 taken 30 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 30 times. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&)::{lambda()#1}::operator()() const: ✓ Branch 1 taken 20 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 20 times. ✗ Branch 5 not taken.	100	base::RecTensor chunk(shard_dst + start * static_cast<size_t>(D),
258		50	{static_cast<int64_t>(end - start), D});
259	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&)::{lambda()#1}::operator()() const: ✓ Branch 1 taken 30 times. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✓ Branch 4 taken 30 times. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&)::{lambda()#1}::operator()() const: ✓ Branch 1 taken 20 times. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✓ Branch 4 taken 20 times.	50	if (client->GetParameter(shard_chunk, chunk) != 0) {
260		✗	return 0;
261			}
262			}
263		44	return 1;
264			}));
265			}
266
267	6/8 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✗ Branch 5 not taken. ✗ Branch 6 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&)::{lambda()#1}::operator()() const: ✓ Branch 5 taken 28 times. ✓ Branch 6 taken 14 times. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 5 taken 48 times. ✓ Branch 6 taken 32 times. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&)::{lambda()#1}::operator()() const: ✓ Branch 5 taken 16 times. ✓ Branch 6 taken 8 times.	146	for (auto& future : futures) {
268	6/16 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&)::{lambda()#1}::operator()() const: ✓ Branch 1 taken 28 times. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✓ Branch 4 taken 28 times. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 1 taken 48 times. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✓ Branch 4 taken 48 times. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&)::{lambda()#1}::operator()() const: ✓ Branch 1 taken 16 times. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✓ Branch 4 taken 16 times.	92	if (!future.get()) {
269	0/24 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken.	✗	LOG(ERROR) << "Failed to get parameters from one of the shards";
270		✗	return -1;
271			}
272			}
273
274	6/8 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✗ Branch 0 not taken. ✗ Branch 1 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&)::{lambda()#1}::operator()() const: ✓ Branch 0 taken 28 times. ✓ Branch 1 taken 14 times. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 0 taken 64 times. ✓ Branch 1 taken 32 times. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&)::{lambda()#1}::operator()() const: ✓ Branch 0 taken 16 times. ✓ Branch 1 taken 8 times.	162	for (int shard_id = 0; shard_id < num_shards_; ++shard_id) {
275	3/8 recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&)::{lambda()#1}::operator()() const: ✓ Branch 3 taken 28 times. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&): ✓ Branch 3 taken 64 times. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetParameter(base::ConstArray<unsigned long> const&, base::RecTensor&)::{lambda()#1}::operator()() const: ✓ Branch 3 taken 16 times. ✗ Branch 4 not taken.	108	ScatterEmbeddingRows(
276			partitioned_results[shard_id], key_index_mapping_[shard_id], values);
277			}
278
279			#ifdef ENABLE_PERF_REPORT
280			auto end_time = std::chrono::high_resolution_clock::now();
281			auto duration =
282			std::chrono::duration_cast<std::chrono::microseconds>(
283			end_time - start_time)
284			.count();
285			double start_us =
286			std::chrono::duration_cast<std::chrono::microseconds>(
287			start_time.time_since_epoch())
288			.count();
289			FlameGraphData fg_data = {
290			"dist_client::GetParameter",
291			start_us,
292			1,
293			static_cast<double>(duration),
294			static_cast<double>(duration)};
295			std::string unique_id =
296			"embread_debug" + std::to_string(recstore::g_trace_id);
297			report_flame_graph("emb_read_flame_map", unique_id.c_str(), fg_data);
298			#endif
299
300		54	return 0;
301		54	}
302
303		62	int PutParameter(const base::ConstArray<uint64_t>& keys,
304			const base::RecTensor& values) override {
305	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 2 taken 6 times. ✗ Branch 3 not taken. ✗ Branch 4 not taken. ✓ Branch 5 taken 6 times. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 2 taken 56 times. ✗ Branch 3 not taken. ✗ Branch 4 not taken. ✓ Branch 5 taken 56 times.	62	if (!IsFloatEmbeddingValues(values, static_cast<int64_t>(keys.Size()))) {
306	0/16 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 11 not taken. ✗ Branch 12 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 11 not taken. ✗ Branch 12 not taken.	✗	LOG(ERROR) << "Keys and values size mismatch: " << keys.Size();
307		✗	return -1;
308			}
309	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✓ Branch 2 taken 6 times. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✓ Branch 2 taken 56 times.	62	if (keys.Size() == 0) {
310		✗	return 0;
311			}
312
313		62	std::vector<std::vector<uint64_t>> partitioned_keys;
314	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 1 taken 6 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 1 taken 56 times. ✗ Branch 2 not taken.	62	PartitionKeys(keys, partitioned_keys);
315	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 1 taken 6 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 1 taken 56 times. ✗ Branch 2 not taken.	62	const int64_t D = values.shape(1);
316
317		62	std::vector<base::RecTensor> partitioned_values;
318	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 1 taken 6 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 1 taken 56 times. ✗ Branch 2 not taken.	62	partitioned_values.reserve(static_cast<size_t>(num_shards_));
319	4/4 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 0 taken 12 times. ✓ Branch 1 taken 6 times. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 0 taken 112 times. ✓ Branch 1 taken 56 times.	186	for (int shard_id = 0; shard_id < num_shards_; ++shard_id) {
320	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 1 taken 12 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 12 times. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 1 taken 112 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 112 times. ✗ Branch 5 not taken.	124	partitioned_values.emplace_back(
321			std::vector<int64_t>{
322		124	static_cast<int64_t>(key_index_mapping_[shard_id].size()), D},
323		124	base::DataType::FLOAT32);
324	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 3 taken 12 times. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 3 taken 112 times. ✗ Branch 4 not taken.	124	GatherEmbeddingRows(
325			values, key_index_mapping_[shard_id], partitioned_values[shard_id]);
326			}
327
328		62	std::vector<std::future<int>> futures;
329	4/4 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 0 taken 12 times. ✓ Branch 1 taken 6 times. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 0 taken 112 times. ✓ Branch 1 taken 56 times.	186	for (int shard_id = 0; shard_id < num_shards_; ++shard_id) {
330	3/4 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 2 not taken. ✓ Branch 3 taken 12 times. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 2 taken 2 times. ✓ Branch 3 taken 110 times.	124	if (partitioned_keys[shard_id].empty()) {
331		2	continue;
332			}
333
334	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 1 taken 12 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 1 taken 110 times. ✗ Branch 2 not taken.	122	auto it = shard_to_client_index_.find(shard_id);
335	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 2 not taken. ✓ Branch 3 taken 12 times. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 2 not taken. ✓ Branch 3 taken 110 times.	122	if (it == shard_to_client_index_.end()) {
336	0/16 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken.	✗	LOG(ERROR) << "No client found for shard " << shard_id;
337		✗	return -1;
338			}
339
340		122	int client_index = it->second;
341		122	auto* client = clients_[client_index].get();
342
343	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 1 taken 12 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 12 times. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 1 taken 110 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 110 times. ✗ Branch 5 not taken.	122	futures.push_back(std::async(
344		786	std::launch::async, [=, &partitioned_keys, &partitioned_values]() {
345		88	const auto& shard_keys_vec = partitioned_keys[shard_id];
346		88	float* shard_src = partitioned_values[shard_id].data_as<float>();
347	6/8 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✓ Branch 1 taken 14 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 14 times. ✓ Branch 5 taken 12 times. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✓ Branch 1 taken 108 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 108 times. ✓ Branch 5 taken 76 times.	332	for (size_t start = 0; start < shard_keys_vec.size();
348		122	start += static_cast<size_t>(max_keys_per_request_)) {
349		122	size_t end =
350		244	std::min(start + static_cast<size_t>(max_keys_per_request_),
351		122	shard_keys_vec.size());
352		244	base::ConstArray<uint64_t> shard_chunk(
353		122	shard_keys_vec.data() + start, static_cast<int>(end - start));
354	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✓ Branch 1 taken 14 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✓ Branch 1 taken 108 times. ✗ Branch 2 not taken.	244	base::RecTensor value_chunk(
355	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✓ Branch 1 taken 14 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✓ Branch 1 taken 108 times. ✗ Branch 2 not taken.	122	shard_src + start * static_cast<size_t>(D),
356		122	{static_cast<int64_t>(end - start), D});
357	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✓ Branch 1 taken 14 times. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✓ Branch 4 taken 14 times. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✓ Branch 1 taken 108 times. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✓ Branch 4 taken 108 times.	122	if (client->PutParameter(shard_chunk, value_chunk) != 0) {
358		✗	return 0;
359			}
360			}
361		88	return 1;
362			}));
363			}
364
365	6/8 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 5 not taken. ✗ Branch 6 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✓ Branch 5 taken 12 times. ✓ Branch 6 taken 6 times. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 5 taken 34 times. ✓ Branch 6 taken 18 times. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✓ Branch 5 taken 76 times. ✓ Branch 6 taken 38 times.	184	for (auto& future : futures) {
366	6/16 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✓ Branch 1 taken 12 times. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✓ Branch 4 taken 12 times. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 1 taken 34 times. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✓ Branch 4 taken 34 times. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✓ Branch 1 taken 76 times. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✓ Branch 4 taken 76 times.	122	if (future.get() != 1) {
367	0/24 recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PutParameter(base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken.	✗	LOG(ERROR) << "Failed to put parameters to one of the shards";
368		✗	return -1;
369			}
370			}
371
372		62	return 0;
373		62	}
374
375		✗	void Command(PSCommand command) override {
376		✗	std::vector<std::future<void>> futures;
377	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::Command(recstore::PSCommand): ✗ Branch 4 not taken. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::Command(recstore::PSCommand): ✗ Branch 4 not taken. ✗ Branch 5 not taken.	✗	for (auto& client : clients_) {
378	0/16 recstore::DistributedShardedClient<BRPCParameterClient>::Command(recstore::PSCommand): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::Command(recstore::PSCommand)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::Command(recstore::PSCommand): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::Command(recstore::PSCommand)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken.	✗	futures.push_back(std::async(std::launch::async, [&client, command]() {
379		✗	client->Command(command);
380			}));
381			}
382	0/8 recstore::DistributedShardedClient<BRPCParameterClient>::Command(recstore::PSCommand): ✗ Branch 5 not taken. ✗ Branch 6 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::Command(recstore::PSCommand)::{lambda()#1}::operator()() const: ✗ Branch 5 not taken. ✗ Branch 6 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::Command(recstore::PSCommand): ✗ Branch 5 not taken. ✗ Branch 6 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::Command(recstore::PSCommand)::{lambda()#1}::operator()() const: ✗ Branch 5 not taken. ✗ Branch 6 not taken.	✗	for (auto& future : futures) {
383	0/8 recstore::DistributedShardedClient<BRPCParameterClient>::Command(recstore::PSCommand): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::Command(recstore::PSCommand)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::Command(recstore::PSCommand): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::Command(recstore::PSCommand)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken.	✗	future.wait();
384			}
385		✗	}
386
387		2	int UpdateParameter(const std::string& table_name,
388			const base::ConstArray<uint64_t>& keys,
389			const base::RecTensor& grads) override {
390	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 2 taken 2 times. ✗ Branch 3 not taken. ✗ Branch 4 not taken. ✓ Branch 5 taken 2 times.	2	if (!IsFloatEmbeddingValues(grads, static_cast<int64_t>(keys.Size()))) {
391	0/16 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 11 not taken. ✗ Branch 12 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 11 not taken. ✗ Branch 12 not taken.	✗	LOG(ERROR) << "UpdateParameter keys/grads size mismatch: " << keys.Size();
392		✗	return -1;
393			}
394	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✓ Branch 2 taken 2 times.	2	if (keys.Size() == 0) {
395		✗	return 0;
396			}
397
398		2	std::vector<std::vector<uint64_t>> partitioned_keys;
399	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 1 taken 2 times. ✗ Branch 2 not taken.	2	PartitionKeys(keys, partitioned_keys);
400	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 1 taken 2 times. ✗ Branch 2 not taken.	2	const int64_t D = grads.shape(1);
401
402		2	std::vector<base::RecTensor> partitioned_grads;
403	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 1 taken 2 times. ✗ Branch 2 not taken.	2	partitioned_grads.reserve(static_cast<size_t>(num_shards_));
404	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 0 not taken. ✗ Branch 1 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 0 taken 4 times. ✓ Branch 1 taken 2 times.	6	for (int shard_id = 0; shard_id < num_shards_; ++shard_id) {
405	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 1 taken 4 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 4 times. ✗ Branch 5 not taken.	4	partitioned_grads.emplace_back(
406			std::vector<int64_t>{
407		4	static_cast<int64_t>(key_index_mapping_[shard_id].size()), D},
408		4	base::DataType::FLOAT32);
409	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 3 taken 4 times. ✗ Branch 4 not taken.	4	GatherEmbeddingRows(
410			grads, key_index_mapping_[shard_id], partitioned_grads[shard_id]);
411			}
412
413		2	std::vector<std::future<int>> futures;
414	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 0 not taken. ✗ Branch 1 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 0 taken 4 times. ✓ Branch 1 taken 2 times.	6	for (int shard_id = 0; shard_id < num_shards_; ++shard_id) {
415	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 2 taken 2 times. ✓ Branch 3 taken 2 times.	4	if (partitioned_keys[shard_id].empty()) {
416		2	continue;
417			}
418
419	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 1 taken 2 times. ✗ Branch 2 not taken.	2	auto it = shard_to_client_index_.find(shard_id);
420	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 2 not taken. ✓ Branch 3 taken 2 times.	2	if (it == shard_to_client_index_.end()) {
421	0/16 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken.	✗	LOG(ERROR) << "No client found for shard " << shard_id;
422		✗	return -1;
423			}
424		2	int client_index = it->second;
425		2	auto* client = clients_[client_index].get();
426
427	3/12 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 1 taken 2 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 2 times. ✗ Branch 5 not taken. ✓ Branch 7 taken 2 times. ✗ Branch 8 not taken.	2	futures.push_back(std::async(
428		✗	std::launch::async, [=, &partitioned_keys, &partitioned_grads]() {
429		✗	const auto& shard_keys_vec = partitioned_keys[shard_id];
430		✗	float* shard_src = partitioned_grads[shard_id].data_as<float>();
431	0/8 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken.	✗	for (size_t start = 0; start < shard_keys_vec.size();
432		✗	start += static_cast<size_t>(max_keys_per_request_)) {
433		✗	size_t end =
434		✗	std::min(start + static_cast<size_t>(max_keys_per_request_),
435		✗	shard_keys_vec.size());
436		✗	base::ConstArray<uint64_t> shard_chunk(
437		✗	shard_keys_vec.data() + start, static_cast<int>(end - start));
438	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken.	✗	base::RecTensor grad_chunk(
439	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken.	✗	shard_src + start * static_cast<size_t>(D),
440		✗	{static_cast<int64_t>(end - start), D});
441		✗	if (client->UpdateParameter(
442	0/8 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✗ Branch 4 not taken.	✗	table_name, shard_chunk, grad_chunk) != 0) {
443		✗	return -1;
444			}
445			}
446		✗	return 0;
447			}));
448			}
449
450	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 5 not taken. ✗ Branch 6 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✗ Branch 5 not taken. ✗ Branch 6 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 5 taken 2 times. ✓ Branch 6 taken 2 times. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✗ Branch 5 not taken. ✗ Branch 6 not taken.	4	for (auto& future : futures) {
451	2/16 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✓ Branch 1 taken 2 times. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✓ Branch 4 taken 2 times. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✗ Branch 4 not taken.	2	if (future.get() != 0) {
452	0/24 recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::UpdateParameter(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, base::ConstArray<unsigned long> const&, base::RecTensor const&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken.	✗	LOG(ERROR) << "Failed to update parameters on one of the shards";
453		✗	return -1;
454			}
455			}
456
457		2	return 0;
458		2	}
459
460		✗	int64_t SparseGradientCommit(const std::string& table,
461			SparseGradientCommitHeader h,
462			const base::ConstArray<uint64_t>& keys,
463			const base::RecTensor& grads) override {
464	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 3 not taken. ✗ Branch 4 not taken.	✗	std::vector<std::vector<uint64_t>> shard_keys(clients_.size());
465	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 3 not taken. ✗ Branch 4 not taken.	✗	std::vector<std::vector<size_t>> indices(clients_.size());
466	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken.	✗	for (size_t i = 0; i < keys.Size(); ++i) {
467	0/8 recstore::DistributedShardedClient<BRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✗ Branch 5 not taken. ✗ Branch 6 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✗ Branch 5 not taken. ✗ Branch 6 not taken.	✗	const auto client = shard_to_client_index_.at(GetShardId(keys[i]));
468	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 3 not taken. ✗ Branch 4 not taken.	✗	shard_keys[client].push_back(keys[i]);
469	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken.	✗	indices[client].push_back(i);
470			}
471		✗	int64_t committed = std::numeric_limits<int64_t>::max();
472			// Every shard receives an end marker, including shards with no rows.
473	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken.	✗	for (size_t i = 0; i < clients_.size(); ++i) {
474	0/8 recstore::DistributedShardedClient<BRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken.	✗	base::RecTensor part({static_cast<int64_t>(indices[i].size()), h.dim},
475			base::DataType::FLOAT32);
476	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken.	✗	if (!indices[i].empty())
477	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken.	✗	GatherEmbeddingRows(grads, indices[i], part);
478		✗	committed = std::min(
479			committed,
480	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::SparseGradientCommit(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::SparseGradientCommitHeader, base::ConstArray<unsigned long> const&, base::RecTensor const&): ✗ Branch 3 not taken. ✗ Branch 4 not taken.	✗	clients_[i]->SparseGradientCommit(
481			table,
482			h,
483		✗	base::ConstArray<uint64_t>(
484		✗	shard_keys[i].data(), shard_keys[i].size()),
485			part));
486			}
487		✗	return committed;
488		✗	}
489
490		10	int InitEmbeddingTable(
491			const std::string& table_name,
492			const recstore::EmbeddingTableConfig& config) override {
493		10	std::vector<std::future<int>> futures;
494	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&): ✗ Branch 4 not taken. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&): ✓ Branch 4 taken 20 times. ✓ Branch 5 taken 10 times.	30	for (auto& client : clients_) {
495	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&): ✓ Branch 1 taken 20 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 20 times. ✗ Branch 5 not taken.	20	futures.push_back(
496		✗	std::async(std::launch::async, [&client, &table_name, &config]() {
497		✗	return client->InitEmbeddingTable(table_name, config);
498			}));
499			}
500
501		10	int tag = -1;
502	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&): ✗ Branch 5 not taken. ✗ Branch 6 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&)::{lambda()#1}::operator()() const: ✗ Branch 5 not taken. ✗ Branch 6 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&): ✓ Branch 5 taken 20 times. ✓ Branch 6 taken 10 times. recstore::DistributedShardedClient<GRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&)::{lambda()#1}::operator()() const: ✗ Branch 5 not taken. ✗ Branch 6 not taken.	30	for (auto& future : futures) {
503	1/8 recstore::DistributedShardedClient<BRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&): ✓ Branch 1 taken 20 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken.	20	const int rc = future.get();
504	1/8 recstore::DistributedShardedClient<BRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&): ✗ Branch 0 not taken. ✗ Branch 1 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&)::{lambda()#1}::operator()() const: ✗ Branch 0 not taken. ✗ Branch 1 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&): ✗ Branch 0 not taken. ✓ Branch 1 taken 20 times. recstore::DistributedShardedClient<GRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&)::{lambda()#1}::operator()() const: ✗ Branch 0 not taken. ✗ Branch 1 not taken.	20	if (rc < 0) {
505	0/24 recstore::DistributedShardedClient<BRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken.	✗	LOG(ERROR) << "InitEmbeddingTable failed on one of the shards";
506		✗	return -1;
507			}
508	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&): ✗ Branch 0 not taken. ✗ Branch 1 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&)::{lambda()#1}::operator()() const: ✗ Branch 0 not taken. ✗ Branch 1 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&): ✓ Branch 0 taken 10 times. ✓ Branch 1 taken 10 times. recstore::DistributedShardedClient<GRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&)::{lambda()#1}::operator()() const: ✗ Branch 0 not taken. ✗ Branch 1 not taken.	20	if (tag < 0) {
509		10	tag = rc;
510	1/8 recstore::DistributedShardedClient<BRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&): ✗ Branch 0 not taken. ✗ Branch 1 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&)::{lambda()#1}::operator()() const: ✗ Branch 0 not taken. ✗ Branch 1 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&): ✗ Branch 0 not taken. ✓ Branch 1 taken 10 times. recstore::DistributedShardedClient<GRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&)::{lambda()#1}::operator()() const: ✗ Branch 0 not taken. ✗ Branch 1 not taken.	10	} else if (tag != rc) {
511	0/24 recstore::DistributedShardedClient<BRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::InitEmbeddingTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, recstore::EmbeddingTableConfig const&)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken.	✗	LOG(ERROR) << "InitEmbeddingTable tag mismatch across shards";
512		✗	return -1;
513			}
514			}
515		10	return tag;
516		10	}
517
518		38	uint64_t PrefetchParameter(const base::ConstArray<uint64_t>& keys) override {
519		2	auto cleanup_state = [this](const DistPrefetchState& state) {
520	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 5 not taken. ✗ Branch 6 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 5 not taken. ✗ Branch 6 not taken.	✗	for (const auto& shard_state : state.shard_states) {
521	0/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 0 not taken. ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 0 not taken. ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken.	✗	if (shard_state.client_index < 0 \|\|
522	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken.	✗	shard_state.client_index >= static_cast<int>(clients_.size())) {
523		✗	continue;
524			}
525		✗	auto* client = clients_[shard_state.client_index].get();
526	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 6 not taken. ✗ Branch 7 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 6 not taken. ✗ Branch 7 not taken.	✗	for (uint64_t child_prefetch_id : shard_state.child_prefetch_ids) {
527	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken.	✗	client->WaitForPrefetch(child_prefetch_id);
528		✗	base::RecTensor discard;
529	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken.	✗	client->GetPrefetchResult(child_prefetch_id, discard);
530			}
531			}
532			};
533
534	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 1 not taken. ✓ Branch 2 taken 2 times. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✓ Branch 2 taken 36 times.	38	if (keys.Size() == 0) {
535	0/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken.	✗	std::lock_guard<std::mutex> lk(prefetch_mu_);
536		✗	uint64_t prefetch_id = next_prefetch_id_++;
537	0/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken.	✗	auto state = std::make_shared<DistPrefetchState>();
538		✗	state->total_keys = 0;
539	0/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken.	✗	prefetch_states_[prefetch_id] = state;
540		✗	return prefetch_id;
541		✗	}
542
543	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✓ Branch 2 taken 2 times. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✓ Branch 2 taken 36 times. ✗ Branch 3 not taken.	38	std::vector<std::vector<uint64_t>> shard_keys(num_shards_);
544	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✓ Branch 2 taken 2 times. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✓ Branch 2 taken 36 times. ✗ Branch 3 not taken.	38	std::vector<std::vector<size_t>> shard_indices(num_shards_);
545	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✓ Branch 1 taken 8 times. ✓ Branch 2 taken 2 times. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✓ Branch 1 taken 416 times. ✓ Branch 2 taken 36 times.	462	for (size_t i = 0; i < keys.Size(); ++i) {
546	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✓ Branch 2 taken 8 times. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✓ Branch 2 taken 416 times. ✗ Branch 3 not taken.	424	const int shard_id = GetShardId(keys[i]);
547	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✓ Branch 3 taken 8 times. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✓ Branch 3 taken 416 times. ✗ Branch 4 not taken.	424	shard_keys[shard_id].push_back(keys[i]);
548	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✓ Branch 2 taken 8 times. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✓ Branch 2 taken 416 times. ✗ Branch 3 not taken.	424	shard_indices[shard_id].push_back(i);
549			}
550
551	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✓ Branch 1 taken 2 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✓ Branch 1 taken 36 times. ✗ Branch 2 not taken.	38	auto state = std::make_shared<DistPrefetchState>();
552		38	state->total_keys = keys.Size();
553
554	6/16 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✓ Branch 1 taken 4 times. ✗ Branch 2 not taken. ✓ Branch 3 taken 4 times. ✓ Branch 4 taken 2 times. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✓ Branch 1 taken 72 times. ✗ Branch 2 not taken. ✓ Branch 3 taken 72 times. ✓ Branch 4 taken 36 times.	190	for (int shard_id = 0; shard_id < num_shards_; ++shard_id) {
555	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 2 not taken. ✓ Branch 3 taken 4 times. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 2 not taken. ✓ Branch 3 taken 72 times.	76	if (shard_keys[shard_id].empty()) {
556		✗	continue;
557			}
558
559	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✓ Branch 1 taken 4 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✓ Branch 1 taken 72 times. ✗ Branch 2 not taken.	76	auto it = shard_to_client_index_.find(shard_id);
560	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 2 not taken. ✓ Branch 3 taken 4 times. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 2 not taken. ✓ Branch 3 taken 72 times.	76	if (it == shard_to_client_index_.end()) {
561	0/32 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken.	✗	LOG(ERROR) << "No client found for shard " << shard_id;
562	0/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 2 not taken. ✗ Branch 3 not taken.	✗	cleanup_state(*state);
563		✗	return 0;
564			}
565
566		76	DistPrefetchShardState shard_state;
567		76	shard_state.shard_id = shard_id;
568		76	shard_state.client_index = it->second;
569		76	shard_state.original_indices = std::move(shard_indices[shard_id]);
570
571		76	const auto& skeys = shard_keys[shard_id];
572	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✓ Branch 1 taken 4 times. ✓ Branch 2 taken 4 times. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✓ Branch 1 taken 100 times. ✓ Branch 2 taken 72 times.	180	for (size_t start = 0; start < skeys.size();
573		104	start += static_cast<size_t>(max_keys_per_request_)) {
574		208	size_t end = std::min(
575		104	start + static_cast<size_t>(max_keys_per_request_), skeys.size());
576		208	base::ConstArray<uint64_t> chunk(
577		104	skeys.data() + start, static_cast<int>(end - start));
578		104	uint64_t child_prefetch_id =
579	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✓ Branch 3 taken 4 times. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✓ Branch 3 taken 100 times. ✗ Branch 4 not taken.	104	clients_[shard_state.client_index]->PrefetchParameter(chunk);
580	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 0 not taken. ✗ Branch 1 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 0 not taken. ✗ Branch 1 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 0 not taken. ✓ Branch 1 taken 4 times. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 0 not taken. ✓ Branch 1 taken 100 times.	104	if (child_prefetch_id == 0) {
581	0/32 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken.	✗	LOG(ERROR) << "PrefetchParameter failed for shard " << shard_id;
582	0/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 2 not taken. ✗ Branch 3 not taken.	✗	cleanup_state(*state);
583		✗	return 0;
584			}
585	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✓ Branch 1 taken 4 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✓ Branch 1 taken 100 times. ✗ Branch 2 not taken.	104	shard_state.child_prefetch_ids.push_back(child_prefetch_id);
586	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✓ Branch 1 taken 4 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✓ Branch 1 taken 100 times. ✗ Branch 2 not taken.	104	shard_state.chunk_sizes.push_back(static_cast<int>(end - start));
587			}
588	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✓ Branch 3 taken 4 times. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✓ Branch 3 taken 72 times. ✗ Branch 4 not taken.	76	state->shard_states.push_back(std::move(shard_state));
589			}
590
591	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✓ Branch 1 taken 2 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✓ Branch 1 taken 36 times. ✗ Branch 2 not taken.	38	std::lock_guard<std::mutex> lk(prefetch_mu_);
592		38	uint64_t prefetch_id = next_prefetch_id_++;
593	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<BRPCParameterClient>::DistPrefetchState const&) const: ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&): ✓ Branch 2 taken 2 times. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PrefetchParameter(base::ConstArray<unsigned long> const&)::{lambda(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&)#1}::operator()(recstore::DistributedShardedClient<GRPCParameterClient>::DistPrefetchState const&) const: ✓ Branch 2 taken 36 times. ✗ Branch 3 not taken.	38	prefetch_states_[prefetch_id] = std::move(state);
594		38	return prefetch_id;
595		38	}
596
597		4	bool IsPrefetchDone(uint64_t prefetch_id) override {
598		4	std::shared_ptr<DistPrefetchState> state;
599			{
600	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::IsPrefetchDone(unsigned long): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::IsPrefetchDone(unsigned long): ✓ Branch 1 taken 4 times. ✗ Branch 2 not taken.	4	std::lock_guard<std::mutex> lk(prefetch_mu_);
601	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::IsPrefetchDone(unsigned long): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::IsPrefetchDone(unsigned long): ✓ Branch 1 taken 4 times. ✗ Branch 2 not taken.	4	auto it = prefetch_states_.find(prefetch_id);
602	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::IsPrefetchDone(unsigned long): ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::IsPrefetchDone(unsigned long): ✓ Branch 2 taken 2 times. ✓ Branch 3 taken 2 times.	4	if (it == prefetch_states_.end()) {
603	4/16 recstore::DistributedShardedClient<BRPCParameterClient>::IsPrefetchDone(unsigned long): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::IsPrefetchDone(unsigned long): ✓ Branch 1 taken 2 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 2 times. ✗ Branch 5 not taken. ✓ Branch 7 taken 2 times. ✗ Branch 8 not taken. ✓ Branch 10 taken 2 times. ✗ Branch 11 not taken.	2	LOG(ERROR) << "Invalid prefetch_id: " << prefetch_id;
604		2	return false;
605			}
606		2	state = it->second;
607	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::IsPrefetchDone(unsigned long): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::IsPrefetchDone(unsigned long): ✓ Branch 1 taken 2 times. ✓ Branch 2 taken 2 times.	4	}
608
609	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::IsPrefetchDone(unsigned long): ✗ Branch 6 not taken. ✗ Branch 7 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::IsPrefetchDone(unsigned long): ✓ Branch 6 taken 4 times. ✓ Branch 7 taken 2 times.	6	for (const auto& shard_state : state->shard_states) {
610		4	auto* client = clients_[shard_state.client_index].get();
611	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::IsPrefetchDone(unsigned long): ✗ Branch 5 not taken. ✗ Branch 6 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::IsPrefetchDone(unsigned long): ✓ Branch 5 taken 4 times. ✓ Branch 6 taken 4 times.	8	for (uint64_t child_prefetch_id : shard_state.child_prefetch_ids) {
612	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::IsPrefetchDone(unsigned long): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::IsPrefetchDone(unsigned long): ✓ Branch 1 taken 4 times. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✓ Branch 4 taken 4 times.	4	if (!client->IsPrefetchDone(child_prefetch_id)) {
613		✗	return false;
614			}
615			}
616			}
617		2	return true;
618		4	}
619
620		74	void WaitForPrefetch(uint64_t prefetch_id) override {
621		74	std::shared_ptr<DistPrefetchState> state;
622			{
623	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::WaitForPrefetch(unsigned long): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::WaitForPrefetch(unsigned long): ✓ Branch 1 taken 74 times. ✗ Branch 2 not taken.	74	std::lock_guard<std::mutex> lk(prefetch_mu_);
624	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::WaitForPrefetch(unsigned long): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::WaitForPrefetch(unsigned long): ✓ Branch 1 taken 74 times. ✗ Branch 2 not taken.	74	auto it = prefetch_states_.find(prefetch_id);
625	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::WaitForPrefetch(unsigned long): ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::WaitForPrefetch(unsigned long): ✗ Branch 2 not taken. ✓ Branch 3 taken 74 times.	74	if (it == prefetch_states_.end()) {
626	0/16 recstore::DistributedShardedClient<BRPCParameterClient>::WaitForPrefetch(unsigned long): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::WaitForPrefetch(unsigned long): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken.	✗	LOG(ERROR) << "Invalid prefetch_id: " << prefetch_id;
627		✗	return;
628			}
629		74	state = it->second;
630	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::WaitForPrefetch(unsigned long): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::WaitForPrefetch(unsigned long): ✓ Branch 1 taken 74 times. ✗ Branch 2 not taken.	74	}
631
632	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::WaitForPrefetch(unsigned long): ✗ Branch 6 not taken. ✗ Branch 7 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::WaitForPrefetch(unsigned long): ✓ Branch 6 taken 148 times. ✓ Branch 7 taken 74 times.	222	for (const auto& shard_state : state->shard_states) {
633		148	auto* client = clients_[shard_state.client_index].get();
634	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::WaitForPrefetch(unsigned long): ✗ Branch 5 not taken. ✗ Branch 6 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::WaitForPrefetch(unsigned long): ✓ Branch 5 taken 204 times. ✓ Branch 6 taken 148 times.	352	for (uint64_t child_prefetch_id : shard_state.child_prefetch_ids) {
635	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::WaitForPrefetch(unsigned long): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::WaitForPrefetch(unsigned long): ✓ Branch 1 taken 204 times. ✗ Branch 2 not taken.	204	client->WaitForPrefetch(child_prefetch_id);
636			}
637			}
638	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::WaitForPrefetch(unsigned long): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::WaitForPrefetch(unsigned long): ✓ Branch 1 taken 74 times. ✗ Branch 2 not taken.	74	}
639
640			bool
641		42	GetPrefetchResult(uint64_t prefetch_id, base::RecTensor& values) override {
642		42	std::shared_ptr<DistPrefetchState> state;
643			{
644	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 1 taken 42 times. ✗ Branch 2 not taken.	42	std::lock_guard<std::mutex> lk(prefetch_mu_);
645	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 1 taken 42 times. ✗ Branch 2 not taken.	42	auto it = prefetch_states_.find(prefetch_id);
646	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 2 taken 4 times. ✓ Branch 3 taken 38 times.	42	if (it == prefetch_states_.end()) {
647	4/16 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 1 taken 4 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 4 times. ✗ Branch 5 not taken. ✓ Branch 7 taken 4 times. ✗ Branch 8 not taken. ✓ Branch 10 taken 4 times. ✗ Branch 11 not taken.	4	LOG(ERROR) << "Invalid prefetch_id: " << prefetch_id;
648		4	return false;
649			}
650		38	state = it->second;
651	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 1 taken 38 times. ✓ Branch 2 taken 4 times.	42	}
652
653	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 1 taken 38 times. ✗ Branch 2 not taken.	38	WaitForPrefetch(prefetch_id);
654
655	3/8 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 1 taken 36 times. ✓ Branch 2 taken 2 times. ✗ Branch 4 not taken. ✓ Branch 5 taken 36 times.	38	const bool discard = values.data() == nullptr && values.dim() == 0;
656	3/12 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 0 not taken. ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 0 taken 38 times. ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✓ Branch 3 taken 38 times. ✗ Branch 4 not taken. ✓ Branch 5 taken 38 times.	76	if (!discard && !EnsureEmbeddingOutput(
657	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 2 taken 38 times. ✗ Branch 3 not taken.	38	values, static_cast<int64_t>(state->total_keys))) {
658		✗	return false;
659			}
660	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 0 not taken. ✗ Branch 1 not taken. ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 0 not taken. ✓ Branch 1 taken 38 times. ✓ Branch 3 taken 38 times. ✗ Branch 4 not taken.	38	const int64_t D = discard ? 0 : values.shape(1);
661
662		38	bool ok_all = true;
663	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 6 not taken. ✗ Branch 7 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 6 taken 76 times. ✓ Branch 7 taken 38 times.	114	for (const auto& shard_state : state->shard_states) {
664		76	auto* client = clients_[shard_state.client_index].get();
665		76	size_t shard_offset = 0;
666	3/10 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✗ Branch 5 not taken. ✗ Branch 6 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 1 taken 104 times. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✓ Branch 5 taken 104 times. ✓ Branch 6 taken 76 times.	284	for (size_t i = 0; i < shard_state.child_prefetch_ids.size(); ++i) {
667		104	const int expected =
668		104	(i < shard_state.chunk_sizes.size()
669	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 0 not taken. ✗ Branch 1 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 0 taken 104 times. ✗ Branch 1 not taken.	104	? shard_state.chunk_sizes[i]
670			: -1);
671		104	base::RecTensor chunk_values;
672	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 0 not taken. ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 0 taken 104 times. ✗ Branch 1 not taken. ✓ Branch 2 taken 104 times. ✗ Branch 3 not taken.	104	if (!discard && expected > 0) {
673	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 1 taken 104 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 104 times. ✗ Branch 5 not taken.	104	chunk_values = base::RecTensor(
674		104	{static_cast<int64_t>(expected), D}, base::DataType::FLOAT32);
675			}
676	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 1 taken 104 times. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✓ Branch 4 taken 104 times.	104	if (!client->GetPrefetchResult(
677		104	shard_state.child_prefetch_ids[i], chunk_values)) {
678		✗	ok_all = false;
679		✗	break;
680			}
681	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 0 not taken. ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 0 taken 104 times. ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✓ Branch 3 taken 104 times.	104	if (discard \|\| expected <= 0) {
682		✗	continue;
683			}
684	2/8 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✗ Branch 4 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 1 taken 104 times. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✓ Branch 4 taken 104 times.	104	if (static_cast<int>(chunk_values.shape(0)) != expected) {
685	0/12 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken.	✗	LOG(ERROR) << "Prefetch chunk size mismatch: got "
686	0/16 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken.	✗	<< chunk_values.shape(0) << ", expected " << expected;
687		✗	ok_all = false;
688		✗	break;
689			}
690	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 1 taken 104 times. ✗ Branch 2 not taken.	104	const float* src = chunk_values.data_as<float>();
691	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 1 taken 104 times. ✗ Branch 2 not taken.	104	float* dst = values.data_as<float>();
692	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 0 not taken. ✗ Branch 1 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 0 taken 424 times. ✓ Branch 1 taken 104 times.	528	for (int row = 0; row < expected; ++row) {
693	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✓ Branch 2 taken 424 times.	424	if (shard_offset >= shard_state.original_indices.size()) {
694	0/12 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken.	✗	LOG(ERROR) << "Prefetch result overflow in shard "
695	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken.	✗	<< shard_state.shard_id;
696		✗	ok_all = false;
697		✗	break;
698			}
699		424	const size_t orig = shard_state.original_indices[shard_offset++];
700		424	std::memcpy(dst + orig * static_cast<size_t>(D),
701		424	src + row * D,
702		424	static_cast<size_t>(D) * sizeof(float));
703			}
704	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 0 not taken. ✗ Branch 1 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 0 not taken. ✓ Branch 1 taken 104 times.	104	if (!ok_all) {
705		✗	break;
706			}
707			}
708	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 0 not taken. ✗ Branch 1 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 0 not taken. ✓ Branch 1 taken 76 times.	76	if (!ok_all) {
709		✗	break;
710			}
711			}
712
713			{
714	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 1 taken 38 times. ✗ Branch 2 not taken.	38	std::lock_guard<std::mutex> lk(prefetch_mu_);
715	1/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetPrefetchResult(unsigned long, base::RecTensor&): ✓ Branch 1 taken 38 times. ✗ Branch 2 not taken.	38	prefetch_states_.erase(prefetch_id);
716		38	}
717		38	return ok_all;
718		42	}
719
720			private:
721			struct ServerConfig {
722			std::string host;
723			int port;
724			int shard;
725			};
726
727			struct DistPrefetchShardState {
728			int shard_id = -1;
729			int client_index = -1;
730			std::vector<size_t> original_indices;
731			std::vector<uint64_t> child_prefetch_ids;
732			std::vector<int> chunk_sizes;
733			};
734
735			struct DistPrefetchState {
736			size_t total_keys = 0;
737			std::vector<DistPrefetchShardState> shard_states;
738			};
739
740			static std::string
741		✗	ShardCheckpointPath(const std::string& path, int shard_id) {
742	0/8 recstore::DistributedShardedClient<BRPCParameterClient>::ShardCheckpointPath(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, int): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::ShardCheckpointPath(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, int): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken.	✗	const std::string prefix =
743	0/8 recstore::DistributedShardedClient<BRPCParameterClient>::ShardCheckpointPath(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, int): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::ShardCheckpointPath(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, int): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken.	✗	path.empty() \|\| path.back() == '/' ? path : path + "/";
744	0/16 recstore::DistributedShardedClient<BRPCParameterClient>::ShardCheckpointPath(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, int): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::ShardCheckpointPath(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, int): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken.	✗	return prefix + "recstore-shard-" + std::to_string(shard_id) + ".bin";
745		✗	}
746
747		✗	bool FanOutCheckpoint(
748			const std::string& path, const std::string& metadata, bool save) {
749	0/12 recstore::DistributedShardedClient<BRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 6 not taken. ✗ Branch 7 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 6 not taken. ✗ Branch 7 not taken.	✗	if (path.empty() \|\| metadata.empty()) {
750	0/12 recstore::DistributedShardedClient<BRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken.	✗	LOG(ERROR) << "Checkpoint path and metadata must be non-empty";
751		✗	return false;
752			}
753		✗	std::vector<std::future<bool>> futures;
754	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool): ✗ Branch 5 not taken. ✗ Branch 6 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool): ✗ Branch 5 not taken. ✗ Branch 6 not taken.	✗	for (const auto& server : server_configs_) {
755	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool): ✗ Branch 1 not taken. ✗ Branch 2 not taken.	✗	const auto it = shard_to_client_index_.find(server.shard);
756	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool): ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool): ✗ Branch 2 not taken. ✗ Branch 3 not taken.	✗	if (it == shard_to_client_index_.end()) {
757	0/16 recstore::DistributedShardedClient<BRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken.	✗	LOG(ERROR) << "No client found for shard " << server.shard;
758		✗	return false;
759			}
760		✗	ClientT* client = clients_[it->second].get();
761	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool): ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool): ✗ Branch 1 not taken. ✗ Branch 2 not taken.	✗	const std::string file = ShardCheckpointPath(path, server.shard);
762	0/16 recstore::DistributedShardedClient<BRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. ✗ Branch 10 not taken. ✗ Branch 11 not taken.	✗	futures.push_back(
763		✗	std::async(std::launch::async, [client, file, metadata, save]() {
764	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)::{lambda()#1}::operator()() const: ✗ Branch 0 not taken. ✗ Branch 1 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)::{lambda()#1}::operator()() const: ✗ Branch 0 not taken. ✗ Branch 1 not taken.	✗	return save ? client->SaveCheckpoint(file, metadata)
765		✗	: client->LoadCheckpoint(file, metadata);
766			}));
767			}
768		✗	bool all_success = true;
769	0/8 recstore::DistributedShardedClient<BRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool): ✗ Branch 4 not taken. ✗ Branch 5 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)::{lambda()#1}::operator()() const: ✗ Branch 4 not taken. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool): ✗ Branch 4 not taken. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)::{lambda()#1}::operator()() const: ✗ Branch 4 not taken. ✗ Branch 5 not taken.	✗	for (auto& future : futures) {
770	0/24 recstore::DistributedShardedClient<BRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 6 not taken. recstore::DistributedShardedClient<BRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 6 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool): ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 6 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::FanOutCheckpoint(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)::{lambda()#1}::operator()() const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 3 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 6 not taken.	✗	all_success = future.get() && all_success;
771			}
772		✗	return all_success;
773		✗	}
774
775		28	void InitializeClients() {
776		28	clients_.clear();
777		28	clients_.reserve(server_configs_.size());
778
779	4/4 recstore::DistributedShardedClient<BRPCParameterClient>::InitializeClients(): ✓ Branch 5 taken 22 times. ✓ Branch 6 taken 12 times. recstore::DistributedShardedClient<GRPCParameterClient>::InitializeClients(): ✓ Branch 5 taken 32 times. ✓ Branch 6 taken 16 times.	82	for (const auto& server_config : server_configs_) {
780	16/32 recstore::DistributedShardedClient<BRPCParameterClient>::InitializeClients(): ✓ Branch 1 taken 22 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 22 times. ✗ Branch 5 not taken. ✓ Branch 7 taken 22 times. ✗ Branch 8 not taken. ✓ Branch 10 taken 22 times. ✗ Branch 11 not taken. ✓ Branch 13 taken 22 times. ✗ Branch 14 not taken. ✓ Branch 16 taken 22 times. ✗ Branch 17 not taken. ✓ Branch 19 taken 22 times. ✗ Branch 20 not taken. ✓ Branch 22 taken 22 times. ✗ Branch 23 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::InitializeClients(): ✓ Branch 1 taken 32 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 32 times. ✗ Branch 5 not taken. ✓ Branch 7 taken 32 times. ✗ Branch 8 not taken. ✓ Branch 10 taken 32 times. ✗ Branch 11 not taken. ✓ Branch 13 taken 32 times. ✗ Branch 14 not taken. ✓ Branch 16 taken 32 times. ✗ Branch 17 not taken. ✓ Branch 19 taken 32 times. ✗ Branch 20 not taken. ✓ Branch 22 taken 32 times. ✗ Branch 23 not taken.	810	json client_config = {{"host", server_config.host},
781		54	{"port", server_config.port},
782		54	{"shard", server_config.shard}};
783
784	4/8 recstore::DistributedShardedClient<BRPCParameterClient>::InitializeClients(): ✓ Branch 1 taken 22 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 22 times. ✗ Branch 5 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::InitializeClients(): ✓ Branch 1 taken 32 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 32 times. ✗ Branch 5 not taken.	54	clients_.push_back(std::make_unique<ClientT>(client_config));
785
786	10/20 recstore::DistributedShardedClient<BRPCParameterClient>::InitializeClients(): ✓ Branch 1 taken 22 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 22 times. ✗ Branch 5 not taken. ✓ Branch 7 taken 22 times. ✗ Branch 8 not taken. ✓ Branch 10 taken 22 times. ✗ Branch 11 not taken. ✓ Branch 13 taken 22 times. ✗ Branch 14 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::InitializeClients(): ✓ Branch 1 taken 32 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 32 times. ✗ Branch 5 not taken. ✓ Branch 7 taken 32 times. ✗ Branch 8 not taken. ✓ Branch 10 taken 32 times. ✗ Branch 11 not taken. ✓ Branch 13 taken 32 times. ✗ Branch 14 not taken.	108	LOG(INFO) << "Created " << transport_name_ << " client for shard "
787	8/16 recstore::DistributedShardedClient<BRPCParameterClient>::InitializeClients(): ✓ Branch 1 taken 22 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 22 times. ✗ Branch 5 not taken. ✓ Branch 7 taken 22 times. ✗ Branch 8 not taken. ✓ Branch 10 taken 22 times. ✗ Branch 11 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::InitializeClients(): ✓ Branch 1 taken 32 times. ✗ Branch 2 not taken. ✓ Branch 4 taken 32 times. ✗ Branch 5 not taken. ✓ Branch 7 taken 32 times. ✗ Branch 8 not taken. ✓ Branch 10 taken 32 times. ✗ Branch 11 not taken.	54	<< server_config.shard << " at " << server_config.host << ":"
788	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::InitializeClients(): ✓ Branch 1 taken 22 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::InitializeClients(): ✓ Branch 1 taken 32 times. ✗ Branch 2 not taken.	54	<< server_config.port;
789			}
790		28	}
791
792		2464	int GetShardId(uint64_t key) const {
793	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetShardId(unsigned long) const: ✓ Branch 1 taken 448 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetShardId(unsigned long) const: ✓ Branch 1 taken 2016 times. ✗ Branch 2 not taken.	2464	if (hash_method_ == "city_hash") {
794		2464	return GetHash(key) % num_shards_;
795	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetShardId(unsigned long) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetShardId(unsigned long) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken.	✗	} else if (hash_method_ == "simple_mod") {
796		✗	return key % num_shards_;
797			} else {
798	0/4 recstore::DistributedShardedClient<BRPCParameterClient>::GetShardId(unsigned long) const: ✗ Branch 2 not taken. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetShardId(unsigned long) const: ✗ Branch 2 not taken. ✗ Branch 3 not taken.	✗	LOG(ERROR) << "Unknown hash method: " << hash_method_
799	0/12 recstore::DistributedShardedClient<BRPCParameterClient>::GetShardId(unsigned long) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::GetShardId(unsigned long) const: ✗ Branch 1 not taken. ✗ Branch 2 not taken. ✗ Branch 4 not taken. ✗ Branch 5 not taken. ✗ Branch 7 not taken. ✗ Branch 8 not taken.	✗	<< ", using city_hash";
800		✗	return GetHash(key) % num_shards_;
801			}
802			}
803
804			void
805		118	PartitionKeys(const base::ConstArray<uint64_t>& keys,
806			std::vector<std::vector<uint64_t>>& partitioned_keys) const {
807	4/4 recstore::DistributedShardedClient<BRPCParameterClient>::PartitionKeys(base::ConstArray<unsigned long> const&, std::vector<std::vector<unsigned long, std::allocator<unsigned long> >, std::allocator<std::vector<unsigned long, std::allocator<unsigned long> > > >&) const: ✓ Branch 5 taken 40 times. ✓ Branch 6 taken 20 times. recstore::DistributedShardedClient<GRPCParameterClient>::PartitionKeys(base::ConstArray<unsigned long> const&, std::vector<std::vector<unsigned long, std::allocator<unsigned long> >, std::allocator<std::vector<unsigned long, std::allocator<unsigned long> > > >&) const: ✓ Branch 5 taken 196 times. ✓ Branch 6 taken 98 times.	354	for (auto& partition : partitioned_key_buffer_) {
808		236	partition.clear();
809			}
810	4/4 recstore::DistributedShardedClient<BRPCParameterClient>::PartitionKeys(base::ConstArray<unsigned long> const&, std::vector<std::vector<unsigned long, std::allocator<unsigned long> >, std::allocator<std::vector<unsigned long, std::allocator<unsigned long> > > >&) const: ✓ Branch 5 taken 40 times. ✓ Branch 6 taken 20 times. recstore::DistributedShardedClient<GRPCParameterClient>::PartitionKeys(base::ConstArray<unsigned long> const&, std::vector<std::vector<unsigned long, std::allocator<unsigned long> >, std::allocator<std::vector<unsigned long, std::allocator<unsigned long> > > >&) const: ✓ Branch 5 taken 196 times. ✓ Branch 6 taken 98 times.	354	for (auto& mapping : key_index_mapping_) {
811		236	mapping.clear();
812			}
813
814	4/4 recstore::DistributedShardedClient<BRPCParameterClient>::PartitionKeys(base::ConstArray<unsigned long> const&, std::vector<std::vector<unsigned long, std::allocator<unsigned long> >, std::allocator<std::vector<unsigned long, std::allocator<unsigned long> > > >&) const: ✓ Branch 1 taken 448 times. ✓ Branch 2 taken 20 times. recstore::DistributedShardedClient<GRPCParameterClient>::PartitionKeys(base::ConstArray<unsigned long> const&, std::vector<std::vector<unsigned long, std::allocator<unsigned long> >, std::allocator<std::vector<unsigned long, std::allocator<unsigned long> > > >&) const: ✓ Branch 1 taken 1592 times. ✓ Branch 2 taken 98 times.	2158	for (size_t i = 0; i < keys.Size(); ++i) {
815		2040	uint64_t key = keys[i];
816	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::PartitionKeys(base::ConstArray<unsigned long> const&, std::vector<std::vector<unsigned long, std::allocator<unsigned long> >, std::allocator<std::vector<unsigned long, std::allocator<unsigned long> > > >&) const: ✓ Branch 1 taken 448 times. ✗ Branch 2 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PartitionKeys(base::ConstArray<unsigned long> const&, std::vector<std::vector<unsigned long, std::allocator<unsigned long> >, std::allocator<std::vector<unsigned long, std::allocator<unsigned long> > > >&) const: ✓ Branch 1 taken 1592 times. ✗ Branch 2 not taken.	2040	int shard_id = GetShardId(key);
817
818	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::PartitionKeys(base::ConstArray<unsigned long> const&, std::vector<std::vector<unsigned long, std::allocator<unsigned long> >, std::allocator<std::vector<unsigned long, std::allocator<unsigned long> > > >&) const: ✓ Branch 2 taken 448 times. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PartitionKeys(base::ConstArray<unsigned long> const&, std::vector<std::vector<unsigned long, std::allocator<unsigned long> >, std::allocator<std::vector<unsigned long, std::allocator<unsigned long> > > >&) const: ✓ Branch 2 taken 1592 times. ✗ Branch 3 not taken.	2040	partitioned_key_buffer_[shard_id].push_back(key);
819	2/4 recstore::DistributedShardedClient<BRPCParameterClient>::PartitionKeys(base::ConstArray<unsigned long> const&, std::vector<std::vector<unsigned long, std::allocator<unsigned long> >, std::allocator<std::vector<unsigned long, std::allocator<unsigned long> > > >&) const: ✓ Branch 2 taken 448 times. ✗ Branch 3 not taken. recstore::DistributedShardedClient<GRPCParameterClient>::PartitionKeys(base::ConstArray<unsigned long> const&, std::vector<std::vector<unsigned long, std::allocator<unsigned long> >, std::allocator<std::vector<unsigned long, std::allocator<unsigned long> > > >&) const: ✓ Branch 2 taken 1592 times. ✗ Branch 3 not taken.	2040	key_index_mapping_[shard_id].push_back(i);
820			}
821
822		118	partitioned_keys = partitioned_key_buffer_;
823		118	}
824
825			const char* transport_name_;
826
827			// Config
828			int num_shards_;
829			int max_keys_per_request_;
830			std::string hash_method_;
831
832			std::vector<ServerConfig> server_configs_;
833
834			// Single-shard clients (one per server entry).
835			std::vector<std::unique_ptr<ClientT>> clients_;
836
837			// Logical shard id -> index in clients_.
838			std::unordered_map<int, int> shard_to_client_index_;
839
840			// Partition buffers (reused).
841			mutable std::vector<std::vector<uint64_t>> partitioned_key_buffer_;
842			mutable std::vector<std::vector<size_t>> key_index_mapping_;
843
844			std::mutex prefetch_mu_;
845			std::unordered_map<uint64_t, std::shared_ptr<DistPrefetchState>>
846			prefetch_states_;
847			uint64_t next_prefetch_id_ = 1;
848			};
849
850			} // namespace recstore
851