Skip to content

Commit 4afb3d3

Browse files
Fix the GetAggregatedData behavior (#1046)
Fix the GetAggregatedData behavior when the aggregated tile root is too far away, for example, the tile on level 14 is aggregated in tile on level 1, it means that the tile could be found on levels 10-14, and tiles on levels 1-9 can't be found anymore. In such a case we iterate up and fetch quadtrees until the aggregated tile root is found in tree subquads. So it could be found later by other APIs, ex. Protect, Release, Remove, etc. Resolves: OLPSUP-11651 Signed-off-by: Mykhailo Kuchma <ext-mykhailo.kuchma@here.com>
1 parent 70fbb64 commit 4afb3d3

File tree

10 files changed

+381
-140
lines changed

10 files changed

+381
-140
lines changed

olp-cpp-sdk-dataservice-read/src/VersionedLayerClientImpl.cpp

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -558,7 +558,10 @@ client::CancellationToken VersionedLayerClientImpl::GetAggregatedData(
558558

559559
auto data_task =
560560
[=](client::CancellationContext context) -> AggregatedDataResponse {
561-
if (request.GetFetchOption() == CacheWithUpdate) {
561+
const auto fetch_option = request.GetFetchOption();
562+
const auto billing_tag = request.GetBillingTag();
563+
564+
if (fetch_option == CacheWithUpdate) {
562565
return {{client::ErrorCode::InvalidArgument,
563566
"CacheWithUpdate option can not be used for versioned "
564567
"layer"}};
@@ -568,17 +571,16 @@ client::CancellationToken VersionedLayerClientImpl::GetAggregatedData(
568571
return {{client::ErrorCode::InvalidArgument, "Tile key is invalid"}};
569572
}
570573

571-
auto version_response =
572-
GetVersion(request.GetBillingTag(), request.GetFetchOption(), context);
574+
auto version_response = GetVersion(billing_tag, fetch_option, context);
573575
if (!version_response.IsSuccessful()) {
574576
return version_response.GetError();
575577
}
576578

577579
auto version = version_response.GetResult().GetVersion();
578580
repository::PartitionsRepository repository(catalog_, settings_,
579581
lookup_client_);
580-
auto partition_response =
581-
repository.GetAggregatedTile(layer_id, request, version, context);
582+
auto partition_response = repository.GetAggregatedTile(
583+
layer_id, std::move(request), version, context);
582584
if (!partition_response.IsSuccessful()) {
583585
return partition_response.GetError();
584586
}
@@ -589,7 +591,8 @@ client::CancellationToken VersionedLayerClientImpl::GetAggregatedData(
589591

590592
auto data_request = DataRequest()
591593
.WithDataHandle(fetch_partition.GetDataHandle())
592-
.WithFetchOption(request.GetFetchOption());
594+
.WithFetchOption(fetch_option)
595+
.WithBillingTag(billing_tag);
593596

594597
repository::DataRepository data_repository(
595598
std::move(catalog), std::move(settings), std::move(lookup_client));

olp-cpp-sdk-dataservice-read/src/repositories/PartitionsRepository.cpp

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -267,9 +267,8 @@ PartitionsResponse PartitionsRepository::GetPartitionById(
267267

268268
const client::OlpClient& client = query_api.GetResult();
269269

270-
PartitionsResponse query_response =
271-
QueryApi::GetPartitionsbyId(client, layer, partitions, version, {},
272-
request.GetBillingTag(), context);
270+
PartitionsResponse query_response = QueryApi::GetPartitionsbyId(
271+
client, layer, partitions, version, {}, request.GetBillingTag(), context);
273272

274273
if (query_response.IsSuccessful() && fetch_option != OnlineOnly) {
275274
OLP_SDK_LOG_DEBUG_F(kLogTag,
@@ -390,13 +389,34 @@ QuadTreeIndexResponse PartitionsRepository::GetQuadTreeIndexForTile(
390389
}
391390

392391
PartitionResponse PartitionsRepository::GetAggregatedTile(
393-
const std::string& layer, const TileRequest& request,
392+
const std::string& layer, TileRequest request,
394393
boost::optional<int64_t> version, client::CancellationContext context) {
395394
auto quad_tree_response =
396395
GetQuadTreeIndexForTile(layer, request, version, context);
397396
if (!quad_tree_response.IsSuccessful()) {
398397
return quad_tree_response.GetError();
399398
}
399+
400+
// When the parent tile is too far away, we iterate up and download metadata
401+
// for parent tiles until we cover aggregated tile root as a subquad. This is
402+
// needed for the users who need to access the aggregated tile root directly.
403+
// Else, we can't find it in cache.
404+
if (request.GetFetchOption() != FetchOptions::CacheOnly) {
405+
const auto& result = quad_tree_response.GetResult();
406+
auto index_data = result.Find(request.GetTileKey(), true);
407+
if (index_data) {
408+
const auto& aggregated_tile_key = index_data.value().tile_key;
409+
auto root = result.GetRootTile();
410+
while (root.Level() > aggregated_tile_key.Level()) {
411+
auto parent = root.Parent();
412+
request.WithTileKey(parent);
413+
// Ignore result for now
414+
GetQuadTreeIndexForTile(layer, request, version, context);
415+
root = parent.ChangedLevelBy(-kAggregateQuadTreeDepth);
416+
}
417+
}
418+
}
419+
400420
return FindPartition(quad_tree_response.GetResult(), request, true);
401421
}
402422

olp-cpp-sdk-dataservice-read/src/repositories/PartitionsRepository.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ class PartitionsRepository {
6868
const std::string& partition);
6969

7070
PartitionResponse GetAggregatedTile(const std::string& layer,
71-
const TileRequest& request,
71+
TileRequest request,
7272
boost::optional<int64_t> version,
7373
client::CancellationContext context);
7474

olp-cpp-sdk-dataservice-read/tests/PartitionsRepositoryTest.cpp

Lines changed: 0 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -962,48 +962,6 @@ TEST_F(PartitionsRepositoryTest, GetAggregatedPartitionForVersionedTile) {
962962
ASSERT_TRUE(response.IsSuccessful()) << response.GetError().GetMessage();
963963
ASSERT_EQ(result.GetPartition(), tile_key.ToHereTile());
964964
}
965-
966-
{
967-
SCOPED_TRACE("Parent tile");
968-
969-
const auto tile_key = olp::geo::TileKey::FromHereTile("23064");
970-
const auto parent_tile_key = tile_key.ChangedLevelBy(-6).ToHereTile();
971-
const auto request =
972-
olp::dataservice::read::TileRequest().WithTileKey(tile_key);
973-
olp::client::CancellationContext context;
974-
975-
auto mock_network = std::make_shared<NetworkMock>();
976-
auto mock_cache = std::make_shared<CacheMock>();
977-
978-
OlpClientSettings settings;
979-
settings.cache = mock_cache;
980-
settings.network_request_handler = mock_network;
981-
982-
EXPECT_CALL(*mock_network, Send(IsGetRequest(kUrlLookupQuery), _, _, _, _))
983-
.WillOnce(ReturnHttpResponse(olp::http::NetworkResponse().WithStatus(
984-
olp::http::HttpStatusCode::OK),
985-
kHttpResponceLookupQuery));
986-
EXPECT_CALL(*mock_network,
987-
Send(IsGetRequest(kQueryQuadTreeIndex), _, _, _, _))
988-
.WillOnce(ReturnHttpResponse(olp::http::NetworkResponse().WithStatus(
989-
olp::http::HttpStatusCode::OK),
990-
kSubQuadsWithParent));
991-
EXPECT_CALL(*mock_cache, Get(_, _)).WillOnce(Return(boost::any()));
992-
EXPECT_CALL(*mock_cache, Put(_, _, _, _)).WillOnce(Return(true));
993-
EXPECT_CALL(*mock_cache, Get(_))
994-
.WillRepeatedly(Return(KeyValueCache::ValueTypePtr()));
995-
EXPECT_CALL(*mock_cache, Put(_, _, _)).WillOnce(Return(true));
996-
997-
olp::client::ApiLookupClient lookup_client(hrn, settings);
998-
repository::PartitionsRepository repository(hrn, settings, lookup_client);
999-
auto response =
1000-
repository.GetAggregatedTile(layer, request, version, context);
1001-
const auto& result = response.GetResult();
1002-
1003-
ASSERT_TRUE(response.IsSuccessful()) << response.GetError().GetMessage();
1004-
ASSERT_EQ(result.GetPartition(), parent_tile_key);
1005-
}
1006-
1007965
{
1008966
SCOPED_TRACE("QuadTree is cached");
1009967

tests/common/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ target_include_directories(olp-cpp-sdk-tests-common PUBLIC
3939
)
4040

4141
target_link_libraries(olp-cpp-sdk-tests-common
42-
PRIVATE
42+
PUBLIC
4343
gmock
4444
olp-cpp-sdk-core
4545
olp-cpp-sdk-dataservice-write

tests/common/ReadDefaultResponses.cpp

Lines changed: 145 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -29,82 +29,110 @@
2929
#include <rapidjson/writer.h>
3030

3131
namespace {
32-
void WriteSubquadsToJson(rapidjson::Document& doc,
33-
const olp::geo::TileKey& root_tile,
34-
const std::vector<std::uint16_t>& sub_quads,
35-
rapidjson::Document::AllocatorType& allocator) {
32+
33+
void WriteSubquadsToJson(
34+
rapidjson::Document& doc, const olp::geo::TileKey& root_tile,
35+
const std::map<std::uint64_t, mockserver::TileMetadata>& sub_quads,
36+
rapidjson::Document::AllocatorType& allocator) {
3637
rapidjson::Value sub_quads_value;
3738
sub_quads_value.SetArray();
3839
for (auto quad : sub_quads) {
39-
const auto partition = root_tile.AddedSubkey64(quad).ToHereTile();
40-
const auto data_handle =
41-
mockserver::ReadDefaultResponses::GenerateDataHandle(partition);
40+
const auto partition = root_tile.AddedSubkey64(quad.first).ToHereTile();
41+
const auto& data_handle = quad.second.data_handle;
42+
const auto version = quad.second.version;
4243

4344
rapidjson::Value item_value;
4445
item_value.SetObject();
45-
olp::serializer::serialize("subQuadKey", std::to_string(quad), item_value,
46-
allocator);
47-
olp::serializer::serialize("version", 0, item_value, allocator);
46+
olp::serializer::serialize("subQuadKey", std::to_string(quad.first),
47+
item_value, allocator);
48+
olp::serializer::serialize("version", version, item_value, allocator);
4849
olp::serializer::serialize("dataHandle", data_handle, item_value,
4950
allocator);
50-
olp::serializer::serialize("dataSize", 100, item_value, allocator);
5151
sub_quads_value.PushBack(std::move(item_value), allocator);
5252
}
5353
doc.AddMember("subQuads", std::move(sub_quads_value), allocator);
5454
}
5555

56-
void WriteParentquadsToJson(rapidjson::Document& doc,
57-
const std::vector<std::uint64_t>& parent_quads,
58-
rapidjson::Document::AllocatorType& allocator) {
56+
void WriteParentquadsToJson(
57+
rapidjson::Document& doc,
58+
const std::map<std::uint64_t, mockserver::TileMetadata>& parent_quads,
59+
rapidjson::Document::AllocatorType& allocator) {
5960
rapidjson::Value parent_quads_value;
6061
parent_quads_value.SetArray();
6162
for (auto parent : parent_quads) {
62-
const auto partition = std::to_string(parent);
63-
const auto data_handle =
64-
mockserver::ReadDefaultResponses::GenerateDataHandle(partition);
63+
const auto partition = std::to_string(parent.first);
64+
const auto version = parent.second.version;
65+
const auto& data_handle = parent.second.data_handle;
6566

6667
rapidjson::Value item_value;
6768
item_value.SetObject();
68-
olp::serializer::serialize("partition", std::to_string(parent), item_value,
69-
allocator);
70-
olp::serializer::serialize("version", 0, item_value, allocator);
69+
olp::serializer::serialize("partition", partition, item_value, allocator);
70+
olp::serializer::serialize("version", version, item_value, allocator);
7171
olp::serializer::serialize("dataHandle", data_handle, item_value,
7272
allocator);
7373
olp::serializer::serialize("dataSize", 100, item_value, allocator);
7474
parent_quads_value.PushBack(std::move(item_value), allocator);
7575
}
7676
doc.AddMember("parentQuads", std::move(parent_quads_value), allocator);
7777
}
78+
79+
std::string GenerateRandomString(size_t length) {
80+
std::string letters =
81+
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
82+
std::random_device device;
83+
std::mt19937 generator(device());
84+
std::uniform_int_distribution<unsigned int> dis(0u, letters.length() - 1);
85+
86+
std::string result;
87+
result.resize(length);
88+
for (auto i = 0u; i < length; ++i) {
89+
result[i] += letters[dis(generator)];
90+
}
91+
92+
return result;
93+
}
94+
95+
void FillSubQuads(std::int32_t depth, std::vector<std::uint16_t>& sub_quads_) {
96+
const auto sub_tile = olp::geo::TileKey::FromRowColumnLevel(0, 0, depth);
97+
const auto start_level_id = sub_tile.ToQuadKey64();
98+
const auto tiles_count = olp::geo::QuadKey64Helper::ChildrenAtLevel(depth);
99+
100+
std::vector<std::uint64_t> layer_ids(tiles_count);
101+
std::iota(layer_ids.begin(), layer_ids.end(), start_level_id);
102+
sub_quads_.insert(sub_quads_.end(), layer_ids.begin(), layer_ids.end());
103+
}
104+
78105
} // namespace
79106

80107
namespace mockserver {
81108

109+
std::string ReadDefaultResponses::GenerateData(size_t length) {
110+
return GenerateRandomString(length);
111+
}
112+
82113
std::string ReadDefaultResponses::GenerateQuadTreeResponse(
83114
olp::geo::TileKey root_tile, std::uint32_t depth,
84115
const std::vector<std::uint32_t>& available_levels) {
85-
std::vector<std::uint16_t> sub_quads;
86-
std::vector<std::uint64_t> parent_quads;
116+
std::map<std::uint64_t, TileMetadata> sub_quads;
117+
std::map<std::uint64_t, TileMetadata> parent_quads;
87118

88119
// generate data
89120
for (auto level : available_levels) {
90121
if (level < root_tile.Level()) {
91-
auto key = root_tile.ChangedLevelTo(level);
92-
parent_quads.push_back(key.ToQuadKey64());
122+
auto key = root_tile.ChangedLevelTo(level).ToQuadKey64();
123+
parent_quads[key] = {GenerateDataHandle(std::to_string(key)), 0};
93124
} else {
94125
const auto level_depth = level - root_tile.Level();
95126
if (level_depth > depth) {
96127
continue;
97128
}
98129

99-
const auto sub_tile =
100-
olp::geo::TileKey::FromRowColumnLevel(0, 0, level_depth);
101-
const auto start_level_id = sub_tile.ToQuadKey64();
102-
const auto tiles_count =
103-
olp::geo::QuadKey64Helper::ChildrenAtLevel(level_depth);
130+
std::vector<std::uint16_t> sub_quads_vector;
131+
FillSubQuads(level_depth, sub_quads_vector);
104132

105-
std::vector<std::uint64_t> layer_ids(tiles_count);
106-
std::iota(layer_ids.begin(), layer_ids.end(), start_level_id);
107-
sub_quads.insert(sub_quads.end(), layer_ids.begin(), layer_ids.end());
133+
for (const auto& sub_quad : sub_quads_vector) {
134+
sub_quads[sub_quad] = {GenerateDataHandle(std::to_string(sub_quad)), 0};
135+
}
108136
}
109137
}
110138

@@ -120,4 +148,89 @@ std::string ReadDefaultResponses::GenerateQuadTreeResponse(
120148
return buffer.GetString();
121149
}
122150

151+
QuadTreeBuilder::QuadTreeBuilder(olp::geo::TileKey root_tile,
152+
boost::optional<int32_t> version)
153+
: root_tile_(root_tile), base_version_(version) {}
154+
155+
QuadTreeBuilder& QuadTreeBuilder::WithParent(olp::geo::TileKey parent,
156+
std::string datahandle,
157+
boost::optional<int32_t> version) {
158+
assert(root_tile_.IsChildOf(parent));
159+
160+
// Make sure to set version when the base_version is set
161+
if (version != boost::none) {
162+
assert(base_version_ != boost::none);
163+
} else if (base_version_) {
164+
version = base_version_;
165+
}
166+
167+
parent_quads_[parent.ToQuadKey64()] = {datahandle, version};
168+
169+
return *this;
170+
}
171+
172+
QuadTreeBuilder& QuadTreeBuilder::FillParents() {
173+
auto key = root_tile_.Parent();
174+
while (key.IsValid()) {
175+
auto quad_key = key.ToQuadKey64();
176+
if (parent_quads_.find(quad_key) == parent_quads_.end()) {
177+
parent_quads_[quad_key] = {GenerateRandomString(32), base_version_};
178+
}
179+
}
180+
return *this;
181+
}
182+
183+
QuadTreeBuilder& QuadTreeBuilder::WithSubQuad(
184+
olp::geo::TileKey tile, std::string datahandle,
185+
boost::optional<int32_t> version) {
186+
assert(tile.IsChildOf(root_tile_) || tile == root_tile_);
187+
assert((tile.Level() - root_tile_.Level()) <= 4);
188+
if (version != boost::none) {
189+
assert(base_version_ != boost::none);
190+
}
191+
192+
auto origin = root_tile_.ChangedLevelTo(tile.Level());
193+
auto sub_quad =
194+
olp::geo::TileKey::FromRowColumnLevel(tile.Row() - origin.Row(),
195+
tile.Column() - origin.Column(),
196+
tile.Level() - root_tile_.Level())
197+
.ToQuadKey64();
198+
199+
sub_quads_[sub_quad] = {datahandle, version};
200+
201+
return *this;
202+
}
203+
204+
QuadTreeBuilder& QuadTreeBuilder::FillSubquads(uint32_t depth) {
205+
assert(depth <= 4);
206+
207+
std::vector<std::uint16_t> sub_quads;
208+
for (uint32_t i = 0; i <= depth; i++) {
209+
FillSubQuads(i, sub_quads);
210+
}
211+
212+
for (const auto& sub_quad : sub_quads) {
213+
if (sub_quads_.find(sub_quad) == sub_quads_.end()) {
214+
sub_quads_[sub_quad] = {GenerateRandomString(32), base_version_};
215+
}
216+
}
217+
218+
return *this;
219+
}
220+
221+
std::string QuadTreeBuilder::BuildJson() const {
222+
rapidjson::Document doc;
223+
auto& allocator = doc.GetAllocator();
224+
doc.SetObject();
225+
WriteSubquadsToJson(doc, root_tile_, sub_quads_, allocator);
226+
WriteParentquadsToJson(doc, parent_quads_, allocator);
227+
228+
rapidjson::StringBuffer buffer;
229+
rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
230+
doc.Accept(writer);
231+
return buffer.GetString();
232+
}
233+
234+
olp::geo::TileKey QuadTreeBuilder::Root() const { return root_tile_; }
235+
123236
} // namespace mockserver

0 commit comments

Comments
 (0)