diff --git a/olp-cpp-sdk-dataservice-read/src/QueryMetadataJob.h b/olp-cpp-sdk-dataservice-read/src/QueryMetadataJob.h index 7ddd66b94..285ae0d7b 100644 --- a/olp-cpp-sdk-dataservice-read/src/QueryMetadataJob.h +++ b/olp-cpp-sdk-dataservice-read/src/QueryMetadataJob.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020 HERE Europe B.V. + * Copyright (C) 2020-2024 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,7 +42,7 @@ using QueryItemsFunc = std::function; template -using FilterItemsFunc = std::function; +using FilterItemsFunc = std::function; using VectorOfTokens = std::vector; @@ -99,9 +99,13 @@ class QueryMetadataJob { accumulated_statistics_ += GetNetworkStatistics(response); if (response.IsSuccessful()) { - auto items = response.MoveResult(); - std::move(items.begin(), items.end(), - std::inserter(query_result_, query_result_.begin())); + if (query_result_.empty()) { + query_result_ = response.MoveResult(); + } else { + auto items = response.MoveResult(); + std::move(items.begin(), items.end(), + std::inserter(query_result_, query_result_.begin())); + } } else { const auto& error = response.GetError(); if (error.GetErrorCode() == client::ErrorCode::Cancelled) { @@ -125,7 +129,7 @@ class QueryMetadataJob { } if (filter_) { - query_result_ = filter_(std::move(query_result_)); + filter_(query_result_); } if (query_result_.empty()) { diff --git a/olp-cpp-sdk-dataservice-read/src/VersionedLayerClientImpl.cpp b/olp-cpp-sdk-dataservice-read/src/VersionedLayerClientImpl.cpp index 232905824..45fbf9cb3 100644 --- a/olp-cpp-sdk-dataservice-read/src/VersionedLayerClientImpl.cpp +++ b/olp-cpp-sdk-dataservice-read/src/VersionedLayerClientImpl.cpp @@ -499,11 +499,12 @@ client::CancellationToken VersionedLayerClientImpl::PrefetchTiles( const bool aggregation_enabled = request.GetDataAggregationEnabled(); - auto filter = [=](repository::SubQuadsResult tiles) mutable { + auto filter = [=](repository::SubQuadsResult& tiles) { if (request_only_input_tiles) { - return repository.FilterTilesByList(request, std::move(tiles)); + repository.FilterTilesByList(request, tiles); + } else { + repository.FilterTilesByLevel(request, tiles); } - return repository.FilterTilesByLevel(request, std::move(tiles)); }; auto query = [=](geo::TileKey root, @@ -512,8 +513,13 @@ client::CancellationToken VersionedLayerClientImpl::PrefetchTiles( root, kQuadTreeDepth, version, inner_context); if (response.IsSuccessful() && aggregation_enabled) { + const auto& tiles = response.GetResult(); auto network_stats = repository.LoadAggregatedSubQuads( - root, filter(response.GetResult()), version, inner_context); + root, + request_only_input_tiles + ? repository.FilterTileKeysByList(request, tiles) + : repository.FilterTileKeysByLevel(request, tiles), + version, inner_context); // append network statistics network_stats += GetNetworkStatistics(response); diff --git a/olp-cpp-sdk-dataservice-read/src/VolatileLayerClientImpl.cpp b/olp-cpp-sdk-dataservice-read/src/VolatileLayerClientImpl.cpp index 914d3cf20..76e27cf59 100644 --- a/olp-cpp-sdk-dataservice-read/src/VolatileLayerClientImpl.cpp +++ b/olp-cpp-sdk-dataservice-read/src/VolatileLayerClientImpl.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2021 HERE Europe B.V. + * Copyright (C) 2019-2024 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -226,11 +226,11 @@ client::CancellationToken VolatileLayerClientImpl::PrefetchTiles( inner_context); }; - auto filter = [=](repository::SubQuadsResult tiles) mutable { + auto filter = [=](repository::SubQuadsResult& tiles) { if (request_only_input_tiles) { - return repository.FilterTilesByList(request, std::move(tiles)); + repository.FilterTilesByList(request, tiles); } else { - return repository.FilterTilesByLevel(request, std::move(tiles)); + repository.FilterTilesByLevel(request, tiles); } }; diff --git a/olp-cpp-sdk-dataservice-read/src/repositories/PrefetchTilesRepository.cpp b/olp-cpp-sdk-dataservice-read/src/repositories/PrefetchTilesRepository.cpp index 6ee9157f9..e5a7fd064 100644 --- a/olp-cpp-sdk-dataservice-read/src/repositories/PrefetchTilesRepository.cpp +++ b/olp-cpp-sdk-dataservice-read/src/repositories/PrefetchTilesRepository.cpp @@ -49,11 +49,14 @@ constexpr std::uint32_t kMaxQuadTreeIndexDepth = 4u; SubQuadsResult FlattenTree(const QuadTreeIndex& tree) { SubQuadsResult result; auto index_data = tree.GetIndexData(); - std::transform(index_data.begin(), index_data.end(), - std::inserter(result, result.end()), - [](const QuadTreeIndex::IndexData& data) { - return std::make_pair(data.tile_key, data.data_handle); - }); + for (auto& data : index_data) { + const auto it = result.lower_bound(data.tile_key); + if (it == result.end() || result.key_comp()(data.tile_key, it->first)) { + result.emplace_hint(it, std::piecewise_construct, + std::forward_as_tuple(data.tile_key), + std::forward_as_tuple(std::move(data.data_handle))); + } + } return result; } @@ -173,8 +176,8 @@ RootTilesForRequest PrefetchTilesRepository::GetSlicedTiles( } client::NetworkStatistics PrefetchTilesRepository::LoadAggregatedSubQuads( - geo::TileKey root, const SubQuadsResult& tiles, std::int64_t version, - client::CancellationContext context) { + geo::TileKey root, const std::vector& tiles, + std::int64_t version, client::CancellationContext context) { // if quad tree isn't cached, no reason to download additional quads QuadTreeIndex quad_tree; client::NetworkStatistics network_stats; @@ -191,7 +194,7 @@ client::NetworkStatistics PrefetchTilesRepository::LoadAggregatedSubQuads( // found in subtiles. In this way we make sure that all tiles within requested // tree have aggregated parent downloaded and cached. This may cause // additional or duplicate download request. - auto root_index = quad_tree.Find(highest_tile_it->first, true); + auto root_index = quad_tree.Find(*highest_tile_it, true); if (root_index) { const auto& aggregated_tile_key = root_index->tile_key; @@ -316,74 +319,101 @@ SubQuadsResponse PrefetchTilesRepository::GetVolatileSubQuads( return result; } -SubQuadsResult PrefetchTilesRepository::FilterTilesByLevel( - const PrefetchTilesRequest& request, SubQuadsResult tiles) { - const auto& tile_keys = request.GetTileKeys(); - - auto skip_tile = [&](const geo::TileKey& tile_key) { - if (tile_key.Level() < request.GetMinLevel()) { - return true; - } - - if (tile_key.Level() > request.GetMaxLevel()) { - return true; - } - - return std::find_if(tile_keys.begin(), tile_keys.end(), - [&tile_key](const geo::TileKey& root_key) { - return (root_key.IsParentOf(tile_key) || - tile_key.IsParentOf(root_key) || - root_key == tile_key); - }) == tile_keys.end(); - }; +static bool skip_tile(const PrefetchTilesRequest& request, + const geo::TileKey& tile_key) { + if (tile_key.Level() < request.GetMinLevel() || + tile_key.Level() > request.GetMaxLevel()) { + return true; + } + for (const geo::TileKey& root_key : request.GetTileKeys()) { + if (root_key == tile_key || root_key.IsParentOf(tile_key) || + tile_key.IsParentOf(root_key)) + return false; + } + return true; +} +void PrefetchTilesRepository::FilterTilesByLevel( + const PrefetchTilesRequest& request, SubQuadsResult& tiles) const { for (auto sub_quad_it = tiles.begin(); sub_quad_it != tiles.end();) { - if (skip_tile(sub_quad_it->first)) { + if (skip_tile(request, sub_quad_it->first)) { sub_quad_it = tiles.erase(sub_quad_it); } else { ++sub_quad_it; } } +} - return tiles; +std::vector PrefetchTilesRepository::FilterTileKeysByLevel( + const PrefetchTilesRequest& request, const SubQuadsResult& tiles) const { + std::vector result; + for (const auto& tile : tiles) { + if (!skip_tile(request, tile.first)) { + result.emplace_back(tile.first); + } + } + return result; } -SubQuadsResult PrefetchTilesRepository::FilterTilesByList( - const PrefetchTilesRequest& request, SubQuadsResult tiles) { - const bool aggregation_enabled = request.GetDataAggregationEnabled(); +void PrefetchTilesRepository::FilterTilesByList( + const PrefetchTilesRequest& request, SubQuadsResult& tiles) const { + SubQuadsResult result; + const bool aggregation_enabled = request.GetDataAggregationEnabled(); const auto& tile_keys = request.GetTileKeys(); if (!aggregation_enabled) { - for (auto it = tiles.begin(); it != tiles.end();) { - if (std::find(tile_keys.begin(), tile_keys.end(), it->first) == - tile_keys.end()) { - it = tiles.erase(it); - } else { - ++it; + for (const auto& tile : tile_keys) { + const auto it = tiles.find(tile); + auto& new_tile = result[tile]; + if (it != tiles.end()) { + new_tile = std::move(it->second); + tiles.erase(it); } } + } else { + auto append_tile = [&](const geo::TileKey& key) { + const auto it = tiles.find(key); + if (it != tiles.end()) { + result.emplace(key, std::move(it->second)); + return true; + } else { + return result.find(key) != result.end(); + } + }; for (const auto& tile : tile_keys) { - if (tiles.find(tile) == tiles.end()) { - tiles[tile] = ""; + auto aggregated_tile = tile; + + while (aggregated_tile.IsValid() && !append_tile(aggregated_tile)) { + aggregated_tile = aggregated_tile.Parent(); + } + + if (!aggregated_tile.IsValid()) { + result[tile].clear(); // To generate Not Found error } } + } + tiles.swap(result); +} - } else { - SubQuadsResult result; +std::vector PrefetchTilesRepository::FilterTileKeysByList( + const PrefetchTilesRequest& request, const SubQuadsResult& tiles) const { + std::vector result; - auto append_tile = [&](const geo::TileKey& key) { - auto tile_it = tiles.find(key); - if (tile_it != tiles.end()) { - result[tile_it->first] = tile_it->second; + if (!request.GetDataAggregationEnabled()) { + result = request.GetTileKeys(); + } else { + auto append_tile = [&tiles, &result](const geo::TileKey& key) { + if (tiles.count(key) == 1) { + result.emplace_back(key); return true; } else { - return false; + return std::find(result.begin(), result.end(), key) != result.end(); } }; - for (const auto& tile : tile_keys) { + for (const auto& tile : request.GetTileKeys()) { auto aggregated_tile = tile; while (aggregated_tile.IsValid() && !append_tile(aggregated_tile)) { @@ -391,14 +421,11 @@ SubQuadsResult PrefetchTilesRepository::FilterTilesByList( } if (!aggregated_tile.IsValid()) { - result[tile] = ""; // To generate Not Found error + result.emplace_back(tile); // To generate Not Found error } } - - tiles.swap(result); } - - return tiles; + return result; } PrefetchTilesRepository::QuadTreeResponse diff --git a/olp-cpp-sdk-dataservice-read/src/repositories/PrefetchTilesRepository.h b/olp-cpp-sdk-dataservice-read/src/repositories/PrefetchTilesRepository.h index f55958338..0826dc2a7 100644 --- a/olp-cpp-sdk-dataservice-read/src/repositories/PrefetchTilesRepository.h +++ b/olp-cpp-sdk-dataservice-read/src/repositories/PrefetchTilesRepository.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2021 HERE Europe B.V. + * Copyright (C) 2019-2024 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -71,6 +72,18 @@ class PrefetchTilesRepository { RootTilesForRequest GetSlicedTiles(const std::vector& tile_keys, std::uint32_t min, std::uint32_t max); + /** + * @brief Filters the input tiles according to the request. + * + * Removes tiles that do not belong to the minimum and maximum levels. + * Removes tiles that are not a child or a parent of the requested tiles. + * + * @param request Your request. + * @param tiles The input tiles. + */ + void FilterTilesByLevel(const PrefetchTilesRequest& request, + SubQuadsResult& tiles) const; + /** * @brief Filters the input tiles according to the request. * @@ -80,10 +93,10 @@ class PrefetchTilesRepository { * @param request Your request. * @param tiles The input tiles. * - * @returns The modified tiles. + * @returns Tile keys matching the request. */ - SubQuadsResult FilterTilesByLevel(const PrefetchTilesRequest& request, - SubQuadsResult tiles); + std::vector FilterTileKeysByLevel( + const PrefetchTilesRequest& request, const SubQuadsResult& tiles) const; /** * @brief Filters the input tiles according to the request. @@ -94,15 +107,28 @@ class PrefetchTilesRepository { * * @param request Your request. * @param tiles The input tiles. + */ + void FilterTilesByList(const PrefetchTilesRequest& request, + SubQuadsResult& tiles) const; + + /** + * @brief Filters the input tiles according to the request. + * + * Removes tile keys that are not requested. + * Adds tile keys that are missing (to notify you that they are not found). + * If aggregated tiles were requested, scans for parents. * - * @returns The modified tiles. + * @param request Your request. + * @param tiles The input tiles. + * + * @returns Tile keys matching the request. */ - SubQuadsResult FilterTilesByList(const PrefetchTilesRequest& request, - SubQuadsResult tiles); + std::vector FilterTileKeysByList( + const PrefetchTilesRequest& request, const SubQuadsResult& tiles) const; client::NetworkStatistics LoadAggregatedSubQuads( - geo::TileKey tile, const SubQuadsResult& tiles, std::int64_t version, - client::CancellationContext context); + geo::TileKey tile, const std::vector& tiles, + std::int64_t version, client::CancellationContext context); SubQuadsResponse GetVersionedSubQuads(geo::TileKey tile, int32_t depth, std::int64_t version, diff --git a/tests/integration/olp-cpp-sdk-dataservice-read/VersionedLayerClientTest.cpp b/tests/integration/olp-cpp-sdk-dataservice-read/VersionedLayerClientTest.cpp index 7de287577..81d62a99f 100644 --- a/tests/integration/olp-cpp-sdk-dataservice-read/VersionedLayerClientTest.cpp +++ b/tests/integration/olp-cpp-sdk-dataservice-read/VersionedLayerClientTest.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2022 HERE Europe B.V. + * Copyright (C) 2019-2024 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -1661,7 +1661,61 @@ TEST_F(DataserviceReadVersionedLayerClientTest, } } -TEST_F(DataserviceReadVersionedLayerClientTest, PrefetchAggregatedTile) { +TEST_F(DataserviceReadVersionedLayerClientTest, + PrefetchAggregatedTile_FilterTileKeysByList) { + constexpr auto kLayerId = "hype-test-prefetch"; + + const auto requested_tile = geo::TileKey::FromHereTile("23618365"); + + read::VersionedLayerClient client(kCatalog, kLayerId, boost::none, settings_); + + { + SCOPED_TRACE("Prefetch aggregated tile"); + + EXPECT_CALL(*network_mock_, + Send(IsGetRequest(URL_QUADKEYS_92259), _, _, _, _)) + .WillOnce(ReturnHttpResponse(GetResponse(http::HttpStatusCode::OK), + HTTP_RESPONSE_QUADKEYS_92259_ROOT_ONLY)); + + auto request = read::PrefetchTilesRequest() + .WithTileKeys({requested_tile}) + .WithDataAggregationEnabled(true); + + auto promise = std::make_shared>(); + auto future = promise->get_future(); + auto token = client.PrefetchTiles( + request, [promise](PrefetchTilesResponse response) { + promise->set_value(std::move(response)); + }); + + ASSERT_NE(future.wait_for(kWaitTimeout), std::future_status::timeout); + PrefetchTilesResponse response = future.get(); + ASSERT_TRUE(response.IsSuccessful()) << response.GetError().GetMessage(); + ASSERT_FALSE(response.GetResult().empty()); + + const auto& result = response.GetResult(); + ASSERT_EQ(result.size(), 1); + + const auto& tile_response = result[0]; + ASSERT_TRUE(tile_response->IsSuccessful()); + ASSERT_TRUE(tile_response->tile_key_.IsParentOf(requested_tile)); + } + { + SCOPED_TRACE("Check that the tile is available as aggregated"); + EXPECT_TRUE(client.IsCached(requested_tile, true)); + EXPECT_FALSE(client.IsCached(requested_tile)); + } + { + SCOPED_TRACE("Check that the tile can be accesed with GetAggregatedData"); + auto future = client.GetAggregatedData( + read::TileRequest().WithTileKey(requested_tile)); + auto result = future.GetFuture().get(); + EXPECT_TRUE(result.IsSuccessful()); + } +} + +TEST_F(DataserviceReadVersionedLayerClientTest, + PrefetchAggregatedTile_FilterTileKeysByLevel) { constexpr auto kLayerId = "hype-test-prefetch"; const auto requested_tile = geo::TileKey::FromHereTile("23618365"); @@ -1678,6 +1732,7 @@ TEST_F(DataserviceReadVersionedLayerClientTest, PrefetchAggregatedTile) { auto request = read::PrefetchTilesRequest() .WithTileKeys({requested_tile}) + .WithMinLevel(11) .WithDataAggregationEnabled(true); auto promise = std::make_shared>();