Skip to content

Commit a921428

Browse files
meravingen123meravingen123
authored
Add Content-Encoding header to requests (#1036)
Add Content-Encoding header to requests in olp::dataservice::write module. Content-Encoding added based on layer settings received from OLP Platform. Resolves: OLPSUP-11354 Signed-off-by: Serhii Lozynskyi <ext-serhii.lozynskyi@here.com>
1 parent 44b2d22 commit a921428

15 files changed

+354
-386
lines changed

‎olp-cpp-sdk-dataservice-write/CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ set(OLP_SDK_DATASERVICE_WRITE_SOURCES
6767
# ./src/BackgroundTaskCollection.h
6868
./src/CancellationTokenList.cpp
6969
./src/CancellationTokenList.h
70+
./src/CatalogSettings.cpp
71+
./src/CatalogSettings.h
7072
# ./src/DefaultFlushEventListener.cpp
7173
# ./src/DefaultFlushEventListener.h
7274
# ./src/FlushEventListener.h

‎olp-cpp-sdk-dataservice-write/src/CatalogSettings.cpp

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
/*
2+
* Copyright (C) 2020 HERE Europe B.V.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
* SPDX-License-Identifier: Apache-2.0
17+
* License-Filename: LICENSE
18+
*/
19+
20+
#include "CatalogSettings.h"
21+
22+
#include <olp/core/cache/CacheSettings.h>
23+
#include <olp/core/client/OlpClientSettingsFactory.h>
24+
#include <boost/format.hpp>
25+
#include "ApiClientLookup.h"
26+
#include "generated/ConfigApi.h"
27+
28+
// clang-format off
29+
#include <generated/serializer/CatalogSerializer.h>
30+
#include <generated/serializer/JsonSerializer.h>
31+
#include <generated/parser/CatalogParser.h>
32+
#include <olp/core/generated/parser/JsonParser.h>
33+
// clang-format on
34+
35+
namespace olp {
36+
namespace dataservice {
37+
namespace write {
38+
39+
CatalogSettings::CatalogSettings(const client::HRN catalog,
40+
client::OlpClientSettings settings)
41+
: catalog_(std::move(catalog)),
42+
cache_(settings.cache),
43+
settings_(std::move(settings)) {
44+
if (!settings_.cache) {
45+
settings_.cache = client::OlpClientSettingsFactory::CreateDefaultCache({});
46+
cache_ = settings_.cache;
47+
}
48+
}
49+
50+
CatalogSettings::LayerSettingsResult CatalogSettings::GetLayerSettingsFromModel(
51+
const model::Catalog& catalog, const std::string& layer_id) {
52+
const auto& layers = catalog.GetLayers();
53+
54+
auto layer_it = std::find_if(
55+
layers.begin(), layers.end(),
56+
[&](const model::Layer& layer) { return layer.GetId() == layer_id; });
57+
58+
CatalogSettings::LayerSettings settings;
59+
if (layer_it != layers.end()) {
60+
return CatalogSettings::LayerSettings{layer_it->GetContentType(),
61+
layer_it->GetContentEncoding()};
62+
}
63+
64+
return client::ApiError(
65+
client::ErrorCode::InvalidArgument,
66+
(boost::format("Layer '%1%' not found in catalog '%2%'") % layer_id %
67+
catalog_.ToString())
68+
.str());
69+
}
70+
71+
CatalogSettings::LayerSettingsResult CatalogSettings::GetLayerSettings(
72+
client::CancellationContext context, BillingTag billing_tag,
73+
const std::string& layer_id) {
74+
const auto catalog_settings_key = catalog_.ToString() + "::catalog";
75+
76+
if (!cache_->Contains(catalog_settings_key)) {
77+
auto lookup_response = ApiClientLookup::LookupApiClient(
78+
catalog_, context, "config", "v1", settings_);
79+
if (!lookup_response.IsSuccessful()) {
80+
return lookup_response.GetError();
81+
}
82+
83+
client::OlpClient client = lookup_response.GetResult();
84+
auto catalog_response = ConfigApi::GetCatalog(client, catalog_.ToString(),
85+
billing_tag, context);
86+
87+
if (!catalog_response.IsSuccessful()) {
88+
return catalog_response.GetError();
89+
}
90+
91+
auto catalog_model = catalog_response.MoveResult();
92+
93+
cache_->Put(
94+
catalog_settings_key, catalog_model,
95+
[&]() { return serializer::serialize<model::Catalog>(catalog_model); },
96+
settings_.default_cache_expiration.count());
97+
98+
return GetLayerSettingsFromModel(catalog_model, layer_id);
99+
}
100+
101+
LayerSettings settings;
102+
103+
const auto& cached_catalog =
104+
cache_->Get(catalog_settings_key, [](const std::string& model) {
105+
return parser::parse<model::Catalog>(model);
106+
});
107+
108+
if (cached_catalog.empty()) {
109+
return client::ApiError(
110+
client::ErrorCode::Unknown,
111+
(boost::format("Cached catalog '%1' is empty") % catalog_.ToString())
112+
.str());
113+
}
114+
115+
const auto& catalog = boost::any_cast<const model::Catalog&>(cached_catalog);
116+
return GetLayerSettingsFromModel(catalog, layer_id);
117+
}
118+
119+
} // namespace write
120+
} // namespace dataservice
121+
} // namespace olp

‎olp-cpp-sdk-dataservice-write/src/CatalogSettings.h

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* Copyright (C) 2020 HERE Europe B.V.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
* SPDX-License-Identifier: Apache-2.0
17+
* License-Filename: LICENSE
18+
*/
19+
20+
#pragma once
21+
22+
#include <string>
23+
24+
#include <olp/core/cache/KeyValueCache.h>
25+
#include <olp/core/client/ApiError.h>
26+
#include <olp/core/client/ApiResponse.h>
27+
#include <olp/core/client/CancellationContext.h>
28+
#include <olp/core/client/HRN.h>
29+
#include <olp/core/client/OlpClientSettings.h>
30+
31+
namespace olp {
32+
namespace dataservice {
33+
namespace write {
34+
35+
namespace model {
36+
class Catalog;
37+
}
38+
39+
class CatalogSettings {
40+
public:
41+
using BillingTag = boost::optional<std::string>;
42+
43+
struct LayerSettings {
44+
std::string content_type;
45+
std::string content_encoding;
46+
};
47+
using LayerSettingsResult =
48+
client::ApiResponse<LayerSettings, client::ApiError>;
49+
50+
CatalogSettings(const client::HRN catalog,
51+
client::OlpClientSettings settings);
52+
53+
LayerSettingsResult GetLayerSettings(client::CancellationContext context,
54+
BillingTag billing_tag,
55+
const std::string& layer_id);
56+
57+
private:
58+
LayerSettingsResult GetLayerSettingsFromModel(const model::Catalog& catalog,
59+
const std::string& layer_id);
60+
61+
client::HRN catalog_;
62+
std::shared_ptr<cache::KeyValueCache> cache_;
63+
client::OlpClientSettings settings_;
64+
};
65+
66+
} // namespace write
67+
} // namespace dataservice
68+
} // namespace olp

‎olp-cpp-sdk-dataservice-write/src/IndexLayerClientImpl.cpp

Lines changed: 27 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,9 @@ namespace write {
4646

4747
IndexLayerClientImpl::IndexLayerClientImpl(client::HRN catalog,
4848
client::OlpClientSettings settings)
49-
: catalog_(std::move(catalog)),
50-
catalog_model_(),
51-
settings_(std::move(settings)),
49+
: catalog_(catalog),
50+
catalog_settings_(catalog, settings),
51+
settings_(settings),
5252
apiclient_config_(nullptr),
5353
apiclient_blob_(nullptr),
5454
apiclient_index_(nullptr),
@@ -118,73 +118,17 @@ olp::client::CancellationToken IndexLayerClientImpl::InitApiClients(
118118
cancel_context->ExecuteOrCancelled(indexApi_function, cancel_function);
119119
};
120120

121-
auto blobApi_function = [=]() -> olp::client::CancellationToken {
122-
return ApiClientLookup::LookupApi(self->apiclient_blob_, "blob", "v1",
123-
self->catalog_, blobApi_callback);
124-
};
125-
126-
auto configApi_callback = [=](ApiClientLookup::ApisResponse apis) {
127-
if (!apis.IsSuccessful()) {
128-
callback(std::move(apis.GetError()));
129-
std::lock_guard<std::mutex> lg{self->mutex_};
130-
self->init_in_progress_ = false;
131-
self->cond_var_.notify_one();
132-
return;
133-
}
134-
self->apiclient_config_->SetBaseUrl(apis.GetResult().at(0).GetBaseUrl());
135-
136-
cancel_context->ExecuteOrCancelled(blobApi_function, cancel_function);
137-
};
138-
139121
ul.unlock();
140122

141-
return ApiClientLookup::LookupApi(apiclient_config_, "config", "v1", catalog_,
142-
configApi_callback);
123+
return ApiClientLookup::LookupApi(self->apiclient_blob_, "blob", "v1",
124+
self->catalog_, blobApi_callback);
143125
}
144126

145127
void IndexLayerClientImpl::CancelPendingRequests() {
146128
pending_requests_->CancelAll();
147129
tokenList_.CancelAll();
148130
}
149131

150-
client::CancellationToken IndexLayerClientImpl::InitCatalogModel(
151-
const model::PublishIndexRequest& /*request*/,
152-
const InitCatalogModelCallback& callback) {
153-
if (!catalog_model_.GetId().empty()) {
154-
callback(boost::none);
155-
return client::CancellationToken();
156-
}
157-
158-
auto self = shared_from_this();
159-
return ConfigApi::GetCatalog(
160-
apiclient_config_, catalog_.ToString(), boost::none,
161-
[=](CatalogResponse catalog_response) mutable {
162-
if (!catalog_response.IsSuccessful()) {
163-
callback(std::move(catalog_response.GetError()));
164-
return;
165-
}
166-
167-
self->catalog_model_ = catalog_response.MoveResult();
168-
169-
callback(boost::none);
170-
});
171-
}
172-
173-
std::string IndexLayerClientImpl::FindContentTypeForLayerId(
174-
const std::string& layer_id) {
175-
std::string content_type;
176-
for (auto layer : catalog_model_.GetLayers()) {
177-
if (layer.GetId() == layer_id) {
178-
// TODO optimization opportunity - cache
179-
// content-type for layer when found for O(1)
180-
// subsequent lookup.
181-
content_type = layer.GetContentType();
182-
break;
183-
}
184-
}
185-
return content_type;
186-
}
187-
188132
client::CancellableFuture<PublishIndexResponse>
189133
IndexLayerClientImpl::PublishIndex(const model::PublishIndexRequest& request) {
190134
auto promise = std::make_shared<std::promise<PublishIndexResponse> >();
@@ -252,22 +196,31 @@ client::CancellationToken IndexLayerClientImpl::PublishIndex(
252196
};
253197

254198
auto put_blob_function =
255-
[=](std::string content_type) -> client::CancellationToken {
199+
[=](std::string content_type,
200+
std::string content_encoding) -> client::CancellationToken {
256201
return BlobApi::PutBlob(*self->apiclient_blob_, request.GetLayerId(),
257-
content_type, data_handle, request.GetData(),
258-
request.GetBillingTag(), put_blob_callback);
202+
content_type, content_encoding, data_handle,
203+
request.GetData(), request.GetBillingTag(),
204+
put_blob_callback);
259205
};
260206

261-
auto init_catalog_model_callback =
262-
[=](boost::optional<client::ApiError> init_catalog_model_error) {
263-
if (init_catalog_model_error) {
207+
auto init_api_client_callback =
208+
[=](boost::optional<client::ApiError> init_api_error) {
209+
if (init_api_error) {
210+
self->tokenList_.RemoveTask(op_id);
211+
callback(PublishIndexResponse(init_api_error.get()));
212+
return;
213+
}
214+
215+
auto layer_settings_response = self->catalog_settings_.GetLayerSettings(
216+
*cancel_context, request.GetBillingTag(), request.GetLayerId());
217+
if (!layer_settings_response.IsSuccessful()) {
264218
self->tokenList_.RemoveTask(op_id);
265-
callback(PublishIndexResponse(init_catalog_model_error.get()));
219+
callback(PublishIndexResponse(layer_settings_response.GetError()));
266220
return;
267221
}
268-
auto content_type =
269-
self->FindContentTypeForLayerId(request.GetLayerId());
270-
if (content_type.empty()) {
222+
auto layer_settings = layer_settings_response.GetResult();
223+
if (layer_settings.content_type.empty()) {
271224
auto errmsg = boost::format(
272225
"Unable to find the Layer ID (%1%) "
273226
"provided in the PublishIndexRequest in the "
@@ -280,23 +233,9 @@ client::CancellationToken IndexLayerClientImpl::PublishIndex(
280233
}
281234

282235
cancel_context->ExecuteOrCancelled(
283-
std::bind(put_blob_function, content_type), cancel_function);
284-
};
285-
286-
auto init_catalog_model_function = [=]() -> client::CancellationToken {
287-
return self->InitCatalogModel(request, init_catalog_model_callback);
288-
};
289-
290-
auto init_api_client_callback =
291-
[=](boost::optional<client::ApiError> init_api_error) {
292-
if (init_api_error) {
293-
self->tokenList_.RemoveTask(op_id);
294-
callback(PublishIndexResponse(init_api_error.get()));
295-
return;
296-
}
297-
298-
cancel_context->ExecuteOrCancelled(init_catalog_model_function,
299-
cancel_function);
236+
std::bind(put_blob_function, layer_settings.content_type,
237+
layer_settings.content_encoding),
238+
cancel_function);
300239
};
301240

302241
auto init_api_client_function = [=]() -> client::CancellationToken {

‎olp-cpp-sdk-dataservice-write/src/IndexLayerClientImpl.h

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232

3333
#include <olp/dataservice/write/IndexLayerClient.h>
3434
#include "generated/model/Catalog.h"
35+
#include "CatalogSettings.h"
3536

3637
namespace olp {
3738
namespace dataservice {
@@ -43,8 +44,6 @@ class PublishDataRequest;
4344

4445
using InitApiClientsCallback =
4546
std::function<void(boost::optional<client::ApiError>)>;
46-
using InitCatalogModelCallback =
47-
std::function<void(boost::optional<client::ApiError>)>;
4847

4948
class IndexLayerClientImpl
5049
: public std::enable_shared_from_this<IndexLayerClientImpl> {
@@ -82,15 +81,9 @@ class IndexLayerClientImpl
8281
std::shared_ptr<client::CancellationContext> cancel_context,
8382
InitApiClientsCallback callback);
8483

85-
client::CancellationToken InitCatalogModel(
86-
const model::PublishIndexRequest& request,
87-
const InitCatalogModelCallback& callback);
88-
89-
std::string FindContentTypeForLayerId(const std::string& layer_id);
90-
9184
private:
9285
client::HRN catalog_;
93-
model::Catalog catalog_model_;
86+
CatalogSettings catalog_settings_;
9487

9588
client::OlpClientSettings settings_;
9689

0 commit comments

Comments
 (0)