Skip to content

Commit 8dff5c1

Browse files
author
hiddenpath
committed
YT-23616: Make implementation of CanonizeYPath method being common
commit_hash:7f3ecc44b4299acc4fc7b0f463eceac61d0b0156
1 parent 0395c50 commit 8dff5c1

File tree

10 files changed

+36
-47
lines changed

10 files changed

+36
-47
lines changed

‎yt/cpp/mapreduce/client/client.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -276,7 +276,7 @@ void TClientBase::Concatenate(
276276

277277
TRichYPath TClientBase::CanonizeYPath(const TRichYPath& path)
278278
{
279-
return NRawClient::CanonizeYPath(ClientRetryPolicy_->CreatePolicyForGenericRequest(), Context_, path);
279+
return NRawClient::CanonizeYPath(RawClient_, path);
280280
}
281281

282282
TVector<TTableColumnarStatistics> TClientBase::GetTableColumnarStatistics(

‎yt/cpp/mapreduce/client/operation.cpp

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -280,8 +280,8 @@ TSimpleOperationIo CreateSimpleOperationIo(
280280
structuredJob,
281281
preparer,
282282
options,
283-
CanonizeStructuredTableList(preparer.GetContext(), GetStructuredInputs(spec)),
284-
CanonizeStructuredTableList(preparer.GetContext(), GetStructuredOutputs(spec)),
283+
CanonizeStructuredTableList(preparer.GetClient()->GetRawClient(), GetStructuredInputs(spec)),
284+
CanonizeStructuredTableList(preparer.GetClient()->GetRawClient(), GetStructuredOutputs(spec)),
285285
hints,
286286
nodeReaderFormat,
287287
GetColumnsUsedInOperation(spec));
@@ -303,8 +303,8 @@ TSimpleOperationIo CreateSimpleOperationIo(
303303
}
304304
};
305305

306-
auto inputs = NRawClient::CanonizeYPaths(/* retryPolicy */ nullptr, preparer.GetContext(), spec.GetInputs());
307-
auto outputs = NRawClient::CanonizeYPaths(/* retryPolicy */ nullptr, preparer.GetContext(), spec.GetOutputs());
306+
auto inputs = NRawClient::CanonizeYPaths(preparer.GetClient()->GetRawClient(), spec.GetInputs());
307+
auto outputs = NRawClient::CanonizeYPaths(preparer.GetClient()->GetRawClient(), spec.GetOutputs());
308308

309309
VerifyHasElements(inputs, "input");
310310
VerifyHasElements(outputs, "output");
@@ -1632,9 +1632,9 @@ void ExecuteMapReduce(
16321632
TMapReduceOperationSpec spec = spec_;
16331633

16341634
TMapReduceOperationIo operationIo;
1635-
auto structuredInputs = CanonizeStructuredTableList(preparer->GetContext(), spec.GetStructuredInputs());
1636-
auto structuredMapOutputs = CanonizeStructuredTableList(preparer->GetContext(), spec.GetStructuredMapOutputs());
1637-
auto structuredOutputs = CanonizeStructuredTableList(preparer->GetContext(), spec.GetStructuredOutputs());
1635+
auto structuredInputs = CanonizeStructuredTableList(preparer->GetClient()->GetRawClient(), spec.GetStructuredInputs());
1636+
auto structuredMapOutputs = CanonizeStructuredTableList(preparer->GetClient()->GetRawClient(), spec.GetStructuredMapOutputs());
1637+
auto structuredOutputs = CanonizeStructuredTableList(preparer->GetClient()->GetRawClient(), spec.GetStructuredOutputs());
16381638

16391639
const bool inferOutputSchema = options.InferOutputSchema_.GetOrElse(preparer->GetContext().Config->InferTableSchema);
16401640

@@ -1898,9 +1898,9 @@ void ExecuteRawMapReduce(
18981898
YT_LOG_DEBUG("Starting raw map-reduce operation (PreparationId: %v)",
18991899
preparer->GetPreparationId());
19001900
TMapReduceOperationIo operationIo;
1901-
operationIo.Inputs = NRawClient::CanonizeYPaths(/* retryPolicy */ nullptr, preparer->GetContext(), spec.GetInputs());
1902-
operationIo.MapOutputs = NRawClient::CanonizeYPaths(/* retryPolicy */ nullptr, preparer->GetContext(), spec.GetMapOutputs());
1903-
operationIo.Outputs = NRawClient::CanonizeYPaths(/* retryPolicy */ nullptr, preparer->GetContext(), spec.GetOutputs());
1901+
operationIo.Inputs = NRawClient::CanonizeYPaths(preparer->GetClient()->GetRawClient(), spec.GetInputs());
1902+
operationIo.MapOutputs = NRawClient::CanonizeYPaths(preparer->GetClient()->GetRawClient(), spec.GetMapOutputs());
1903+
operationIo.Outputs = NRawClient::CanonizeYPaths(preparer->GetClient()->GetRawClient(), spec.GetOutputs());
19041904

19051905
VerifyHasElements(operationIo.Inputs, "inputs");
19061906
VerifyHasElements(operationIo.Outputs, "outputs");
@@ -1947,8 +1947,8 @@ void ExecuteSort(
19471947
{
19481948
YT_LOG_DEBUG("Starting sort operation (PreparationId: %v)",
19491949
preparer->GetPreparationId());
1950-
auto inputs = NRawClient::CanonizeYPaths(/* retryPolicy */ nullptr, preparer->GetContext(), spec.Inputs_);
1951-
auto output = NRawClient::CanonizeYPath(nullptr, preparer->GetContext(), spec.Output_);
1950+
auto inputs = NRawClient::CanonizeYPaths(preparer->GetClient()->GetRawClient(), spec.Inputs_);
1951+
auto output = NRawClient::CanonizeYPath(preparer->GetClient()->GetRawClient(), spec.Output_);
19521952

19531953
if (options.CreateOutputTables_) {
19541954
CheckInputTablesExist(*preparer, inputs);
@@ -1996,8 +1996,8 @@ void ExecuteMerge(
19961996
{
19971997
YT_LOG_DEBUG("Starting merge operation (PreparationId: %v)",
19981998
preparer->GetPreparationId());
1999-
auto inputs = NRawClient::CanonizeYPaths(/* retryPolicy */ nullptr, preparer->GetContext(), spec.Inputs_);
2000-
auto output = NRawClient::CanonizeYPath(nullptr, preparer->GetContext(), spec.Output_);
1999+
auto inputs = NRawClient::CanonizeYPaths(preparer->GetClient()->GetRawClient(), spec.Inputs_);
2000+
auto output = NRawClient::CanonizeYPath(preparer->GetClient()->GetRawClient(), spec.Output_);
20012001

20022002
if (options.CreateOutputTables_) {
20032003
CheckInputTablesExist(*preparer, inputs);
@@ -2046,7 +2046,7 @@ void ExecuteErase(
20462046
{
20472047
YT_LOG_DEBUG("Starting erase operation (PreparationId: %v)",
20482048
preparer->GetPreparationId());
2049-
auto tablePath = NRawClient::CanonizeYPath(nullptr, preparer->GetContext(), spec.TablePath_);
2049+
auto tablePath = NRawClient::CanonizeYPath(preparer->GetClient()->GetRawClient(), spec.TablePath_);
20502050

20512051
TNode specNode = BuildYsonNodeFluently()
20522052
.BeginMap()
@@ -2082,8 +2082,8 @@ void ExecuteRemoteCopy(
20822082
{
20832083
YT_LOG_DEBUG("Starting remote copy operation (PreparationId: %v)",
20842084
preparer->GetPreparationId());
2085-
auto inputs = NRawClient::CanonizeYPaths(/* retryPolicy */ nullptr, preparer->GetContext(), spec.Inputs_);
2086-
auto output = NRawClient::CanonizeYPath(nullptr, preparer->GetContext(), spec.Output_);
2085+
auto inputs = NRawClient::CanonizeYPaths(preparer->GetClient()->GetRawClient(), spec.Inputs_);
2086+
auto output = NRawClient::CanonizeYPath(preparer->GetClient()->GetRawClient(), spec.Output_);
20872087

20882088
if (options.CreateOutputTables_) {
20892089
CreateOutputTable(*preparer, output);

‎yt/cpp/mapreduce/client/operation_preparer.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -394,7 +394,7 @@ TJobPreparer::TJobPreparer(
394394
{
395395
396396
CreateStorage();
397-
auto cypressFileList = NRawClient::CanonizeYPaths(/* retryPolicy */ nullptr, OperationPreparer_.GetContext(), spec.Files_);
397+
auto cypressFileList = NRawClient::CanonizeYPaths(RawClient_, spec.Files_);
398398
399399
for (const auto& file : cypressFileList) {
400400
UseFileInCypress(file);

‎yt/cpp/mapreduce/client/py_helpers.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ TStructuredJobTableList NodeToStructuredTablePaths(const TNode& node, const TOpe
5151
paths.emplace_back(inputNode.AsString());
5252
}
5353
}
54-
paths = NRawClient::CanonizeYPaths(/* retryPolicy */ nullptr, preparer.GetContext(), paths);
54+
paths = NRawClient::CanonizeYPaths(preparer.GetClient()->GetRawClient(), paths);
5555
TStructuredJobTableList result(intermediateTableCount, TStructuredJobTable::Intermediate(TUnspecifiedTableStructure()));
5656
for (const auto& path : paths) {
5757
result.emplace_back(TStructuredJobTable{TUnspecifiedTableStructure(), path});

‎yt/cpp/mapreduce/client/skiff.cpp

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -280,8 +280,6 @@ TFormat CreateSkiffFormat(const NSkiff::TSkiffSchemaPtr& schema) {
280280

281281
NSkiff::TSkiffSchemaPtr CreateSkiffSchemaIfNecessary(
282282
const IRawClientPtr& rawClient,
283-
const TClientContext& context,
284-
const IClientRetryPolicyPtr& clientRetryPolicy,
285283
const TTransactionId& transactionId,
286284
ENodeReaderFormat nodeReaderFormat,
287285
const TVector<TRichYPath>& tablePaths,
@@ -306,7 +304,7 @@ NSkiff::TSkiffSchemaPtr CreateSkiffSchemaIfNecessary(
306304

307305
auto nodes = NRawClient::BatchTransform(
308306
rawClient,
309-
NRawClient::CanonizeYPaths(clientRetryPolicy->CreatePolicyForGenericRequest(), context, tablePaths),
307+
NRawClient::CanonizeYPaths(rawClient, tablePaths),
310308
[&] (IRawBatchRequestPtr batch, const TRichYPath& path) {
311309
auto getOptions = TGetOptions()
312310
.AttributeFilter(

‎yt/cpp/mapreduce/client/skiff.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,6 @@ TFormat CreateSkiffFormat(const NSkiff::TSkiffSchemaPtr& schema);
6060

6161
NSkiff::TSkiffSchemaPtr CreateSkiffSchemaIfNecessary(
6262
const IRawClientPtr& rawClient,
63-
const TClientContext& context,
64-
const IClientRetryPolicyPtr& clientRetryPolicy,
6563
const TTransactionId& transactionId,
6664
ENodeReaderFormat nodeReaderFormat,
6765
const TVector<TRichYPath>& tablePaths,

‎yt/cpp/mapreduce/client/structured_table_formats.cpp

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -122,8 +122,6 @@ namespace NDetail {
122122

123123
NSkiff::TSkiffSchemaPtr TryCreateSkiffSchema(
124124
const IRawClientPtr& rawClient,
125-
const TClientContext& context,
126-
const IClientRetryPolicyPtr& clientRetryPolicy,
127125
const TTransactionId& transactionId,
128126
const TVector<TRichYPath>& tables,
129127
const TOperationOptions& options,
@@ -137,8 +135,6 @@ NSkiff::TSkiffSchemaPtr TryCreateSkiffSchema(
137135
}
138136
return CreateSkiffSchemaIfNecessary(
139137
rawClient,
140-
context,
141-
clientRetryPolicy,
142138
transactionId,
143139
nodeReaderFormat,
144140
tables,
@@ -214,14 +210,14 @@ TStructuredJobTableList ToStructuredJobTableList(const TVector<TStructuredTableP
214210
return result;
215211
}
216212

217-
TStructuredJobTableList CanonizeStructuredTableList(const TClientContext& context, const TVector<TStructuredTablePath>& tableList)
213+
TStructuredJobTableList CanonizeStructuredTableList(const IRawClientPtr& rawClient, const TVector<TStructuredTablePath>& tableList)
218214
{
219215
TVector<TRichYPath> toCanonize;
220216
toCanonize.reserve(tableList.size());
221217
for (const auto& table : tableList) {
222218
toCanonize.emplace_back(table.RichYPath);
223219
}
224-
const auto canonized = NRawClient::CanonizeYPaths(/* retryPolicy */ nullptr, context, toCanonize);
220+
const auto canonized = NRawClient::CanonizeYPaths(rawClient, toCanonize);
225221
Y_ABORT_UNLESS(canonized.size() == tableList.size());
226222

227223
TStructuredJobTableList result;
@@ -437,8 +433,6 @@ std::pair<TFormat, TMaybe<TSmallJobFile>> TFormatBuilder::CreateNodeFormat(
437433
}
438434
skiffSchema = TryCreateSkiffSchema(
439435
RawClient_,
440-
Context_,
441-
ClientRetryPolicy_,
442436
TransactionId_,
443437
tableList,
444438
OperationOptions_,

‎yt/cpp/mapreduce/client/structured_table_formats.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ using TStructuredJobTableList = TVector<TStructuredJobTable>;
6969
TString JobTablePathString(const TStructuredJobTable& jobTable);
7070
TStructuredJobTableList ToStructuredJobTableList(const TVector<TStructuredTablePath>& tableList);
7171

72-
TStructuredJobTableList CanonizeStructuredTableList(const TClientContext& context, const TVector<TStructuredTablePath>& tableList);
72+
TStructuredJobTableList CanonizeStructuredTableList(const IRawClientPtr& rawClient, const TVector<TStructuredTablePath>& tableList);
7373
TVector<TRichYPath> GetPathList(
7474
const TStructuredJobTableList& tableList,
7575
const TMaybe<TVector<TTableSchema>>& schemaInferenceResult,

‎yt/cpp/mapreduce/raw_client/raw_requests.cpp

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -253,25 +253,26 @@ TCheckPermissionResponse ParseCheckPermissionResponse(const TNode& node)
253253
}
254254

255255
TRichYPath CanonizeYPath(
256-
const IRequestRetryPolicyPtr& retryPolicy,
257-
const TClientContext& context,
256+
const IRawClientPtr& rawClient,
258257
const TRichYPath& path)
259258
{
260-
return CanonizeYPaths(retryPolicy, context, {path}).front();
259+
return CanonizeYPaths(rawClient, {path}).front();
261260
}
262261

263262
TVector<TRichYPath> CanonizeYPaths(
264-
const IRequestRetryPolicyPtr& retryPolicy,
265-
const TClientContext& context,
263+
const IRawClientPtr& rawClient,
266264
const TVector<TRichYPath>& paths)
267265
{
268-
THttpRawBatchRequest batch(context, retryPolicy);
266+
auto batch = rawClient->CreateRawBatchRequest();
267+
269268
TVector<NThreading::TFuture<TRichYPath>> futures;
270269
futures.reserve(paths.size());
271-
for (int i = 0; i < static_cast<int>(paths.size()); ++i) {
272-
futures.push_back(batch.CanonizeYPath(paths[i]));
270+
for (const auto& path : paths) {
271+
futures.push_back(batch->CanonizeYPath(path));
273272
}
274-
batch.ExecuteBatch();
273+
274+
batch->ExecuteBatch();
275+
275276
TVector<TRichYPath> result;
276277
result.reserve(futures.size());
277278
for (auto& future : futures) {

‎yt/cpp/mapreduce/raw_client/raw_requests.h

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,13 +34,11 @@ TCheckPermissionResponse ParseCheckPermissionResponse(const TNode& node);
3434
////////////////////////////////////////////////////////////////////////////////
3535

3636
TRichYPath CanonizeYPath(
37-
const IRequestRetryPolicyPtr& retryPolicy,
38-
const TClientContext& context,
37+
const IRawClientPtr& rawClient,
3938
const TRichYPath& path);
4039

4140
TVector<TRichYPath> CanonizeYPaths(
42-
const IRequestRetryPolicyPtr& retryPolicy,
43-
const TClientContext& context,
41+
const IRawClientPtr& rawClient,
4442
const TVector<TRichYPath>& paths);
4543

4644
NHttpClient::IHttpResponsePtr SkyShareTable(

0 commit comments

Comments
 (0)