Skip to content

Commit 3721d5e

Browse files
MrLolthe1stMrLolthe1st
committed
UdfResolver analytics
commit_hash:393ec43ede4bf529dd68413165afce4bb49a43db
1 parent 8e53255 commit 3721d5e

20 files changed

+172
-45
lines changed

‎yql/essentials/core/facade/yql_facade.cpp

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
#include <yql/essentials/providers/common/udf_resolve/yql_simple_udf_resolver.h>
3030
#include <yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.h>
3131
#include <yql/essentials/providers/common/udf_resolve/yql_udf_resolver_with_index.h>
32+
#include <yql/essentials/providers/common/udf_resolve/yql_udf_resolver_logger.h>
3233
#include <yql/essentials/providers/common/arrow_resolve/yql_simple_arrow_resolver.h>
3334
#include <yql/essentials/providers/common/config/yql_setting.h>
3435
#include <yql/essentials/core/qplayer/udf_resolver/yql_qplayer_udf_resolver.h>
@@ -218,6 +219,10 @@ void TProgramFactory::SetArrowResolver(IArrowResolver::TPtr arrowResolver) {
218219
ArrowResolver_ = arrowResolver;
219220
}
220221

222+
void TProgramFactory::SetUdfResolverLogfile(const TString& path) {
223+
UdfResolverLogfile_ = path;
224+
}
225+
221226
void TProgramFactory::SetUrlListerManager(IUrlListerManagerPtr urlListerManager) {
222227
UrlListerManager_ = std::move(urlListerManager);
223228
}
@@ -249,7 +254,16 @@ TProgramPtr TProgramFactory::Create(
249254
TUdfIndexPackageSet::TPtr udfIndexPackageSet = (UdfIndexPackageSet_ && hiddenMode == EHiddenMode::Disable) ? UdfIndexPackageSet_->Clone() : nullptr;
250255
IModuleResolver::TPtr moduleResolver = Modules_ ? Modules_->CreateMutableChild() : nullptr;
251256
IUrlListerManagerPtr urlListerManager = UrlListerManager_ ? UrlListerManager_->Clone() : nullptr;
252-
auto udfResolver = udfIndex ? NCommon::CreateUdfResolverWithIndex(udfIndex, UdfResolver_, FileStorage_) : UdfResolver_;
257+
258+
auto udfResolver = UdfResolver_;
259+
260+
if (UdfResolverLogfile_) {
261+
udfResolver = NCommon::CreateUdfResolverDecoratorWithLogger(udfResolver, *UdfResolverLogfile_, sessionId);
262+
}
263+
264+
if (udfIndex) {
265+
udfResolver = NCommon::CreateUdfResolverWithIndex(udfIndex, udfResolver, FileStorage_);
266+
}
253267

254268
// make UserDataTable_ copy here
255269
return new TProgram(FunctionRegistry_, randomProvider, timeProvider, NextUniqueId_, DataProvidersInit_,

‎yql/essentials/core/facade/yql_facade.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ class TProgramFactory: public TThrRefBase, private TMoveOnly
6464
void SetUrlPreprocessing(IUrlPreprocessing::TPtr urlPreprocessing);
6565
void EnableRangeComputeFor();
6666
void SetArrowResolver(IArrowResolver::TPtr arrowResolver);
67+
void SetUdfResolverLogfile(const TString& path);
6768

6869
TProgramPtr Create(
6970
const TFile& file,
@@ -101,6 +102,7 @@ class TProgramFactory: public TThrRefBase, private TMoveOnly
101102
TString Runner_;
102103
bool EnableRangeComputeFor_ = false;
103104
IArrowResolver::TPtr ArrowResolver_;
105+
TMaybe<TString> UdfResolverLogfile_;
104106
};
105107

106108
///////////////////////////////////////////////////////////////////////////////

‎yql/essentials/core/qplayer/udf_resolver/yql_qplayer_udf_resolver.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ class TResolver : public IUdfResolver {
3838
}
3939

4040
bool LoadMetadata(const TVector<TImport*>& imports,
41-
const TVector<TFunction*>& functions, TExprContext& ctx, NUdf::ELogLevel logLevel) const final {
41+
const TVector<TFunction*>& functions, TExprContext& ctx, NUdf::ELogLevel logLevel, THoldingFileStorage& storage) const final {
4242
if (QContext_.CanRead()) {
4343
for (auto& f : functions) {
4444
auto key = MakeKey(f);
@@ -53,7 +53,7 @@ class TResolver : public IUdfResolver {
5353
return true;
5454
}
5555

56-
auto res = Inner_->LoadMetadata(imports, functions, ctx, logLevel);
56+
auto res = Inner_->LoadMetadata(imports, functions, ctx, logLevel, storage);
5757
if (res && QContext_.CanWrite()) {
5858
// calculate hash for each function and store it
5959
for (const auto& f : functions) {
@@ -66,12 +66,12 @@ class TResolver : public IUdfResolver {
6666
return res;
6767
}
6868

69-
TResolveResult LoadRichMetadata(const TVector<TImport>& imports, NUdf::ELogLevel logLevel) const final {
69+
TResolveResult LoadRichMetadata(const TVector<TImport>& imports, NUdf::ELogLevel logLevel, THoldingFileStorage& storage) const final {
7070
if (QContext_.CanRead()) {
7171
ythrow yexception() << "Can't replay LoadRichMetadata";
7272
}
7373

74-
return Inner_->LoadRichMetadata(imports, logLevel);
74+
return Inner_->LoadRichMetadata(imports, logLevel, storage);
7575
}
7676

7777
bool ContainsModule(const TStringBuf& moduleName) const final {

‎yql/essentials/core/type_ann/type_ann_core.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -486,7 +486,7 @@ namespace NTypeAnnImpl {
486486
imports.push_back(&x.second);
487487
}
488488

489-
if (!Types.UdfResolver->LoadMetadata(imports, functions, Expr, Types.RuntimeLogLevel)) {
489+
if (!Types.UdfResolver->LoadMetadata(imports, functions, Expr, Types.RuntimeLogLevel, Types.UserDataStorage->GetHoldingFileStorage())) {
490490
return false;
491491
}
492492

‎yql/essentials/core/yql_holding_file_storage.cpp

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,16 @@ TFileLinkPtr THoldingFileStorage::FreezeFile(const TUserDataBlock& block) {
2525
return FindOrPutData(block);
2626
}
2727

28+
29+
TFileLinkPtr THoldingFileStorage::GetFrozenBlock(const TUserDataBlock& block) const {
30+
if (block.FrozenFile) {
31+
return block.FrozenFile;
32+
}
33+
auto it = Links_.find(block);
34+
Y_ENSURE (it != Links_.end());
35+
return it->second;
36+
}
37+
2838
NThreading::TFuture<std::function<TFileLinkPtr()>> THoldingFileStorage::FreezeFileAsync(const TUserDataBlock& block) {
2939
if (block.FrozenFile) {
3040
return MakeFutureWithConstantAction(block.FrozenFile);

‎yql/essentials/core/yql_holding_file_storage.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ class THoldingFileStorage {
2323
// if we have single url but with different tokens we can potentially download several different versions of url
2424
TFileLinkPtr FreezeFile(const TUserDataBlock& block);
2525

26+
TFileLinkPtr GetFrozenBlock(const TUserDataBlock& block) const;
27+
2628
// downloaded file will be registered in cache after function invocation
2729
NThreading::TFuture<std::function<TFileLinkPtr()>> FreezeFileAsync(const TUserDataBlock& block);
2830

‎yql/essentials/core/yql_udf_index.cpp

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -290,25 +290,25 @@ void TUdfIndex::RegisterResources(const TVector<TResourceInfo::TPtr>& resources,
290290
}
291291
}
292292

293-
void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TVector<TString>& paths, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry, NUdf::ELogLevel logLevel) {
293+
void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TVector<TString>& paths, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry, THoldingFileStorage& storage, NUdf::ELogLevel logLevel) {
294294
TMap<TString, TString> pathsWithMd5;
295295
for (const auto& path : paths) {
296296
pathsWithMd5[path] = "";
297297
}
298-
LoadRichMetadataToUdfIndex(resolver, pathsWithMd5, isTrusted, mode, registry, logLevel);
298+
LoadRichMetadataToUdfIndex(resolver, pathsWithMd5, isTrusted, mode, registry, storage, logLevel);
299299
}
300300

301-
void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TMap<TString, TString>& pathsWithMd5, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry, NUdf::ELogLevel logLevel) {
301+
void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TMap<TString, TString>& pathsWithMd5, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry, THoldingFileStorage& storage, NUdf::ELogLevel logLevel) {
302302
TVector<TString> paths;
303303
paths.reserve(pathsWithMd5.size());
304304
for (const auto& p : pathsWithMd5) {
305305
paths.push_back(p.first);
306306
}
307-
const TResolveResult resolveResult = LoadRichMetadata(resolver, paths, logLevel);
307+
const TResolveResult resolveResult = LoadRichMetadata(resolver, paths, storage, logLevel);
308308
AddResolveResultToRegistry(resolveResult, pathsWithMd5, isTrusted, mode, registry);
309309
}
310310

311-
void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TVector<TUserDataBlock>& blocks, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry, NUdf::ELogLevel logLevel) {
311+
void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TVector<TUserDataBlock>& blocks, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry, THoldingFileStorage& storage, NUdf::ELogLevel logLevel) {
312312
TVector<TUserDataBlock> blocksResolve;
313313
blocksResolve.reserve(blocks.size());
314314
// we can work with file path only
@@ -342,16 +342,17 @@ void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TVector<TUse
342342
br.Data = path;
343343
br.Usage.Set(EUserDataBlockUsage::Udf);
344344
br.CustomUdfPrefix = b.CustomUdfPrefix;
345+
br.FrozenFile = b.FrozenFile;
345346
blocksResolve.emplace_back(br);
346347
}
347-
const TResolveResult resolveResult = LoadRichMetadata(resolver, blocksResolve, logLevel);
348+
const TResolveResult resolveResult = LoadRichMetadata(resolver, blocksResolve, storage, logLevel);
348349
AddResolveResultToRegistry(resolveResult, pathsWithMd5, isTrusted, mode, registry);
349350
}
350351

351-
void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TUserDataBlock& block, TUdfIndex::EOverrideMode mode, TUdfIndex& registry, NUdf::ELogLevel logLevel) {
352+
void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TUserDataBlock& block, TUdfIndex::EOverrideMode mode, TUdfIndex& registry, THoldingFileStorage& storage, NUdf::ELogLevel logLevel) {
352353
TVector<TUserDataBlock> blocks({ block });
353354
const bool isTrusted = false;
354-
LoadRichMetadataToUdfIndex(resolver, blocks, isTrusted, mode, registry, logLevel);
355+
LoadRichMetadataToUdfIndex(resolver, blocks, isTrusted, mode, registry, storage, logLevel);
355356
}
356357

357358
} // namespace NYql

‎yql/essentials/core/yql_udf_index.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -139,9 +139,9 @@ class TUdfIndex : public TThrRefBase {
139139
TMap<TString, TSet<TString>> ICaseModules_;
140140
};
141141

142-
void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TVector<TString>& paths, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry, NUdf::ELogLevel logLevel = NUdf::ELogLevel::Info);
143-
void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TMap<TString, TString>& pathsWithMd5, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry, NUdf::ELogLevel logLevel = NUdf::ELogLevel::Info);
144-
void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TVector<TUserDataBlock>& blocks, TUdfIndex::EOverrideMode mode, TUdfIndex& registry, NUdf::ELogLevel logLevel = NUdf::ELogLevel::Info);
145-
void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TUserDataBlock& block, TUdfIndex::EOverrideMode mode, TUdfIndex& registry, NUdf::ELogLevel logLevel = NUdf::ELogLevel::Info);
142+
void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TVector<TString>& paths, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry, THoldingFileStorage& storage, NUdf::ELogLevel logLevel = NUdf::ELogLevel::Info);
143+
void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TMap<TString, TString>& pathsWithMd5, bool isTrusted, TUdfIndex::EOverrideMode mode, TUdfIndex& registry, THoldingFileStorage& storage, NUdf::ELogLevel logLevel = NUdf::ELogLevel::Info);
144+
void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TVector<TUserDataBlock>& blocks, TUdfIndex::EOverrideMode mode, TUdfIndex& registry, THoldingFileStorage& storage, NUdf::ELogLevel logLevel = NUdf::ELogLevel::Info);
145+
void LoadRichMetadataToUdfIndex(const IUdfResolver& resolver, const TUserDataBlock& block, TUdfIndex::EOverrideMode mode, TUdfIndex& registry, THoldingFileStorage& storage, NUdf::ELogLevel logLevel = NUdf::ELogLevel::Info);
146146

147147
}

‎yql/essentials/core/yql_udf_resolver.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
namespace NYql {
44

5-
TResolveResult LoadRichMetadata(const IUdfResolver& resolver, const TVector<TUserDataBlock>& blocks, NUdf::ELogLevel logLevel) {
5+
TResolveResult LoadRichMetadata(const IUdfResolver& resolver, const TVector<TUserDataBlock>& blocks, THoldingFileStorage& storage, NUdf::ELogLevel logLevel) {
66
TVector<IUdfResolver::TImport> imports;
77
imports.reserve(blocks.size());
88
std::transform(blocks.begin(), blocks.end(), std::back_inserter(imports), [](auto& b) {
@@ -13,10 +13,10 @@ TResolveResult LoadRichMetadata(const IUdfResolver& resolver, const TVector<TUse
1313
return import;
1414
});
1515

16-
return resolver.LoadRichMetadata(imports, logLevel);
16+
return resolver.LoadRichMetadata(imports, logLevel, storage);
1717
}
1818

19-
TResolveResult LoadRichMetadata(const IUdfResolver& resolver, const TVector<TString>& paths, NUdf::ELogLevel logLevel) {
19+
TResolveResult LoadRichMetadata(const IUdfResolver& resolver, const TVector<TString>& paths, THoldingFileStorage& storage, NUdf::ELogLevel logLevel) {
2020
TVector<TUserDataBlock> blocks;
2121
blocks.reserve(paths.size());
2222
std::transform(paths.begin(), paths.end(), std::back_inserter(blocks), [](auto& p) {
@@ -27,7 +27,7 @@ TResolveResult LoadRichMetadata(const IUdfResolver& resolver, const TVector<TStr
2727
return b;
2828
});
2929

30-
return LoadRichMetadata(resolver, blocks, logLevel);
30+
return LoadRichMetadata(resolver, blocks, storage, logLevel);
3131
}
3232

3333
}

‎yql/essentials/core/yql_udf_resolver.h

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#include "yql_user_data.h"
44
#include <yql/essentials/providers/common/proto/udf_resolver.pb.h>
55

6+
#include <yql/essentials/core/yql_holding_file_storage.h>
67
#include <yql/essentials/public/issue/yql_issue.h>
78
#include <yql/essentials/public/udf/udf_log.h>
89

@@ -68,13 +69,13 @@ class IUdfResolver : public TThrRefBase {
6869
*/
6970
virtual TMaybe<TFilePathWithMd5> GetSystemModulePath(const TStringBuf& moduleName) const = 0;
7071
virtual bool LoadMetadata(const TVector<TImport*>& imports,
71-
const TVector<TFunction*>& functions, TExprContext& ctx, NUdf::ELogLevel logLevel) const = 0;
72+
const TVector<TFunction*>& functions, TExprContext& ctx, NUdf::ELogLevel logLevel, THoldingFileStorage& storage) const = 0;
7273

73-
virtual TResolveResult LoadRichMetadata(const TVector<TImport>& imports, NUdf::ELogLevel logLevel) const = 0;
74+
virtual TResolveResult LoadRichMetadata(const TVector<TImport>& imports, NUdf::ELogLevel logLevel, THoldingFileStorage& storage) const = 0;
7475
virtual bool ContainsModule(const TStringBuf& moduleName) const = 0;
7576
};
7677

77-
TResolveResult LoadRichMetadata(const IUdfResolver& resolver, const TVector<TUserDataBlock>& blocks, NUdf::ELogLevel logLevel = NUdf::ELogLevel::Info);
78-
TResolveResult LoadRichMetadata(const IUdfResolver& resolver, const TVector<TString>& paths, NUdf::ELogLevel logLevel = NUdf::ELogLevel::Info);
78+
TResolveResult LoadRichMetadata(const IUdfResolver& resolver, const TVector<TUserDataBlock>& blocks, THoldingFileStorage& storage, NUdf::ELogLevel logLevel = NUdf::ELogLevel::Info);
79+
TResolveResult LoadRichMetadata(const IUdfResolver& resolver, const TVector<TString>& paths, THoldingFileStorage& storage, NUdf::ELogLevel logLevel = NUdf::ELogLevel::Info);
7980

8081
}

0 commit comments

Comments
 (0)