Skip to content

Commit a92340d

Browse files
vitstnvitstn
authored
Support of replay of isolated views with inner udfs (#10199)
1 parent 5fb7165 commit a92340d

File tree

7 files changed

+70
-27
lines changed

7 files changed

+70
-27
lines changed

‎ydb/library/yql/core/qplayer/udf_resolver/yql_qplayer_udf_resolver.cpp

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ namespace NYql::NCommon {
1111
namespace {
1212

1313
const TString UdfResolver_LoadMetadata = "UdfResolver_LoadMetadata";
14+
const TString UdfResolver_ContainsModule = "UdfResolver_ContainsModule";
1415

1516
TString MakeHash(const TString& str) {
1617
SHA256_CTX sha;
@@ -75,10 +76,20 @@ class TResolver : public IUdfResolver {
7576

7677
bool ContainsModule(const TStringBuf& moduleName) const final {
7778
if (QContext_.CanRead()) {
78-
ythrow yexception() << "Can't replay ContainsModule";
79+
auto res = QContext_.GetReader()->Get({UdfResolver_ContainsModule, TString(moduleName)}).GetValueSync();
80+
if (!res) {
81+
ythrow yexception() << "Missing replay data";
82+
}
83+
84+
return res->Value == "1";
85+
}
86+
87+
auto ret = Inner_->ContainsModule(moduleName);
88+
if (QContext_.CanWrite()) {
89+
QContext_.GetWriter()->Put({UdfResolver_ContainsModule, TString(moduleName)}, ret ? "1" : "0").GetValueSync();
7990
}
8091

81-
return Inner_->ContainsModule(moduleName);
92+
return ret;
8293
}
8394

8495
private:

‎ydb/library/yql/providers/common/provider/yql_provider.cpp

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -967,7 +967,9 @@ bool FillUsedFilesImpl(
967967
auto key = TUserDataKey::Udf(alias);
968968
if (const auto block = types.UserDataStorage->FindUserDataBlock(key)) {
969969
files[key] = *block;
970-
YQL_ENSURE(block->FrozenFile);
970+
if (!types.QContext.CanRead()) {
971+
YQL_ENSURE(block->FrozenFile);
972+
}
971973
} else {
972974
// Check alias clash with user files
973975
if (files.contains(TUserDataStorage::ComposeUserDataKey(alias))) {
@@ -977,7 +979,10 @@ bool FillUsedFilesImpl(
977979

978980
if (!alias.StartsWith(NKikimr::NMiniKQL::StaticModulePrefix) && !files.contains(key)) {
979981
// CreateFakeFileLink calculates md5 for file, let's do it once
980-
sysBlock.FrozenFile = CreateFakeFileLink(sysBlock.Data, pathWithMd5->Md5);
982+
if (!types.QContext.CanRead()) {
983+
sysBlock.FrozenFile = CreateFakeFileLink(sysBlock.Data, pathWithMd5->Md5);
984+
}
985+
981986
files[key] = sysBlock;
982987
types.UserDataStorage->AddUserDataBlock(key, sysBlock);
983988
}
@@ -1194,8 +1199,11 @@ bool FreezeUsedFilesSync(const TExprNode& node, TUserDataTable& files, const TTy
11941199
return false;
11951200
}
11961201

1197-
auto future = FreezeUserDataTableIfNeeded(types.UserDataStorage, files, urlDownloadFilter);
1198-
files = future.GetValueSync()();
1202+
if (!types.QContext.CanRead()) {
1203+
auto future = FreezeUserDataTableIfNeeded(types.UserDataStorage, files, urlDownloadFilter);
1204+
files = future.GetValueSync()();
1205+
}
1206+
11991207
return true;
12001208
}
12011209

‎ydb/library/yql/providers/yt/provider/yql_yt_load_table_meta.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ class TYtLoadTableMetadataTransformer : public TGraphTransformerBase {
102102
if (State_->Types->IsReadOnly || State_->Types->UseTableMetaFromGraph || tableDesc.HasWriteLock || !HasModifyIntents(tableDesc.Intents)) {
103103
// Intents/views can be updated since evaluation phase
104104
if (!tableDesc.FillViews(
105-
clusterAndTable.first, clusterAndTable.second, ctx,
105+
clusterAndTable.first, clusterAndTable.second, State_->Types->QContext, ctx,
106106
State_->Types->Modules.get(), State_->Types->UrlListerManager.Get(), *State_->Types->RandomProvider,
107107
State_->Configuration->ViewIsolation.Get().GetOrElse(false),
108108
State_->Types->UdfResolver
@@ -231,7 +231,7 @@ class TYtLoadTableMetadataTransformer : public TGraphTransformerBase {
231231

232232
if (0 == LoadCtx->Epoch) {
233233
if (!tableDesc.Fill(
234-
cluster, tableName, ctx,
234+
cluster, tableName, State_->Types->QContext, ctx,
235235
State_->Types->Modules.get(), State_->Types->UrlListerManager.Get(), *State_->Types->RandomProvider,
236236
State_->Configuration->ViewIsolation.Get().GetOrElse(false),
237237
State_->Types->UdfResolver

‎ydb/library/yql/providers/yt/provider/yql_yt_provider.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
namespace NYql {
1515

1616
bool TYtTableDescription::Fill(
17-
const TString& cluster, const TString& table, TExprContext& ctx,
17+
const TString& cluster, const TString& table, const TQContext& qContext, TExprContext& ctx,
1818
IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider,
1919
bool allowViewIsolation, IUdfResolver::TPtr udfResolver) {
2020
const TStructExprType* type = RowSpec ? RowSpec->GetType() : nullptr;
@@ -29,7 +29,7 @@ bool TYtTableDescription::Fill(
2929
}
3030

3131
if (!TYtTableDescriptionBase::Fill(TString{YtProviderName}, cluster,
32-
table, type, Meta->SqlView, Meta->SqlViewSyntaxVersion, Meta->Attrs, ctx,
32+
table, type, Meta->SqlView, Meta->SqlViewSyntaxVersion, qContext, Meta->Attrs, ctx,
3333
moduleResolver, urlListerManager, randomProvider, allowViewIsolation, udfResolver)) {
3434
return false;
3535
}
@@ -244,11 +244,11 @@ void TYtTableDescription::SetConstraintsReady() {
244244
}
245245

246246
bool TYtTableDescription::FillViews(
247-
const TString& cluster, const TString& table, TExprContext& ctx,
247+
const TString& cluster, const TString& table, const TQContext& qContext, TExprContext& ctx,
248248
IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider,
249249
bool allowViewIsolation, IUdfResolver::TPtr udfResolver) {
250250
return TYtTableDescriptionBase::FillViews(
251-
TString{YtProviderName}, cluster, table, Meta->Attrs, ctx,
251+
TString{YtProviderName}, cluster, table, Meta->Attrs, qContext, ctx,
252252
moduleResolver, urlListerManager, randomProvider, allowViewIsolation, udfResolver);
253253
}
254254

‎ydb/library/yql/providers/yt/provider/yql_yt_provider.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,15 +48,15 @@ struct TYtTableDescription: public TYtTableDescriptionBase {
4848
bool RowSpecSortReady = false;
4949

5050
bool Fill(
51-
const TString& cluster, const TString& table, TExprContext& ctx,
51+
const TString& cluster, const TString& table, const TQContext& qContext, TExprContext& ctx,
5252
IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider,
5353
bool allowViewIsolation, IUdfResolver::TPtr udfResolver);
5454
void ToYson(NYson::TYsonWriter& writer, const TString& cluster, const TString& table, const TString& view) const;
5555
bool Validate(TPosition pos, TStringBuf cluster, TStringBuf tableName, bool withQB,
5656
const THashMap<std::pair<TString, TString>, TString>& anonymousLabels, TExprContext& ctx) const;
5757
void SetConstraintsReady();
5858
bool FillViews(
59-
const TString& cluster, const TString& table, TExprContext& ctx,
59+
const TString& cluster, const TString& table, const TQContext& qContext, TExprContext& ctx,
6060
IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider,
6161
bool allowViewIsolation, IUdfResolver::TPtr udfResolver);
6262
};

‎ydb/library/yql/providers/yt/provider/yql_yt_table_desc.cpp

Lines changed: 31 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include <ydb/library/yql/core/issue/protos/issue_id.pb.h>
55
#include <ydb/library/yql/core/yql_expr_optimize.h>
66
#include <ydb/library/yql/core/yql_expr_type_annotation.h>
7+
#include <ydb/library/yql/core/qplayer/storage/interface/yql_qstorage.h>
78
#include <ydb/library/yql/core/issue/yql_issue.h>
89
#include <ydb/library/yql/sql/sql.h>
910
#include <ydb/library/yql/utils/yql_panic.h>
@@ -15,6 +16,7 @@ namespace NYql {
1516
namespace {
1617

1718
const TString RAW_VIEW_SQL = "select * from self_raw";
19+
const TString YtView_Component = "YtView";
1820

1921
TExprNode::TPtr BuildProtoRemapper(const TMap<TString, TString>& protoFields, TExprContext& ctx) {
2022
auto rowArg = ctx.NewArgument(TPosition(), TStringBuf("row"));
@@ -150,6 +152,7 @@ TExprNode::TPtr BuildIgnoreTypeV3Remapper(const TStructExprType* rowType, TExprC
150152
}
151153

152154
TExprNode::TPtr CompileViewSql(const TString& provider, const TString& cluster, const TString& sql, ui16 syntaxVersion,
155+
const TString& viewId, const TQContext& qContext,
153156
TExprContext& ctx, IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager,
154157
IRandomProvider& randomProvider, bool enableViewIsolation, IUdfResolver::TPtr udfResolver)
155158
{
@@ -159,9 +162,22 @@ TExprNode::TPtr CompileViewSql(const TString& provider, const TString& cluster,
159162
settings.ClusterMapping[settings.DefaultCluster] = cluster.empty() ? "data" : provider;
160163
settings.SyntaxVersion = syntaxVersion;
161164
settings.V0Behavior = NSQLTranslation::EV0Behavior::Disable;
162-
settings.FileAliasPrefix = "view_" + randomProvider.GenUuid4().AsGuidString() + "/";
163-
if (!enableViewIsolation) {
164-
settings.FileAliasPrefix.clear(); // disable FileAliasPrefix while preserving number of randomProvider calls
165+
if (qContext.CanRead() && enableViewIsolation) {
166+
auto res = qContext.GetReader()->Get({YtView_Component, viewId}).GetValueSync();
167+
if (!res) {
168+
ythrow yexception() << "Missing replay data";
169+
}
170+
171+
settings.FileAliasPrefix = res->Value;
172+
} else {
173+
settings.FileAliasPrefix = "view_" + randomProvider.GenUuid4().AsGuidString() + "/";
174+
if (!enableViewIsolation) {
175+
settings.FileAliasPrefix.clear(); // disable FileAliasPrefix while preserving number of randomProvider calls
176+
}
177+
178+
if (enableViewIsolation && qContext.CanWrite()) {
179+
qContext.GetWriter()->Put({YtView_Component, viewId}, settings.FileAliasPrefix).GetValueSync();
180+
}
165181
}
166182

167183
NYql::TAstParseResult sqlRes = NSQLTranslation::SqlToYql(sql, settings);
@@ -268,12 +284,14 @@ TExprNode::TPtr CompileViewSql(const TString& provider, const TString& cluster,
268284
} // unnamed
269285

270286

271-
bool TYtViewDescription::Fill(const TString& provider, const TString& cluster, const TString& sql, ui16 syntaxVersion, TExprContext& ctx,
287+
bool TYtViewDescription::Fill(const TString& provider, const TString& cluster, const TString& sql, ui16 syntaxVersion,
288+
const TString& viewId, const TQContext& qContext, TExprContext& ctx,
272289
IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider, bool enableViewIsolation,
273290
IUdfResolver::TPtr udfResolver)
274291
{
275292
Sql = sql;
276-
CompiledSql = CompileViewSql(provider, cluster, sql, syntaxVersion, ctx, moduleResolver, urlListerManager, randomProvider, enableViewIsolation, udfResolver);
293+
CompiledSql = CompileViewSql(provider, cluster, sql, syntaxVersion, viewId, qContext,
294+
ctx, moduleResolver, urlListerManager, randomProvider, enableViewIsolation, udfResolver);
277295
return bool(CompiledSql);
278296
}
279297

@@ -283,7 +301,8 @@ void TYtViewDescription::CleanupCompiledSQL()
283301
}
284302

285303
bool TYtTableDescriptionBase::Fill(const TString& provider, const TString& cluster, const TString& table,
286-
const TStructExprType* type, const TString& viewSql, ui16 syntaxVersion, const THashMap<TString, TString>& metaAttrs,
304+
const TStructExprType* type, const TString& viewSql, ui16 syntaxVersion, const TQContext& qContext,
305+
const THashMap<TString, TString>& metaAttrs,
287306
TExprContext& ctx, IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider, bool enableViewIsolation,
288307
IUdfResolver::TPtr udfResolver)
289308
{
@@ -364,13 +383,14 @@ bool TYtTableDescriptionBase::Fill(const TString& provider, const TString& clust
364383
}
365384

366385
// (3) views
367-
if (!FillViews(provider, cluster, table, metaAttrs, ctx, moduleResolver, urlListerManager, randomProvider, enableViewIsolation, udfResolver)) {
386+
if (!FillViews(provider, cluster, table, metaAttrs, qContext, ctx, moduleResolver, urlListerManager, randomProvider, enableViewIsolation, udfResolver)) {
368387
return false;
369388
}
370389

371390
if (viewSql) {
372391
if (!View) {
373-
if (!View.ConstructInPlace().Fill(provider, cluster, viewSql, syntaxVersion, ctx, moduleResolver, urlListerManager, randomProvider, enableViewIsolation, udfResolver)) {
392+
auto viewId = cluster + "/" + table;
393+
if (!View.ConstructInPlace().Fill(provider, cluster, viewSql, syntaxVersion, viewId, qContext, ctx, moduleResolver, urlListerManager, randomProvider, enableViewIsolation, udfResolver)) {
374394
ctx.AddError(TIssue(TPosition(),
375395
TStringBuilder() << "Can't load sql view, table: " << cluster << '.' << table));
376396
return false;
@@ -382,7 +402,7 @@ bool TYtTableDescriptionBase::Fill(const TString& provider, const TString& clust
382402
}
383403

384404
bool TYtTableDescriptionBase::FillViews(const TString& provider, const TString& cluster, const TString& table,
385-
const THashMap<TString, TString>& metaAttrs, TExprContext& ctx, IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager,
405+
const THashMap<TString, TString>& metaAttrs, const TQContext& qContext, TExprContext& ctx, IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager,
386406
IRandomProvider& randomProvider, bool allowViewIsolation, IUdfResolver::TPtr udfResolver)
387407
{
388408
for (auto& view: Views) {
@@ -410,7 +430,8 @@ bool TYtTableDescriptionBase::FillViews(const TString& provider, const TString&
410430
}
411431
}
412432

413-
if (!viewDesc.Fill(provider, cluster, viewSql, syntaxVersion, ctx, moduleResolver, urlListerManager, randomProvider, allowViewIsolation, udfResolver)) {
433+
auto viewId = cluster + "/" + table + "/" + view.first;
434+
if (!viewDesc.Fill(provider, cluster, viewSql, syntaxVersion, viewId, qContext, ctx, moduleResolver, urlListerManager, randomProvider, allowViewIsolation, udfResolver)) {
414435
ctx.AddError(TIssue(TPosition(),
415436
TStringBuilder() << "Can't load sql view " << viewSql.Quote()
416437
<< ", table: " << cluster << '.' << table

‎ydb/library/yql/providers/yt/provider/yql_yt_table_desc.h

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818

1919
namespace NYql {
2020

21+
class TQContext;
22+
2123
enum class TYtTableIntent: ui32 {
2224
Read = 1 << 0,
2325
View = 1 << 1, // Read via view
@@ -49,7 +51,8 @@ struct TYtViewDescription {
4951
TExprNode::TPtr CompiledSql; // contains Read! to self/self_raw tables
5052
const TTypeAnnotationNode* RowType = nullptr; // Filled only if scheme requested
5153

52-
bool Fill(const TString& provider, const TString& cluster, const TString& sql, ui16 syntaxVersion, TExprContext& ctx,
54+
bool Fill(const TString& provider, const TString& cluster, const TString& sql, ui16 syntaxVersion,
55+
const TString& viewId, const TQContext& qContext, TExprContext& ctx,
5356
IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider,
5457
bool enableViewIsolation, IUdfResolver::TPtr udfResolver);
5558
void CleanupCompiledSQL();
@@ -71,12 +74,12 @@ struct TYtTableDescriptionBase {
7174
bool IgnoreTypeV3 = false;
7275

7376
bool Fill(const TString& provider, const TString& cluster, const TString& table, const TStructExprType* type,
74-
const TString& viewSql, ui16 syntaxVersion, const THashMap<TString, TString>& metaAttrs, TExprContext& ctx,
77+
const TString& viewSql, ui16 syntaxVersion, const TQContext& qContext, const THashMap<TString, TString>& metaAttrs, TExprContext& ctx,
7578
IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider,
7679
bool enableViewIsolation, IUdfResolver::TPtr udfResolver);
7780
void CleanupCompiledSQL();
7881
bool FillViews(const TString& provider, const TString& cluster, const TString& table, const THashMap<TString, TString>& metaAttrs,
79-
TExprContext& ctx, IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider,
82+
const TQContext& qContext, TExprContext& ctx, IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider,
8083
bool enableViewIsolation, IUdfResolver::TPtr udfResolver);
8184
};
8285

0 commit comments

Comments
 (0)