Skip to content

Commit 365b065

Browse files
author
avevad
committed
YQL-19123 More SQL features for Variant type
More SQL features for Variant type commit_hash:51c15343e2d24190ec59085888dfa3fd008cafcc
1 parent bc7efd5 commit 365b065

File tree

9 files changed

+477
-12
lines changed

9 files changed

+477
-12
lines changed

‎yql/essentials/core/type_ann/type_ann_core.cpp

Lines changed: 184 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -946,6 +946,30 @@ namespace NTypeAnnImpl {
946946
return result;
947947
}
948948

949+
TMaybe<ui32> FindOrReportMissingMember(TStringBuf memberName, const TTupleExprType& tupleType, TString& errStr) {
950+
ui32 index = 0;
951+
if (!TryFromString(memberName, index)) {
952+
errStr = TStringBuilder() << "Failed to convert to integer: " << memberName;
953+
return Nothing();
954+
}
955+
if (index >= tupleType.GetSize()) {
956+
errStr = TStringBuilder()
957+
<< "Index out of range. Index: "
958+
<< index << ", size: " << tupleType.GetSize();
959+
return Nothing();
960+
}
961+
return index;
962+
}
963+
964+
TMaybe<ui32> FindOrReportMissingMember(TStringBuf memberName, TPositionHandle pos, const TTupleExprType& tupleType, TExprContext& ctx) {
965+
TString errStr;
966+
auto result = FindOrReportMissingMember(memberName, tupleType, errStr);
967+
if (!result) {
968+
ctx.AddError(TIssue(ctx.GetPosition(pos), errStr));
969+
}
970+
return result;
971+
}
972+
949973
IGraphTransformer::TStatus MemberWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
950974
if (!EnsureArgsCount(*input, 2, ctx.Expr)) {
951975
return IGraphTransformer::TStatus::Error;
@@ -8534,6 +8558,50 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
85348558
return IGraphTransformer::TStatus::Ok;
85358559
}
85368560

8561+
IGraphTransformer::TStatus SqlVariantItemWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
8562+
if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
8563+
return IGraphTransformer::TStatus::Error;
8564+
}
8565+
8566+
if (IsNull(input->Head())) {
8567+
output = input->HeadPtr();
8568+
return IGraphTransformer::TStatus::Repeat;
8569+
}
8570+
8571+
if (input->Head().GetTypeAnn() && input->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
8572+
auto optType = input->Head().GetTypeAnn()->Cast<TOptionalExprType>();
8573+
if (!EnsureVariantType(input->Head().Pos(), *optType->GetItemType(), ctx.Expr)) {
8574+
return IGraphTransformer::TStatus::Error;
8575+
}
8576+
auto varType = optType->GetItemType()->Cast<TVariantExprType>();
8577+
auto isOptionalItem = false;
8578+
if (varType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
8579+
auto tupType = varType->GetUnderlyingType()->Cast<TTupleExprType>();
8580+
if (tupType->GetSize() > 0 && tupType->GetItems()[0]->GetKind() == ETypeAnnotationKind::Optional) {
8581+
isOptionalItem = true;
8582+
}
8583+
} else {
8584+
auto strType = varType->GetUnderlyingType()->Cast<TStructExprType>();
8585+
if (strType->GetSize() > 0 && strType->GetItems()[0]->GetItemType()->GetKind() == ETypeAnnotationKind::Optional) {
8586+
isOptionalItem = true;
8587+
}
8588+
}
8589+
output = ctx.Expr.Builder(input->Pos())
8590+
.Callable(isOptionalItem ? "FlatMap" : "Map")
8591+
.Add(0, input->HeadPtr())
8592+
.Lambda(1)
8593+
.Param("var")
8594+
.Callable("VariantItem")
8595+
.Arg(0, "var")
8596+
.Seal()
8597+
.Seal()
8598+
.Seal().Build();
8599+
} else {
8600+
output = ctx.Expr.RenameNode(*input, "VariantItem");
8601+
}
8602+
return IGraphTransformer::TStatus::Repeat;
8603+
}
8604+
85378605
IGraphTransformer::TStatus VariantItemWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
85388606
Y_UNUSED(output);
85398607
if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
@@ -8591,6 +8659,112 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
85918659
return IGraphTransformer::TStatus::Ok;
85928660
}
85938661

8662+
IGraphTransformer::TStatus SqlVisitWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
8663+
if (!EnsureMinArgsCount(*input, 2, ctx.Expr)) {
8664+
return IGraphTransformer::TStatus::Error;
8665+
}
8666+
8667+
if (IsNull(input->Head())) {
8668+
output = input->HeadPtr();
8669+
return IGraphTransformer::TStatus::Repeat;
8670+
}
8671+
8672+
if (input->Head().GetTypeAnn() && input->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
8673+
auto appendOtherArgs = [&] (auto &builder) -> auto {
8674+
for (size_t pos = 1; pos < input->ChildrenSize(); pos++) {
8675+
builder.Add(pos, input->ChildPtr(pos));
8676+
}
8677+
return builder;
8678+
};
8679+
8680+
auto optType = input->Head().GetTypeAnn()->Cast<TOptionalExprType>();
8681+
if (!EnsureVariantType(input->Pos(), *optType->GetItemType(), ctx.Expr)) {
8682+
return IGraphTransformer::TStatus::Error;
8683+
}
8684+
auto varType = optType->GetItemType()->Cast<TVariantExprType>();
8685+
const TTupleExprType* tupType = nullptr;
8686+
const TStructExprType* strType = nullptr;
8687+
if (varType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
8688+
tupType = varType->GetUnderlyingType()->Cast<TTupleExprType>();
8689+
} else {
8690+
strType = varType->GetUnderlyingType()->Cast<TStructExprType>();
8691+
}
8692+
8693+
bool isOptionalResult = false;
8694+
bool repeat = false;
8695+
for (size_t idx = 1; idx < input->ChildrenSize(); idx++) {
8696+
auto child = input->ChildPtr(idx);
8697+
if (child->IsAtom()) {
8698+
const TTypeAnnotationNode* itemType;
8699+
if (tupType) {
8700+
auto pos = FindOrReportMissingMember(child->Content(), child->Pos(), *tupType, ctx.Expr);
8701+
if (!pos) {
8702+
return IGraphTransformer::TStatus::Error;
8703+
}
8704+
itemType = tupType->GetItems()[*pos];
8705+
} else {
8706+
auto pos = FindOrReportMissingMember(child->Content(), child->Pos(), *strType, ctx.Expr);
8707+
if (!pos) {
8708+
return IGraphTransformer::TStatus::Error;
8709+
}
8710+
itemType = strType->GetItems()[*pos]->GetItemType();
8711+
}
8712+
8713+
idx++;
8714+
if (idx == input->ChildrenSize()) {
8715+
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), "Expected lambda after this argument"));
8716+
return IGraphTransformer::TStatus::Error;
8717+
}
8718+
8719+
auto status = ConvertToLambda(input->ChildRef(idx), ctx.Expr, 1);
8720+
if (status.Level != IGraphTransformer::TStatus::Ok) {
8721+
return status;
8722+
}
8723+
8724+
auto& lambda = input->ChildRef(idx);
8725+
if (!UpdateLambdaAllArgumentsTypes(lambda, {itemType}, ctx.Expr)) {
8726+
return IGraphTransformer::TStatus::Error;
8727+
}
8728+
8729+
if (!lambda->GetTypeAnn()) {
8730+
repeat = true;
8731+
continue;
8732+
}
8733+
8734+
if (lambda->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
8735+
isOptionalResult = true;
8736+
}
8737+
} else {
8738+
if (!EnsureComputable(*child, ctx.Expr)) {
8739+
return IGraphTransformer::TStatus::Error;
8740+
}
8741+
if (child->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
8742+
isOptionalResult = true;
8743+
}
8744+
}
8745+
}
8746+
8747+
if (repeat) {
8748+
return IGraphTransformer::TStatus::Repeat;
8749+
}
8750+
8751+
output = appendOtherArgs(ctx.Expr.Builder(input->Pos())
8752+
.Callable(isOptionalResult ? "FlatMap" : "Map")
8753+
.Add(0, input->HeadPtr())
8754+
.Lambda(1)
8755+
.Param("var")
8756+
.Callable("Visit")
8757+
.Arg(0, "var"))
8758+
.Seal()
8759+
.Seal()
8760+
.Seal().Build();
8761+
} else {
8762+
output = ctx.Expr.RenameNode(*input, "Visit");
8763+
}
8764+
8765+
return IGraphTransformer::TStatus::Repeat;
8766+
}
8767+
85948768
IGraphTransformer::TStatus VisitWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
85958769
Y_UNUSED(output);
85968770
if (!EnsureMinArgsCount(*input, 2, ctx.Expr)) {
@@ -8625,21 +8799,13 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
86258799
const TTypeAnnotationNode* itemType;
86268800
ui32 itemIndex;
86278801
if (tupleType) {
8628-
ui32 index = 0;
8629-
if (!TryFromString(child->Content(), index)) {
8630-
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() << "Failed to convert to integer: " << child->Content()));
8631-
return IGraphTransformer::TStatus::Error;
8632-
}
8633-
8634-
if (index >= tupleType->GetSize()) {
8635-
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder()
8636-
<< "Index out of range. Index: "
8637-
<< index << ", size: " << tupleType->GetSize()));
8802+
auto pos = FindOrReportMissingMember(child->Content(), child->Pos(), *tupleType, ctx.Expr);
8803+
if (!pos) {
86388804
return IGraphTransformer::TStatus::Error;
86398805
}
86408806

8641-
itemType = tupleType->GetItems()[index];
8642-
itemIndex = index;
8807+
itemType = tupleType->GetItems()[*pos];
8808+
itemIndex = *pos;
86438809
} else {
86448810
auto pos = FindOrReportMissingMember(child->Content(), child->Pos(), *structType, ctx.Expr);
86458811
if (!pos) {
@@ -8763,6 +8929,10 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
87638929

87648930
variantType = itemType->Cast<TVariantExprType>();
87658931
}
8932+
else if (IsNull(input->Head())) {
8933+
output = input->HeadPtr();
8934+
return IGraphTransformer::TStatus::Repeat;
8935+
}
87668936
else {
87678937
if (!EnsureVariantType(input->Head(), ctx.Expr)) {
87688938
return IGraphTransformer::TStatus::Error;
@@ -12421,7 +12591,9 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
1242112591
Functions["FlattenMembersType"] = &TypeArgWrapper<ETypeArgument::FlattenMembers>;
1242212592
Functions["VariantUnderlyingType"] = &TypeArgWrapper<ETypeArgument::VariantUnderlying>;
1242312593
Functions["Guess"] = &GuessWrapper;
12594+
Functions["SqlVariantItem"] = &SqlVariantItemWrapper;
1242412595
Functions["VariantItem"] = &VariantItemWrapper;
12596+
Functions["SqlVisit"] = &SqlVisitWrapper;
1242512597
Functions["Visit"] = &VisitWrapper;
1242612598
Functions["Way"] = &WayWrapper;
1242712599
Functions["SqlAccess"] = &SqlAccessWrapper;

‎yql/essentials/sql/v1/builtin.cpp

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3010,6 +3010,7 @@ struct TBuiltinFuncData {
30103010
{"callableresulttype", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("CallableResultType", 1, 1) },
30113011
{"callableargumenttype", BuildSimpleBuiltinFactoryCallback<TYqlCallableArgumentType>() },
30123012
{"variantunderlyingtype", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("VariantUnderlyingType", 1, 1) },
3013+
{"variantitem", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("SqlVariantItem", 1, 1) },
30133014
{"fromysonsimpletype", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("FromYsonSimpleType", 2, 2) },
30143015
{"currentutcdate", BuildNamedDepsArgcBuiltinFactoryCallback<TCallNodeDepArgs>(0, "CurrentUtcDate", 0, -1) },
30153016
{"currentutcdatetime", BuildNamedDepsArgcBuiltinFactoryCallback<TCallNodeDepArgs>(0, "CurrentUtcDatetime", 0, -1) },
@@ -3705,6 +3706,54 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec
37053706
BuildTuple(pos, {BuildQuotedAtom(pos, ""), args[1]}),
37063707
};
37073708
return new TCallNodeImpl(pos, "FlattenMembers", 2, 2, flattenMembersArgs);
3709+
} else if (normalizedName == "visit" || normalizedName == "visitordefault") {
3710+
bool withDefault = normalizedName == "visitordefault";
3711+
TNodePtr variant;
3712+
TVector<TNodePtr> labels, handlers;
3713+
TMaybe<TNodePtr> dflt;
3714+
if (mustUseNamed && *mustUseNamed) {
3715+
*mustUseNamed = false;
3716+
auto &positional = *args[0]->GetTupleNode();
3717+
if (positional.GetTupleSize() != (withDefault ? 2 : 1)) {
3718+
return new TInvalidBuiltin(pos, TStringBuilder() << name
3719+
<< " requires exactly " << (withDefault ? 2 : 1) << " positional arguments when named args are used");
3720+
}
3721+
auto &named = *args[1]->GetStructNode();
3722+
variant = positional.GetTupleElement(0);
3723+
auto &namedExprs = named.GetExprs();
3724+
labels.reserve(namedExprs.size());
3725+
handlers.reserve(namedExprs.size());
3726+
for (size_t idx = 0; idx < namedExprs.size(); idx++) {
3727+
labels.push_back(BuildQuotedAtom(pos, namedExprs[idx]->GetLabel()));
3728+
handlers.push_back(namedExprs[idx]);
3729+
}
3730+
if (withDefault) {
3731+
dflt = positional.GetTupleElement(positional.GetTupleSize() - 1);
3732+
}
3733+
} else {
3734+
variant = args[0];
3735+
size_t defaultSuffix = withDefault ? 1 : 0;
3736+
labels.reserve(args.size() - 1 - defaultSuffix);
3737+
handlers.reserve(args.size() - 1 - defaultSuffix);
3738+
for (size_t idx = 0; idx + 1 < args.size() - defaultSuffix; idx++) {
3739+
labels.push_back(BuildQuotedAtom(pos, ToString(idx)));
3740+
handlers.push_back(args[idx + 1]);
3741+
}
3742+
if (withDefault) {
3743+
dflt = args.back();
3744+
}
3745+
}
3746+
TVector<TNodePtr> resultArgs;
3747+
resultArgs.reserve(1 + labels.size() + handlers.size());
3748+
resultArgs.emplace_back(std::move(variant));
3749+
for (size_t idx = 0; idx < labels.size(); idx++) {
3750+
resultArgs.emplace_back(std::move(labels[idx]));
3751+
resultArgs.emplace_back(std::move(handlers[idx]));
3752+
}
3753+
if (dflt.Defined()) {
3754+
resultArgs.emplace_back(std::move(dflt->Get()));
3755+
}
3756+
return new TCallNodeImpl(pos, "SqlVisit", 1, -1, resultArgs);
37083757
} else if (normalizedName == "sqlexternalfunction") {
37093758
return new TCallNodeImpl(pos, "SqlExternalFunction", args);
37103759
} else {

‎yql/essentials/tests/sql/sql2yql/canondata/result.json

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6908,20 +6908,41 @@
69086908
"uri": "https://{canondata_backend}/1937429/434276f26b2857be3c5ad3fdbbf877d2bf775ac5/resource.tar.gz#test_sql2yql.test_expr-uuid_order_/sql.yql"
69096909
}
69106910
],
6911+
"test_sql2yql.test[expr-variant_builtins_opt]": [
6912+
{
6913+
"checksum": "00f7f7ba82bd2474f7b1bfd7771323f9",
6914+
"size": 3847,
6915+
"uri": "https://{canondata_backend}/1936842/4ad1682e9f564ef555f5e0180b972a1adad2fc43/resource.tar.gz#test_sql2yql.test_expr-variant_builtins_opt_/sql.yql"
6916+
}
6917+
],
69116918
"test_sql2yql.test[expr-variant_list_sort]": [
69126919
{
69136920
"checksum": "abd93bfb65a1cb17235be843d4cbd2c3",
69146921
"size": 1481,
69156922
"uri": "https://{canondata_backend}/1936273/4a1b39013e1bae40e722cff8ccef8829784964e2/resource.tar.gz#test_sql2yql.test_expr-variant_list_sort_/sql.yql"
69166923
}
69176924
],
6925+
"test_sql2yql.test[expr-variant_struct_builtins]": [
6926+
{
6927+
"checksum": "cb90540556b74bdbb9a95959c62a3cbf",
6928+
"size": 5780,
6929+
"uri": "https://{canondata_backend}/1936842/4ad1682e9f564ef555f5e0180b972a1adad2fc43/resource.tar.gz#test_sql2yql.test_expr-variant_struct_builtins_/sql.yql"
6930+
}
6931+
],
69186932
"test_sql2yql.test[expr-variant_struct_comp]": [
69196933
{
69206934
"checksum": "3f85119441280a8729ef912910ac7745",
69216935
"size": 9664,
69226936
"uri": "https://{canondata_backend}/1936273/4a1b39013e1bae40e722cff8ccef8829784964e2/resource.tar.gz#test_sql2yql.test_expr-variant_struct_comp_/sql.yql"
69236937
}
69246938
],
6939+
"test_sql2yql.test[expr-variant_tuple_builtins]": [
6940+
{
6941+
"checksum": "41d01afdf58dcca513a44186a279efcb",
6942+
"size": 5682,
6943+
"uri": "https://{canondata_backend}/1936842/4ad1682e9f564ef555f5e0180b972a1adad2fc43/resource.tar.gz#test_sql2yql.test_expr-variant_tuple_builtins_/sql.yql"
6944+
}
6945+
],
69256946
"test_sql2yql.test[expr-variant_tuple_comp]": [
69266947
{
69276948
"checksum": "62f3d0d5c99c037db94a4f8f3c3c9d1d",
@@ -24821,16 +24842,31 @@
2482124842
"uri": "file://test_sql_format.test_expr-uuid_order_/formatted.sql"
2482224843
}
2482324844
],
24845+
"test_sql_format.test[expr-variant_builtins_opt]": [
24846+
{
24847+
"uri": "file://test_sql_format.test_expr-variant_builtins_opt_/formatted.sql"
24848+
}
24849+
],
2482424850
"test_sql_format.test[expr-variant_list_sort]": [
2482524851
{
2482624852
"uri": "file://test_sql_format.test_expr-variant_list_sort_/formatted.sql"
2482724853
}
2482824854
],
24855+
"test_sql_format.test[expr-variant_struct_builtins]": [
24856+
{
24857+
"uri": "file://test_sql_format.test_expr-variant_struct_builtins_/formatted.sql"
24858+
}
24859+
],
2482924860
"test_sql_format.test[expr-variant_struct_comp]": [
2483024861
{
2483124862
"uri": "file://test_sql_format.test_expr-variant_struct_comp_/formatted.sql"
2483224863
}
2483324864
],
24865+
"test_sql_format.test[expr-variant_tuple_builtins]": [
24866+
{
24867+
"uri": "file://test_sql_format.test_expr-variant_tuple_builtins_/formatted.sql"
24868+
}
24869+
],
2483424870
"test_sql_format.test[expr-variant_tuple_comp]": [
2483524871
{
2483624872
"uri": "file://test_sql_format.test_expr-variant_tuple_comp_/formatted.sql"

0 commit comments

Comments
 (0)