Skip to content

Commit dcc9572

Browse files
vitstnvitstn
committed
Refactor out codec for table format
init commit_hash:65402bd8880a077306c1ded09b6d1aa8ea03cd1e
1 parent a487ac9 commit dcc9572

File tree

12 files changed

+2056
-1649
lines changed

12 files changed

+2056
-1649
lines changed

‎yql/essentials/minikql/protobuf_udf/ut/value_builder_ut.cpp

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
#include <yql/essentials/providers/common/codec/yql_codec.h>
77
#include <yql/essentials/providers/common/codec/yql_codec_buf.h>
8+
#include <yt/yql/providers/yt/codec/yt_codec.h>
89
#include <yql/essentials/minikql/mkql_alloc.h>
910
#include <yql/essentials/minikql/mkql_node.h>
1011
#include <yql/essentials/minikql/mkql_type_builder.h>
@@ -58,12 +59,12 @@ struct TSetup {
5859
template <typename TProto>
5960
TString YsonToProtoText(TSetup& setup, NUdf::TProtoInfo& info, TStringBuf yson) {
6061
TStringStream err;
61-
auto val = NCommon::ParseYsonValue(
62+
auto val = ParseYsonValueInTableFormat(
6263
setup.HolderFactory,
6364
NYT::NodeToYsonString(NYT::NodeFromYsonString(yson), ::NYson::EYsonFormat::Binary),
6465
static_cast<NKikimr::NMiniKQL::TStructType*>(info.StructType),
6566
0,
66-
&err, true);
67+
&err);
6768
if (!val) {
6869
throw yexception() << err.Str();
6970
}
@@ -120,7 +121,7 @@ TString ProtoTextToYson(TSetup& setup, NUdf::TProtoInfo& info, TStringBuf protoT
120121
auto value = FillValueFromProto(proto, &setup.ValueBuilder, info);
121122
TTestWriter out;
122123
NCommon::TOutputBuf buf(out, nullptr);
123-
NCommon::WriteYsonValueInTableFormat(buf, static_cast<NKikimr::NMiniKQL::TStructType*>(info.StructType), 0, value, true);
124+
WriteYsonValueInTableFormat(buf, static_cast<NKikimr::NMiniKQL::TStructType*>(info.StructType), 0, value, true);
124125
buf.Finish();
125126

126127
return NYT::NodeToYsonString(NYT::NodeFromYsonString(out.Str()), ::NYson::EYsonFormat::Text);

‎yql/essentials/minikql/protobuf_udf/ut/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ SRCS(
1111
PEERDIR(
1212
yt/yql/providers/yt/lib/schema
1313
yt/yql/providers/yt/common
14+
yt/yql/providers/yt/codec
1415
yql/essentials/public/udf/service/exception_policy
1516
yql/essentials/minikql
1617
yql/essentials/public/udf

‎yql/essentials/providers/common/codec/ya.make

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ PEERDIR(
1919
library/cpp/yson
2020
library/cpp/json
2121
library/cpp/enumbitset
22-
yt/yt/library/decimal
2322
)
2423

2524
YQL_LAST_ABI_VERSION()

‎yql/essentials/providers/common/codec/yql_codec.cpp

Lines changed: 89 additions & 1602 deletions
Large diffs are not rendered by default.

‎yql/essentials/providers/common/codec/yql_codec.h

Lines changed: 3 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -63,43 +63,15 @@ struct TCodecContext {
6363
void SkipYson(char cmd, TInputBuf& buf);
6464
void CopyYson(char cmd, TInputBuf& buf, TVector<char>& yson);
6565
void CopyYsonWithAttrs(char cmd, TInputBuf& buf, TVector<char>& yson);
66-
NKikimr::NUdf::TUnboxedValue ReadYsonValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags, const NKikimr::NMiniKQL::THolderFactory& holderFactory, char cmd, TInputBuf& buf, bool isTableFormat);
66+
TStringBuf ReadNextString(char cmd, TInputBuf& buf);
67+
NKikimr::NUdf::TUnboxedValue ReadYsonValue(NKikimr::NMiniKQL::TType* type, const NKikimr::NMiniKQL::THolderFactory& holderFactory, char cmd, TInputBuf& buf);
6768

6869
TMaybe<NKikimr::NUdf::TUnboxedValue> ParseYsonValue(const NKikimr::NMiniKQL::THolderFactory& holderFactory,
69-
const TStringBuf& yson, NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags, IOutputStream* err, bool isTableFormat);
70-
TMaybe<NKikimr::NUdf::TUnboxedValue> ParseYsonNode(const NKikimr::NMiniKQL::THolderFactory& holderFactory,
71-
const NYT::TNode& node, NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags, IOutputStream* err);
70+
const TStringBuf& yson, NKikimr::NMiniKQL::TType* type, IOutputStream* err);
7271

7372
TMaybe<NKikimr::NUdf::TUnboxedValue> ParseYsonNodeInResultFormat(const NKikimr::NMiniKQL::THolderFactory& holderFactory,
7473
const NYT::TNode& node, NKikimr::NMiniKQL::TType* type, IOutputStream* err);
7574

76-
extern "C" void ReadYsonContainerValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags,
77-
const NKikimr::NMiniKQL::THolderFactory& holderFactory, NKikimr::NUdf::TUnboxedValue& value, NCommon::TInputBuf& buf,
78-
bool wrapOptional);
79-
80-
void SkipSkiffField(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags, TInputBuf& buf);
81-
82-
NKikimr::NUdf::TUnboxedValue ReadSkiffNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags,
83-
const NKikimr::NMiniKQL::THolderFactory& holderFactory, TInputBuf& buf);
84-
85-
NKikimr::NUdf::TUnboxedValue ReadSkiffData(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags, NCommon::TInputBuf& buf);
86-
extern "C" void ReadContainerNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags,
87-
const NKikimr::NMiniKQL::THolderFactory& holderFactory, NKikimr::NUdf::TUnboxedValue& value, NCommon::TInputBuf& buf,
88-
bool wrapOptional);
89-
90-
extern "C" void WriteYsonContainerValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags,
91-
const NKikimr::NUdf::TUnboxedValuePod& value, NCommon::TOutputBuf& buf);
92-
93-
void WriteSkiffData(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags, const NKikimr::NUdf::TUnboxedValuePod& value, NCommon::TOutputBuf& buf);
94-
95-
void WriteSkiffNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags,
96-
const NKikimr::NUdf::TUnboxedValuePod& value, NCommon::TOutputBuf& buf);
97-
98-
extern "C" void WriteContainerNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags,
99-
const NKikimr::NUdf::TUnboxedValuePod& value, NCommon::TOutputBuf& buf);
100-
101-
void WriteYsonValueInTableFormat(TOutputBuf& buf, NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags, const NKikimr::NUdf::TUnboxedValuePod& value, bool topLevel);
102-
10375
TExprNode::TPtr ValueToExprLiteral(const TTypeAnnotationNode* type, const NKikimr::NUdf::TUnboxedValuePod& value, TExprContext& ctx,
10476
TPositionHandle pos = {});
10577

‎yt/yql/providers/yt/codec/codegen/ya.make.inc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ PEERDIR(
1515
yql/essentials/parser/pg_wrapper/interface
1616
yql/essentials/utils
1717
yt/yql/providers/yt/codec/codegen
18+
yt/yql/providers/yt/codec
1819
)
1920

2021
IF (NOT MKQL_DISABLE_CODEGEN)

‎yt/yql/providers/yt/codec/codegen/yt_codec_cg.cpp

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include <yql/essentials/parser/pg_wrapper/interface/codec.h>
55
#include <yql/essentials/providers/common/codec/yql_codec_buf.h>
66
#include <yql/essentials/providers/common/codec/yql_codec_type_flags.h>
7+
#include <yt/yql/providers/yt/codec/yt_codec.h>
78

89
#ifndef MKQL_DISABLE_CODEGEN
910
#include <yql/essentials/minikql/mkql_node.h>
@@ -171,7 +172,7 @@ class TYtCodecCgWriter : public IYtCodecCgWriter {
171172
const auto valType = Type::getInt128Ty(context);
172173
const auto flagsConst = ConstantInt::get(Type::getInt64Ty(context), nativeYtTypeFlags);
173174
if (nativeYtTypeFlags) {
174-
const auto funcAddr = ConstantInt::get(Type::getInt64Ty(context), (ui64)&NYql::NCommon::WriteContainerNativeYtValue);
175+
const auto funcAddr = ConstantInt::get(Type::getInt64Ty(context), (ui64)&NYql::WriteContainerNativeYtValue);
175176
const auto funType = FunctionType::get(Type::getVoidTy(context), {
176177
Type::getInt64Ty(context), Type::getInt64Ty(context), PointerType::getUnqual(valType),
177178
PointerType::getUnqual(Type::getInt8Ty(context))
@@ -180,7 +181,7 @@ class TYtCodecCgWriter : public IYtCodecCgWriter {
180181
const auto funcPtr = CastInst::Create(Instruction::IntToPtr, funcAddr, PointerType::getUnqual(funType), "ptr", Block_);
181182
CallInst::Create(funType, funcPtr, { typeConst, flagsConst, elemPtr, buf }, "", Block_);
182183
} else {
183-
const auto funcAddr = ConstantInt::get(Type::getInt64Ty(context), (ui64)&NYql::NCommon::WriteYsonContainerValue);
184+
const auto funcAddr = ConstantInt::get(Type::getInt64Ty(context), (ui64)&NYql::WriteYsonContainerValue);
184185
const auto funType = FunctionType::get(Type::getVoidTy(context), {
185186
Type::getInt64Ty(context), Type::getInt64Ty(context), PointerType::getUnqual(valType),
186187
PointerType::getUnqual(Type::getInt8Ty(context))
@@ -792,7 +793,7 @@ class TYtCodecCgReader : public IYtCodecCgReader {
792793
void GenerateContainer(Value* velemPtr, Value* buf, TType* type, bool wrapOptional, ui64 nativeYtTypeFlags) {
793794
auto& context = Codegen_->GetContext();
794795

795-
const auto funcAddr = ConstantInt::get(Type::getInt64Ty(context), nativeYtTypeFlags ? (ui64)&NCommon::ReadContainerNativeYtValue : (ui64)&NCommon::ReadYsonContainerValue);
796+
const auto funcAddr = ConstantInt::get(Type::getInt64Ty(context), nativeYtTypeFlags ? (ui64)&ReadContainerNativeYtValue : (ui64)&ReadYsonContainerValue);
796797
const auto typeConst = ConstantInt::get(Type::getInt64Ty(context), (ui64)type);
797798
const auto holderFactoryConst = ConstantInt::get(Type::getInt64Ty(context), (ui64)&HolderFactory_);
798799
const auto wrapConst = ConstantInt::get(Type::getInt1Ty(context), wrapOptional);

‎yt/yql/providers/yt/codec/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ PEERDIR(
3333
yt/yql/providers/yt/common
3434
yt/yql/providers/yt/lib/mkql_helpers
3535
yt/yql/providers/yt/lib/skiff
36+
yt/yt/library/decimal
3637
yql/essentials/providers/common/codec/yt_arrow_converter_interface
3738
)
3839

0 commit comments

Comments
 (0)