diff --git a/CMakeLists.txt b/CMakeLists.txt index 5c4a7aed1ec..686aabe9ccd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,8 +61,7 @@ if (YDB_SDK_EXAMPLES) endif() if (YDB_SDK_TESTS) - add_subdirectory(tests/unit) - add_subdirectory(tests/integration) + add_subdirectory(tests) endif() if (YDB_SDK_INSTALL) diff --git a/cmake/protos_public_headers.txt b/cmake/protos_public_headers.txt index 5920de5ea17..8bf8cb074e3 100644 --- a/cmake/protos_public_headers.txt +++ b/cmake/protos_public_headers.txt @@ -1,6 +1,4 @@ -src/api/grpc/draft/ydb_datastreams_v1.grpc.pb.h src/api/grpc/draft/ydb_datastreams_v1.pb.h -src/api/grpc/ydb_topic_v1.grpc.pb.h src/api/grpc/ydb_topic_v1.pb.h src/api/protos/annotations/validation.pb.h src/api/protos/draft/datastreams.pb.h @@ -19,6 +17,4 @@ src/api/protos/ydb_issue_message.pb.h src/api/protos/ydb_export.pb.h src/api/protos/ydb_coordination.pb.h src/api/protos/ydb_status_codes.pb.h -src/api/protos/draft/ydb_replication.pb.h -src/library/operation_id/protos/operation_id.pb.h -src/library/yql_common/issue/protos/issue_severity.pb.h \ No newline at end of file +src/api/protos/draft/ydb_replication.pb.h \ No newline at end of file diff --git a/examples/basic_example/basic_example.cpp b/examples/basic_example/basic_example.cpp index a69c2e1031a..13d5930b9c3 100644 --- a/examples/basic_example/basic_example.cpp +++ b/examples/basic_example/basic_example.cpp @@ -472,7 +472,7 @@ void StreamQuerySelect(TQueryClient client) { } } } - return TStatus(EStatus::SUCCESS, NYql::TIssues()); + return TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues()); })); } diff --git a/include/ydb-cpp-sdk/client/datastreams/datastreams.h b/include/ydb-cpp-sdk/client/datastreams/datastreams.h index 0e031374e59..b995d75b2fb 100644 --- a/include/ydb-cpp-sdk/client/datastreams/datastreams.h +++ b/include/ydb-cpp-sdk/client/datastreams/datastreams.h @@ -2,7 +2,7 @@ #include -#include +#include namespace NYdb::NDataStreams::V1 { diff --git a/include/ydb-cpp-sdk/client/discovery/discovery.h b/include/ydb-cpp-sdk/client/discovery/discovery.h index dd8c86427b8..81301e189ce 100644 --- a/include/ydb-cpp-sdk/client/discovery/discovery.h +++ b/include/ydb-cpp-sdk/client/discovery/discovery.h @@ -100,7 +100,7 @@ struct TNodeInfo { class TNodeRegistrationResult : public TStatus { public: - TNodeRegistrationResult() : TStatus(EStatus::GENERIC_ERROR, NYql::TIssues()) {} + TNodeRegistrationResult() : TStatus(EStatus::GENERIC_ERROR, NYdb::NIssue::TIssues()) {} TNodeRegistrationResult(TStatus&& status, const Ydb::Discovery::NodeRegistrationResult& proto); uint32_t GetNodeId() const; const std::string& GetDomainPath() const; diff --git a/include/ydb-cpp-sdk/client/draft/ydb_replication.h b/include/ydb-cpp-sdk/client/draft/ydb_replication.h index 7528395245d..3cb919dd856 100644 --- a/include/ydb-cpp-sdk/client/draft/ydb_replication.h +++ b/include/ydb-cpp-sdk/client/draft/ydb_replication.h @@ -17,10 +17,6 @@ namespace NYdb { class TProtoAccessor; } -namespace NYql { - class TIssues; -} - namespace NYdb::NReplication { class TDescribeReplicationResult; @@ -94,9 +90,9 @@ class TErrorState { class TImpl; public: - explicit TErrorState(NYql::TIssues&& issues); + explicit TErrorState(NYdb::NIssue::TIssues&& issues); - const NYql::TIssues& GetIssues() const; + const NYdb::NIssue::TIssues& GetIssues() const; private: std::shared_ptr Impl_; diff --git a/include/ydb-cpp-sdk/client/draft/ydb_view.h b/include/ydb-cpp-sdk/client/draft/ydb_view.h index 91510050586..e2318c33be4 100644 --- a/include/ydb-cpp-sdk/client/draft/ydb_view.h +++ b/include/ydb-cpp-sdk/client/draft/ydb_view.h @@ -11,10 +11,6 @@ namespace NYdb { class TProtoAccessor; } -namespace NYql { - class TIssues; -} - namespace NYdb::NView { class TDescribeViewResult; diff --git a/include/ydb-cpp-sdk/client/types/status/status.h b/include/ydb-cpp-sdk/client/types/status/status.h index 85b8793ab97..dd20f8964a4 100644 --- a/include/ydb-cpp-sdk/client/types/status/status.h +++ b/include/ydb-cpp-sdk/client/types/status/status.h @@ -3,7 +3,7 @@ #include #include -#include +#include #include @@ -15,11 +15,11 @@ struct TPlainStatus; //! Represents status of call class TStatus { public: - TStatus(EStatus statusCode, NYql::TIssues&& issues); + TStatus(EStatus statusCode, NYdb::NIssue::TIssues&& issues); TStatus(TPlainStatus&& plain); EStatus GetStatus() const; - const NYql::TIssues& GetIssues() const; + const NYdb::NIssue::TIssues& GetIssues() const; bool IsSuccess() const; bool IsTransportError() const; const std::string& GetEndpoint() const; diff --git a/include/ydb-cpp-sdk/library/yql_common/issue/yql_issue.h b/include/ydb-cpp-sdk/library/issue/yql_issue.h similarity index 83% rename from include/ydb-cpp-sdk/library/yql_common/issue/yql_issue.h rename to include/ydb-cpp-sdk/library/issue/yql_issue.h index 30ccd1869de..01779537a2e 100644 --- a/include/ydb-cpp-sdk/library/yql_common/issue/yql_issue.h +++ b/include/ydb-cpp-sdk/library/issue/yql_issue.h @@ -1,7 +1,5 @@ #pragma once -#include "yql_issue_id.h" - #include #include #include @@ -13,11 +11,23 @@ #include #include -#include #include #include -namespace NYql { +namespace NYdb::NIssue { + +using TIssueCode = uint32_t; +constexpr TIssueCode DEFAULT_ERROR = 0; +constexpr TIssueCode UNEXPECTED_ERROR = 1; + +enum class ESeverity : uint32_t { + Fatal = 0, + Error = 1, + Warning = 2, + Info = 3, +}; + +std::string SeverityToString(ESeverity severity); void SanitizeNonAscii(std::string& s); @@ -56,37 +66,6 @@ struct TPosition { } }; -class TTextWalker { -public: - TTextWalker(TPosition& position, bool utf8Aware) - : Position(position) - , Utf8Aware(utf8Aware) - , HaveCr(false) - , LfCount(0) - { - } - - static inline bool IsUtf8Intermediate(char c) { - return (c & 0xC0) == 0x80; - } - - template - TTextWalker& Advance(const T& buf) { - for (char c : buf) { - Advance(c); - } - return *this; - } - - TTextWalker& Advance(char c); - -private: - TPosition& Position; - const bool Utf8Aware; - bool HaveCr; - uint32_t LfCount; -}; - struct TRange { TPosition Position; TPosition EndPosition; @@ -118,12 +97,12 @@ class TIssue; using TIssuePtr = TIntrusivePtr; class TIssue: public TThrRefBase { std::vector> Children_; - TString Message; + std::string Message; public: TPosition Position; TPosition EndPosition; TIssueCode IssueCode = 0U; - ESeverity Severity = TSeverityIds::S_ERROR; + ESeverity Severity = ESeverity::Error; TIssue() = default; @@ -163,14 +142,7 @@ class TIssue: public TThrRefBase { && IssueCode == other.IssueCode; } - ui64 Hash() const noexcept { - return CombineHashes( - CombineHashes( - (size_t)CombineHashes(IntHash(Position.Row), IntHash(Position.Column)), - std::hash{}(Position.File) - ), - (size_t)CombineHashes((size_t)IntHash(static_cast(IssueCode)), std::hash{}(Message))); - } + uint64_t Hash() const noexcept; TIssue& SetCode(TIssueCode id, ESeverity severity) { IssueCode = id; @@ -197,7 +169,8 @@ class TIssue: public TThrRefBase { } TIssue& AddSubIssue(TIntrusivePtr issue) { - Severity = (ESeverity)std::min((uint32_t)issue->GetSeverity(), (uint32_t)Severity); + Severity = static_cast(std::min(static_cast(issue->GetSeverity()), + static_cast(Severity))); Children_.push_back(issue); return *this; } @@ -215,7 +188,7 @@ class TIssue: public TThrRefBase { } // Unsafe method. Doesn't call SanitizeNonAscii(Message) - TString* MutableMessage() { + std::string* MutableMessage() { return &Message; } @@ -359,20 +332,20 @@ std::optional TryParseTerminationMessage(std::string_view& message); } // namespace NYql template <> -void Out(IOutputStream& out, const NYql::TPosition& pos); +void Out(IOutputStream& out, const NYdb::NIssue::TPosition& pos); template <> -void Out(IOutputStream& out, const NYql::TRange& pos); +void Out(IOutputStream& out, const NYdb::NIssue::TRange& pos); template <> -void Out(IOutputStream& out, const NYql::TIssue& error); +void Out(IOutputStream& out, const NYdb::NIssue::TIssue& error); template <> -void Out(IOutputStream& out, const NYql::TIssues& error); +void Out(IOutputStream& out, const NYdb::NIssue::TIssues& error); template <> -struct THash { - inline size_t operator()(const NYql::TIssue& err) const { +struct THash { + inline size_t operator()(const NYdb::NIssue::TIssue& err) const { return err.Hash(); } }; diff --git a/include/ydb-cpp-sdk/library/yql_common/issue/yql_issue_id.h b/include/ydb-cpp-sdk/library/yql_common/issue/yql_issue_id.h deleted file mode 100644 index 88111b104e2..00000000000 --- a/include/ydb-cpp-sdk/library/yql_common/issue/yql_issue_id.h +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include - -namespace NYql { - -using TIssueCode = uint32_t; -using ESeverity = NYql::TSeverityIds::ESeverityId; -const TIssueCode DEFAULT_ERROR = 0; -const TIssueCode UNEXPECTED_ERROR = 1; - -std::string SeverityToString(ESeverity severity); - -} diff --git a/include/ydb-cpp-sdk/library/yql_common/utils/utf8.h b/include/ydb-cpp-sdk/library/yql_common/utils/utf8.h deleted file mode 100644 index 5c28353416a..00000000000 --- a/include/ydb-cpp-sdk/library/yql_common/utils/utf8.h +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once - -#include -#include - -namespace NYql { - -bool IsUtf8(const std::string_view& str); - -unsigned char WideCharSize(char head); - -std::optional RoundToNearestValidUtf8(const std::string_view& str, bool roundDown); -std::optional NextValidUtf8(const std::string_view& str); -std::optional NextLexicographicString(const std::string_view& str); - -} diff --git a/src/client/coordination/coordination.cpp b/src/client/coordination/coordination.cpp index ba457838f3d..1fba2be26b9 100644 --- a/src/client/coordination/coordination.cpp +++ b/src/client/coordination/coordination.cpp @@ -660,8 +660,8 @@ class TSessionContext : public TThrRefBase { Ydb::StatusIds::StatusCode protoStatus, const google::protobuf::RepeatedPtrField& protoIssues) const { - NYql::TIssues issues; - NYql::IssuesFromMessage(protoIssues, issues); + NYdb::NIssue::TIssues issues; + NYdb::NIssue::IssuesFromMessage(protoIssues, issues); return TPlainStatus(static_cast(protoStatus), std::move(issues)); } @@ -674,7 +674,7 @@ class TSessionContext : public TThrRefBase { return TStatus(std::forward(source)); } - TStatus MakeStatus(EStatus status, NYql::TIssues&& issues) const { + TStatus MakeStatus(EStatus status, NYdb::NIssue::TIssues&& issues) const { return TStatus(TPlainStatus(status, std::move(issues))); } diff --git a/src/client/datastreams/datastreams.cpp b/src/client/datastreams/datastreams.cpp index a17009b9b95..31cfdea4ca9 100644 --- a/src/client/datastreams/datastreams.cpp +++ b/src/client/datastreams/datastreams.cpp @@ -4,11 +4,13 @@ #include #undef INCLUDE_YDB_INTERNAL_H -#include -#include +#include +#include #include +#include + namespace NYdb::NDataStreams::V1 { TPartitioningSettingsBuilder TCreateStreamSettings::BeginConfigurePartitioningSettings() { diff --git a/src/client/draft/ydb_replication.cpp b/src/client/draft/ydb_replication.cpp index b2be0c7eed1..ced46984194 100644 --- a/src/client/draft/ydb_replication.cpp +++ b/src/client/draft/ydb_replication.cpp @@ -4,8 +4,8 @@ #include #undef INCLUDE_YDB_INTERNAL_H -#include -#include +#include +#include #include #include #include @@ -93,27 +93,26 @@ const TStats& TRunningState::GetStats() const { class TErrorState::TImpl { public: - NYql::TIssues Issues; + NYdb::NIssue::TIssues Issues; - explicit TImpl(NYql::TIssues&& issues) + explicit TImpl(NYdb::NIssue::TIssues&& issues) : Issues(std::move(issues)) { } }; -TErrorState::TErrorState(NYql::TIssues&& issues) +TErrorState::TErrorState(NYdb::NIssue::TIssues&& issues) : Impl_(std::make_shared(std::move(issues))) { } -const NYql::TIssues& TErrorState::GetIssues() const { +const NYdb::NIssue::TIssues& TErrorState::GetIssues() const { return Impl_->Issues; } -template -NYql::TIssues IssuesFromMessage(const ::google::protobuf::RepeatedPtrField& message) { - NYql::TIssues issues; - NYql::IssuesFromMessage(message, issues); +NYdb::NIssue::TIssues IssuesFromMessage(const ::google::protobuf::RepeatedPtrField& message) { + NYdb::NIssue::TIssues issues; + NYdb::NIssue::IssuesFromMessage(message, issues); return issues; } diff --git a/src/client/draft/ydb_scripting.cpp b/src/client/draft/ydb_scripting.cpp index a4c761ee30e..8c99a857d7f 100644 --- a/src/client/draft/ydb_scripting.cpp +++ b/src/client/draft/ydb_scripting.cpp @@ -72,8 +72,8 @@ class TYqlResultPartIterator::TReaderImpl { self->Finished_ = true; promise.SetValue({ TStatus(TPlainStatus(grpcStatus, self->Endpoint_)) }); } else { - NYql::TIssues issues; - NYql::IssuesFromMessage(self->Response_.issues(), issues); + NYdb::NIssue::TIssues issues; + NYdb::NIssue::IssuesFromMessage(self->Response_.issues(), issues); EStatus clientStatus = static_cast(self->Response_.status()); TPlainStatus plainStatus{ clientStatus, std::move(issues), self->Endpoint_, {} }; TStatus status{ std::move(plainStatus) }; diff --git a/src/client/federated_topic/impl/federated_read_session.cpp b/src/client/federated_topic/impl/federated_read_session.cpp index b2f54db0201..30c85ac9d3e 100644 --- a/src/client/federated_topic/impl/federated_read_session.cpp +++ b/src/client/federated_topic/impl/federated_read_session.cpp @@ -248,7 +248,7 @@ std::vector TFederatedReadSessionImpl::GetEvents(bool { std::lock_guard guard(Lock); if (Closing) { - return {NTopic::TSessionClosedEvent(FederationState->Status.GetStatus(), NYql::TIssues(FederationState->Status.GetIssues()))}; + return {NTopic::TSessionClosedEvent(FederationState->Status.GetStatus(), NYdb::NIssue::TIssues(FederationState->Status.GetIssues()))}; } // TODO!!! handle aborting or closing state // via handler on SessionClosedEvent { diff --git a/src/client/federated_topic/impl/federated_write_session.cpp b/src/client/federated_topic/impl/federated_write_session.cpp index be491f04987..cb4bbba85c7 100644 --- a/src/client/federated_topic/impl/federated_write_session.cpp +++ b/src/client/federated_topic/impl/federated_write_session.cpp @@ -285,7 +285,7 @@ std::shared_ptr TFederatedWriteSessionImpl::OnFederationS if (!FederationState->Status.IsSuccess()) { // The observer became stale, it won't try to get federation state anymore due to retry policy, // so there's no reason to keep the write session alive. - CloseImpl(FederationState->Status.GetStatus(), NYql::TIssues(FederationState->Status.GetIssues())); + CloseImpl(FederationState->Status.GetStatus(), NYdb::NIssue::TIssues(FederationState->Status.GetIssues())); return {}; } @@ -303,7 +303,7 @@ std::shared_ptr TFederatedWriteSessionImpl::OnFederationS } else { std::string message = "Failed to select database: no available database"; LOG_LAZY(Log, TLOG_ERR, GetLogPrefixImpl() << message << ". Status: " << status); - CloseImpl(status, NYql::TIssues{NYql::TIssue(message)}); + CloseImpl(status, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue(message)}); } return {}; } @@ -341,7 +341,7 @@ void TFederatedWriteSessionImpl::ScheduleFederationStateUpdateImpl(TDuration del UpdateStateDelayContext = Connections->CreateContext(); if (!UpdateStateDelayContext) { - CloseImpl(EStatus::TRANSPORT_UNAVAILABLE, NYql::TIssues{NYql::TIssue("Could not update federation state")}); + CloseImpl(EStatus::TRANSPORT_UNAVAILABLE, NYdb::NIssue::TIssues{NYdb::NIssue::TIssue("Could not update federation state")}); // TODO log DRIVER_IS_STOPPING_DESCRIPTION return; } @@ -424,7 +424,7 @@ bool TFederatedWriteSessionImpl::MaybeWriteImpl() { return true; } -void TFederatedWriteSessionImpl::CloseImpl(EStatus statusCode, NYql::TIssues&& issues) { +void TFederatedWriteSessionImpl::CloseImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues) { CloseImpl(TPlainStatus(statusCode, std::move(issues))); } @@ -462,7 +462,7 @@ bool TFederatedWriteSessionImpl::Close(TDuration timeout) { NThreading::WaitAny(futures).Wait(timeout); with_lock(Lock) { - CloseImpl(EStatus::SUCCESS, NYql::TIssues{}); + CloseImpl(EStatus::SUCCESS, NYdb::NIssue::TIssues{}); return MessageQueuesAreEmptyImpl(); } } diff --git a/src/client/federated_topic/impl/federated_write_session.h b/src/client/federated_topic/impl/federated_write_session.h index 9cfbce6aac4..878503b6e8c 100644 --- a/src/client/federated_topic/impl/federated_write_session.h +++ b/src/client/federated_topic/impl/federated_write_session.h @@ -93,7 +93,7 @@ class TFederatedWriteSessionImpl : public NTopic::TContinuationTokenIssuer, void WriteInternal(NTopic::TContinuationToken&&, TWrappedWriteMessage&& message); bool MaybeWriteImpl(); - void CloseImpl(EStatus statusCode, NYql::TIssues&& issues); + void CloseImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues); void CloseImpl(NTopic::TSessionClosedEvent const& ev); bool MessageQueuesAreEmptyImpl() const; diff --git a/src/client/helpers/CMakeLists.txt b/src/client/helpers/CMakeLists.txt index 62e334acc4e..cb1ad203935 100644 --- a/src/client/helpers/CMakeLists.txt +++ b/src/client/helpers/CMakeLists.txt @@ -6,7 +6,6 @@ target_link_libraries(client-helpers client-ydb_types-credentials-oauth2 client-iam-common client-ydb_types-credentials - yql-public-issue-protos ) target_sources(client-helpers diff --git a/src/client/impl/ydb_internal/common/types.h b/src/client/impl/ydb_internal/common/types.h index 2191300e212..8d7dea5e7fe 100644 --- a/src/client/impl/ydb_internal/common/types.h +++ b/src/client/impl/ydb_internal/common/types.h @@ -5,7 +5,7 @@ #include #include -#include +#include #include diff --git a/src/client/impl/ydb_internal/db_driver_state/state.cpp b/src/client/impl/ydb_internal/db_driver_state/state.cpp index 8114b756d4a..05affd18c0d 100644 --- a/src/client/impl/ydb_internal/db_driver_state/state.cpp +++ b/src/client/impl/ydb_internal/db_driver_state/state.cpp @@ -92,7 +92,7 @@ void TDbDriverState::SignalDiscoveryCompleted() { TPeriodicCb CreatePeriodicDiscoveryTask(TDbDriverState::TPtr driverState) { auto weak = std::weak_ptr(driverState); - return [weak](NYql::TIssues&&, EStatus status) { + return [weak](NYdb::NIssue::TIssues&&, EStatus status) { if (status != EStatus::SUCCESS) { return false; } diff --git a/src/client/impl/ydb_internal/grpc_connections/actions.cpp b/src/client/impl/ydb_internal/grpc_connections/actions.cpp index 65a152240f2..0be7ef9fa95 100644 --- a/src/client/impl/ydb_internal/grpc_connections/actions.cpp +++ b/src/client/impl/ydb_internal/grpc_connections/actions.cpp @@ -86,7 +86,7 @@ TPeriodicAction::TPeriodicAction( } void TPeriodicAction::OnAlarm() { - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; if (!UserResponseCb_(std::move(issues), EStatus::SUCCESS)) { return; } @@ -105,8 +105,8 @@ void TPeriodicAction::OnAlarm() { } void TPeriodicAction::OnError() { - NYql::TIssues issues; - issues.AddIssue(NYql::TIssue("Deferred timer interrupted")); + NYdb::NIssue::TIssues issues; + issues.AddIssue(NYdb::NIssue::TIssue("Deferred timer interrupted")); UserResponseCb_(std::move(issues), EStatus::CLIENT_INTERNAL_ERROR); } diff --git a/src/client/impl/ydb_internal/grpc_connections/actions.h b/src/client/impl/ydb_internal/grpc_connections/actions.h index 029035068af..948c4bac2fa 100644 --- a/src/client/impl/ydb_internal/grpc_connections/actions.h +++ b/src/client/impl/ydb_internal/grpc_connections/actions.h @@ -130,7 +130,7 @@ class TGRpcErrorResponse if (!Endpoint_.empty()) { std::string msg = "Grpc error response on endpoint "; msg += Endpoint_; - status.Issues.AddIssue(NYql::TIssue(msg)); + status.Issues.AddIssue(NYdb::NIssue::TIssue(msg)); } this->UserResponseCb_(nullptr, status); diff --git a/src/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp b/src/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp index d6705e09887..3ecf2d3fb43 100644 --- a/src/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp +++ b/src/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp @@ -162,7 +162,7 @@ TGRpcConnectionsImpl::TGRpcConnectionsImpl(std::shared_ptr p #ifndef YDB_GRPC_BYPASS_CHANNEL_POOL if (params->GetSocketIdleTimeout() != TDuration::Max()) { auto channelPoolUpdateWrapper = [this] - (NYql::TIssues&&, EStatus status) mutable + (NYdb::NIssue::TIssues&&, EStatus status) mutable { if (status != EStatus::SUCCESS) { return false; @@ -195,7 +195,7 @@ TGRpcConnectionsImpl::~TGRpcConnectionsImpl() { void TGRpcConnectionsImpl::AddPeriodicTask(TPeriodicCb&& cb, TDuration period) { std::shared_ptr context; if (!TryCreateContext(context)) { - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; cb(std::move(issues), EStatus::CLIENT_INTERNAL_ERROR); } else { auto action = MakeIntrusive( @@ -208,7 +208,7 @@ void TGRpcConnectionsImpl::AddPeriodicTask(TPeriodicCb&& cb, TDuration period) { } void TGRpcConnectionsImpl::ScheduleOneTimeTask(TSimpleCb&& fn, TDuration timeout) { - auto cbLow = [this, fn = std::move(fn)](NYql::TIssues&&, EStatus status) mutable { + auto cbLow = [this, fn = std::move(fn)](NYdb::NIssue::TIssues&&, EStatus status) mutable { if (status != EStatus::SUCCESS) { return false; } @@ -233,7 +233,7 @@ void TGRpcConnectionsImpl::ScheduleOneTimeTask(TSimpleCb&& fn, TDuration timeout if (timeout) { AddPeriodicTask(std::move(cbLow), timeout); } else { - cbLow(NYql::TIssues(), EStatus::SUCCESS); + cbLow(NYdb::NIssue::TIssues(), EStatus::SUCCESS); } } diff --git a/src/client/impl/ydb_internal/grpc_connections/grpc_connections.h b/src/client/impl/ydb_internal/grpc_connections/grpc_connections.h index a0c33090c51..30ba3e1d655 100644 --- a/src/client/impl/ydb_internal/grpc_connections/grpc_connections.h +++ b/src/client/impl/ydb_internal/grpc_connections/grpc_connections.h @@ -14,7 +14,7 @@ #include #include -#include +#include namespace NYdb { @@ -303,8 +303,8 @@ class TGRpcConnectionsImpl action->Start(); } else { - NYql::TIssues opIssues; - NYql::IssuesFromMessage(operation->issues(), opIssues); + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(operation->issues(), opIssues); userResponseCb(operation, TPlainStatus{static_cast(operation->status()), std::move(opIssues), status.Endpoint, std::move(status.Metadata)}); } @@ -628,7 +628,7 @@ class TGRpcConnectionsImpl discoveryStatus = TPlainStatus(EStatus::UNAVAILABLE, errString.Str()); } else { errString <<"."; - discoveryStatus.Issues.AddIssues({NYql::TIssue(errString.Str())}); + discoveryStatus.Issues.AddIssues({NYdb::NIssue::TIssue(errString.Str())}); } dbState->StatCollector.IncReqFailNoEndpoint(); callback( diff --git a/src/client/impl/ydb_internal/plain_status/status.cpp b/src/client/impl/ydb_internal/plain_status/status.cpp index effbd99b832..757321b27c5 100644 --- a/src/client/impl/ydb_internal/plain_status/status.cpp +++ b/src/client/impl/ydb_internal/plain_status/status.cpp @@ -54,7 +54,7 @@ TPlainStatus::TPlainStatus( Status = EStatus::SUCCESS; } if (!msg.empty()) { - Issues.AddIssue(NYql::TIssue(msg)); + Issues.AddIssue(NYdb::NIssue::TIssue(msg)); } for (const auto& [name, value] : grpcStatus.ServerTrailingMetadata) { Metadata.emplace( diff --git a/src/client/impl/ydb_internal/plain_status/status.h b/src/client/impl/ydb_internal/plain_status/status.h index 295fea794ae..768231d012a 100644 --- a/src/client/impl/ydb_internal/plain_status/status.h +++ b/src/client/impl/ydb_internal/plain_status/status.h @@ -5,7 +5,7 @@ #include #include -#include +#include #include @@ -15,7 +15,7 @@ namespace NYdb { struct TPlainStatus { EStatus Status; - NYql::TIssues Issues; + NYdb::NIssue::TIssues Issues; std::string Endpoint; std::multimap Metadata; Ydb::CostInfo ConstInfo; @@ -24,12 +24,12 @@ struct TPlainStatus { : Status(EStatus::SUCCESS) { } - TPlainStatus(EStatus status, NYql::TIssues&& issues) + TPlainStatus(EStatus status, NYdb::NIssue::TIssues&& issues) : Status(status) , Issues(std::move(issues)) { } - TPlainStatus(EStatus status, NYql::TIssues&& issues, const std::string& endpoint, + TPlainStatus(EStatus status, NYdb::NIssue::TIssues&& issues, const std::string& endpoint, std::multimap&& metadata) : Status(status) , Issues(std::move(issues)) @@ -41,7 +41,7 @@ struct TPlainStatus { : Status(status) { if (!message.empty()) { - Issues.AddIssue(NYql::TIssue(message)); + Issues.AddIssue(NYdb::NIssue::TIssue(message)); } } diff --git a/src/client/impl/ydb_internal/session_pool/CMakeLists.txt b/src/client/impl/ydb_internal/session_pool/CMakeLists.txt index de74a59ac5e..91577a39695 100644 --- a/src/client/impl/ydb_internal/session_pool/CMakeLists.txt +++ b/src/client/impl/ydb_internal/session_pool/CMakeLists.txt @@ -5,8 +5,7 @@ target_link_libraries(impl-ydb_internal-session_pool PUBLIC threading-future api-protos client-impl-ydb_endpoints - client-ydb_types-operation - yql-public-issue-protos + client-ydb_types-operation ) target_sources(impl-ydb_internal-session_pool PRIVATE diff --git a/src/client/impl/ydb_internal/session_pool/session_pool.cpp b/src/client/impl/ydb_internal/session_pool/session_pool.cpp index dabb9c12e9a..810e05d49aa 100644 --- a/src/client/impl/ydb_internal/session_pool/session_pool.cpp +++ b/src/client/impl/ydb_internal/session_pool/session_pool.cpp @@ -253,7 +253,7 @@ void TSessionPool::Drain(std::function&& TPeriodicCb TSessionPool::CreatePeriodicTask(std::weak_ptr weakClient, TKeepAliveCmd&& cmd, TDeletePredicate&& deletePredicate) { - auto periodicCb = [this, weakClient, cmd=std::move(cmd), deletePredicate=std::move(deletePredicate)](NYql::TIssues&&, EStatus status) { + auto periodicCb = [this, weakClient, cmd=std::move(cmd), deletePredicate=std::move(deletePredicate)](NYdb::NIssue::TIssues&&, EStatus status) { if (status != EStatus::SUCCESS) { return false; } diff --git a/src/client/operation/operation.cpp b/src/client/operation/operation.cpp index 95907b967b8..0c815473fe1 100644 --- a/src/client/operation/operation.cpp +++ b/src/client/operation/operation.cpp @@ -35,8 +35,8 @@ class TOperationClient::TImpl : public TClientImplCommonissues(), opIssues); + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(response->issues(), opIssues); TStatus st(static_cast(response->status()), std::move(opIssues)); promise.SetValue(std::move(st)); } else { @@ -88,15 +88,15 @@ class TOperationClient::TImpl : public TClientImplCommonissues(), opIssues); + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(response->issues(), opIssues); TStatus st(static_cast(response->status()), std::move(opIssues)); std::vector operations; operations.reserve(response->operations_size()); for (auto& operation : *response->mutable_operations()) { - NYql::TIssues opIssues; - NYql::IssuesFromMessage(operation.issues(), opIssues); + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(operation.issues(), opIssues); operations.emplace_back( TStatus(static_cast(operation.status()), std::move(opIssues)), std::move(operation)); diff --git a/src/client/persqueue_public/impl/read_session.cpp b/src/client/persqueue_public/impl/read_session.cpp index dd45ffcc1f0..45bda6e817e 100644 --- a/src/client/persqueue_public/impl/read_session.cpp +++ b/src/client/persqueue_public/impl/read_session.cpp @@ -53,7 +53,7 @@ TReadSession::~TReadSession() { { TDeferredActions deferred; - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; issues.AddIssue("Aborted"); EventsQueue->Close(TSessionClosedEvent(EStatus::ABORTED, std::move(issues)), deferred); } @@ -95,7 +95,7 @@ void TReadSession::Start() { } bool TReadSession::ValidateSettings() { - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; if (Settings.Topics_.empty()) { issues.AddIssue("Empty topics list."); } @@ -264,7 +264,7 @@ void TReadSession::OnClusterDiscovery(const TStatus& status, const Ydb::PersQueu } } - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; EStatus errorStatus = EStatus::INTERNAL_ERROR; for (size_t topicIndex = 0; topicIndex < Settings.Topics_.size(); ++topicIndex) { const TTopicReadSettings& topicSettings = Settings.Topics_[topicIndex]; @@ -408,7 +408,7 @@ bool TReadSession::Close(TDuration timeout) { if (result) { Cancel(timeoutContext); - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; issues.AddIssue("Session was gracefully closed"); EventsQueue->Close(TSessionClosedEvent(EStatus::SUCCESS, std::move(issues)), deferred); } else { @@ -417,7 +417,7 @@ bool TReadSession::Close(TDuration timeout) { session->Abort(); } - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; issues.AddIssue(TStringBuilder() << "Session was closed after waiting " << timeout); EventsQueue->Close(TSessionClosedEvent(EStatus::TIMEOUT, std::move(issues)), deferred); } @@ -450,7 +450,7 @@ void TReadSession::AbortImpl(TDeferredActions&) { } } -void TReadSession::AbortImpl(EStatus statusCode, NYql::TIssues&& issues, TDeferredActions& deferred) { +void TReadSession::AbortImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues, TDeferredActions& deferred) { Y_ABORT_UNLESS(Lock.IsLocked()); auto closeEvent = TSessionClosedEvent(statusCode, std::move(issues)); LOG_LAZY(Log, TLOG_NOTICE, GetLogPrefix() << "Aborting read session. Description: " << closeEvent.DebugString()); @@ -462,7 +462,7 @@ void TReadSession::AbortImpl(EStatus statusCode, NYql::TIssues&& issues, TDeferr void TReadSession::AbortImpl(EStatus statusCode, const std::string& message, TDeferredActions& deferred) { Y_ABORT_UNLESS(Lock.IsLocked()); - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; issues.AddIssue(message); AbortImpl(statusCode, std::move(issues), deferred); diff --git a/src/client/persqueue_public/impl/read_session.h b/src/client/persqueue_public/impl/read_session.h index de1d6d19d0b..1903995f76b 100644 --- a/src/client/persqueue_public/impl/read_session.h +++ b/src/client/persqueue_public/impl/read_session.h @@ -94,7 +94,7 @@ class TReadSession : public IReadSession, void AbortImpl(TDeferredActions& deferred); void AbortImpl(TSessionClosedEvent&& closeEvent, TDeferredActions& deferred); - void AbortImpl(EStatus statusCode, NYql::TIssues&& issues, TDeferredActions& deferred); + void AbortImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues, TDeferredActions& deferred); void AbortImpl(EStatus statusCode, const std::string& message, TDeferredActions& deferred); void MakeCountersIfNeeded(); diff --git a/src/client/persqueue_public/impl/write_session_impl.cpp b/src/client/persqueue_public/impl/write_session_impl.cpp index 67d5e95143c..4b9dd0eec00 100644 --- a/src/client/persqueue_public/impl/write_session_impl.cpp +++ b/src/client/persqueue_public/impl/write_session_impl.cpp @@ -196,7 +196,7 @@ void TWriteSessionImpl::OnCdsResponse( return; } - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; EStatus errorStatus = EStatus::INTERNAL_ERROR; { std::lock_guard guard(Lock); @@ -727,7 +727,7 @@ TWriteSessionImpl::TProcessSrvMessageResult TWriteSessionImpl::ProcessServerMess SessionEstablished = false; result.HandleResult = OnErrorImpl({ static_cast(ServerMessage->status()), - {NYql::TIssue{ServerMessage->DebugString()}} + {NYdb::NIssue::TIssue{ServerMessage->DebugString()}} }); result.Ok = false; break; @@ -1258,7 +1258,7 @@ bool TWriteSessionImpl::Close(TDuration closeTimeout) { } { std::lock_guard guard(Lock); - CloseImpl(EStatus::SUCCESS, NYql::TIssues{}); + CloseImpl(EStatus::SUCCESS, NYdb::NIssue::TIssues{}); needSetSeqNoValue = !InitSeqNoSetDone && (InitSeqNoSetDone = true); } if (needSetSeqNoValue) { @@ -1362,7 +1362,7 @@ void TWriteSessionImpl::AbortImpl() { } } -void TWriteSessionImpl::CloseImpl(EStatus statusCode, NYql::TIssues&& issues) { +void TWriteSessionImpl::CloseImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues) { Y_ABORT_UNLESS(Lock.IsLocked()); LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Write session will now close"); @@ -1373,7 +1373,7 @@ void TWriteSessionImpl::CloseImpl(EStatus statusCode, NYql::TIssues&& issues) { void TWriteSessionImpl::CloseImpl(EStatus statusCode, const std::string& message) { Y_ABORT_UNLESS(Lock.IsLocked()); - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; issues.AddIssue(message); CloseImpl(statusCode, std::move(issues)); } @@ -1392,7 +1392,7 @@ TWriteSessionImpl::~TWriteSessionImpl() { { std::lock_guard guard(Lock); if (!Aborting.load()) { - CloseImpl(EStatus::SUCCESS, NYql::TIssues{}); + CloseImpl(EStatus::SUCCESS, NYdb::NIssue::TIssues{}); needClose = !InitSeqNoSetDone && (InitSeqNoSetDone = true); } diff --git a/src/client/persqueue_public/impl/write_session_impl.h b/src/client/persqueue_public/impl/write_session_impl.h index f527aee6826..82a8bbdb7ac 100644 --- a/src/client/persqueue_public/impl/write_session_impl.h +++ b/src/client/persqueue_public/impl/write_session_impl.h @@ -364,7 +364,7 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, ui64 GetIdImpl(ui64 seqNo); void SendImpl(); void AbortImpl(); - void CloseImpl(EStatus statusCode, NYql::TIssues&& issues); + void CloseImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues); void CloseImpl(EStatus statusCode, const std::string& message); void CloseImpl(TPlainStatus&& status); diff --git a/src/client/proto/CMakeLists.txt b/src/client/proto/CMakeLists.txt index e240c9962a6..cd2b8590e6b 100644 --- a/src/client/proto/CMakeLists.txt +++ b/src/client/proto/CMakeLists.txt @@ -8,7 +8,6 @@ target_link_libraries(client-ydb_proto PUBLIC library-operation_id client-ydb_params client-ydb_value - yql-public-issue-protos ) target_sources(client-ydb_proto PRIVATE diff --git a/src/client/query/client.cpp b/src/client/query/client.cpp index 422b40bb4a7..887b3242011 100644 --- a/src/client/query/client.cpp +++ b/src/client/query/client.cpp @@ -105,8 +105,8 @@ class TQueryClient::TImpl: public TClientImplCommon, public (Ydb::Operations::Operation* response, TPlainStatus status) mutable { try { if (response) { - NYql::TIssues opIssues; - NYql::IssuesFromMessage(response->issues(), opIssues); + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(response->issues(), opIssues); TStatus executeScriptStatus(TPlainStatus{static_cast(response->status()), std::move(opIssues), status.Endpoint, std::move(status.Metadata)}); promise.SetValue(TScriptExecutionOperation(TStatus(std::move(executeScriptStatus)), std::move(*response))); @@ -147,8 +147,8 @@ class TQueryClient::TImpl: public TClientImplCommon, public (Ydb::Query::RollbackTransactionResponse* response, TPlainStatus status) mutable { try { if (response) { - NYql::TIssues opIssues; - NYql::IssuesFromMessage(response->issues(), opIssues); + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(response->issues(), opIssues); TStatus rollbackTxStatus(TPlainStatus{static_cast(response->status()), std::move(opIssues), status.Endpoint, std::move(status.Metadata)}); @@ -183,8 +183,8 @@ class TQueryClient::TImpl: public TClientImplCommon, public (Ydb::Query::CommitTransactionResponse* response, TPlainStatus status) mutable { try { if (response) { - NYql::TIssues opIssues; - NYql::IssuesFromMessage(response->issues(), opIssues); + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(response->issues(), opIssues); TStatus commitTxStatus(TPlainStatus{static_cast(response->status()), std::move(opIssues), status.Endpoint, std::move(status.Metadata)}); @@ -222,8 +222,8 @@ class TQueryClient::TImpl: public TClientImplCommon, public (Ydb::Query::BeginTransactionResponse* response, TPlainStatus status) mutable { try { if (response) { - NYql::TIssues opIssues; - NYql::IssuesFromMessage(response->issues(), opIssues); + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(response->issues(), opIssues); TStatus beginTxStatus(TPlainStatus{static_cast(response->status()), std::move(opIssues), status.Endpoint, std::move(status.Metadata)}); @@ -261,8 +261,8 @@ class TQueryClient::TImpl: public TClientImplCommon, public auto extractor = [promise] (FetchScriptResultsResponse* response, TPlainStatus status) mutable { if (response) { - NYql::TIssues opIssues; - NYql::IssuesFromMessage(response->issues(), opIssues); + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(response->issues(), opIssues); TStatus st(static_cast(response->status()), std::move(opIssues)); if (st.IsSuccess()) { @@ -378,8 +378,8 @@ class TQueryClient::TImpl: public TClientImplCommon, public auto extractor = [promise, self] (Ydb::Query::CreateSessionResponse* resp, TPlainStatus status) mutable { if (resp) { if (resp->status() != Ydb::StatusIds::SUCCESS) { - NYql::TIssues opIssues; - NYql::IssuesFromMessage(resp->issues(), opIssues); + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(resp->issues(), opIssues); TStatus st(static_cast(resp->status()), std::move(opIssues)); promise.SetValue(TCreateSessionResult(std::move(st), TSession())); } else { diff --git a/src/client/query/impl/client_session.cpp b/src/client/query/impl/client_session.cpp index 4d37337dfd8..40b5ab7c97c 100644 --- a/src/client/query/impl/client_session.cpp +++ b/src/client/query/impl/client_session.cpp @@ -4,7 +4,7 @@ #include #undef INCLUDE_YDB_INTERNAL_H -#include +#include namespace NYdb::NQuery { @@ -36,8 +36,8 @@ void TSession::TImpl::MakeImplAsync(TStreamProcessorPtr ptr, TSession::TImpl::NewSmartShared(ptr, std::move(args), st); } else { - NYql::TIssues opIssues; - NYql::IssuesFromMessage(resp->issues(), opIssues); + NYdb::NIssue::TIssues opIssues; + NYdb::NIssue::IssuesFromMessage(resp->issues(), opIssues); TStatus st(static_cast(resp->status()), std::move(opIssues)); args->Promise.SetValue(TCreateSessionResult(std::move(st), TSession())); } diff --git a/src/client/query/impl/exec_query.cpp b/src/client/query/impl/exec_query.cpp index 3dd9a896c01..6e8541a927c 100644 --- a/src/client/query/impl/exec_query.cpp +++ b/src/client/query/impl/exec_query.cpp @@ -69,8 +69,8 @@ class TExecuteQueryIterator::TReaderImpl { self->Finished_ = true; promise.SetValue({TStatus(TPlainStatus(grpcStatus, self->Endpoint_)), {}, {}}); } else { - NYql::TIssues issues; - NYql::IssuesFromMessage(self->Response_.issues(), issues); + NYdb::NIssue::TIssues issues; + NYdb::NIssue::IssuesFromMessage(self->Response_.issues(), issues); EStatus clientStatus = static_cast(self->Response_.status()); TPlainStatus plainStatus{clientStatus, std::move(issues), self->Endpoint_, {}}; TStatus status{std::move(plainStatus)}; @@ -141,7 +141,7 @@ struct TExecuteQueryBuffer : public TThrRefBase, TNonCopyable { TPromise Promise_; TExecuteQueryIterator Iterator_; - std::vector Issues_; + std::vector Issues_; std::vector ResultSets_; std::optional Stats_; std::optional Tx_; @@ -161,7 +161,7 @@ struct TExecuteQueryBuffer : public TThrRefBase, TNonCopyable { std::swap(self->Stats_, stats); if (part.EOS()) { - std::vector issues; + std::vector issues; std::vector resultProtos; std::optional tx; @@ -175,7 +175,7 @@ struct TExecuteQueryBuffer : public TThrRefBase, TNonCopyable { } self->Promise_.SetValue(TExecuteQueryResult( - TStatus(EStatus::SUCCESS, NYql::TIssues(std::move(issues))), + TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues(std::move(issues))), std::move(resultSets), std::move(stats), std::move(tx) diff --git a/src/client/table/impl/CMakeLists.txt b/src/client/table/impl/CMakeLists.txt index b93290b0461..5bbeecf5243 100644 --- a/src/client/table/impl/CMakeLists.txt +++ b/src/client/table/impl/CMakeLists.txt @@ -10,7 +10,6 @@ target_link_libraries(client-ydb_table-impl client-impl-ydb_endpoints impl-ydb_internal-session_pool client-ydb_table-query_stats - yql-public-issue-protos PRIVATE OpenSSL::SSL ) diff --git a/src/client/table/impl/readers.cpp b/src/client/table/impl/readers.cpp index 04ff0216f43..505e1f2cb0c 100644 --- a/src/client/table/impl/readers.cpp +++ b/src/client/table/impl/readers.cpp @@ -39,8 +39,8 @@ TAsyncSimpleStreamPart TTablePartIterator::TReaderImpl::ReadNext(std TStatus(TPlainStatus(grpcStatus, self->Endpoint_)), snapshot}); } else { - NYql::TIssues issues; - NYql::IssuesFromMessage(self->Response_.issues(), issues); + NYdb::NIssue::TIssues issues; + NYdb::NIssue::IssuesFromMessage(self->Response_.issues(), issues); EStatus clientStatus = static_cast(self->Response_.status()); promise.SetValue({TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())), TStatus(clientStatus, std::move(issues)), @@ -75,8 +75,8 @@ TAsyncScanQueryPart TScanQueryPartIterator::TReaderImpl::ReadNext(std::shared_pt self->Finished_ = true; promise.SetValue({TStatus(TPlainStatus(grpcStatus, self->Endpoint_))}); } else { - NYql::TIssues issues; - NYql::IssuesFromMessage(self->Response_.issues(), issues); + NYdb::NIssue::TIssues issues; + NYdb::NIssue::IssuesFromMessage(self->Response_.issues(), issues); EStatus clientStatus = static_cast(self->Response_.status()); // TODO: Add headers for streaming calls. TPlainStatus plainStatus{clientStatus, std::move(issues), self->Endpoint_, {}}; diff --git a/src/client/table/impl/table_client.cpp b/src/client/table/impl/table_client.cpp index c75479503d4..a60f7e99cac 100644 --- a/src/client/table/impl/table_client.cpp +++ b/src/client/table/impl/table_client.cpp @@ -214,7 +214,7 @@ void TTableClient::TImpl::StartPeriodicHostScanTask() { // and ready to accept new one std::pair winner = {0, 0}; - auto periodicCb = [weak, winner](NYql::TIssues&&, EStatus status) mutable -> bool { + auto periodicCb = [weak, winner](NYdb::NIssue::TIssues&&, EStatus status) mutable -> bool { if (status != EStatus::SUCCESS) { return false; @@ -831,8 +831,8 @@ TAsyncReadRowsResult TTableClient::TImpl::ReadRows(const std::string& path, TVal // if there is no response status contains transport errors if (response) { Ydb::StatusIds::StatusCode msgStatus = response->status(); - NYql::TIssues issues; - NYql::IssuesFromMessage(response->issues(), issues); + NYdb::NIssue::TIssues issues; + NYdb::NIssue::IssuesFromMessage(response->issues(), issues); status = TPlainStatus(static_cast(msgStatus), std::move(issues)); resultSet = std::move(*response->mutable_result_set()); } diff --git a/src/client/table/table.cpp b/src/client/table/table.cpp index 3151b9fac12..2ff8ae897c8 100644 --- a/src/client/table/table.cpp +++ b/src/client/table/table.cpp @@ -1890,7 +1890,7 @@ TAsyncDataQueryResult TSession::ExecuteDataQuery(const std::string& query, const TAsyncPrepareQueryResult TSession::PrepareDataQuery(const std::string& query, const TPrepareDataQuerySettings& settings) { auto maybeQuery = SessionImpl_->GetQueryFromCache(query, Client_->Settings_.AllowRequestMigration_); if (maybeQuery) { - TStatus status(EStatus::SUCCESS, NYql::TIssues()); + TStatus status(EStatus::SUCCESS, NYdb::NIssue::TIssues()); TDataQuery dataQuery(*this, query, maybeQuery->QueryId, maybeQuery->ParameterTypes); TPrepareQueryResult result(std::move(status), dataQuery, true); return MakeFuture(result); diff --git a/src/client/topic/codecs/CMakeLists.txt b/src/client/topic/codecs/CMakeLists.txt index c597804dd0b..e8069c8debd 100644 --- a/src/client/topic/codecs/CMakeLists.txt +++ b/src/client/topic/codecs/CMakeLists.txt @@ -3,7 +3,6 @@ _ydb_sdk_add_library(client-ydb_topic-codecs) target_link_libraries(client-ydb_topic-codecs PUBLIC yutil streams-zstd - yql-public-issue-protos api-grpc-draft api-grpc api-protos diff --git a/src/client/topic/impl/common.cpp b/src/client/topic/impl/common.cpp index 58e3f406b46..160682edfac 100644 --- a/src/client/topic/impl/common.cpp +++ b/src/client/topic/impl/common.cpp @@ -55,7 +55,7 @@ ERetryErrorClass GetRetryErrorClassV2(EStatus status) { } } -std::string IssuesSingleLineString(const NYql::TIssues& issues) { +std::string IssuesSingleLineString(const NYdb::NIssue::TIssues& issues) { return SubstGlobalCopy(issues.ToString(), '\n', ' '); } @@ -65,11 +65,11 @@ void Cancel(NYdbGrpc::IQueueClientContextPtr& context) { } } -NYql::TIssues MakeIssueWithSubIssues(const std::string& description, const NYql::TIssues& subissues) { - NYql::TIssues issues; - NYql::TIssue issue(description); - for (const NYql::TIssue& i : subissues) { - issue.AddSubIssue(MakeIntrusive(i)); +NYdb::NIssue::TIssues MakeIssueWithSubIssues(const std::string& description, const NYdb::NIssue::TIssues& subissues) { + NYdb::NIssue::TIssues issues; + NYdb::NIssue::TIssue issue(description); + for (const auto& i : subissues) { + issue.AddSubIssue(MakeIntrusive(i)); } issues.AddIssue(std::move(issue)); return issues; diff --git a/src/client/topic/impl/common.h b/src/client/topic/impl/common.h index 88c7f858f64..cca058c759e 100644 --- a/src/client/topic/impl/common.h +++ b/src/client/topic/impl/common.h @@ -17,9 +17,9 @@ ERetryErrorClass GetRetryErrorClassV2(EStatus status); void Cancel(NYdbGrpc::IQueueClientContextPtr& context); -NYql::TIssues MakeIssueWithSubIssues(const std::string& description, const NYql::TIssues& subissues); +NYdb::NIssue::TIssues MakeIssueWithSubIssues(const std::string& description, const NYdb::NIssue::TIssues& subissues); -std::string IssuesSingleLineString(const NYql::TIssues& issues); +std::string IssuesSingleLineString(const NYdb::NIssue::TIssues& issues); template size_t CalcDataSize(const typename TEvent::TEvent& event) { @@ -61,8 +61,8 @@ bool IsErrorMessage(const TMessage& serverMessage) { template TPlainStatus MakeErrorFromProto(const TMessage& serverMessage) { - NYql::TIssues issues; - NYql::IssuesFromMessage(serverMessage.issues(), issues); + NYdb::NIssue::TIssues issues; + NYdb::NIssue::IssuesFromMessage(serverMessage.issues(), issues); return TPlainStatus(static_cast(serverMessage.status()), std::move(issues)); } diff --git a/src/client/topic/impl/read_session.cpp b/src/client/topic/impl/read_session.cpp index 539d235c138..0c80f72ca6f 100644 --- a/src/client/topic/impl/read_session.cpp +++ b/src/client/topic/impl/read_session.cpp @@ -101,7 +101,7 @@ void TReadSession::CreateClusterSessionsImpl(TDeferredActions& deferred) } bool TReadSession::ValidateSettings() { - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; if (Settings.Topics_.empty()) { issues.AddIssue("Empty topics list."); } @@ -225,14 +225,14 @@ bool TReadSession::Close(TDuration timeout) { if (result) { Cancel(timeoutContext); - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; issues.AddIssue("Session was gracefully closed"); EventsQueue->Close(TSessionClosedEvent(EStatus::SUCCESS, std::move(issues)), deferred); } else { ++*Settings.Counters_->Errors; session->Abort(); - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; issues.AddIssue(TStringBuilder() << "Session was closed after waiting " << timeout); EventsQueue->Close(TSessionClosedEvent(EStatus::TIMEOUT, std::move(issues)), deferred); } @@ -293,7 +293,7 @@ void TReadSession::AbortImpl(TSessionClosedEvent&& closeEvent, TDeferredActions< AbortImpl(deferred); } -void TReadSession::AbortImpl(EStatus statusCode, NYql::TIssues&& issues, TDeferredActions& deferred) { +void TReadSession::AbortImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues, TDeferredActions& deferred) { Y_ABORT_UNLESS(Lock.IsLocked()); AbortImpl(TSessionClosedEvent(statusCode, std::move(issues)), deferred); @@ -302,17 +302,17 @@ void TReadSession::AbortImpl(EStatus statusCode, NYql::TIssues&& issues, TDeferr void TReadSession::AbortImpl(EStatus statusCode, const std::string& message, TDeferredActions& deferred) { Y_ABORT_UNLESS(Lock.IsLocked()); - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; issues.AddIssue(message); AbortImpl(statusCode, std::move(issues), deferred); } -void TReadSession::Abort(EStatus statusCode, NYql::TIssues&& issues) { +void TReadSession::Abort(EStatus statusCode, NYdb::NIssue::TIssues&& issues) { Abort(TSessionClosedEvent(statusCode, std::move(issues))); } void TReadSession::Abort(EStatus statusCode, const std::string& message) { - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; issues.AddIssue(message); Abort(statusCode, std::move(issues)); } diff --git a/src/client/topic/impl/read_session.h b/src/client/topic/impl/read_session.h index 8c314ed0854..54d044127f7 100644 --- a/src/client/topic/impl/read_session.h +++ b/src/client/topic/impl/read_session.h @@ -54,12 +54,12 @@ class TReadSession : public IReadSession { void SetupCountersLogger(); // Shutdown. - void Abort(EStatus statusCode, NYql::TIssues&& issues); + void Abort(EStatus statusCode, NYdb::NIssue::TIssues&& issues); void Abort(EStatus statusCode, const std::string& message); void AbortImpl(TDeferredActions& deferred); void AbortImpl(TSessionClosedEvent&& closeEvent, TDeferredActions& deferred); - void AbortImpl(EStatus statusCode, NYql::TIssues&& issues, TDeferredActions& deferred); + void AbortImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues, TDeferredActions& deferred); void AbortImpl(EStatus statusCode, const std::string& message, TDeferredActions& deferred); private: diff --git a/src/client/topic/impl/read_session_impl.h b/src/client/topic/impl/read_session_impl.h index c9e0a3500a1..40e1b4535f6 100644 --- a/src/client/topic/impl/read_session_impl.h +++ b/src/client/topic/impl/read_session_impl.h @@ -147,7 +147,7 @@ class TDeferredActions { void DeferReadFromProcessor(const typename IProcessor::TPtr& processor, TServerMessage* dst, typename IProcessor::TReadCallback callback); void DeferStartExecutorTask(const typename IAExecutor::TPtr& executor, typename IAExecutor::TFunction&& task); void DeferAbortSession(TCallbackContextPtr cbContext, TASessionClosedEvent&& closeEvent); - void DeferAbortSession(TCallbackContextPtr cbContext, EStatus statusCode, NYql::TIssues&& issues); + void DeferAbortSession(TCallbackContextPtr cbContext, EStatus statusCode, NYdb::NIssue::TIssues&& issues); void DeferAbortSession(TCallbackContextPtr cbContext, EStatus statusCode, const std::string& message); void DeferAbortSession(TCallbackContextPtr cbContext, TPlainStatus&& status); void DeferReconnection(TCallbackContextPtr cbContext, TPlainStatus&& status); @@ -1124,12 +1124,12 @@ class TSingleClusterReadSessionImpl : public TEnableSelfContext callback); void AbortSession(TASessionClosedEvent&& closeEvent); - void AbortSession(EStatus statusCode, NYql::TIssues&& issues) { + void AbortSession(EStatus statusCode, NYdb::NIssue::TIssues&& issues) { AbortSession(TASessionClosedEvent(statusCode, std::move(issues))); } void AbortSession(EStatus statusCode, const std::string& message) { - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; issues.AddIssue(message); AbortSession(statusCode, std::move(issues)); } @@ -1172,7 +1172,7 @@ class TSingleClusterReadSessionImpl : public TEnableSelfContext& deferred); - void BreakConnectionAndReconnectImpl(EStatus statusCode, NYql::TIssues&& issues, TDeferredActions& deferred) { + void BreakConnectionAndReconnectImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues, TDeferredActions& deferred) { BreakConnectionAndReconnectImpl(TPlainStatus(statusCode, std::move(issues)), deferred); } diff --git a/src/client/topic/impl/read_session_impl.ipp b/src/client/topic/impl/read_session_impl.ipp index a247cce4ac6..a64cba5866d 100644 --- a/src/client/topic/impl/read_session_impl.ipp +++ b/src/client/topic/impl/read_session_impl.ipp @@ -2910,13 +2910,13 @@ void TDeferredActions::DeferAbortSession(TCallbackContextP } template -void TDeferredActions::DeferAbortSession(TCallbackContextPtr cbContext, EStatus statusCode, NYql::TIssues&& issues) { +void TDeferredActions::DeferAbortSession(TCallbackContextPtr cbContext, EStatus statusCode, NYdb::NIssue::TIssues&& issues) { DeferAbortSession(std::move(cbContext), TASessionClosedEvent(statusCode, std::move(issues))); } template void TDeferredActions::DeferAbortSession(TCallbackContextPtr cbContext, EStatus statusCode, const std::string& message) { - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; issues.AddIssue(message); DeferAbortSession(std::move(cbContext), statusCode, std::move(issues)); } diff --git a/src/client/topic/impl/transaction.cpp b/src/client/topic/impl/transaction.cpp index 8358bb93b5d..d912bc54fd6 100644 --- a/src/client/topic/impl/transaction.cpp +++ b/src/client/topic/impl/transaction.cpp @@ -8,7 +8,7 @@ TTransactionId MakeTransactionId(const NTable::TTransaction& tx) return {tx.GetSession().GetId(), tx.GetId()}; } -TStatus MakeStatus(EStatus code, NYql::TIssues&& issues) +TStatus MakeStatus(EStatus code, NYdb::NIssue::TIssues&& issues) { return {code, std::move(issues)}; } diff --git a/src/client/topic/impl/write_session_impl.cpp b/src/client/topic/impl/write_session_impl.cpp index cd57cc1d293..016d3c5860a 100644 --- a/src/client/topic/impl/write_session_impl.cpp +++ b/src/client/topic/impl/write_session_impl.cpp @@ -1035,7 +1035,7 @@ TWriteSessionImpl::TProcessSrvMessageResult TWriteSessionImpl::ProcessServerMess SessionEstablished = false; result.HandleResult = OnErrorImpl({ static_cast(ServerMessage->status()), - {NYql::TIssue{ServerMessage->DebugString()}} + {NYdb::NIssue::TIssue{ServerMessage->DebugString()}} }); result.Ok = false; break; @@ -1591,7 +1591,7 @@ bool TWriteSessionImpl::Close(TDuration closeTimeout) { } { std::lock_guard guard(Lock); - CloseImpl(EStatus::SUCCESS, NYql::TIssues{}); + CloseImpl(EStatus::SUCCESS, NYdb::NIssue::TIssues{}); needSetSeqNoValue = !InitSeqNoSetDone && (InitSeqNoSetDone = true); } if (needSetSeqNoValue) { @@ -1705,7 +1705,7 @@ void TWriteSessionImpl::CancelTransactions() Txs.clear(); } -void TWriteSessionImpl::CloseImpl(EStatus statusCode, NYql::TIssues&& issues) { +void TWriteSessionImpl::CloseImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues) { Y_ABORT_UNLESS(Lock.IsLocked()); LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Write session will now close"); @@ -1716,7 +1716,7 @@ void TWriteSessionImpl::CloseImpl(EStatus statusCode, NYql::TIssues&& issues) { void TWriteSessionImpl::CloseImpl(EStatus statusCode, const std::string& message) { Y_ABORT_UNLESS(Lock.IsLocked()); - NYql::TIssues issues; + NYdb::NIssue::TIssues issues; issues.AddIssue(message); CloseImpl(statusCode, std::move(issues)); } @@ -1735,7 +1735,7 @@ TWriteSessionImpl::~TWriteSessionImpl() { { std::lock_guard guard(Lock); if (!Aborting.load()) { - CloseImpl(EStatus::SUCCESS, NYql::TIssues{}); + CloseImpl(EStatus::SUCCESS, NYdb::NIssue::TIssues{}); needClose = !InitSeqNoSetDone && (InitSeqNoSetDone = true); } diff --git a/src/client/topic/impl/write_session_impl.h b/src/client/topic/impl/write_session_impl.h index 0e078b5246a..56d9692ba33 100644 --- a/src/client/topic/impl/write_session_impl.h +++ b/src/client/topic/impl/write_session_impl.h @@ -406,7 +406,7 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, uint64_t GetIdImpl(uint64_t seqNo); void SendImpl(); void AbortImpl(); - void CloseImpl(EStatus statusCode, NYql::TIssues&& issues); + void CloseImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues); void CloseImpl(EStatus statusCode, const std::string& message); void CloseImpl(TPlainStatus&& status); diff --git a/src/client/topic/ut/describe_topic_ut.cpp b/src/client/topic/ut/describe_topic_ut.cpp index a6d3ec0e1a1..4ab9c395855 100644 --- a/src/client/topic/ut/describe_topic_ut.cpp +++ b/src/client/topic/ut/describe_topic_ut.cpp @@ -379,7 +379,7 @@ namespace NYdb::NTopic::NTests { bool allowUpdateRow; bool allowDescribeSchema; EStatus status; - NYql::TIssueCode issueCode; + NYdb::NIssue::TIssueCode issueCode; }; std::vector expectations{ @@ -397,7 +397,7 @@ namespace NYdb::NTopic::NTests { auto result = RunPermissionTest(setup, userId++, existing, update, describe); auto resultStatus = result.GetStatus(); auto line = TStringBuilder() << "=== status=" << resultStatus; - NYql::TIssueCode resultIssue = 0; + NYdb::NIssue::TIssueCode resultIssue = 0; if (!result.GetIssues().Empty()) { resultIssue = result.GetIssues().begin()->GetCode(); line << " issueCode=" << resultIssue; diff --git a/src/client/types/core_facility/core_facility.h b/src/client/types/core_facility/core_facility.h index bb09a813602..33556b59821 100644 --- a/src/client/types/core_facility/core_facility.h +++ b/src/client/types/core_facility/core_facility.h @@ -4,7 +4,7 @@ #include namespace NYdb { -using TPeriodicCb = std::function; +using TPeriodicCb = std::function; // !!!Experimental!!! // Allows to communicate with sdk core diff --git a/src/client/types/credentials/login/login.cpp b/src/client/types/credentials/login/login.cpp index fae7b3a20ee..f6b7b3534ea 100644 --- a/src/client/types/credentials/login/login.cpp +++ b/src/client/types/credentials/login/login.cpp @@ -77,7 +77,7 @@ TLoginCredentialsProvider::TLoginCredentialsProvider(std::weak_ptr bool { + auto periodicTask = [facility, this](NYdb::NIssue::TIssues&&, EStatus status) -> bool { if (status != EStatus::SUCCESS) { return false; } diff --git a/src/client/types/status/status.cpp b/src/client/types/status/status.cpp index 99a2ad135fe..ee7db9149a0 100644 --- a/src/client/types/status/status.cpp +++ b/src/client/types/status/status.cpp @@ -29,7 +29,7 @@ class TStatus::TImpl { } }; -TStatus::TStatus(EStatus statusCode, NYql::TIssues&& issues) +TStatus::TStatus(EStatus statusCode, NYdb::NIssue::TIssues&& issues) : Impl_(std::make_shared(TPlainStatus{statusCode, std::move(issues)})) { } @@ -37,7 +37,7 @@ TStatus::TStatus(TPlainStatus&& plain) : Impl_(std::make_shared(std::move(plain))) { } -const NYql::TIssues& TStatus::GetIssues() const { +const NYdb::NIssue::TIssues& TStatus::GetIssues() const { return Impl_->Status.Issues; } diff --git a/src/client/value/value.cpp b/src/client/value/value.cpp index 5ab92492277..a54a9139692 100644 --- a/src/client/value/value.cpp +++ b/src/client/value/value.cpp @@ -12,7 +12,7 @@ #include -#include +#include #include #include @@ -958,7 +958,7 @@ TDecimalValue::TDecimalValue(const Ydb::Value& valueProto, const TDecimalType& d TDecimalValue::TDecimalValue(const std::string& decimalString, ui8 precision, ui8 scale) : DecimalType_(precision, scale) { - NYql::NDecimal::TInt128 val = NYql::NDecimal::FromString(decimalString, precision, scale); + NYdb::NDecimal::TInt128 val = NYdb::NDecimal::FromString(decimalString, precision, scale); static_assert(sizeof(val) == 16, "wrong TInt128 size"); char* buf = reinterpret_cast(&val); Low_ = *(uint64_t*)buf; @@ -966,8 +966,8 @@ TDecimalValue::TDecimalValue(const std::string& decimalString, ui8 precision, ui } std::string TDecimalValue::ToString() const { - NYql::NDecimal::TInt128 val = NYql::NDecimal::FromHalfs(Low_, Hi_); - return NYql::NDecimal::ToString(val, DecimalType_.Precision, DecimalType_.Scale); + NYdb::NDecimal::TInt128 val = NYdb::NDecimal::FromHalfs(Low_, Hi_); + return NYdb::NDecimal::ToString(val, DecimalType_.Precision, DecimalType_.Scale); } //////////////////////////////////////////////////////////////////////////////// diff --git a/src/library/CMakeLists.txt b/src/library/CMakeLists.txt index a1f8aeb5274..dbebd28275a 100644 --- a/src/library/CMakeLists.txt +++ b/src/library/CMakeLists.txt @@ -2,7 +2,9 @@ if (YDB_SDK_TESTS) add_subdirectory(benchmark) endif() +add_subdirectory(decimal) add_subdirectory(grpc/client) +add_subdirectory(issue) add_subdirectory(json_value) add_subdirectory(jwt) add_subdirectory(malloc/api) @@ -15,5 +17,4 @@ add_subdirectory(string_utils/base64) add_subdirectory(string_utils/helpers) add_subdirectory(string_utils/misc) add_subdirectory(uuid) -add_subdirectory(yql_common) add_subdirectory(yson_value) diff --git a/src/library/yql_common/decimal/CMakeLists.txt b/src/library/decimal/CMakeLists.txt similarity index 88% rename from src/library/yql_common/decimal/CMakeLists.txt rename to src/library/decimal/CMakeLists.txt index a403425ec3c..edbf8f0917b 100644 --- a/src/library/yql_common/decimal/CMakeLists.txt +++ b/src/library/decimal/CMakeLists.txt @@ -7,7 +7,6 @@ target_link_libraries(yql-public-decimal PUBLIC target_sources(yql-public-decimal PRIVATE yql_decimal.cpp - yql_decimal_serialize.cpp ) _ydb_sdk_install_targets(TARGETS yql-public-decimal) diff --git a/src/library/decimal/yql_decimal.cpp b/src/library/decimal/yql_decimal.cpp new file mode 100644 index 00000000000..0eb36911ff7 --- /dev/null +++ b/src/library/decimal/yql_decimal.cpp @@ -0,0 +1,241 @@ +#include "yql_decimal.h" + +#include +#include +#include + +namespace NYdb::NDecimal { + +static const TUint128 Ten(10U); + +TUint128 GetDivider(ui8 scale) { + TUint128 d(1U); + while (scale--) + d *= Ten; + return d; +} + +bool IsError(TInt128 v) { + return v > Nan() || v < -Inf(); +} + +bool IsNan(TInt128 v) { + return v == Nan(); +} + +bool IsInf(TInt128 v) { + return v == Inf() || v == -Inf(); +} + +bool IsNormal(TInt128 v) { + return v < Inf() && v > -Inf(); +} + +const char* ToString(TInt128 val, ui8 precision, ui8 scale) { + if (!precision || precision > MaxPrecision || scale > precision) { + return ""; + } + + if (val == Inf()) + return "inf"; + if (val == -Inf()) + return "-inf"; + if (val == Nan()) + return "nan"; + + if (!IsNormal(val)) { + return nullptr; + } + + if (!val) { + return "0"; + } + + const bool neg = val < 0; + TUint128 v = neg ? -val : val; + + // log_{10}(2^120) ~= 36.12, 37 decimal places + // plus dot, zero before dot, sign and zero byte at the end + static thread_local char str[40]; + auto end = str + sizeof(str); + *--end = 0; + + auto s = end; + + do { + if (!precision--) { + return ""; + } + + + const auto digit = ui8(v % Ten); + if (digit || !scale || s != end) { + *--s = "0123456789"[digit]; + } + + if (scale && !--scale && s != end) { + *--s = '.'; + } + } while (v /= Ten); + + if (scale) { + do { + if (!precision--) { + return nullptr; + } + + *--s = '0'; + } while (--scale); + + *--s = '.'; + } + + if (*s == '.') { + *--s = '0'; + } + + if (neg) { + *--s = '-'; + } + + return s; +} + +namespace { + bool IsNan(const char* s) { + return (s[0] == 'N' || s[0] == 'n') && (s[1] == 'A' || s[1] == 'a') && (s[2] == 'N' || s[2] == 'n'); + } + + bool IsInf(const char* s) { + return (s[0] == 'I' || s[0] == 'i') && (s[1] == 'N' || s[1] == 'n') && (s[2] == 'F' || s[2] == 'f'); + } +} + + +TInt128 FromString(const std::string_view& str, ui8 precision, ui8 scale) { + if (scale > precision) + return Err(); + + auto s = str.data(); + auto l = str.size(); + + if (!s || !l) + return Err(); + + const bool neg = '-' == *s; + if (neg || '+' == *s) { + ++s; + --l; + } + + if (3U == l) { + if (IsInf(s)) + return neg ? -Inf() : Inf(); + if (IsNan(s)) + return Nan(); + } + + TUint128 v = 0U; + auto integral = precision - scale; + + for (bool dot = false; l; --l) { + if (*s == '.') { + if (dot) + return Err(); + + ++s; + dot = true; + continue; + } + + if (dot) { + if (scale) + --scale; + else + break; + } + + const char c = *s++; + if (!std::isdigit(c)) + return Err(); + + v *= Ten; + v += c - '0'; + + if (!dot && v && !integral--) { + return neg ? -Inf() : Inf(); + } + } + + if (l--) { + const char c = *s++; + if (!std::isdigit(c)) + return Err(); + + bool plus = c > '5'; + if (!plus && c == '5') { + for (plus = v & 1; !plus && l; --l) { + const char c = *s++; + if (!std::isdigit(c)) + return Err(); + + plus = c != '0'; + } + } + + while (l--) + if (!std::isdigit(*s++)) + return Err(); + + if (plus) + if (++v >= GetDivider(precision)) + v = Inf(); + } + + while (scale--) + v *= Ten; + + return neg ? -v : v; +} + +TInt128 FromHalfs(ui64 lo, i64 hi) { + ui64 half[2] = {lo, static_cast(hi)}; + TInt128 val128; + std::memcpy(&val128, half, sizeof(val128)); + return val128; +} + +bool IsValid(const std::string_view& str) { + auto s = str.data(); + auto l = str.size(); + + if (!s || !l) + return false; + + if ('-' == *s || '+' == *s) { + ++s; + --l; + } + + if (3U == l && (IsInf(s) || IsNan(s))) { + return true; + } + + for (bool dot = false; l--;) { + const char c = *s++; + if (c == '.') { + if (dot) + return false; + + dot = true; + continue; + } + + if (!std::isdigit(c)) + return false; + } + + return true; +} + +} diff --git a/src/library/decimal/yql_decimal.h b/src/library/decimal/yql_decimal.h new file mode 100644 index 00000000000..d768a1e1b3f --- /dev/null +++ b/src/library/decimal/yql_decimal.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include "yql_wide_int.h" + +#include +#include +#include + +namespace NYdb::NDecimal { + +#ifdef _win_ +#ifndef DONT_USE_NATIVE_INT128 +#define DONT_USE_NATIVE_INT128 +#endif +#endif + +#ifdef DONT_USE_NATIVE_INT128 +using TInt128 = TWide; +using TUint128 = TWide; +#else +using TInt128 = signed __int128; +using TUint128 = unsigned __int128; +#endif + +constexpr ui8 MaxPrecision = 35; + +static_assert(sizeof(TInt128) == 16, "Wrong size of TInt128, expected 16"); + +inline constexpr TInt128 Inf() { + return TInt128(100000000000000000ULL) * TInt128(1000000000000000000ULL); +} + +inline constexpr TInt128 Nan() { + return Inf() + TInt128(1); +} + +inline constexpr TInt128 Err() { + return Nan() + TInt128(1); +} + +bool IsError(TInt128 v); +bool IsNan(TInt128 v); +bool IsInf(TInt128 v); + +bool IsNormal(TInt128 v); + +const char* ToString(TInt128 v, ui8 precision, ui8 scale = 0); +TInt128 FromString(const std::string_view& str, ui8 precision, ui8 scale = 0); +TInt128 FromHalfs(ui64 lo, i64 hi); + +bool IsValid(const std::string_view& str); + +} diff --git a/src/library/yql_common/decimal/yql_wide_int.h b/src/library/decimal/yql_wide_int.h similarity index 99% rename from src/library/yql_common/decimal/yql_wide_int.h rename to src/library/decimal/yql_wide_int.h index 0f650213b90..00a9cbe575a 100644 --- a/src/library/yql_common/decimal/yql_wide_int.h +++ b/src/library/decimal/yql_wide_int.h @@ -6,7 +6,7 @@ #include #include -namespace NYql { +namespace NYdb::NDecimal { #ifndef _win_ typedef __int128 i128_t; diff --git a/src/library/yql_common/issue/CMakeLists.txt b/src/library/issue/CMakeLists.txt similarity index 78% rename from src/library/yql_common/issue/CMakeLists.txt rename to src/library/issue/CMakeLists.txt index e8f0e1df096..2d81c602eba 100644 --- a/src/library/yql_common/issue/CMakeLists.txt +++ b/src/library/issue/CMakeLists.txt @@ -1,5 +1,3 @@ -add_subdirectory(protos) - _ydb_sdk_add_library(yql-public-issue) target_link_libraries(yql-public-issue @@ -9,18 +7,15 @@ target_link_libraries(yql-public-issue colorizer resource api-protos - yql-public-issue-protos - yql-utils enum_serialization_runtime string_utils-misc ) target_sources(yql-public-issue PRIVATE - yql_issue_id.cpp + utf8.cpp yql_issue_message.cpp yql_issue.cpp - out.cpp ) _ydb_sdk_install_targets(TARGETS yql-public-issue) diff --git a/src/library/issue/utf8.cpp b/src/library/issue/utf8.cpp new file mode 100644 index 00000000000..39ca0eec4eb --- /dev/null +++ b/src/library/issue/utf8.cpp @@ -0,0 +1,58 @@ +#include "utf8.h" + +#include + +namespace NYdb::NIssue { + +namespace { + +unsigned char GetRange(unsigned char c) { + // Referring to DFA of http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ + // With new mapping 1 -> 0x10, 7 -> 0x20, 9 -> 0x40, such that AND operation can test multiple types. + static const unsigned char type[] = { + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10, + 0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40, + 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, + 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, + 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, + 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, + }; + return type[c]; +} + +} + +bool IsUtf8(const std::string_view& str) { + for (auto it = str.cbegin(); str.cend() != it;) { +#define COPY() if (str.cend() != it) { c = *it++; } else { return false; } +#define TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) +#define TAIL() COPY(); TRANS(0x70) + auto c = *it++; + if (!(c & 0x80)) + continue; + + bool result = true; + switch (GetRange(static_cast(c))) { + case 2: TAIL(); break; + case 3: TAIL(); TAIL(); break; + case 4: COPY(); TRANS(0x50); TAIL(); break; + case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); break; + case 6: TAIL(); TAIL(); TAIL(); break; + case 10: COPY(); TRANS(0x20); TAIL(); break; + case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); break; + default: return false; + } + + if (!result) return false; +#undef COPY +#undef TRANS +#undef TAIL + } + return true; +} + +} diff --git a/src/library/issue/utf8.h b/src/library/issue/utf8.h new file mode 100644 index 00000000000..66141bf2998 --- /dev/null +++ b/src/library/issue/utf8.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +namespace NYdb::NIssue { + +bool IsUtf8(const std::string_view& str); + +} diff --git a/src/library/yql_common/issue/yql_issue.cpp b/src/library/issue/yql_issue.cpp similarity index 79% rename from src/library/yql_common/issue/yql_issue.cpp rename to src/library/issue/yql_issue.cpp index 2bb6b15caf3..ef139fd5885 100644 --- a/src/library/yql_common/issue/yql_issue.cpp +++ b/src/library/issue/yql_issue.cpp @@ -1,10 +1,11 @@ -#include +#include "utf8.h" -#include +#include -#include #include +#include + #include #include #include @@ -12,13 +13,27 @@ #include #include #include + #include #include -namespace NYql { +namespace NYdb::NIssue { + +std::string SeverityToString(ESeverity severity) { + switch (severity) { + case ESeverity::Fatal: + return "Fatal"; + case ESeverity::Error: + return "Error"; + case ESeverity::Warning: + return "Warning"; + case ESeverity::Info: + return "Info"; + } +} void SanitizeNonAscii(std::string& s) { - if (!NYql::IsUtf8(s)) { + if (!NYdb::NIssue::IsUtf8(s)) { std::string escaped; escaped.reserve(s.size()); const unsigned char* i = reinterpret_cast(s.data()); @@ -39,34 +54,13 @@ void SanitizeNonAscii(std::string& s) { } } -TTextWalker& TTextWalker::Advance(char c) { - if (c == '\n') { - HaveCr = false; - ++LfCount; - return *this; - } - - - if (c == '\r' && !HaveCr) { - HaveCr = true; - return *this; - } - - uint32_t charDistance = 1; - if (Utf8Aware && IsUtf8Intermediate(c)) { - charDistance = 0; - } - - // either not '\r' or second '\r' - if (LfCount) { - Position.Row += LfCount; - Position.Column = charDistance; - LfCount = 0; - } else { - Position.Column += charDistance + (HaveCr && c != '\r'); - } - HaveCr = (c == '\r'); - return *this; +uint64_t TIssue::Hash() const noexcept { + return CombineHashes( + CombineHashes( + (size_t)CombineHashes(IntHash(Position.Row), IntHash(Position.Column)), + std::hash{}(Position.File) + ), + (size_t)CombineHashes((size_t)IntHash(static_cast(IssueCode)), std::hash{}(Message))); } void TIssue::PrintTo(IOutputStream& out, bool oneLine) const { @@ -83,14 +77,14 @@ void TIssue::PrintTo(IOutputStream& out, bool oneLine) const { } } -void WalkThroughIssues(const TIssue& topIssue, bool leafOnly, std::function fn, std::function afterChildrenFn) { +void WalkThroughIssues(const TIssue& topIssue, bool leafOnly, std::function fn, std::function afterChildrenFn) { enum class EFnType { Main, AfterChildren, }; const bool hasAfterChildrenFn = bool(afterChildrenFn); - TStack> issuesStack; + TStack> issuesStack; if (hasAfterChildrenFn) { issuesStack.push(std::make_tuple(0, &topIssue, EFnType::AfterChildren)); } @@ -122,7 +116,7 @@ void WalkThroughIssues(const TIssue& topIssue, bool leafOnly, std::function(alloca(indentation)); memset(whitespaces, ' ', indentation); out.Write(whitespaces, indentation); @@ -131,12 +125,12 @@ Y_NO_INLINE void Indent(IOutputStream& out, ui32 indentation) { void ProgramLinesWithErrors( const std::string& programText, const std::vector& errors, - std::map& lines) + std::map& lines) { - std::vector rows; + std::vector rows; for (const auto& topIssue: errors) { - WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 /*level*/) { - for (ui32 row = issue.Position.Row; row <= issue.EndPosition.Row; row++) { + WalkThroughIssues(topIssue, false, [&](const TIssue& issue, uint16_t /*level*/) { + for (uint32_t row = issue.Position.Row; row <= issue.EndPosition.Row; row++) { rows.push_back(row); } }); @@ -145,9 +139,9 @@ void ProgramLinesWithErrors( auto prog = StringSplitter(programText).Split('\n'); auto progIt = prog.begin(), progEnd = prog.end(); - ui32 progRow = 1; + uint32_t progRow = 1; - for (ui32 row: rows) { + for (uint32_t row: rows) { while (progRow < row && progIt != progEnd) { ++progRow; ++progIt; @@ -169,7 +163,7 @@ void TIssues::PrintTo(IOutputStream& out, bool oneLine) const out << "["; } for (const auto& topIssue: Issues_) { - WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 level) { + WalkThroughIssues(topIssue, false, [&](const TIssue& issue, uint16_t level) { if (level > 0) { out << " subissue: { "; } else { @@ -177,7 +171,7 @@ void TIssues::PrintTo(IOutputStream& out, bool oneLine) const } issue.PrintTo(out, true); }, - [&](const TIssue&, ui16) { + [&](const TIssue&, uint16_t) { out << " }"; }); } @@ -186,7 +180,7 @@ void TIssues::PrintTo(IOutputStream& out, bool oneLine) const } } else { for (const auto& topIssue: Issues_) { - WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 level) { + WalkThroughIssues(topIssue, false, [&](const TIssue& issue, uint16_t level) { auto shift = level * 4; Indent(out, shift); out << issue << Endl; @@ -202,16 +196,16 @@ void TIssues::PrintWithProgramTo( { using namespace NColorizer; - std::map lines; + std::map lines; ProgramLinesWithErrors(programText, Issues_, lines); for (const TIssue& topIssue: Issues_) { - WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 level) { + WalkThroughIssues(topIssue, false, [&](const TIssue& issue, uint16_t level) { auto shift = level * 4; Indent(out, shift); out << DarkGray() << programFilename << Old() << ':'; out << Purple() << issue.Range() << Old(); - auto color = (issue.GetSeverity() == TSeverityIds::S_WARNING) ? Yellow() : LightRed(); + auto color = (issue.GetSeverity() == ESeverity::Warning) ? Yellow() : LightRed(); auto severityName = SeverityToString(issue.GetSeverity()); out << color << ": "<< severityName << ": " << issue.GetMessage() << Old() << '\n'; Indent(out, shift); @@ -234,9 +228,9 @@ TIssue ExceptionToIssue(const std::exception& e, const TPosition& pos) { auto issue = TIssue(parsedPos.value_or(pos), messageBuf); const TErrorException* errorException = dynamic_cast(&e); if (errorException) { - issue.SetCode(errorException->GetCode(), ESeverity::TSeverityIds_ESeverityId_S_ERROR); + issue.SetCode(errorException->GetCode(), ESeverity::Error); } else { - issue.SetCode(UNEXPECTED_ERROR, ESeverity::TSeverityIds_ESeverityId_S_FATAL); + issue.SetCode(UNEXPECTED_ERROR, ESeverity::Fatal); } return issue; } @@ -268,7 +262,7 @@ std::optional TryParseTerminationMessage(std::string_view& message) { NUtils::GetNext(s, ':', file); NUtils::GetNext(s, ':', row); NUtils::GetNext(s, ':', column); - ui32 rowValue, columnValue; + uint32_t rowValue, columnValue; if (file && row && column && TryFromString(*row, rowValue) && TryFromString(*column, columnValue)) { message = StripStringLeft(s); return TPosition(columnValue, rowValue, std::string(*file)); @@ -281,7 +275,7 @@ std::optional TryParseTerminationMessage(std::string_view& message) { } // namspace NYql template <> -void Out(IOutputStream& out, const NYql::TPosition& pos) { +void Out(IOutputStream& out, const NYdb::NIssue::TPosition& pos) { out << (pos.File.empty() ? "
" : pos.File); if (pos) { out << ":" << pos.Row << ':' << pos.Column; @@ -289,7 +283,7 @@ void Out(IOutputStream& out, const NYql::TPosition& pos) { } template<> -void Out(IOutputStream & out, const NYql::TRange & range) { +void Out(IOutputStream & out, const NYdb::NIssue::TRange & range) { if (range.IsRange()) { out << '[' << range.Position << '-' << range.EndPosition << ']'; } else { @@ -298,11 +292,11 @@ void Out(IOutputStream & out, const NYql::TRange & range) { } template <> -void Out(IOutputStream& out, const NYql::TIssue& error) { +void Out(IOutputStream& out, const NYdb::NIssue::TIssue& error) { error.PrintTo(out); } template <> -void Out(IOutputStream& out, const NYql::TIssues& error) { +void Out(IOutputStream& out, const NYdb::NIssue::TIssues& error) { error.PrintTo(out); } diff --git a/src/library/issue/yql_issue_message.cpp b/src/library/issue/yql_issue_message.cpp new file mode 100644 index 00000000000..65fae800443 --- /dev/null +++ b/src/library/issue/yql_issue_message.cpp @@ -0,0 +1,93 @@ +#include "yql_issue_message.h" + +#include + +#include +#include +#include + +#include + +namespace NYdb::NIssue { + +TIssue IssueFromMessage(const Ydb::Issue::IssueMessage& issueMessage) { + TIssue topIssue; + std::deque> queue; + queue.push_front(std::make_pair(&topIssue, &issueMessage)); + while (!queue.empty()) { + TIssue& issue = *queue.back().first; + const auto& message = *queue.back().second; + queue.pop_back(); + TPosition position(message.position().column(), message.position().row(), message.position().file()); + TPosition endPosition(message.end_position().column(), message.end_position().row()); + if (position.HasValue()) { + if (endPosition.HasValue()) { + issue = TIssue(position, endPosition, message.message()); + } else { + issue = TIssue(position, message.message()); + } + } else { + issue = TIssue(message.message()); + } + + for (const auto& subMessage : message.issues()) { + auto subIssue = new TIssue(); + issue.AddSubIssue(subIssue); + queue.push_front(std::make_pair(subIssue, &subMessage)); + } + + issue.SetCode(message.issue_code(), static_cast(message.severity())); + } + return topIssue; +} + +void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField &message, TIssues &issues) { + issues.Clear(); + if (message.size()) { + issues.Reserve(message.size()); + for (auto &x : message) { + issues.AddIssue(IssueFromMessage(x)); + } + } +} + +void IssueToMessage(const TIssue& topIssue, Ydb::Issue::IssueMessage* issueMessage) { + std::deque> queue; + queue.push_front(std::make_pair(&topIssue, issueMessage)); + while (!queue.empty()) { + const TIssue& issue = *queue.back().first; + auto& message = *queue.back().second; + queue.pop_back(); + if (issue.Position) { + auto& position = *message.mutable_position(); + position.set_row(issue.Position.Row); + position.set_column(issue.Position.Column); + position.set_file(NYdb::TStringType{issue.Position.File}); + } + if (issue.EndPosition) { + auto& endPosition = *message.mutable_end_position(); + endPosition.set_row(issue.EndPosition.Row); + endPosition.set_column(issue.EndPosition.Column); + } + message.set_message(NYdb::TStringType{issue.GetMessage()}); + message.set_issue_code(issue.GetCode()); + message.set_severity(static_cast(issue.GetSeverity())); + + for (auto subIssue : issue.GetSubIssues()) { + Ydb::Issue::IssueMessage* subMessage = message.add_issues(); + queue.push_front(std::make_pair(subIssue.Get(), subMessage)); + } + } +} + +void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField *message) { + message->Clear(); + if (!issues) + return; + message->Reserve(issues.Size()); + for (const auto &issue : issues) { + IssueToMessage(issue, message->Add()); + } +} + +} diff --git a/src/library/issue/yql_issue_message.h b/src/library/issue/yql_issue_message.h new file mode 100644 index 00000000000..0629b90b2cf --- /dev/null +++ b/src/library/issue/yql_issue_message.h @@ -0,0 +1,15 @@ +#pragma once + +#include + +#include + +namespace NYdb::NIssue { + +TIssue IssueFromMessage(const Ydb::Issue::IssueMessage& issueMessage); +void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField& message, TIssues& issues); + +void IssueToMessage(const TIssue& topIssue, Ydb::Issue::IssueMessage* message); +void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField* message); + +} diff --git a/src/library/proto_output/proto_output.cpp b/src/library/proto_output/proto_output.cpp index a8c4cf3eb09..99abf83294d 100644 --- a/src/library/proto_output/proto_output.cpp +++ b/src/library/proto_output/proto_output.cpp @@ -1,8 +1,11 @@ -#include +#include +#include #include #include +#include + #include namespace { @@ -23,6 +26,23 @@ namespace { } } +Y_DECLARE_OUT_SPEC(, Ydb::Issue::IssueMessage, stream, value) { + google::protobuf::TextFormat::Printer printer; + printer.SetSingleLineMode(true); + printer.SetUseUtf8StringEscaping(true); + + NYdb::TStringType str; + printer.PrintToString(value, &str); + + // Copied from text_format.h + // Single line mode currently might have an extra space at the end. + if (str.size() > 0 && str[str.size() - 1] == ' ') { + str.resize(str.size() - 1); + } + + stream << "{ " << str << " }"; +} + Y_DECLARE_OUT_SPEC(, Ydb::VariantType, stream, value) { stream << "{ " << ShortUtf8DebugString(value) << " }"; } diff --git a/src/library/string_utils/helpers/CMakeLists.txt b/src/library/string_utils/helpers/CMakeLists.txt index 77cea504012..ae96855924c 100644 --- a/src/library/string_utils/helpers/CMakeLists.txt +++ b/src/library/string_utils/helpers/CMakeLists.txt @@ -1,7 +1,8 @@ _ydb_sdk_add_library(string_utils-helpers) -target_sources(string_utils-helpers PRIVATE - ${YDB_SDK_SOURCE_DIR}/src/library/string_utils/helpers/helpers.cpp +target_sources(string_utils-helpers + PRIVATE + helpers.cpp ) _ydb_sdk_install_targets(TARGETS string_utils-helpers) diff --git a/src/library/yql_common/CMakeLists.txt b/src/library/yql_common/CMakeLists.txt deleted file mode 100644 index a59772ad800..00000000000 --- a/src/library/yql_common/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -add_subdirectory(decimal) -add_subdirectory(issue) -add_subdirectory(utils) diff --git a/src/library/yql_common/decimal/yql_decimal.cpp b/src/library/yql_common/decimal/yql_decimal.cpp deleted file mode 100644 index b2829e09ac6..00000000000 --- a/src/library/yql_common/decimal/yql_decimal.cpp +++ /dev/null @@ -1,438 +0,0 @@ -#include "yql_decimal.h" - -#include -#include -#include - -namespace NYql { -namespace NDecimal { - -static const TUint128 Ten(10U); - -TUint128 GetDivider(ui8 scale) { - TUint128 d(1U); - while (scale--) - d *= Ten; - return d; -} - -bool IsError(TInt128 v) { - return v > Nan() || v < -Inf(); -} - -bool IsNan(TInt128 v) { - return v == Nan(); -} - -bool IsInf(TInt128 v) { - return v == Inf() || v == -Inf(); -} - -bool IsNormal(TInt128 v) { - return v < Inf() && v > -Inf(); -} - -bool IsComparable(TInt128 v) { - return v <= Inf() && v >= -Inf(); -} - -const char* ToString(TInt128 val, ui8 precision, ui8 scale) { - if (!precision || precision > MaxPrecision || scale > precision) { - return ""; - } - - if (val == Inf()) - return "inf"; - if (val == -Inf()) - return "-inf"; - if (val == Nan()) - return "nan"; - - if (!IsNormal(val)) { - return nullptr; - } - - if (!val) { - return "0"; - } - - const bool neg = val < 0; - TUint128 v = neg ? -val : val; - - // log_{10}(2^120) ~= 36.12, 37 decimal places - // plus dot, zero before dot, sign and zero byte at the end - static thread_local char str[40]; - auto end = str + sizeof(str); - *--end = 0; - - auto s = end; - - do { - if (!precision--) { - return ""; - } - - - const auto digit = ui8(v % Ten); - if (digit || !scale || s != end) { - *--s = "0123456789"[digit]; - } - - if (scale && !--scale && s != end) { - *--s = '.'; - } - } while (v /= Ten); - - if (scale) { - do { - if (!precision--) { - return nullptr; - } - - *--s = '0'; - } while (--scale); - - *--s = '.'; - } - - if (*s == '.') { - *--s = '0'; - } - - if (neg) { - *--s = '-'; - } - - return s; -} - -namespace { - bool IsNan(const char* s) { - return (s[0] == 'N' || s[0] == 'n') && (s[1] == 'A' || s[1] == 'a') && (s[2] == 'N' || s[2] == 'n'); - } - - bool IsInf(const char* s) { - return (s[0] == 'I' || s[0] == 'i') && (s[1] == 'N' || s[1] == 'n') && (s[2] == 'F' || s[2] == 'f'); - } -} - - -TInt128 FromString(const std::string_view& str, ui8 precision, ui8 scale) { - if (scale > precision) - return Err(); - - auto s = str.data(); - auto l = str.size(); - - if (!s || !l) - return Err(); - - const bool neg = '-' == *s; - if (neg || '+' == *s) { - ++s; - --l; - } - - if (3U == l) { - if (IsInf(s)) - return neg ? -Inf() : Inf(); - if (IsNan(s)) - return Nan(); - } - - TUint128 v = 0U; - auto integral = precision - scale; - - for (bool dot = false; l; --l) { - if (*s == '.') { - if (dot) - return Err(); - - ++s; - dot = true; - continue; - } - - if (dot) { - if (scale) - --scale; - else - break; - } - - const char c = *s++; - if (!std::isdigit(c)) - return Err(); - - v *= Ten; - v += c - '0'; - - if (!dot && v && !integral--) { - return neg ? -Inf() : Inf(); - } - } - - if (l--) { - const char c = *s++; - if (!std::isdigit(c)) - return Err(); - - bool plus = c > '5'; - if (!plus && c == '5') { - for (plus = v & 1; !plus && l; --l) { - const char c = *s++; - if (!std::isdigit(c)) - return Err(); - - plus = c != '0'; - } - } - - while (l--) - if (!std::isdigit(*s++)) - return Err(); - - if (plus) - if (++v >= GetDivider(precision)) - v = Inf(); - } - - while (scale--) - v *= Ten; - - return neg ? -v : v; -} - -TInt128 FromStringEx(const std::string_view& str, ui8 precision, ui8 scale) { - if (scale > precision) - return Err(); - - const auto s = str.data(); - - for (auto ptr = s + str.size() - 1U; ptr > s; --ptr) { - if (*ptr == 'E' || *ptr == 'e') { - const auto len = ptr - s; - if (!len) - return Err(); - - const auto exp = std::atoi(++ptr); - if (!exp) - return Err(); - - const int p = precision, s = int(scale) + exp; - const auto r = exp > 0 ? - FromString(str.substr(0, len), precision, std::min(s, p)): - FromString(str.substr(0, len), std::min(p - exp, int(MaxPrecision)), std::max(s, 0)); - - if (IsNan(r)) { - return Err(); - } - - if (const auto e = exp > 0 ? std::max(0, s - p) : std::min(0, s)) { - if (r && IsNormal(r)) { - if (exp > 0) - return Mul(r, GetDivider(+e)); - if (exp < 0) - return Div(r, GetDivider(-e)); - } - } - - return r; - } - } - - return FromString(str, precision, scale); -} - -bool IsValid(const std::string_view& str) { - auto s = str.data(); - auto l = str.size(); - - if (!s || !l) - return false; - - if ('-' == *s || '+' == *s) { - ++s; - --l; - } - - if (3U == l && (IsInf(s) || IsNan(s))) { - return true; - } - - for (bool dot = false; l--;) { - const char c = *s++; - if (c == '.') { - if (dot) - return false; - - dot = true; - continue; - } - - if (!std::isdigit(c)) - return false; - } - - return true; -} - -TInt128 Mod(TInt128 a, TInt128 b) { - if (!b || !(IsNormal(a) && IsNormal(b))) - return Nan(); - - return a % b; -} - -TInt128 Div(TInt128 a, TInt128 b) { - if (IsNan(a) || IsNan(b)) - return Nan(); - - if (!b) { - if (a > 0) - return Inf(); - else if (a < 0) - return -Inf(); - else - return Nan(); - } else if (IsInf(b)) { - return IsInf(a) ? Nan() : TInt128(0); - } else if (IsInf(a)) { - return b > 0 ? a : -a; - } - - if (b & 1) - a = TUint128(a) << 1U; - else - b >>= 1; - - auto d = a / b; - - if (d & 1) { - if (const auto m = a % b) { - if (m > 0) ++d; - // else --d; - } else { - if (d & 2) ++d; - } - } - - return d >>= 1; -} - -namespace { - -using TInt256 = TWide; - -TInt128 Normalize(const TInt256& v) { - static const TInt256 PInf256(+Inf()), NInf256(-Inf()); - - if (v > PInf256) - return +Inf(); - if (v < NInf256) - return -Inf(); - return *reinterpret_cast(&v); -} - -constexpr auto HalfBitSize = sizeof(TUint128) << 2U; - -TUint128 GetUpperHalf(const TUint128& v) { - return v >> HalfBitSize; -} - -TUint128 GetLowerHalf(const TUint128& v) { - return v & TUint128(0xFFFFFFFFFFFFFFFFULL); -} - -TInt256 WidenMul(const TInt128& lhs, const TInt128& rhs) { - const bool nl = lhs < 0; - const bool nr = rhs < 0; - - const TUint128 l = nl ? -lhs : +lhs; - const TUint128 r = nr ? -rhs : +rhs; - - const TUint128 lh[] = {GetLowerHalf(l), GetUpperHalf(l)}; - const TUint128 rh[] = {GetLowerHalf(r), GetUpperHalf(r)}; - - const TUint128 prods[] = {lh[0] * rh[0], lh[0] * rh[1], lh[1] * rh[0], lh[1] * rh[1]}; - - const TUint128 fourthQ = GetLowerHalf(prods[0]); - const TUint128 thirdQ = GetUpperHalf(prods[0]) + GetLowerHalf(prods[1]) + GetLowerHalf(prods[2]); - const TUint128 secondQ = GetUpperHalf(thirdQ) + GetUpperHalf(prods[1]) + GetUpperHalf(prods[2]) + GetLowerHalf(prods[3]); - const TUint128 firstQ = GetUpperHalf(secondQ) + GetUpperHalf(prods[3]); - - const TInt256 combine((firstQ << HalfBitSize) | GetLowerHalf(secondQ), (thirdQ << HalfBitSize) | fourthQ); - return nl == nr ? +combine : -combine; -} - -template -TInt256 Div(TInt256&& a, TInt256&& b) { - if (MayOddDivider && b & 1) - a <<= 1; - else - b >>= 1; - - auto d = a / b; - - if (d & 1) { - if (const auto m = a % b) { - if (m > 0) ++d; - // else --d; - } else { - if (d & 2) ++d; - } - } - - return d >>= 1; -} - -} - -TInt128 Mul(TInt128 a, TInt128 b) { - if (IsNan(a) || IsNan(b)) - return Nan(); - - if (IsInf(a)) - return !b ? Nan() : (b > 0 ? a : -a); - - if (IsInf(b)) - return !a ? Nan() : (a > 0 ? b : -b); - - return Normalize(WidenMul(a, b)); -} - -TInt128 MulAndDivNormalMultiplier(TInt128 a, TInt128 b, TInt128 c) { - if (IsNan(a) || IsNan(c)) - return Nan(); - - if (!c) { - if (a > 0) - return Inf(); - else if (a < 0) - return -Inf(); - else - return Nan(); - } else if (IsInf(c)) { - return IsInf(a) ? Nan() : TInt128(0); - } else if (IsInf(a)) { - return c > 0 ? a : -a; - } - - return Normalize(Div(WidenMul(a, b), TInt256(c))); -} - -TInt128 MulAndDivNormalDivider(TInt128 a, TInt128 b, TInt128 c) { - if (IsNan(a) || IsNan(b)) - return Nan(); - - if (IsInf(a)) - return !b ? Nan() : (b > 0 ? a : -a); - - if (IsInf(b)) - return !a ? Nan() : (a > 0 ? b : -b); - - return Normalize(Div(WidenMul(a, b), TInt256(c))); -} - -} -} diff --git a/src/library/yql_common/decimal/yql_decimal.h b/src/library/yql_common/decimal/yql_decimal.h deleted file mode 100644 index 771d1691da7..00000000000 --- a/src/library/yql_common/decimal/yql_decimal.h +++ /dev/null @@ -1,342 +0,0 @@ -#pragma once - -#include -#include "yql_wide_int.h" - -#include -#include -#include - -namespace NYql { -namespace NDecimal { - -#ifdef _win_ -#ifndef DONT_USE_NATIVE_INT128 -#define DONT_USE_NATIVE_INT128 -#endif -#endif - -#ifdef DONT_USE_NATIVE_INT128 -using TInt128 = TWide; -using TUint128 = TWide; -#else -using TInt128 = signed __int128; -using TUint128 = unsigned __int128; -#endif - -template struct TDivider; -#if defined(__clang__) && defined(DONT_USE_NATIVE_INT128) -template<> struct TDivider<0> { static inline constexpr TUint128 Value = 1U; }; -template struct TDivider { static inline constexpr TInt128 Value = TDivider::Value * 10U; }; -#else -template<> struct TDivider<0> { static constexpr TUint128 Value = 1U; }; -template struct TDivider { static constexpr TUint128 Value = TDivider::Value * 10U; }; -#endif - -constexpr ui8 MaxPrecision = 35; - -static_assert(sizeof(TInt128) == 16, "Wrong size of TInt128, expected 16"); - -inline constexpr TInt128 Inf() { - return TInt128(100000000000000000ULL) * TInt128(1000000000000000000ULL); -} - -inline constexpr TInt128 Nan() { - return Inf() + TInt128(1); -} - -inline constexpr TInt128 Err() { - return Nan() + TInt128(1); -} - -TUint128 GetDivider(ui8 scale); - -template -inline constexpr TUint128 GetDivider() { - return TDivider::Value; -} - -template -inline constexpr std::pair GetBounds() { - return std::make_pair(-GetDivider() + (IncLow ? 1 : 0), +GetDivider() - (DecHigh ? 1 : 0)); -} - -bool IsError(TInt128 v); -bool IsNan(TInt128 v); -bool IsInf(TInt128 v); - -bool IsNormal(TInt128 v); -bool IsComparable(TInt128 v); - -template -inline bool IsNormal(TInt128 v) { - const auto& b = GetBounds(); - return v > b.first && v < b.second; -} - -const char* ToString(TInt128 v, ui8 precision, ui8 scale = 0); -TInt128 FromString(const std::string_view& str, ui8 precision, ui8 scale = 0); - -// Accept string representation with exponent. -TInt128 FromStringEx(const std::string_view& str, ui8 precision, ui8 scale); - -template -inline TInt128 FromProto(const TMkqlProto& val) { - ui64 half[2] = {val.GetLow128(), val.GetHi128()}; - TInt128 val128; - std::memcpy(&val128, half, sizeof(val128)); - return val128; -} - -template -inline constexpr TValue YtDecimalNan() { - return std::numeric_limits::max(); -} - -template<> -inline constexpr TInt128 YtDecimalNan() { - return ~(TInt128(1) << 127); -} - -template -inline constexpr TValue YtDecimalInf() { - return YtDecimalNan() - 1; -} - -template -inline TInt128 FromYtDecimal(TValue val) { - static_assert(std::is_same::value || std::is_signed::value, "Expected signed value"); - if (YtDecimalNan() == val) { - return Nan(); - } else if (YtDecimalInf() == val) { - return Inf(); - } else if (-YtDecimalInf() == val) { - return -Inf(); - } else { - return TInt128(val); - } -} - -template -inline TValue ToYtDecimal(TInt128 val) { - static_assert(std::is_same::value || std::is_signed::value, "Expected signed value"); - if (IsNormal(val)) { - return (TValue)val; - } else if (val == Inf()) { - return YtDecimalInf(); - } else if (val == -Inf()) { - return -YtDecimalInf(); - } - return YtDecimalNan(); -} - -inline TInt128 FromHalfs(ui64 lo, i64 hi) { - ui64 half[2] = {lo, static_cast(hi)}; - TInt128 val128; - std::memcpy(&val128, half, sizeof(val128)); - return val128; -} - -inline std::pair MakePair(const TInt128 v) { - std::pair r; - std::memcpy(&r, &v, sizeof(v)); - return r; - static_assert(sizeof(r) == sizeof(v), "Bad pair size."); -} - -bool IsValid(const std::string_view& str); - -// Round to nearest, ties to even. -TInt128 Div(TInt128 a, TInt128 b); // a/b -TInt128 Mul(TInt128 a, TInt128 b); // a*b -TInt128 Mod(TInt128 a, TInt128 b); // a%b - -// a*b/c Only for non zero even normal positive divider. -TInt128 MulAndDivNormalDivider(TInt128 a, TInt128 b, TInt128 c); -// a*b/c Only for non zero normal positive multiplier. -TInt128 MulAndDivNormalMultiplier(TInt128 a, TInt128 b, TInt128 c); - -struct TDecimal { - TInt128 Value = 0; - - TDecimal() = default; - - template - TDecimal(T t): Value(t) { } - - explicit operator TInt128() const { - return Value; - } - - TDecimal& operator+=(TDecimal right) { - const auto l = Value; - const auto r = right.Value; - const auto a = l + r; - if (IsNormal(l) && IsNormal(r) && IsNormal(a)) { - Value = a; - } else if (IsNan(l) || IsNan(r) || !a /* inf - inf*/) { - Value = Nan(); - } else { - Value = a > 0 - ? +Inf() - : -Inf(); - } - return *this; - } - - TDecimal& operator*=(TDecimal right) { - Value = Mul(Value, right.Value); - return *this; - } - - TDecimal& operator/=(TDecimal right) { - Value = Div(Value, right.Value); - return *this; - } - - friend TDecimal operator+(TDecimal left, TDecimal right) { - left += right; - return left; - } - - friend TDecimal operator*(TDecimal left, TDecimal right) { - left *= right; - return left; - } - - friend TDecimal operator/(TDecimal left, TDecimal right) { - left /= right; - return left; - } -}; - -template -class TDecimalMultiplicator { -protected: - const TInt128 Bound; - -public: - TDecimalMultiplicator( - ui8 precision, - [[maybe_unused]] ui8 scale = 0) - : Bound(GetDivider(precision)) - { - } - - TInt128 Do(TInt128 left, TRight right) const { - TInt128 mul = Mul(left, right); - - if (mul > -Bound && mul < +Bound) - return mul; - - return IsNan(mul) ? Nan() : (mul > 0 ? +Inf() : -Inf()); - } -}; - -template<> -class TDecimalMultiplicator { -protected: - const TInt128 Bound; - const TInt128 Divider; - -public: - TDecimalMultiplicator( - ui8 precision, - ui8 scale) - : Bound(GetDivider(precision)) - , Divider(GetDivider(scale)) - { } - - TInt128 Do(TInt128 left, TInt128 right) const { - TInt128 mul = Divider > 1 ? - MulAndDivNormalDivider(left, right, Divider): - Mul(left, right); - - if (mul > -Bound && mul < +Bound) - return mul; - - return IsNan(mul) ? Nan() : (mul > 0 ? +Inf() : -Inf()); - } -}; - -template -class TDecimalDivisor { -public: - TDecimalDivisor( - [[maybe_unused]] ui8 precision = 0, - [[maybe_unused]] ui8 scale = 0) - { - } - - TInt128 Do(TInt128 left, TRight right) const { - return Div(left, right); - } -}; - -template<> -class TDecimalDivisor { -protected: - const TInt128 Bound; - const TInt128 Divider; - -public: - TDecimalDivisor( - ui8 precision, - ui8 scale) - : Bound(GetDivider(precision)) - , Divider(GetDivider(scale)) - { } - - TInt128 Do(TInt128 left, TInt128 right) const { - TInt128 div = MulAndDivNormalMultiplier(left, Divider, right); - if (div > -Bound && div < +Bound) { - return div; - } - - return IsNan(div) ? Nan() : (div > 0 ? +Inf() : -Inf()); - } -}; - -template -class TDecimalRemainder { -protected: - const TInt128 Bound; - const TInt128 Divider; - -public: - TDecimalRemainder( - ui8 precision, - ui8 scale) - : Bound(NYql::NDecimal::GetDivider(precision - scale)) - , Divider(NYql::NDecimal::GetDivider(scale)) - { } - - TInt128 Do(TInt128 left, TRight right) const { - if constexpr (std::is_signed::value) { - if (TInt128(right) >= +Bound || TInt128(right) <= -Bound) - return left; - } else { - if (TInt128(right) >= Bound) - return left; - } - - return Mod(left, Mul(Divider, right)); - } -}; - -template<> -class TDecimalRemainder { -public: - TDecimalRemainder( - [[maybe_unused]] ui8 precision = 0, - [[maybe_unused]] ui8 scale = 0) - { - } - - TInt128 Do(TInt128 left, TInt128 right) const { - return NYql::NDecimal::Mod(left, right); - } -}; - -} -} diff --git a/src/library/yql_common/decimal/yql_decimal_serialize.cpp b/src/library/yql_common/decimal/yql_decimal_serialize.cpp deleted file mode 100644 index 5fb844f5f89..00000000000 --- a/src/library/yql_common/decimal/yql_decimal_serialize.cpp +++ /dev/null @@ -1,82 +0,0 @@ -#include "yql_decimal_serialize.h" - -#include - -namespace NYql { -namespace NDecimal { - -size_t Serialize(TInt128 value, char* buf) { - if (value == -Nan()) { - *buf = 0x00; - return 1U; - } - - if (value == -Inf()) { - *buf = 0x01; - return 1U; - } - - if (value == +Inf()) { - *buf = 0xFE; - return 1U; - } - - if (value == +Nan()) { - *buf = 0xFF; - return 1U; - } - - auto size = sizeof(value); - auto p = reinterpret_cast(&value) + size - 1U; - - if (*(p - 1U) & 0x80) { - while (size > 1U && ~0 == *--p) - --size; - *buf = 0x80 - size; - } else { - while (size > 1U && 0 == *--p) - --size; - *buf = 0x7F + size; - } - - for (auto i = 1U; i < size; ++i) { - *++buf = *p--; - } - - return size; -} - -std::pair Deserialize(const char* b, size_t len) { - if (!b || len == 0U) - return std::make_pair(Err(), 0U); - - const auto mark = ui8(*b); - const bool neg = mark < 0x80u; - if (mark == 0x00u || mark == 0xFFu) - return std::make_pair(neg ? -Nan() : +Nan(), 1U); - if (mark == 0x01u || mark == 0xFEu) - return std::make_pair(neg ? -Inf() : +Inf(), 1U); - - if (mark < 0x70u || mark > 0x8Fu) { - return std::make_pair(Err(), 0U); - } - - const auto used = neg ? 0x80u - mark : mark - 0x7Fu; - if (len < used) - return std::make_pair(Err(), 0U); - - TInt128 v; - const auto size = sizeof(v); - auto p = reinterpret_cast(&v) + size; - - for (auto fill = size - used + 2U; --fill;) - *--p = neg ? ~0 : 0; - - for (auto copy = used; --copy;) - *--p = *++b; - - return std::make_pair(v, used); -} - -} -} diff --git a/src/library/yql_common/decimal/yql_decimal_serialize.h b/src/library/yql_common/decimal/yql_decimal_serialize.h deleted file mode 100644 index 2f540ac1e66..00000000000 --- a/src/library/yql_common/decimal/yql_decimal_serialize.h +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -#include "yql_decimal.h" - -namespace NYql { -namespace NDecimal { - -// big-endian 16 bytes buffer. -size_t Serialize(TInt128 v, char* buff); -std::pair Deserialize(const char* buff, size_t len); - -} -} diff --git a/src/library/yql_common/issue/out.cpp b/src/library/yql_common/issue/out.cpp deleted file mode 100644 index 620198fe9d5..00000000000 --- a/src/library/yql_common/issue/out.cpp +++ /dev/null @@ -1,42 +0,0 @@ -#include -#include - -#include - -#include - -#include - -Y_DECLARE_OUT_SPEC(, Ydb::Issue::IssueMessage, stream, value) { - google::protobuf::TextFormat::Printer printer; - printer.SetSingleLineMode(true); - printer.SetUseUtf8StringEscaping(true); - - NYdb::TStringType str; - printer.PrintToString(value, &str); - - // Copied from text_format.h - // Single line mode currently might have an extra space at the end. - if (str.size() > 0 && str[str.size() - 1] == ' ') { - str.resize(str.size() - 1); - } - - stream << "{ " << str << " }"; -} - -Y_DECLARE_OUT_SPEC(, NYql::NIssue::NProto::IssueMessage, stream, value) { - google::protobuf::TextFormat::Printer printer; - printer.SetSingleLineMode(true); - printer.SetUseUtf8StringEscaping(true); - - NYdb::TStringType str; - printer.PrintToString(value, &str); - - // Copied from text_format.h - // Single line mode currently might have an extra space at the end. - if (str.size() > 0 && str[str.size() - 1] == ' ') { - str.resize(str.size() - 1); - } - - stream << "{ " << str << " }"; -} diff --git a/src/library/yql_common/issue/protos/CMakeLists.txt b/src/library/yql_common/issue/protos/CMakeLists.txt deleted file mode 100644 index 9e16b329830..00000000000 --- a/src/library/yql_common/issue/protos/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -file(GLOB_RECURSE YQL_PUBLIC_ISSUE_PROTOS - *.proto -) - -_ydb_sdk_add_proto_library(yql-public-issue-protos - SOURCES ${YQL_PUBLIC_ISSUE_PROTOS} -) - -target_include_directories(yql-public-issue-protos PRIVATE - ${YDB_SDK_BINARY_DIR}/ydb-cpp-sdk -) - -_ydb_sdk_install_targets(TARGETS yql-public-issue-protos) diff --git a/src/library/yql_common/issue/protos/issue_message.proto b/src/library/yql_common/issue/protos/issue_message.proto deleted file mode 100644 index d581a5d8f07..00000000000 --- a/src/library/yql_common/issue/protos/issue_message.proto +++ /dev/null @@ -1,17 +0,0 @@ -package NYql.NIssue.NProto; -option java_package = "com.yandex.yql.issue.proto"; - -message IssueMessage { - message Position { - optional uint32 row = 1; - optional uint32 column = 2; - optional string file = 3; - } - - optional Position position = 1; - optional string message = 2; - optional Position end_position = 3; - optional uint32 issue_code = 4; - optional uint32 severity = 5; - repeated IssueMessage issues = 6; -} diff --git a/src/library/yql_common/issue/protos/issue_severity.proto b/src/library/yql_common/issue/protos/issue_severity.proto deleted file mode 100644 index 70fd61f5d9a..00000000000 --- a/src/library/yql_common/issue/protos/issue_severity.proto +++ /dev/null @@ -1,14 +0,0 @@ -syntax = "proto3"; - -package NYql; - -option java_package = "com.yandex.yql.proto"; - -message TSeverityIds { - enum ESeverityId { - S_FATAL = 0; - S_ERROR = 1; - S_WARNING = 2; - S_INFO = 3; - } -} diff --git a/src/library/yql_common/issue/yql_issue_id.cpp b/src/library/yql_common/issue/yql_issue_id.cpp deleted file mode 100644 index 9677234e8e8..00000000000 --- a/src/library/yql_common/issue/yql_issue_id.cpp +++ /dev/null @@ -1,12 +0,0 @@ -#include - -#include - -namespace NYql { - -std::string SeverityToString(ESeverity severity) { - auto ret = NYql::TSeverityIds::ESeverityId_Name(severity); - return ret.empty() ? "Unknown" : NUtils::ToTitle(ret.substr(2)); //remove prefix "S_" -} - -}; diff --git a/src/library/yql_common/issue/yql_issue_message.cpp b/src/library/yql_common/issue/yql_issue_message.cpp deleted file mode 100644 index 3bbef566d1a..00000000000 --- a/src/library/yql_common/issue/yql_issue_message.cpp +++ /dev/null @@ -1,150 +0,0 @@ -#include "yql_issue_message.h" - -#include - -#include -#include - -#include -#include -#include - -#include - -namespace NYql { - -using namespace NIssue::NProto; - -template -TIssue IssueFromMessage(const TIssueMessage& issueMessage) { - TIssue topIssue; - std::deque> queue; - queue.push_front(std::make_pair(&topIssue, &issueMessage)); - while (!queue.empty()) { - TIssue& issue = *queue.back().first; - const auto& message = *queue.back().second; - queue.pop_back(); - TPosition position(message.position().column(), message.position().row(), message.position().file()); - TPosition endPosition(message.end_position().column(), message.end_position().row()); - if (position.HasValue()) { - if (endPosition.HasValue()) { - issue = TIssue(position, endPosition, message.message()); - } else { - issue = TIssue(position, message.message()); - } - } else { - issue = TIssue(message.message()); - } - - for (const auto& subMessage : message.issues()) { - auto subIssue = new TIssue(); - issue.AddSubIssue(subIssue); - queue.push_front(std::make_pair(subIssue, &subMessage)); - } - - issue.SetCode(message.issue_code(), static_cast(message.severity())); - } - return topIssue; -} - -template -void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField &message, TIssues &issues) { - issues.Clear(); - if (message.size()) { - issues.Reserve(message.size()); - for (auto &x : message) - issues.AddIssue(IssueFromMessage(x)); - } -} - -template -void IssueToMessage(const TIssue& topIssue, TIssueMessage* issueMessage) { - std::deque> queue; - queue.push_front(std::make_pair(&topIssue, issueMessage)); - while (!queue.empty()) { - const TIssue& issue = *queue.back().first; - auto& message = *queue.back().second; - queue.pop_back(); - if (issue.Position) { - auto& position = *message.mutable_position(); - position.set_row(issue.Position.Row); - position.set_column(issue.Position.Column); - position.set_file(NYdb::TStringType{issue.Position.File}); - } - if (issue.EndPosition) { - auto& endPosition = *message.mutable_end_position(); - endPosition.set_row(issue.EndPosition.Row); - endPosition.set_column(issue.EndPosition.Column); - } - message.set_message(NYdb::TStringType{issue.GetMessage()}); - message.set_issue_code(issue.GetCode()); - message.set_severity(issue.GetSeverity()); - - for (auto subIssue : issue.GetSubIssues()) { - TIssueMessage* subMessage = message.add_issues(); - queue.push_front(std::make_pair(subIssue.Get(), subMessage)); - } - } -} - -template -void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField *message) { - message->Clear(); - if (!issues) - return; - message->Reserve(issues.Size()); - for (const auto &issue : issues) { - IssueToMessage(issue, message->Add()); - } -} - -template -TIssue IssueFromMessage(const Ydb::Issue::IssueMessage& issueMessage); -template -TIssue IssueFromMessage(const NYql::NIssue::NProto::IssueMessage& issueMessage); - -template -void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField& message, TIssues& issues); -template -void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField& message, TIssues& issues); - -template -void IssueToMessage(const TIssue& topIssue, Ydb::Issue::IssueMessage* issueMessage); -template -void IssueToMessage(const TIssue& topIssue, NYql::NIssue::NProto::IssueMessage* issueMessage); - -template -void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField* message); -template -void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField* message); - -NIssue::NProto::IssueMessage IssueToMessage(const TIssue& topIssue) { - NIssue::NProto::IssueMessage issueMessage; - IssueToMessage(topIssue, &issueMessage); - return issueMessage; -} - -std::string IssueToBinaryMessage(const TIssue& issue) { - NYdb::TStringType result; - Ydb::Issue::IssueMessage protobuf; - IssueToMessage(issue, &protobuf); - Y_UNUSED(protobuf.SerializeToString(&result)); - return result; -} - -TIssue IssueFromBinaryMessage(const std::string& binaryMessage) { - Ydb::Issue::IssueMessage protobuf; - if (!protobuf.ParseFromString(NYdb::TStringType{binaryMessage})) { - ythrow yexception() << "unable to parse binary string as issue protobuf"; - } - return IssueFromMessage(protobuf); -} - -} - -Y_DECLARE_OUT_SPEC(, google::protobuf::RepeatedPtrField, stream, issues) { - stream << JoinSeq("", issues); -} -Y_DECLARE_OUT_SPEC(, google::protobuf::RepeatedPtrField, stream, issues) { - stream << JoinSeq("", issues); -} \ No newline at end of file diff --git a/src/library/yql_common/issue/yql_issue_message.h b/src/library/yql_common/issue/yql_issue_message.h deleted file mode 100644 index a8d84015362..00000000000 --- a/src/library/yql_common/issue/yql_issue_message.h +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once - -#include - -#include - -namespace NYql { - -namespace NIssue { -namespace NProto { -class IssueMessage; -} -} - -template -TIssue IssueFromMessage(const TIssueMessage& issueMessage); -template -void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField& message, TIssues& issues); - -template -std::string IssuesFromMessageAsString(const ::google::protobuf::RepeatedPtrField& message) { - TIssues issues; - IssuesFromMessage(message, issues); - return issues.ToOneLineString(); -} - -NIssue::NProto::IssueMessage IssueToMessage(const TIssue& topIssue); - -template -void IssueToMessage(const TIssue& topIssue, TIssueMessage* message); -template -void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField* message); - -std::string IssueToBinaryMessage(const TIssue& issue); -TIssue IssueFromBinaryMessage(const std::string& binaryMessage); - -} diff --git a/src/library/yql_common/utils/CMakeLists.txt b/src/library/yql_common/utils/CMakeLists.txt deleted file mode 100644 index 529e45c3bcb..00000000000 --- a/src/library/yql_common/utils/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -_ydb_sdk_add_library(yql-utils) - -target_link_libraries(yql-utils PUBLIC - yutil - string_utils-quote - threading-future -) - -target_sources(yql-utils PRIVATE - utf8.cpp -) - -_ydb_sdk_install_targets(TARGETS yql-utils) diff --git a/src/library/yql_common/utils/utf8.cpp b/src/library/yql_common/utils/utf8.cpp deleted file mode 100644 index d32e9e1b87e..00000000000 --- a/src/library/yql_common/utils/utf8.cpp +++ /dev/null @@ -1,260 +0,0 @@ -#include - -#include - -#include -#include - -namespace NYql { - -namespace { - -unsigned char GetRange(unsigned char c) { - // Referring to DFA of http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ - // With new mapping 1 -> 0x10, 7 -> 0x20, 9 -> 0x40, such that AND operation can test multiple types. - static const unsigned char type[] = { - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10, - 0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40, - 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, - 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, - 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, - 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, - }; - return type[c]; -} - -struct TByteRange { - ui8 First = 0; - ui8 Last = 0; -}; - -struct TUtf8Ranges { - size_t BytesCount = 0; - TByteRange Bytes[4] = {}; -}; - -// see https://lemire.me/blog/2018/05/09/how-quickly-can-you-check-that-a-string-is-valid-unicode-utf-8 -inline static const std::vector Utf8Ranges = { - { 1, { {0x00, 0x7f}, {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, } }, - { 2, { {0xc2, 0xdf}, {0x80, 0xbf}, {0x00, 0x00}, {0x00, 0x00}, } }, - { 3, { {0xe0, 0xe0}, {0xa0, 0xbf}, {0x80, 0xbf}, {0x00, 0x00}, } }, - { 3, { {0xe1, 0xec}, {0x80, 0xbf}, {0x80, 0xbf}, {0x00, 0x00}, } }, - { 3, { {0xed, 0xed}, {0x80, 0x9f}, {0x80, 0xbf}, {0x00, 0x00}, } }, - { 3, { {0xee, 0xef}, {0x80, 0xbf}, {0x80, 0xbf}, {0x00, 0x00}, } }, - { 4, { {0xf0, 0xf0}, {0x90, 0xbf}, {0x80, 0xbf}, {0x80, 0xbf}, } }, - { 4, { {0xf1, 0xf3}, {0x80, 0xbf}, {0x80, 0xbf}, {0x80, 0xbf}, } }, - { 4, { {0xf4, 0xf4}, {0x80, 0x8f}, {0x80, 0xbf}, {0x80, 0xbf}, } }, -}; - -std::optional RoundBadUtf8(size_t range, std::string_view inputString, size_t pos, - bool roundDown) -{ - Y_ENSURE(range > 0); - Y_ENSURE(range < Utf8Ranges.size()); - - const std::string prefix{inputString.substr(0, pos)}; - std::string_view suffix = inputString.substr(pos, Utf8Ranges[range].BytesCount); - - std::string newSuffix; - if (roundDown) { - std::optional lastNonMin; - for (size_t i = 0; i < suffix.size(); ++i) { - if (Utf8Ranges[range].Bytes[i].First < ui8(suffix[i])) { - lastNonMin = i; - } - } - - if (!lastNonMin) { - for (size_t i = 0; i < Utf8Ranges[range - 1].BytesCount; ++i) { - newSuffix.push_back(Utf8Ranges[range - 1].Bytes[i].Last); - } - } else { - for (size_t i = 0; i < Utf8Ranges[range].BytesCount; ++i) { - if (i < *lastNonMin) { - ui8 c = suffix[i]; - newSuffix.push_back(c); - } else if (i == *lastNonMin) { - ui8 c = suffix[i]; - newSuffix.push_back(std::min(c - 1, Utf8Ranges[range].Bytes[i].Last)); - } else { - newSuffix.push_back(Utf8Ranges[range].Bytes[i].Last); - } - } - } - } else { - std::optional lastNonMax; - bool valid = true; - for (size_t i = 0; i < suffix.size(); ++i) { - ui8 last = Utf8Ranges[range].Bytes[i].Last; - ui8 first = Utf8Ranges[range].Bytes[i].First; - ui8 curr = ui8(suffix[i]); - - valid = valid && curr <= last && curr >= first; - if (curr < last) { - lastNonMax = i; - } - } - - if (valid) { - newSuffix = suffix; - for (size_t i = suffix.size(); i < Utf8Ranges[range].BytesCount; ++i) { - newSuffix.push_back(Utf8Ranges[range].Bytes[i].First); - } - } else if (!lastNonMax) { - return NextValidUtf8(prefix); - } else { - for (size_t i = 0; i < Utf8Ranges[range].BytesCount; ++i) { - if (i < *lastNonMax) { - ui8 c = suffix[i]; - newSuffix.push_back(c); - } else if (i == *lastNonMax) { - ui8 c = suffix[i]; - newSuffix.push_back(std::max(c + 1, Utf8Ranges[range].Bytes[i].First)); - } else { - newSuffix.push_back(Utf8Ranges[range].Bytes[i].First); - } - } - } - - } - return prefix + newSuffix; -} - -} - -bool IsUtf8(const std::string_view& str) { - for (auto it = str.cbegin(); str.cend() != it;) { -#define COPY() if (str.cend() != it) { c = *it++; } else { return false; } -#define TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) -#define TAIL() COPY(); TRANS(0x70) - auto c = *it++; - if (!(c & 0x80)) - continue; - - bool result = true; - switch (GetRange(static_cast(c))) { - case 2: TAIL(); break; - case 3: TAIL(); TAIL(); break; - case 4: COPY(); TRANS(0x50); TAIL(); break; - case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); break; - case 6: TAIL(); TAIL(); TAIL(); break; - case 10: COPY(); TRANS(0x20); TAIL(); break; - case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); break; - default: return false; - } - - if (!result) return false; -#undef COPY -#undef TRANS -#undef TAIL - } - return true; -} - -unsigned char WideCharSize(char head) { - switch (GetRange(static_cast(head))) { - case 0: return 1; - case 2: return 2; - case 3: return 3; - case 4: return 3; - case 5: return 4; - case 6: return 4; - case 10: return 3; - case 11: return 4; - default: return 0; - } -} - -std::optional RoundToNearestValidUtf8(const std::string_view& str, bool roundDown) { - const size_t ss = str.size(); - for (size_t pos = 0; pos < ss; ) { - ui8 c = str[pos]; - - for (size_t i = 0; i < Utf8Ranges.size(); ++i) { - auto& range = Utf8Ranges[i]; - - if (c < range.Bytes[0].First) { - return RoundBadUtf8(i, str, pos, roundDown); - } - - if (c <= range.Bytes[0].Last) { - // valid UTF8 code point start - for (size_t j = 1; j < range.BytesCount; ++j) { - if (pos + j >= ss) { - return RoundBadUtf8(i, str, pos, roundDown); - } - ui8 cur = str[pos + j]; - if (!(cur >= range.Bytes[j].First && cur <= range.Bytes[j].Last)) { - return RoundBadUtf8(i, str, pos, roundDown); - } - } - - pos += range.BytesCount; - break; - } else if (i + 1 == Utf8Ranges.size()) { - if (!roundDown) { - return NextValidUtf8(str.substr(0, pos)); - } - return RoundBadUtf8(i, str, pos, roundDown); - } - } - } - return std::string(str); -} - -std::optional NextValidUtf8(const std::string_view& str) { - Y_ENSURE(IsUtf8(str)); - auto wide = UTF8ToUTF32(str); - bool incremented = false; - size_t toDrop = 0; - for (auto it = wide.rbegin(); it != wide.rend(); ++it) { - auto& c = *it; - if (c < 0x10ffff) { - c = (c == 0xd7ff) ? 0xe000 : (c + 1); - incremented = true; - break; - } else { - ++toDrop; - } - } - - if (!incremented) { - return {}; - } - - Y_ENSURE(toDrop < wide.size()); - wide.resize(wide.size() - toDrop); - - std::string result = WideToUTF8(wide); - return std::string(result.data(), result.size()); -} - -std::optional NextLexicographicString(const std::string_view& str) { - bool incremented = false; - size_t toDrop = 0; - std::string result{str}; - for (auto it = result.rbegin(); it != result.rend(); ++it) { - auto& c = *it; - if (ui8(c) < 0xff) { - ++c; - incremented = true; - break; - } else { - ++toDrop; - } - } - - if (!incremented) { - return {}; - } - - Y_ENSURE(toDrop < result.size()); - result.resize(result.size() - toDrop); - return result; -} - -} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 00000000000..729c6ee0778 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(integration) +add_subdirectory(unit) diff --git a/tests/integration/basic_example_it/basic_example.cpp b/tests/integration/basic_example_it/basic_example.cpp index fe517037690..f1a857b6926 100644 --- a/tests/integration/basic_example_it/basic_example.cpp +++ b/tests/integration/basic_example_it/basic_example.cpp @@ -461,7 +461,7 @@ static TStatus ScanQuerySelect(TTableClient client, const std::string& path, std vectorResultSet.push_back(rs); } } - return TStatus(EStatus::SUCCESS, NYql::TIssues()); + return TStatus(EStatus::SUCCESS, NYdb::NIssue::TIssues()); } /////////////////////////////////////////////////////////////////////////////// diff --git a/tests/integration/bulk_upsert_simple_it/bulk_upsert.cpp b/tests/integration/bulk_upsert_simple_it/bulk_upsert.cpp index 90bf26207a4..d0e104ace00 100644 --- a/tests/integration/bulk_upsert_simple_it/bulk_upsert.cpp +++ b/tests/integration/bulk_upsert_simple_it/bulk_upsert.cpp @@ -144,7 +144,7 @@ TStatistic Select(TTableClient& client, const std::string& path) { if (parser.ColumnsCount() != 3 || parser.RowsCount() != 1) { throw TYdbErrorException(TStatus(EStatus::GENERIC_ERROR, - {NYql::TIssue("The number of columns should be: 3.\nThe number of rows should be: 1")})); + {NYdb::NIssue::TIssue("The number of columns should be: 3.\nThe number of rows should be: 1")})); } if (parser.TryNextRow()) { diff --git a/tests/unit/client/draft/ydb_view_ut.cpp b/tests/unit/client/draft/ydb_view_ut.cpp index af7f058b366..94336d1995f 100644 --- a/tests/unit/client/draft/ydb_view_ut.cpp +++ b/tests/unit/client/draft/ydb_view_ut.cpp @@ -10,7 +10,7 @@ using namespace NYdb::NView; Y_UNIT_TEST_SUITE(ViewClient) { Y_UNIT_TEST(Basic) { - TString addr = "localhost:2135"; + TString addr = "localhost:2000"; TViewDummyService viewService; auto server = StartGrpcServer(addr, viewService); diff --git a/tests/unit/library/CMakeLists.txt b/tests/unit/library/CMakeLists.txt index b02abc2e9fc..1e4ccd6cd50 100644 --- a/tests/unit/library/CMakeLists.txt +++ b/tests/unit/library/CMakeLists.txt @@ -1,4 +1,5 @@ +add_subdirectory(decimal) +add_subdirectory(issue) add_subdirectory(grpc_client) add_subdirectory(json_value) add_subdirectory(operation_id) -add_subdirectory(yql_common) diff --git a/tests/unit/library/decimal/CMakeLists.txt b/tests/unit/library/decimal/CMakeLists.txt new file mode 100644 index 00000000000..6675cc27ab5 --- /dev/null +++ b/tests/unit/library/decimal/CMakeLists.txt @@ -0,0 +1,11 @@ +add_ydb_test(NAME yql-public-decimal_ut + SOURCES + yql_decimal_ut.cpp + yql_wide_int_ut.cpp + LINK_LIBRARIES + yutil + cpp-testing-unittest_main + yql-public-decimal + LABELS + unit +) diff --git a/tests/unit/library/yql_common/decimal/yql_decimal_ut.cpp b/tests/unit/library/decimal/yql_decimal_ut.cpp similarity index 50% rename from tests/unit/library/yql_common/decimal/yql_decimal_ut.cpp rename to tests/unit/library/decimal/yql_decimal_ut.cpp index 2fec102f610..8052da4c293 100644 --- a/tests/unit/library/yql_common/decimal/yql_decimal_ut.cpp +++ b/tests/unit/library/decimal/yql_decimal_ut.cpp @@ -1,10 +1,9 @@ -#include -#include +#include #include -namespace NYql { -namespace NDecimal { +namespace NYdb::NDecimal { + Y_UNIT_TEST_SUITE(TYqlDecimalTest) { void SimplePositiveTest(TInt128 v, ui8 precision, ui8 scale, const std::string& expected) { std::string result = ToString(v, precision, scale); @@ -18,44 +17,6 @@ Y_UNIT_TEST_SUITE(TYqlDecimalTest) { UNIT_ASSERT_VALUES_EQUAL(result, ""); } - void SimpleSerializeAndDeserialize(TInt128 v, size_t expectedSize) { - char buff[sizeof(TInt128)]; - const auto s = Serialize(v, buff); - UNIT_ASSERT_VALUES_EQUAL(s, expectedSize); - const auto& des = Deserialize(buff, expectedSize); - UNIT_ASSERT_VALUES_EQUAL(des.second, expectedSize); - UNIT_ASSERT(des.first == v); - const auto& e = Deserialize(buff, expectedSize - 1); - UNIT_ASSERT(e.first == Err()); - } - - template - void CheckMulAndRescale(const std::string_view& lhs, const std::string_view& rhs, const std::string_view& expected) { - const auto l = FromString(lhs, Precision, Scale); - const auto r = FromString(rhs, Precision, Scale); - const auto m = MulAndDivNormalDivider(l, r, GetDivider()); - const auto result = ToString(m, Precision, Scale); - UNIT_ASSERT_VALUES_EQUAL(result, expected); - } - - template - void CheckDivAndRescale(const std::string_view& lhs, const std::string_view& rhs, const std::string_view& expected) { - const auto l = FromString(lhs, Precision, Scale); - const auto r = FromString(rhs, Precision, Scale); - const auto m = MulAndDivNormalMultiplier(l, GetDivider(), r); - const auto result = ToString(m, Precision, Scale); - UNIT_ASSERT_VALUES_EQUAL(result, expected); - } - - template - void CheckMul(const std::string_view& lhs, const std::string_view& rhs, const std::string_view& expected) { - const auto l = FromString(lhs, Precision, Scale); - const auto r = FromString(rhs, Precision, Scale); - const auto m = Mul(l, r); - const auto result = ToString(m, Precision, Scale); - UNIT_ASSERT_VALUES_EQUAL(result, expected); - } - Y_UNIT_TEST(TestZeroFormat) { UNIT_ASSERT_VALUES_EQUAL(ToString(0, 1, 0), "0"); UNIT_ASSERT_VALUES_EQUAL(ToString(0, 15, 6), "0"); @@ -201,34 +162,6 @@ Y_UNIT_TEST_SUITE(TYqlDecimalTest) { UNIT_ASSERT(IsError(FromString("+7.039493804E1", 35, 5))); // letter in tail after scale } - Y_UNIT_TEST(TestFormStringEx) { - UNIT_ASSERT(FromStringEx("NAN", 13, 1) == Nan()); - UNIT_ASSERT(FromStringEx("+inf", 11, 7) == Inf()); - UNIT_ASSERT(FromStringEx("-inf", 7, 7) == -Inf()); - - UNIT_ASSERT(FromStringEx("0.1E3", 10, 1) == 1000); - UNIT_ASSERT(FromStringEx("0.51e-3", 10, 3) == 1); - - UNIT_ASSERT(FromStringEx("1E30", 10, 0) == Inf()); - UNIT_ASSERT(FromStringEx("1e-30", 10, 0) == 0); - UNIT_ASSERT(FromStringEx("-1E+99", 10, 2) == -Inf()); - UNIT_ASSERT(FromStringEx("-1e-99", 10, 2) == 0); - UNIT_ASSERT(FromStringEx("-510e-3", 1, 0) == -1); - UNIT_ASSERT(FromStringEx("+99E3", 5, 0) == 99000); - } - - Y_UNIT_TEST(TestFormStringExInvalidValues) { - UNIT_ASSERT(IsError(FromStringEx("", 35, 15))); // empty - UNIT_ASSERT(IsError(FromStringEx("12.2.3", 35, 15))); // double dot - UNIT_ASSERT(IsError(FromStringEx("+-12", 35, 15))); // extra sign - UNIT_ASSERT(IsError(FromStringEx("463786378O74674", 35, 15))); // letter inside - - UNIT_ASSERT(IsError(FromStringEx("E2", 35, 15))); // empty - UNIT_ASSERT(IsError(FromStringEx("E2E4", 35, 15))); // empty - UNIT_ASSERT(IsError(FromStringEx("12E0", 35, 15))); // zero isn't avail - UNIT_ASSERT(IsError(FromStringEx("NANE5", 35, 15))); // nan with exp - } - Y_UNIT_TEST(TestSpecialAsString) { UNIT_ASSERT(IsValid("Nan")); UNIT_ASSERT(IsValid("INF")); @@ -257,89 +190,6 @@ Y_UNIT_TEST_SUITE(TYqlDecimalTest) { UNIT_ASSERT(ToString(i, MaxPrecision, 0) == nullptr); } } - - Y_UNIT_TEST(TestSerializeAndDeserialize) { - SimpleSerializeAndDeserialize(-Nan(), 1U); - SimpleSerializeAndDeserialize(-Inf(), 1U); - - SimpleSerializeAndDeserialize(-Inf() + 1, 16U); - SimpleSerializeAndDeserialize(-Inf() + 2, 16U); - - SimpleSerializeAndDeserialize(-65537, 4U); - SimpleSerializeAndDeserialize(-65536, 3U); - - SimpleSerializeAndDeserialize(-257, 3U); - SimpleSerializeAndDeserialize(-256, 2U); - - SimpleSerializeAndDeserialize(-3, 2U); - SimpleSerializeAndDeserialize(-2, 2U); - - SimpleSerializeAndDeserialize(-1, 1U); - SimpleSerializeAndDeserialize(0, 1U); - - SimpleSerializeAndDeserialize(+1, 2U); - SimpleSerializeAndDeserialize(+2, 2U); - - SimpleSerializeAndDeserialize(+255, 2U); - SimpleSerializeAndDeserialize(+256, 3U); - - SimpleSerializeAndDeserialize(+65535, 3U); - SimpleSerializeAndDeserialize(+65536, 4U); - - SimpleSerializeAndDeserialize(+Inf() - 2, 16U); - SimpleSerializeAndDeserialize(+Inf() - 1, 16U); - - SimpleSerializeAndDeserialize(+Inf(), 1U); - SimpleSerializeAndDeserialize(+Nan(), 1U); - } - - Y_UNIT_TEST(TestMulAndRescale) { - CheckMulAndRescale<35,35>("0.99999999999999999999999999999999999", "-0.99999999999999999999999999999999999", "-0.99999999999999999999999999999999998"); - CheckMulAndRescale<35,35>("-0.99999999999999999999999999999999999", "0.33333333333333333333333333333333333", "-0.33333333333333333333333333333333333"); - CheckMulAndRescale<35,35>("0.33333333333333333333333333333333333", "0.33333333333333333333333333333333333", "0.11111111111111111111111111111111111"); - CheckMulAndRescale<35,35>("0.99999999999999999999999999999999999", "0.000000000000001", "0.000000000000001"); - CheckMulAndRescale<35,35>("0.99999999999999999999999999999999999", "0.00000000000000101010101", "0.00000000000000101010101"); - CheckMulAndRescale<35,35>("0.12345678901234567890123456789012345", "0.12345678901234567890123456789012345", "0.01524157875323883675049535156256668"); - - CheckMulAndRescale<35,34>("9.9999999999999999999999999999999999", "-1.9999999999999999999999999999999999", "-inf"); - CheckMulAndRescale<35,34>("3.3333333333333333333333333333333333", "3.3333333333333333333333333333333333", "inf"); - CheckMulAndRescale<35,34>("3.3333333333333333333333333333333333", "1.3333333333333333333333333333333333", "4.4444444444444444444444444444444443"); - CheckMulAndRescale<35,34>("-1.3333333333333333333333333333333333", "1.3333333333333333333333333333333333", "-1.7777777777777777777777777777777777"); - - CheckMulAndRescale<35,34>("-7", "0", "0"); - CheckMulAndRescale<35,34>("inf", "nan", "nan"); - CheckMulAndRescale<35,34>("inf", "0", "nan"); - CheckMulAndRescale<35,34>("-inf", "-inf", "inf"); - } - - Y_UNIT_TEST(TestDivAndRescale) { - CheckDivAndRescale<35,35>("-0.99999999999999999999999999999999999", "0.33333333333333333333333333333333333", "-inf"); - CheckDivAndRescale<35,35>("0.33333333333333333333333333333333333", "-0.33333333333333333333333333333333333", "-inf"); - CheckDivAndRescale<35,35>("0.12345678901234567890123456789012345", "0.12345678901234567890123456789012345", "inf"); - - CheckDivAndRescale<35,34>("9.9999999999999999999999999999999999", "-1.9999999999999999999999999999999999", "-5.0000000000000000000000000000000002"); - CheckDivAndRescale<35,34>("3.3333333333333333333333333333333333", "3.3333333333333333333333333333333333", "1"); - CheckDivAndRescale<35,34>("3.3333333333333333333333333333333333", "1.3333333333333333333333333333333333", "2.5"); - CheckDivAndRescale<35,34>("-1.7777777777777777777777777777777777", "1.3333333333333333333333333333333333", "-1.3333333333333333333333333333333333"); - - CheckDivAndRescale<35,34>("-7", "0", "-inf"); - CheckDivAndRescale<35,34>("inf", "0", "inf"); - CheckDivAndRescale<35,34>("inf", "0", "inf"); - CheckDivAndRescale<35,34>("-inf", "inf", "nan"); - } - - Y_UNIT_TEST(TestWideMul) { - CheckMul<35>("999999999999999", "99999999999999999999", "99999999999999899999000000000000001"); - CheckMul<35>("9999999999999999", "99999999999999999999", "inf"); - CheckMul<35>("-99999999999999999999999999999999999", "10000000000000000000000000000000000", "-inf"); - CheckMul<35>("-99999999999999999999999999999999999", "-1", "99999999999999999999999999999999999"); - CheckMul<35>("-99999999999999999999999999999999999", "-2", "inf"); - - CheckMul<35>("nan", "0", "nan"); - CheckMul<35>("inf", "-inf", "-inf"); - CheckMul<35>("inf", "nan", "nan"); - } } } -} diff --git a/tests/unit/library/yql_common/decimal/yql_wide_int_ut.cpp b/tests/unit/library/decimal/yql_wide_int_ut.cpp similarity index 99% rename from tests/unit/library/yql_common/decimal/yql_wide_int_ut.cpp rename to tests/unit/library/decimal/yql_wide_int_ut.cpp index 9481d4cce3f..d19427f8489 100644 --- a/tests/unit/library/yql_common/decimal/yql_wide_int_ut.cpp +++ b/tests/unit/library/decimal/yql_wide_int_ut.cpp @@ -1,7 +1,8 @@ -#include +#include #include -namespace NYql { +namespace NYdb::NDecimal { + Y_UNIT_TEST_SUITE(TYqlWideIntTest) { template void TestUnary(const T aa) { diff --git a/tests/unit/library/issue/CMakeLists.txt b/tests/unit/library/issue/CMakeLists.txt new file mode 100644 index 00000000000..49e833e9cf7 --- /dev/null +++ b/tests/unit/library/issue/CMakeLists.txt @@ -0,0 +1,12 @@ +add_ydb_test(NAME yql-public-issue_ut + SOURCES + utf8_ut.cpp + yql_issue_ut.cpp + LINK_LIBRARIES + yutil + cpp-testing-unittest_main + yql-public-issue + unicode-normalization + LABELS + unit +) diff --git a/tests/unit/library/issue/utf8_ut.cpp b/tests/unit/library/issue/utf8_ut.cpp new file mode 100644 index 00000000000..283ca65cffc --- /dev/null +++ b/tests/unit/library/issue/utf8_ut.cpp @@ -0,0 +1,18 @@ +#include + +#include + +Y_UNIT_TEST_SUITE(TUtf8Tests) { + Y_UNIT_TEST(Simple) { + UNIT_ASSERT(NYdb::NIssue::IsUtf8("")); + UNIT_ASSERT(NYdb::NIssue::IsUtf8("\x01_ASCII_\x7F")); + UNIT_ASSERT(NYdb::NIssue::IsUtf8("Привет!")); + UNIT_ASSERT(NYdb::NIssue::IsUtf8("\xF0\x9F\x94\xA2")); + + UNIT_ASSERT(!NYdb::NIssue::IsUtf8("\xf5\x80\x80\x80")); + UNIT_ASSERT(!NYdb::NIssue::IsUtf8("\xed\xa6\x80")); + UNIT_ASSERT(!NYdb::NIssue::IsUtf8("\xF0\x9F\x94")); + UNIT_ASSERT(!NYdb::NIssue::IsUtf8("\xE3\x85\xB6\xE7\x9C\xB0\xE3\x9C\xBA\xE2\xAA\x96\xEE\xA2\x8C\xEC\xAF\xB8\xE1\xB2\xBB\xEC\xA3\x9C\xE3\xAB\x8B\xEC\x95\x92\xE1\x8A\xBF\xE2\x8E\x86\xEC\x9B\x8D\xE2\x8E\xAE\xE3\x8A\xA3\xE0\xAC\xBC\xED\xB6\x85")); + UNIT_ASSERT(!NYdb::NIssue::IsUtf8("\xc0\xbe\xd0\xb1\xd0\xbd\xd0\xbe\xd0\xb2\xd0\xbb\xd0\xb5\xd0\xbd\xd0\xb8\xd1\x8e")); + } +} diff --git a/tests/unit/library/issue/yql_issue_ut.cpp b/tests/unit/library/issue/yql_issue_ut.cpp new file mode 100644 index 00000000000..7797729ebc6 --- /dev/null +++ b/tests/unit/library/issue/yql_issue_ut.cpp @@ -0,0 +1,101 @@ +#include + +#include +#include +#include + +#include + +#include + +#include +#include +#include + +#include + +using namespace google::protobuf; +using namespace NYdb::NIssue; + +Y_UNIT_TEST_SUITE(IssueTest) { + Y_UNIT_TEST(Ascii) { + TIssue issue1("тест abc"); + UNIT_ASSERT_VALUES_EQUAL(issue1.GetMessage(), "тест abc"); + TIssue issue2("\xFF abc"); + UNIT_ASSERT_VALUES_EQUAL(issue2.GetMessage(), "? abc"); + TIssue issue3(""); + UNIT_ASSERT_VALUES_EQUAL(issue3.GetMessage(), ""); + TIssue issue4("abc"); + UNIT_ASSERT_VALUES_EQUAL(issue4.GetMessage(), "abc"); + } +} + +Y_UNIT_TEST_SUITE(ToMessage) { + Y_UNIT_TEST(NonUtf8) { + const std::string nonUtf8String = "\x7f\xf8\xf7\xff\xf8\x1f\xff\xf2\xaf\xbf\xfe\xfa\xf5\x7f\xfe\xfa\x27\x20\x7d\x20\x5d\x2e"; + UNIT_ASSERT(!IsUtf(nonUtf8String)); + TIssue issue; + issue.SetMessage(nonUtf8String); + + Ydb::Issue::IssueMessage msg; + IssueToMessage(issue, &msg); + NYdb::TStringType serialized; + UNIT_ASSERT(msg.SerializeToString(&serialized)); + Ydb::Issue::IssueMessage msg2; + UNIT_ASSERT(msg2.ParseFromString(serialized)); + } +} + +Y_UNIT_TEST_SUITE(ToOneLineStringTest) { + Y_UNIT_TEST(OneMessageTest) { + TIssues issues; + issues.AddIssue(TPosition(12, 34, "file.abc"), "error"); + UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "{ file.abc:34:12: Error: error }"); + } + + Y_UNIT_TEST(SubIssuesTest) { + TIssue issue(TPosition(12, 34, "file.abc"), "error"); + TIssue subissue("suberror"); + subissue.AddSubIssue(MakeIntrusive("subsuberror")); + issue.AddSubIssue(MakeIntrusive(subissue)); + + TIssues issues; + issues.AddIssue(issue); + UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "{ file.abc:34:12: Error: error subissue: {
: Error: suberror subissue: {
: Error: subsuberror } } }"); + } + + Y_UNIT_TEST(ManyIssuesTest) { + TIssue issue(TPosition(12, 34, "file.abc"), "error\n"); + issue.AddSubIssue(MakeIntrusive("suberror")); + TIssues issues; + issues.AddIssue(issue); + issues.AddIssue(TPosition(100, 2, "abc.file"), "my\nmessage"); + UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "[ { file.abc:34:12: Error: error subissue: {
: Error: suberror } } { abc.file:2:100: Error: my message } ]"); + } +} + +Y_UNIT_TEST_SUITE(EscapeNonUtf8) { + Y_UNIT_TEST(Escape) { + const std::string nonUtf8String = "\xfe\xfa\xf5\xc2"; + UNIT_ASSERT(!IsUtf(nonUtf8String)); + + // Check that our escaping correctly processes unicode pairs + const std::string toNormalize = "Ёлка"; + const std::string nfd = WideToUTF8(Normalize(UTF8ToWide(toNormalize))); // dots over 'ё' will be separate unicode symbol + const std::string nfc = WideToUTF8(Normalize(UTF8ToWide(toNormalize))); // dots over 'ё' will be with with their letter + UNIT_ASSERT_STRINGS_UNEQUAL(nfc, nfd); + std::pair nonUtf8Messages[] = { + { nonUtf8String, "????" }, + { TStringBuilder() << nonUtf8String << "Failed to parse file " << nonUtf8String << "עברית" << nonUtf8String, "????Failed to parse file ????עברית????" }, + { nfd, nfd }, + { nfc, nfc }, + { TStringBuilder() << nfc << nonUtf8String << nfd, TStringBuilder() << nfc << "????" << nfd }, + { TStringBuilder() << nfd << nonUtf8String << nfc, TStringBuilder() << nfd << "????" << nfc }, + }; + + for (const auto& [src, dst] : nonUtf8Messages) { + TIssue issue(src); + UNIT_ASSERT_STRINGS_EQUAL(issue.GetMessage(), dst); + } + } +} diff --git a/tests/unit/library/yql_common/CMakeLists.txt b/tests/unit/library/yql_common/CMakeLists.txt deleted file mode 100644 index af676437779..00000000000 --- a/tests/unit/library/yql_common/CMakeLists.txt +++ /dev/null @@ -1,44 +0,0 @@ -add_ydb_test(NAME yql-utils_ut - SOURCES - utils/utf8_ut.cpp - LINK_LIBRARIES - yutil - cpp-testing-unittest_main - yql-utils - LABELS - unit -) - -add_ydb_test(NAME yql-public-issue_ut - SOURCES - issue/yql_issue_ut.cpp - LINK_LIBRARIES - yutil - cpp-testing-unittest_main - yql-public-issue - unicode-normalization - LABELS - unit -) - -add_ydb_test(NAME yql-public-decimal_ut - SOURCES - decimal/yql_decimal_ut.cpp - LINK_LIBRARIES - yutil - cpp-testing-unittest_main - yql-public-decimal - LABELS - unit -) - -add_ydb_test(NAME yql-public-wide_int_ut - SOURCES - decimal/yql_wide_int_ut.cpp - LINK_LIBRARIES - yutil - cpp-testing-unittest_main - yql-public-decimal - LABELS - unit -) diff --git a/tests/unit/library/yql_common/issue/yql_issue_ut.cpp b/tests/unit/library/yql_common/issue/yql_issue_ut.cpp deleted file mode 100644 index 1f0d2d02b1f..00000000000 --- a/tests/unit/library/yql_common/issue/yql_issue_ut.cpp +++ /dev/null @@ -1,252 +0,0 @@ -#include -#include - -#include -#include -#include -#include - -#include - -#include - -#include -#include -#include - -#include -#include - -using namespace google::protobuf; -using namespace NYql; - -void ensureMessageTypesSame(const Descriptor* a, const Descriptor* b, std::unordered_set* visitedTypes); -void ensureFieldDescriptorsSame(const FieldDescriptor* a, const FieldDescriptor* b, std::unordered_set* visitedTypes) { - UNIT_ASSERT(a); - UNIT_ASSERT(b); - - UNIT_ASSERT_VALUES_EQUAL(FieldDescriptor::TypeName(a->type()), FieldDescriptor::TypeName(b->type())); - UNIT_ASSERT_VALUES_EQUAL((int)a->label(), (int)b->label()); - UNIT_ASSERT_VALUES_EQUAL(a->name(), b->name()); - UNIT_ASSERT_VALUES_EQUAL(a->number(), b->number()); - UNIT_ASSERT_VALUES_EQUAL(a->is_repeated(), b->is_repeated()); - UNIT_ASSERT_VALUES_EQUAL(a->is_packed(), b->is_packed()); - UNIT_ASSERT_VALUES_EQUAL(a->index(), b->index()); - if (a->type() == FieldDescriptor::TYPE_MESSAGE || a->type() == FieldDescriptor::TYPE_GROUP) { - ensureMessageTypesSame(a->message_type(), b->message_type(), visitedTypes); - } -} - -void ensureMessageTypesSame(const Descriptor* a, const Descriptor* b, std::unordered_set* visitedTypes) { - UNIT_ASSERT(a); - UNIT_ASSERT(b); - if (!visitedTypes->insert(a->name()).second) { - return; - } - - UNIT_ASSERT_VALUES_EQUAL(a->name(), b->name()); - UNIT_ASSERT_VALUES_EQUAL(a->field_count(), b->field_count()); - - for (int i = 0; i < a->field_count(); i++) { - ensureFieldDescriptorsSame(a->field(i), b->field(i), visitedTypes); - } -} - -Y_UNIT_TEST_SUITE(IssueTest) { - Y_UNIT_TEST(Ascii) { - TIssue issue1("тест abc"); - UNIT_ASSERT_VALUES_EQUAL(issue1.GetMessage(), "тест abc"); - TIssue issue2("\xFF abc"); - UNIT_ASSERT_VALUES_EQUAL(issue2.GetMessage(), "? abc"); - TIssue issue3(""); - UNIT_ASSERT_VALUES_EQUAL(issue3.GetMessage(), ""); - TIssue issue4("abc"); - UNIT_ASSERT_VALUES_EQUAL(issue4.GetMessage(), "abc"); - } -} - -Y_UNIT_TEST_SUITE(IssueProtoTest) { - Y_UNIT_TEST(KikimrYqlSameLayout) { - Ydb::Issue::IssueMessage yqlIssue; - NYql::NIssue::NProto::IssueMessage kikimrIssue; - std::unordered_set visitedTypes; - ensureMessageTypesSame(yqlIssue.GetDescriptor(), kikimrIssue.GetDescriptor(), &visitedTypes); - } - - Y_UNIT_TEST(BinarySerialization) { - TIssue issueTo("root_issue"); - std::string bin = IssueToBinaryMessage(issueTo); - TIssue issueFrom = IssueFromBinaryMessage(bin); - UNIT_ASSERT_EQUAL(issueTo, issueFrom); - } - - Y_UNIT_TEST(WrongBinStringException) { - UNIT_ASSERT_EXCEPTION(IssueFromBinaryMessage("qqq"), yexception); - } -} - - -Y_UNIT_TEST_SUITE(TextWalkerTest) { - using namespace std::string_view_literals; - Y_UNIT_TEST(BasicTest) { - TPosition pos; - pos.Row = 1; - - TTextWalker walker(pos, false); - walker.Advance("a\r\taa"sv); - - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(5, 1)); - walker.Advance("\na"sv); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(1, 2)); - } - - Y_UNIT_TEST(CrLfTest) { - TPosition pos; - pos.Row = 1; - - TTextWalker walker(pos, false); - walker.Advance("a\raa\r"sv); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 1)); - walker.Advance('\n'); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 1)); - walker.Advance("\r\r\ra"sv); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 2)); - walker.Advance('\r'); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 2)); - walker.Advance('\n'); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 2)); - walker.Advance('a'); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(1, 3)); - } - - Y_UNIT_TEST(UnicodeTest) { - { - TPosition pos; - pos.Row = 1; - - TTextWalker walker(pos, false); - walker.Advance(TStringBuf("привет")); - - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(12, 1)); - } - - { - TPosition pos; - pos.Row = 1; - - TTextWalker walker(pos, true); - walker.Advance(TStringBuf("привет")); - - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(6, 1)); - } - } -} - -Y_UNIT_TEST_SUITE(ToOneLineStringTest) { - Y_UNIT_TEST(OneMessageTest) { - TIssues issues; - issues.AddIssue(TPosition(12, 34, "file.abc"), "error"); - UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "{ file.abc:34:12: Error: error }"); - } - - Y_UNIT_TEST(SubIssuesTest) { - TIssue issue(TPosition(12, 34, "file.abc"), "error"); - TIssue subissue("suberror"); - subissue.AddSubIssue(MakeIntrusive("subsuberror")); - issue.AddSubIssue(MakeIntrusive(subissue)); - - TIssues issues; - issues.AddIssue(issue); - UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "{ file.abc:34:12: Error: error subissue: {
: Error: suberror subissue: {
: Error: subsuberror } } }"); - } - - Y_UNIT_TEST(ManyIssuesTest) { - TIssue issue(TPosition(12, 34, "file.abc"), "error\n"); - issue.AddSubIssue(MakeIntrusive("suberror")); - TIssues issues; - issues.AddIssue(issue); - issues.AddIssue(TPosition(100, 2, "abc.file"), "my\nmessage"); - UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "[ { file.abc:34:12: Error: error subissue: {
: Error: suberror } } { abc.file:2:100: Error: my message } ]"); - } -} - -Y_UNIT_TEST_SUITE(ToStreamTest) { - template - void CheckSerializationToStream(const TIssues& issues, const std::string& expected) { - google::protobuf::RepeatedPtrField protoIssues; - IssuesToMessage(issues, &protoIssues); - TStringBuilder stream; - stream << protoIssues; - UNIT_ASSERT_STRINGS_EQUAL(stream, expected); - }; - - Y_UNIT_TEST(OneMessageTest) { - TIssues issues; - issues.AddIssue("error"); - CheckSerializationToStream(issues, "{ message: \"error\" severity: 1 }"); - CheckSerializationToStream(issues, "{ message: \"error\" issue_code: 0 severity: 1 }"); - } - - Y_UNIT_TEST(SubIssuesTest) { - TIssue issue(TPosition(12, 34, "file.abc"), "error"); - TIssue subissue("suberror"); - subissue.AddSubIssue(MakeIntrusive("subsuberror")); - issue.AddSubIssue(MakeIntrusive(subissue)); - TIssues issues; - issues.AddIssue(issue); - CheckSerializationToStream(issues, "{ position { row: 34 column: 12 file: \"file.abc\" } message: \"error\" end_position { row: 34 column: 12 } severity: 1 issues { message: \"suberror\" severity: 1 issues { message: \"subsuberror\" severity: 1 } } }"); - CheckSerializationToStream(issues, "{ position { row: 34 column: 12 file: \"file.abc\" } message: \"error\" end_position { row: 34 column: 12 } issue_code: 0 severity: 1 issues { message: \"suberror\" issue_code: 0 severity: 1 issues { message: \"subsuberror\" issue_code: 0 severity: 1 } } }"); - } - - Y_UNIT_TEST(ManyIssuesTest) { - TIssue issue(TPosition(12, 34, "file.abc"), "error"); - issue.AddSubIssue(MakeIntrusive("suberror")); - TIssues issues; - issues.AddIssue(issue); - issues.AddIssue(TPosition(100, 2, "abc.file"), "my message"); - CheckSerializationToStream(issues, "{ position { row: 34 column: 12 file: \"file.abc\" } message: \"error\" end_position { row: 34 column: 12 } severity: 1 issues { message: \"suberror\" severity: 1 } }{ position { row: 2 column: 100 file: \"abc.file\" } message: \"my message\" end_position { row: 2 column: 100 } severity: 1 }"); - CheckSerializationToStream(issues, "{ position { row: 34 column: 12 file: \"file.abc\" } message: \"error\" end_position { row: 34 column: 12 } issue_code: 0 severity: 1 issues { message: \"suberror\" issue_code: 0 severity: 1 } }{ position { row: 2 column: 100 file: \"abc.file\" } message: \"my message\" end_position { row: 2 column: 100 } issue_code: 0 severity: 1 }"); - } -} - -Y_UNIT_TEST_SUITE(ToMessage) { - Y_UNIT_TEST(NonUtf8) { - const std::string nonUtf8String = "\x7f\xf8\xf7\xff\xf8\x1f\xff\xf2\xaf\xbf\xfe\xfa\xf5\x7f\xfe\xfa\x27\x20\x7d\x20\x5d\x2e"; - UNIT_ASSERT(!IsUtf(nonUtf8String)); - TIssue issue; - issue.SetMessage(nonUtf8String); - - Ydb::Issue::IssueMessage msg; - IssueToMessage(issue, &msg); - NYdb::TStringType serialized; - UNIT_ASSERT(msg.SerializeToString(&serialized)); - Ydb::Issue::IssueMessage msg2; - UNIT_ASSERT(msg2.ParseFromString(serialized)); - } -} - -Y_UNIT_TEST_SUITE(EscapeNonUtf8) { - Y_UNIT_TEST(Escape) { - const std::string nonUtf8String = "\xfe\xfa\xf5\xc2"; - UNIT_ASSERT(!IsUtf(nonUtf8String)); - - // Check that our escaping correctly processes unicode pairs - const std::string toNormalize = "Ёлка"; - const std::string nfd = WideToUTF8(Normalize(UTF8ToWide(toNormalize))); // dots over 'ё' will be separate unicode symbol - const std::string nfc = WideToUTF8(Normalize(UTF8ToWide(toNormalize))); // dots over 'ё' will be with with their letter - UNIT_ASSERT_STRINGS_UNEQUAL(nfc, nfd); - std::pair nonUtf8Messages[] = { - { nonUtf8String, "????" }, - { TStringBuilder() << nonUtf8String << "Failed to parse file " << nonUtf8String << "עברית" << nonUtf8String, "????Failed to parse file ????עברית????" }, - { nfd, nfd }, - { nfc, nfc }, - { TStringBuilder() << nfc << nonUtf8String << nfd, TStringBuilder() << nfc << "????" << nfd }, - { TStringBuilder() << nfd << nonUtf8String << nfc, TStringBuilder() << nfd << "????" << nfc }, - }; - - for (const auto& [src, dst] : nonUtf8Messages) { - TIssue issue(src); - UNIT_ASSERT_STRINGS_EQUAL(issue.GetMessage(), dst); - } - } -} diff --git a/tests/unit/library/yql_common/utils/utf8_ut.cpp b/tests/unit/library/yql_common/utils/utf8_ut.cpp deleted file mode 100644 index d4ce27e6746..00000000000 --- a/tests/unit/library/yql_common/utils/utf8_ut.cpp +++ /dev/null @@ -1,99 +0,0 @@ -#include - -#include - -Y_UNIT_TEST_SUITE(TUtf8Tests) { - Y_UNIT_TEST(Simple) { - UNIT_ASSERT(NYql::IsUtf8("")); - UNIT_ASSERT(NYql::IsUtf8("\x01_ASCII_\x7F")); - UNIT_ASSERT(NYql::IsUtf8("Привет!")); - UNIT_ASSERT(NYql::IsUtf8("\xF0\x9F\x94\xA2")); - - UNIT_ASSERT(!NYql::IsUtf8("\xf5\x80\x80\x80")); - UNIT_ASSERT(!NYql::IsUtf8("\xed\xa6\x80")); - UNIT_ASSERT(!NYql::IsUtf8("\xF0\x9F\x94")); - UNIT_ASSERT(!NYql::IsUtf8("\xE3\x85\xB6\xE7\x9C\xB0\xE3\x9C\xBA\xE2\xAA\x96\xEE\xA2\x8C\xEC\xAF\xB8\xE1\xB2\xBB\xEC\xA3\x9C\xE3\xAB\x8B\xEC\x95\x92\xE1\x8A\xBF\xE2\x8E\x86\xEC\x9B\x8D\xE2\x8E\xAE\xE3\x8A\xA3\xE0\xAC\xBC\xED\xB6\x85")); - UNIT_ASSERT(!NYql::IsUtf8("\xc0\xbe\xd0\xb1\xd0\xbd\xd0\xbe\xd0\xb2\xd0\xbb\xd0\xb5\xd0\xbd\xd0\xb8\xd1\x8e")); - } - - Y_UNIT_TEST(CharSize) { - UNIT_ASSERT_VALUES_EQUAL(NYql::WideCharSize(' '), 1); - UNIT_ASSERT_VALUES_EQUAL(NYql::WideCharSize('\x00'), 1); - UNIT_ASSERT_VALUES_EQUAL(NYql::WideCharSize('\x7F'), 1); - UNIT_ASSERT_VALUES_EQUAL(NYql::WideCharSize('\xD1'), 2); - UNIT_ASSERT_VALUES_EQUAL(NYql::WideCharSize('\xF0'), 4); - UNIT_ASSERT_VALUES_EQUAL(NYql::WideCharSize('\xFF'), 0); - } - - Y_UNIT_TEST(RoundingDown) { - auto checkDown = [](std::string_view in, std::string_view out) { - auto res = NYql::RoundToNearestValidUtf8(in, true); - UNIT_ASSERT(res); - UNIT_ASSERT(NYql::IsUtf8(*res)); - UNIT_ASSERT_VALUES_EQUAL(*res, out); - UNIT_ASSERT(*res <= in); - }; - checkDown("привет", "привет"); - checkDown("тест\x80", "тест\x7f"); - checkDown("привет\xf5", "привет\xf4\x8f\xbf\xbf"); - checkDown("тест2\xee\x80\x7f", "тест2\xed\x9f\xbf"); - checkDown("ага\xf0\xaa\xaa\xff", "ага\xf0\xaa\xaa\xbf"); - } - - Y_UNIT_TEST(RoundingUp) { - auto checkUp = [](std::string_view in, std::string_view out) { - auto res = NYql::RoundToNearestValidUtf8(in, false); - UNIT_ASSERT(res); - UNIT_ASSERT(NYql::IsUtf8(*res)); - UNIT_ASSERT_VALUES_EQUAL(*res, out); - UNIT_ASSERT(*res >= in); - }; - - checkUp("", ""); - checkUp("привет", "привет"); - checkUp("а\xf6", "б"); - checkUp("\xf4\x8f\xbf\xbfа\xf4\x8f\xbf\xbf\xf5", "\xf4\x8f\xbf\xbfб"); - UNIT_ASSERT(!NYql::RoundToNearestValidUtf8("\xf4\x8f\xbf\xbf\xf5", false)); - UNIT_ASSERT(!NYql::RoundToNearestValidUtf8("\xf5", false)); - checkUp("тест\x80", "тест\xc2\x80"); - checkUp("тест\xdf", "тест\xdf\x80"); - checkUp("тест\xf0\x90\xff", "тест\xf0\x91\x80\x80"); - checkUp("ааа\xff", "ааб"); - } - - Y_UNIT_TEST(NextValid) { - auto checkNext = [](std::string_view in, std::string_view out) { - auto res = NYql::NextValidUtf8(in); - UNIT_ASSERT(res); - UNIT_ASSERT(NYql::IsUtf8(*res)); - UNIT_ASSERT_VALUES_EQUAL(*res, out); - UNIT_ASSERT(*res > in); - }; - - UNIT_ASSERT(!NYql::NextValidUtf8("")); - checkNext("привет", "привеу"); - checkNext("а", "б"); - checkNext(std::string_view("\x00", 1), "\x01"); - checkNext("\xf4\x8f\xbf\xbfа\xf4\x8f\xbf\xbf", "\xf4\x8f\xbf\xbfб"); - UNIT_ASSERT(!NYql::NextValidUtf8("\xf4\x8f\xbf\xbf")); - UNIT_ASSERT(!NYql::NextValidUtf8("\xf4\x8f\xbf\xbf\xf4\x8f\xbf\xbf")); - } - - Y_UNIT_TEST(NextValidString) { - auto checkNext = [](std::string_view in, std::string_view out) { - auto res = NYql::NextLexicographicString(in); - UNIT_ASSERT(res); - UNIT_ASSERT_VALUES_EQUAL(*res, out); - UNIT_ASSERT(*res > in); - }; - - UNIT_ASSERT(!NYql::NextLexicographicString("")); - checkNext("привет", "привеу"); - checkNext("а", "б"); - checkNext(std::string_view("\x00", 1), "\x01"); - checkNext("\xf4\x8f\xbf\xbfа\xf4\x8f\xbf\xbf", "\xf4\x8f\xbf\xbfа\xf4\x8f\xbf\xc0"); - UNIT_ASSERT(!NYql::NextLexicographicString("\xff")); - UNIT_ASSERT(!NYql::NextLexicographicString("\xff\xff")); - checkNext(std::string_view("x\x00\xff\xff", 4), "x\x01"); - } -}