Skip to content

Commit 4f23b8f

Browse files
robot-pigletblinkov
robot-piglet
authored and
blinkov
committed
Intermediate changes
commit_hash:e2da3ad430fabaa84a74178b1f2103b09ac69ae7
1 parent c4662ad commit 4f23b8f

27 files changed

+202
-131
lines changed

‎library/cpp/neh/http_common.cpp

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,5 +231,32 @@ namespace NNeh {
231231
return NeedGetRequestFor(scheme) || NeedPostRequestFor(scheme);
232232
}
233233
}
234+
235+
HttpCodes GetHttpCode(const IRequest::TResponseError& error) {
236+
switch (error) {
237+
case IRequest::TResponseError::BadRequest:
238+
return HttpCodes::HTTP_BAD_REQUEST;
239+
case IRequest::TResponseError::Forbidden:
240+
return HttpCodes::HTTP_FORBIDDEN;
241+
case IRequest::TResponseError::NotExistService:
242+
return HttpCodes::HTTP_NOT_FOUND;
243+
case IRequest::TResponseError::TooManyRequests:
244+
return HttpCodes::HTTP_TOO_MANY_REQUESTS;
245+
case IRequest::TResponseError::InternalError:
246+
return HttpCodes::HTTP_INTERNAL_SERVER_ERROR;
247+
case IRequest::TResponseError::NotImplemented:
248+
return HttpCodes::HTTP_NOT_IMPLEMENTED;
249+
case IRequest::TResponseError::BadGateway:
250+
return HttpCodes::HTTP_BAD_GATEWAY;
251+
case IRequest::TResponseError::ServiceUnavailable:
252+
return HttpCodes::HTTP_SERVICE_UNAVAILABLE;
253+
case IRequest::TResponseError::BandwidthLimitExceeded:
254+
return HttpCodes::HTTP_BANDWIDTH_LIMIT_EXCEEDED;
255+
case IRequest::TResponseError::MaxResponseError:
256+
ythrow yexception() << TStringBuf("unknow type of error");
257+
258+
Y_UNREACHABLE();
259+
}
260+
}
234261
}
235262

‎library/cpp/neh/http_common.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#include <util/stream/mem.h>
88
#include <util/stream/output.h>
99
#include <library/cpp/deprecated/atomic/atomic.h>
10+
#include <library/cpp/http/misc/httpcodes.h>
1011

1112
#include "location.h"
1213
#include "neh.h"
@@ -298,4 +299,6 @@ namespace NNeh {
298299

299300
bool IsHttpScheme(TStringBuf scheme);
300301
}
302+
303+
HttpCodes GetHttpCode(const IRequest::TResponseError&);
301304
}

‎library/cpp/neh/https.cpp

Lines changed: 1 addition & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1394,37 +1394,7 @@ namespace NNeh {
13941394
return;
13951395
}
13961396

1397-
switch (*error) {
1398-
case IRequest::TResponseError::BadRequest:
1399-
os << HttpCodeStrEx(HttpCodes::HTTP_BAD_REQUEST);
1400-
break;
1401-
case IRequest::TResponseError::Forbidden:
1402-
os << HttpCodeStrEx(HttpCodes::HTTP_FORBIDDEN);
1403-
break;
1404-
case IRequest::TResponseError::NotExistService:
1405-
os << HttpCodeStrEx(HttpCodes::HTTP_NOT_FOUND);
1406-
break;
1407-
case IRequest::TResponseError::TooManyRequests:
1408-
os << HttpCodeStrEx(HttpCodes::HTTP_TOO_MANY_REQUESTS);
1409-
break;
1410-
case IRequest::TResponseError::InternalError:
1411-
os << HttpCodeStrEx(HttpCodes::HTTP_INTERNAL_SERVER_ERROR);
1412-
break;
1413-
case IRequest::TResponseError::NotImplemented:
1414-
os << HttpCodeStrEx(HttpCodes::HTTP_NOT_IMPLEMENTED);
1415-
break;
1416-
case IRequest::TResponseError::BadGateway:
1417-
os << HttpCodeStrEx(HttpCodes::HTTP_BAD_GATEWAY);
1418-
break;
1419-
case IRequest::TResponseError::ServiceUnavailable:
1420-
os << HttpCodeStrEx(HttpCodes::HTTP_SERVICE_UNAVAILABLE);
1421-
break;
1422-
case IRequest::TResponseError::BandwidthLimitExceeded:
1423-
os << HttpCodeStrEx(HttpCodes::HTTP_BANDWIDTH_LIMIT_EXCEEDED);
1424-
break;
1425-
case IRequest::TResponseError::MaxResponseError:
1426-
ythrow yexception() << TStringBuf("unknow type of error");
1427-
}
1397+
os << HttpCodeStrEx(GetHttpCode(*error));
14281398
}
14291399

14301400
public:

‎yql/essentials/tests/common/test_framework/yql_utils.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ def do_custom_query_check(res, sql_query):
5252

5353
def do_custom_error_check(res, sql_query):
5454
err_string = None
55-
custom_error = re.search(r"/\* custom error:(.*)\*/", sql_query)
55+
custom_error = re.search(r"/\* custom error:(.*?)\*/", sql_query, re.DOTALL)
5656
if custom_error:
5757
err_string = custom_error.group(1).strip()
5858
assert err_string, 'Expected custom error check in test.\nTest error: %s' % res.std_err

‎yql/essentials/udfs/common/python/bindings/py_callable_ut.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#include "ut3/py_test_engine.h"
1+
#include "py_test_engine.h"
22

33
#include <library/cpp/testing/unittest/registar.h>
44

‎yql/essentials/udfs/common/python/bindings/py_cast.cpp

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
#include <library/cpp/containers/stack_vector/stack_vec.h>
2424

25+
#include <util/generic/scope.h>
2526
#include <util/string/join.h>
2627
#include <util/string/builder.h>
2728

@@ -252,7 +253,7 @@
252253
template <> \
253254
bool TryPyCast(PyObject* value, Type& result) { \
254255
if (PyUnicode_Check(value)) { \
255-
const TPyObjectPtr utf8(PyUnicode_AsUTF8String(value)); \
256+
const TPyObjectPtr utf8(AsUtf8StringOrThrow(value)); \
256257
char* str = nullptr; \
257258
Py_ssize_t size = 0; \
258259
int rc = PyBytes_AsStringAndSize(utf8.Get(), &str, &size); \
@@ -279,6 +280,22 @@ namespace NPython {
279280

280281
using namespace NKikimr;
281282

283+
namespace {
284+
285+
NPython::TPyObjectPtr AsUtf8StringOrThrow(PyObject* obj) {
286+
auto* utf8String = PyUnicode_AsUTF8String(obj);
287+
if (!utf8String) {
288+
Y_ENSURE(PyErr_Occurred());
289+
Y_DEFER {
290+
PyErr_Clear();
291+
};
292+
throw yexception() << "Failed to convert the string to UTF-8 format. Original message is:\n" << GetLastErrorAsString() << "\n";
293+
}
294+
return NPython::TPyObjectPtr(utf8String);
295+
}
296+
297+
} // namespace
298+
282299
inline void ThrowCastTypeException(PyObject* value, TStringBuf toType) {
283300
throw yexception() << "Can't cast object '" << Py_TYPE(value)->tp_name << "' to " << toType
284301
<< "; Object repr: " << PyObjectRepr(value);
@@ -548,7 +565,7 @@ NUdf::TUnboxedValue FromPyData(
548565
case NUdf::TDataType<NUdf::TUtf8>::Id:
549566
case NUdf::TDataType<NUdf::TJson>::Id:
550567
if (PyUnicode_Check(value)) {
551-
const TPyObjectPtr uif8(PyUnicode_AsUTF8String(value));
568+
const TPyObjectPtr uif8(AsUtf8StringOrThrow(value));
552569
return ctx->ValueBuilder->NewString(PyCast<NUdf::TStringRef>(uif8.Get()));
553570
}
554571
throw yexception() << "Python object " << PyObjectRepr(value) << " has invalid value for unicode";
@@ -557,7 +574,7 @@ NUdf::TUnboxedValue FromPyData(
557574
case NUdf::TDataType<NUdf::TJson>::Id:
558575
case NUdf::TDataType<NUdf::TUtf8>::Id: {
559576
if (PyUnicode_Check(value)) {
560-
const TPyObjectPtr utf8(PyUnicode_AsUTF8String(value));
577+
const TPyObjectPtr utf8(AsUtf8StringOrThrow(value));
561578
return ctx->ValueBuilder->NewString(PyCast<NUdf::TStringRef>(utf8.Get()));
562579
}
563580

‎yql/essentials/udfs/common/python/bindings/py_cast_ut.cpp

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,39 @@
1-
#include "ut3/py_test_engine.h"
1+
#include "py_test_engine.h"
22

33
#include <library/cpp/testing/unittest/registar.h>
4+
#include <util/string/strip.h>
45

56
using namespace NPython;
67

8+
namespace {
9+
template <typename TType>
10+
void TestBadUtf8Encode() {
11+
#if PY_MAJOR_VERSION == 2
12+
// In Python 2, strings can encode single surrogate pairs, so this issue does not occur there.
13+
return;
14+
#endif // PY_MAJOR_VERSION == 2
15+
16+
TPythonTestEngine engine;
17+
18+
constexpr char programToRun[] = R"(
19+
def Test():
20+
return "\uDC00"
21+
)";
22+
constexpr char expectedError[] = R"(
23+
Failed to convert the string to UTF-8 format. Original message is:
24+
UnicodeEncodeError: 'utf-8' codec can't encode character '\udc00' in position 0: surrogates not allowed
25+
)";
26+
27+
UNIT_ASSERT_EXCEPTION_CONTAINS(
28+
engine.ToMiniKQL<TType>(
29+
StripString(TString(programToRun)),
30+
[](const NUdf::TUnboxedValuePod& value) {
31+
Y_UNUSED(value);
32+
}),
33+
yexception, StripString(TString(expectedError)));
34+
}
35+
} // namespace
36+
737
Y_UNIT_TEST_SUITE(TPyCastTest) {
838
Y_UNIT_TEST(FromPyStrToInt) {
939
TPythonTestEngine engine;
@@ -87,4 +117,11 @@ Y_UNIT_TEST_SUITE(TPyCastTest) {
87117
yexception, "Cast error object " RETVAL " to Long");
88118
}
89119

90-
}
120+
Y_UNIT_TEST(BadFromPythonUtf8) {
121+
TestBadUtf8Encode<NUdf::TUtf8>();
122+
}
123+
124+
Y_UNIT_TEST(BadFromPythonJson) {
125+
TestBadUtf8Encode<NUdf::TJson>();
126+
}
127+
} // Y_UNIT_TEST_SUITE(TPyCastTest)

‎yql/essentials/udfs/common/python/bindings/py_decimal_ut.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#include "ut3/py_test_engine.h"
1+
#include "py_test_engine.h"
22

33
#include <library/cpp/testing/unittest/registar.h>
44

‎yql/essentials/udfs/common/python/bindings/py_dict_ut.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#include "ut3/py_test_engine.h"
1+
#include "py_test_engine.h"
22

33
#include <yql/essentials/public/udf/udf_ut_helpers.h>
44

‎yql/essentials/udfs/common/python/bindings/py_list_ut.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#include "ut3/py_test_engine.h"
1+
#include "py_test_engine.h"
22

33
#include <yql/essentials/public/udf/udf_ut_helpers.h>
44

0 commit comments

Comments
 (0)