Issue #170 Fix src/library/charset tests (#261)

Author: tsayukov

src/library/charset/ci_string_ut.cpp

@@ -7,12 +7,13 @@ bool CanBeEncoded(std::u16string_view text, ECharset encoding) {
     wchar16 decodeBuf[BUFSIZE];
 
     while (!text.empty()) {
-        std::u16string_view src = text;
-        src.remove_prefix(LEN);
+        std::u16string_view src = text.substr(0, LEN);
+        text.remove_prefix(std::min(LEN, text.size()));
         std::string_view encoded = NDetail::NBaseOps::Recode(src, encodeBuf, encoding);
         std::u16string_view decoded = NDetail::NBaseOps::Recode(encoded, decodeBuf, encoding);
-        if (decoded != src)
+        if (decoded != src) {
             return false;
+        }
     }
 
     return true;

src/library/charset/lite/ut/ya.make

@@ -1,6 +1,7 @@
 add_subdirectory(cache)
 add_subdirectory(case_insensitive_string)
 add_subdirectory(cgiparam)
+add_subdirectory(charset)
 add_subdirectory(containers)
 add_subdirectory(coroutine)
 add_subdirectory(json)

src/library/charset/ut/ya.make

@@ -0,0 +1,49 @@
+add_ydb_test(NAME library-charset-ci_string_ut
+  SOURCES
+    ci_string_ut.cpp
+  INCLUDE_DIRS
+    ${YDB_SDK_SOURCE_DIR}/src/library/charset
+  LINK_LIBRARIES
+    library-charset-lite
+    cpp-testing-unittest_main
+)
+
+add_ydb_test(NAME library-charset-codepage_ut
+  SOURCES
+    codepage_ut.cpp
+  INCLUDE_DIRS
+    ${YDB_SDK_SOURCE_DIR}/src/library/charset
+  LINK_LIBRARIES
+    library-charset
+    cpp-testing-unittest_main
+)
+
+add_ydb_test(NAME library-charset-iconv_ut
+  SOURCES
+    iconv_ut.cpp
+  INCLUDE_DIRS
+    ${YDB_SDK_SOURCE_DIR}/src/library/charset
+  LINK_LIBRARIES
+    library-charset
+    cpp-testing-unittest_main
+)
+
+add_ydb_test(NAME library-charset-recyr_int_ut
+  SOURCES
+    recyr_int_ut.cpp
+  INCLUDE_DIRS
+    ${YDB_SDK_SOURCE_DIR}/src/library/charset
+  LINK_LIBRARIES
+    library-charset
+    cpp-testing-unittest_main
+)
+
+add_ydb_test(NAME library-charset-wide_ut
+  SOURCES
+    wide_ut.cpp
+  INCLUDE_DIRS
+    ${YDB_SDK_SOURCE_DIR}/src/library/charset
+  LINK_LIBRARIES
+    library-charset
+    cpp-testing-unittest_main
+)

src/library/charset/wide.cpp

@@ -0,0 +1,75 @@
+#include "ci_string.h"
+
+#include <src/library/testing/unittest/registar.h>
+
+Y_UNIT_TEST_SUITE(TCiStringTest) {
+    Y_UNIT_TEST(TestOperators) {
+        TCiString s("0123456");
+
+        const auto x = "x";
+        const auto y = "y";
+        const auto z = "z";
+
+        // operator +=
+        s += TCiString(x);
+        UNIT_ASSERT(s == "0123456x");
+
+        s += y;
+        UNIT_ASSERT(s == "0123456xy");
+
+        s += *z;
+        UNIT_ASSERT(s == "0123456xyz");
+
+        // operator +
+        s = "0123456";
+        s = s + TCiString(x);
+        UNIT_ASSERT(s == "0123456x");
+
+        s = s + y;
+        UNIT_ASSERT(s == "0123456xy");
+
+        s = s + *z;
+        UNIT_ASSERT(s == "0123456xyz");
+
+        // operator !=
+        s = "012345";
+        const auto xyz = "xyz";
+        UNIT_ASSERT(s != TCiString(xyz));
+        UNIT_ASSERT(s != xyz);
+        UNIT_ASSERT(xyz != s);
+
+        // operator <
+        UNIT_ASSERT_EQUAL(s < TCiString(xyz), true);
+        UNIT_ASSERT_EQUAL(s < xyz, true);
+        UNIT_ASSERT_EQUAL(xyz < s, false);
+
+        // operator <=
+        UNIT_ASSERT_EQUAL(s <= TCiString(xyz), true);
+        UNIT_ASSERT_EQUAL(s <= xyz, true);
+        UNIT_ASSERT_EQUAL(xyz <= s, false);
+
+        // operator >
+        UNIT_ASSERT_EQUAL(s > TCiString(xyz), false);
+        UNIT_ASSERT_EQUAL(s > xyz, false);
+        UNIT_ASSERT_EQUAL(xyz > s, true);
+
+        // operator >=
+        UNIT_ASSERT_EQUAL(s >= TCiString(xyz), false);
+        UNIT_ASSERT_EQUAL(s >= xyz, false);
+        UNIT_ASSERT_EQUAL(xyz >= s, true);
+    }
+
+    Y_UNIT_TEST(TestOperatorsCI) {
+        TCiString s("ABCD");
+        UNIT_ASSERT(s > "abc0123456xyz");
+        UNIT_ASSERT("abc0123456xyz" < s);
+        UNIT_ASSERT(s == "abcd");
+        UNIT_ASSERT("abcd" == s);
+    }
+
+    Y_UNIT_TEST(TestSpecial) {
+        TCiString ss = "0123456"; // type 'TCiString' is used as is
+        size_t hash_val = THash<TCiString>{}(ss);
+        UNIT_ASSERT(hash_val == 1489244);
+    }
+}

tests/library/CMakeLists.txt

@@ -1,10 +1,10 @@
 #include "codepage.h"
-#include "wide.h"
 
-#include <src/library/testing/unittest/registar.h>
+#include <ydb-cpp-sdk/util/system/yassert.h>
 
 #include <src/util/charset/utf8.h>
-#include <ydb-cpp-sdk/util/system/yassert.h>
+
+#include <src/library/testing/unittest/registar.h>
 
 #if defined(_MSC_VER)
 #pragma warning(disable : 4309) /*truncation of constant value*/
@@ -72,14 +72,15 @@ void TCodepageTest::TestUTF() {
         size_t rune_len;
         size_t ref_len = 0;
 
-        if (i < 0x80)
+        if (i < 0x80) {
             ref_len = 1;
-        else if (i < 0x800)
+        } else if (i < 0x800) {
             ref_len = 2;
-        else if (i < 0x10000)
+        } else if (i < 0x10000) {
             ref_len = 3;
-        else
+        } else {
             ref_len = 4;
+        }
 
         RECODE_RESULT res = SafeWriteUTF8Char(i, rune_len, buffer, buffer + 32);
         UNIT_ASSERT(res == RECODE_OK);
@@ -143,7 +144,7 @@ void TCodepageTest::TestUTF() {
     };
     for (size_t i = 0; i < Y_ARRAY_SIZE(badStrings); ++i) {
         wchar32 rune;
-        const ui8* p = (const ui8*)badStrings[i];
+        const ui8* p = reinterpret_cast<const ui8*>(badStrings[i]);
         size_t len;
         RECODE_RESULT res = SafeReadUTF8Char(rune, len, p, p + strlen(badStrings[i]));
         UNIT_ASSERT(res == RECODE_BROKENSYMBOL);

tests/library/charset/CMakeLists.txt

@@ -5,8 +5,8 @@
 #include <src/library/testing/unittest/registar.h>
 
 static void TestIconv(const std::string& utf8, const std::string& other, ECharset enc) {
-    TUtf16String wide0 = CharToWide(utf8, CODES_UTF8);
-    TUtf16String wide1 = CharToWide(other, enc);
+    std::u16string wide0 = CharToWide(utf8, CODES_UTF8);
+    std::u16string wide1 = CharToWide(other, enc);
 
     UNIT_ASSERT(wide0 == wide1);
 
@@ -25,13 +25,13 @@ static void TestIconv(const std::string& utf8, const std::string& other, ECharse
     size_t read = 0;
     size_t written = 0;
 
-    RECODE_RESULT res = RecodeToUnicode(enc, other.c_str(), wide1.begin(), other.size(), wide1.size(), read, written);
+    RECODE_RESULT res = RecodeToUnicode(enc, other.c_str(), wide1.data(), other.size(), wide1.size(), read, written);
     UNIT_ASSERT(res == RECODE_OK);
     UNIT_ASSERT(read == other.size());
     UNIT_ASSERT(written == wide1.size());
     UNIT_ASSERT(wide0 == wide1);
 
-    res = RecodeFromUnicode(enc, wide0.c_str(), temp.begin(), wide0.size(), temp.size(), read, written);
+    res = RecodeFromUnicode(enc, wide0.c_str(), temp.data(), wide0.size(), temp.size(), read, written);
     UNIT_ASSERT(res == RECODE_OK);
     UNIT_ASSERT(read == wide0.size());
     UNIT_ASSERT(written == other.size());

tests/library/charset/ci_string_ut.cpp

@@ -2,10 +2,11 @@
 #include "recyr.hh"
 #include "wide.h"
 
-#include <src/library/testing/unittest/registar.h>
+#include <ydb-cpp-sdk/util/system/yassert.h>
 
 #include <src/util/charset/utf8.h>
-#include <ydb-cpp-sdk/util/system/yassert.h>
+
+#include <src/library/testing/unittest/registar.h>
 
 class TRecyr_intTest: public TTestBase {
 private:
@@ -83,8 +84,9 @@ void TRecyr_intTest::TestUTFFromUnknownPlane() {
     DecodeUnknownPlane(charbuffer, charbufferend, CODES_UTF8);
 
     UNIT_ASSERT(charbufferend == charbuffer + samplelen);
-    for (size_t i = 0; i < samplelen; ++i)
+    for (size_t i = 0; i < samplelen; ++i) {
         UNIT_ASSERT(sampletext[i] == charbuffer[i]);
+    }
 
     // Now, concatenate the thing with an explicit character and retest
     res = RecodeToUnicode(CODES_UNKNOWNPLANE, bytebuffer, charbuffer, writtenbytes, BUFFER_SIZE, readbytes, writtenchars);
@@ -114,9 +116,9 @@ void TRecyr_intTest::TestUTFFromUnknownPlane() {
 
     // test TChar version
     // bytebuffer of len writtenbytes contains sampletext of len samplelen chars in utf8
-    TUtf16String wtr = CharToWide(std::string_view(bytebuffer, writtenbytes), CODES_UNKNOWNPLANE);
-    TChar* strend = wtr.begin() + wtr.size();
-    DecodeUnknownPlane(wtr.begin(), strend, CODES_UTF8);
+    std::u16string wtr = CharToWide(std::string_view(bytebuffer, writtenbytes), CODES_UNKNOWNPLANE);
+    TChar* strend = wtr.data() + wtr.size();
+    DecodeUnknownPlane(wtr.data(), strend, CODES_UTF8);
     wtr.resize(strend - wtr.data(), 'Q');
     UNIT_ASSERT_VALUES_EQUAL(wtr.size(), samplelen);
     for (size_t i = 0; i < wtr.size(); ++i) {