Introduce compatibility test (between last stable and main) (#9828)

Author: maximyurchuk

ydb/tests/functional/compatibility/test_compatibility.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+from ydb.tests.library.common import yatest_common
+from ydb.tests.library.harness.kikimr_cluster import KiKiMR
+from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator
+from ydb.tests.library.harness.param_constants import kikimr_driver_path
+from ydb.tests.library.common.types import Erasure
+from ydb.tests.oss.ydb_sdk_import import ydb
+
+
+class TestCompatibility(object):
+    @classmethod
+    def setup_class(cls):
+        last_stable_path = yatest_common.binary_path("ydb/tests/library/compatibility/ydbd-last-stable")
+        binary_paths = [kikimr_driver_path(), last_stable_path]
+        cls.cluster = KiKiMR(KikimrConfigGenerator(erasure=Erasure.MIRROR_3_DC, binary_paths=binary_paths))
+        cls.cluster.start()
+        cls.endpoint = "%s:%s" % (
+            cls.cluster.nodes[1].host, cls.cluster.nodes[1].port
+        )
+        cls.driver = ydb.Driver(
+            ydb.DriverConfig(
+                database='/Root',
+                endpoint=cls.endpoint
+            )
+        )
+        cls.driver.wait()
+
+    @classmethod
+    def teardown_class(cls):
+        if hasattr(cls, 'driver'):
+            cls.driver.stop()
+
+        if hasattr(cls, 'cluster'):
+            cls.cluster.stop(kill=True)  # TODO fix
+
+    def test_simple(self):
+        session = ydb.retry_operation_sync(lambda: self.driver.table_client.session().create())
+
+        with ydb.SessionPool(self.driver, size=1) as pool:
+            with pool.checkout() as session:
+                session.execute_scheme(
+                    "create table `sample_table` (id Uint64, value Uint64, payload Utf8, PRIMARY KEY(id)) WITH (AUTO_PARTITIONING_BY_SIZE = ENABLED, AUTO_PARTITIONING_PARTITION_SIZE_MB = 1);"
+                )
+                id_ = 0
+
+                upsert_count = 200
+                iteration_count = 1
+                for i in range(iteration_count):
+                    rows = []
+                    for j in range(upsert_count):
+                        row = {}
+                        row["id"] = id_
+                        row["value"] = 1
+                        row["payload"] = "DEADBEEF" * 1024 * 16  # 128 kb
+                        rows.append(row)
+                        id_ += 1
+
+                    column_types = ydb.BulkUpsertColumns()
+                    column_types.add_column("id", ydb.PrimitiveType.Uint64)
+                    column_types.add_column("value", ydb.PrimitiveType.Uint64)
+                    column_types.add_column("payload", ydb.PrimitiveType.Utf8)
+                    self.driver.table_client.bulk_upsert(
+                        "Root/sample_table", rows, column_types
+                    )
+
+                query = "SELECT SUM(value) from sample_table"
+                result_sets = session.transaction().execute(
+                    query, commit_tx=True
+                )
+                for row in result_sets[0].rows:
+                    print(" ".join([str(x) for x in list(row.values())]))
+
+                assert len(result_sets) == 1
+                assert len(result_sets[0].rows) == 1
+                result = list(result_sets[0].rows[0].values())
+                assert len(result) == 1
+                assert result[0] == upsert_count * iteration_count

ydb/tests/functional/compatibility/ya.make

@@ -0,0 +1,21 @@
+PY3TEST()
+ENV(YDB_DRIVER_BINARY="ydb/apps/ydbd/ydbd")
+
+TEST_SRCS(
+    test_compatibility.py
+)
+
+TIMEOUT(3600)
+SIZE(LARGE)
+TAG(ya:fat)
+
+DEPENDS(
+    ydb/apps/ydbd
+    ydb/tests/library/compatibility
+)
+
+PEERDIR(
+    ydb/tests/library
+)
+
+END()

ydb/tests/functional/ya.make

@@ -7,6 +7,7 @@ RECURSE(
     canonical
     clickbench
     cms
+    compatibility
     dynumber
     encryption
     hive

ydb/tests/library/compatibility/downloader/__main__.py

@@ -0,0 +1,31 @@
+import os
+import stat
+import sys
+
+import boto3
+from botocore import UNSIGNED
+from botocore.client import Config
+
+AWS_ENDPOINT = "https://storage.yandexcloud.net"
+AWS_BUCKET = "ydb-builds"
+
+
+def main():
+    s3_client = boto3.client(
+        service_name="s3",
+        endpoint_url=AWS_ENDPOINT,
+        config=Config(signature_version=UNSIGNED)
+    )
+
+    s3_bucket = AWS_BUCKET
+    remote_src = sys.argv[1]
+    local_dst = sys.argv[2]
+    s3_client.download_file(s3_bucket, remote_src, local_dst)
+
+    # chmod +x
+    st = os.stat(local_dst)
+    os.chmod(local_dst, st.st_mode | stat.S_IEXEC)
+
+
+if __name__ == "__main__":
+    main()

ydb/tests/library/compatibility/downloader/ya.make

@@ -0,0 +1,10 @@
+PY3_PROGRAM()
+    PEERDIR(
+      contrib/python/boto3
+      contrib/python/botocore
+    )
+
+    PY_SRCS(
+       __main__.py
+     )
+END()

ydb/tests/library/compatibility/ya.make

@@ -0,0 +1,10 @@
+UNION()
+
+RUN_PROGRAM(
+    ydb/tests/library/compatibility/downloader stable-24-3/relwithdebinfo/ydbd ydbd-last-stable
+    OUT_NOAUTO ydbd-last-stable
+)
+
+END()
+
+RECURSE(downloader)

ydb/tests/library/ya.make

@@ -115,4 +115,5 @@ PEERDIR(
 
 END()
 
+RECURSE(compatibility)
 RECURSE_FOR_TESTS(ut)