Add policity for check canonical results. Check some as warning (#12654)

Author: iddqdex

ydb/tests/olap/lib/allure_utils.py

@@ -3,8 +3,9 @@
 from ydb.tests.olap.lib.ydb_cluster import YdbCluster
 from ydb.tests.olap.lib.results_processor import ResultsProcessor
 from urllib.parse import urlencode
-from datetime import datetime, UTC
+from datetime import datetime
 from copy import deepcopy
+from pytz import timezone
 
 
 def _set_monitoring(test_info: dict[str, str], start_time: float, end_time: float) -> None:
@@ -55,10 +56,11 @@ def _set_logs_command(test_info: dict[str, str], start_time: float, end_time: fl
         if node.role == YdbCluster.Node.Role.STORAGE:
             hosts.append(node.host)
     hosts_cmd = ' '.join([f'-H {h}' for h in hosts])
-    start = datetime.fromtimestamp(start_time, UTC).isoformat()
-    end = datetime.fromtimestamp(end_time, UTC).isoformat()
+    tz = timezone('Europe/Moscow')
+    start = datetime.fromtimestamp(start_time, tz).isoformat()
+    end = datetime.fromtimestamp(end_time, tz).isoformat()
     time_cmd = f'-S "{start}" -U "{end}"'
-    cmd = f"parallel-ssh {hosts_cmd} -o . 'unified_agent select {time_cmd} -s kikimr'"
+    cmd = f"parallel-ssh {hosts_cmd} -o . 'ulimit -n 100500;unified_agent select {time_cmd} -s kikimr'"
     test_info['logs_command'] = f'<code>{cmd}</code>'
 
 

ydb/tests/olap/lib/ya.make

@@ -11,6 +11,7 @@ PY3_LIBRARY()
     PEERDIR(
         contrib/python/allure-pytest
         contrib/python/allure-python-commons
+        contrib/python/pytz
         contrib/python/requests
         library/python/testing/yatest_common
         ydb/public/api/client/yc_public/iam

ydb/tests/olap/lib/ydb_cli.py

@@ -6,7 +6,7 @@
 import re
 from ydb.tests.olap.lib.ydb_cluster import YdbCluster
 from ydb.tests.olap.lib.utils import get_external_param
-from enum import StrEnum
+from enum import StrEnum, Enum
 from types import TracebackType
 
 
@@ -16,6 +16,12 @@ class WorkloadType(StrEnum):
     TPC_DS = 'tpcds'
 
 
+class CheckCanonicalPolicy(Enum):
+    NO = 0
+    WARNING = 1
+    ERROR = 2
+
+
 class YdbCliHelper:
     @staticmethod
     def get_cli_command() -> list[str]:
@@ -46,6 +52,7 @@ def __init__(self):
             self.stdout: str = ''
             self.stderr: str = ''
             self.error_message: str = ''
+            self.warning_message: str = ''
             self.plans: Optional[list[YdbCliHelper.QueryPlan]] = None
             self.explain_plan: Optional[YdbCliHelper.QueryPlan] = None
             self.errors_by_iter: dict[int, str] = {}
@@ -63,7 +70,7 @@ def __init__(self,
                      query_num: int,
                      iterations: int,
                      timeout: float,
-                     check_canonical: bool,
+                     check_canonical: CheckCanonicalPolicy,
                      query_syntax: str,
                      scale: Optional[int]):
             def _get_output_path(ext: str) -> str:
@@ -90,6 +97,13 @@ def _add_error(self, msg: Optional[str]):
                 else:
                     self.result.error_message = msg
 
+        def _add_warning(self, msg: Optional[str]):
+            if msg is not None and len(msg) > 0:
+                if len(self.result.warning_message) > 0:
+                    self.result.warning_message += f'\n\n{msg}'
+                else:
+                    self.result.warning_message = msg
+
         def _process_returncode(self, returncode) -> None:
             begin_str = f'{self.query_num}:'
             end_str = 'Query text:'
@@ -147,7 +161,10 @@ def _load_stats(self):
                     self.result.stats[q] = {}
                 self.result.stats[q][signal['sensor']] = signal['value']
             if self.result.stats.get(f'Query{self.query_num:02d}', {}).get("DiffsCount", 0) > 0:
-                self._add_error('There is diff in query results')
+                if self.check_canonical == CheckCanonicalPolicy.WARNING:
+                    self._add_warning('There is diff in query results')
+                else:
+                    self._add_error('There is diff in query results')
 
         def _load_query_out(self) -> None:
             if (os.path.exists(self._query_output_path)):
@@ -196,7 +213,7 @@ def _get_cmd(self) -> list[str]:
             query_preffix = get_external_param('query-prefix', '')
             if query_preffix:
                 cmd += ['--query-settings', query_preffix]
-            if self.check_canonical:
+            if self.check_canonical != CheckCanonicalPolicy.NO:
                 cmd.append('--check-canonical')
             if self.query_syntax:
                 cmd += ['--syntax', self.query_syntax]
@@ -230,7 +247,7 @@ def process(self) -> YdbCliHelper.WorkloadRunResult:
 
     @staticmethod
     def workload_run(workload_type: WorkloadType, path: str, query_num: int, iterations: int = 5,
-                     timeout: float = 100., check_canonical: bool = False, query_syntax: str = '',
+                     timeout: float = 100., check_canonical: CheckCanonicalPolicy = CheckCanonicalPolicy.NO, query_syntax: str = '',
                      scale: Optional[int] = None) -> YdbCliHelper.WorkloadRunResult:
         return YdbCliHelper.WorkloadProcessor(
             workload_type,

ydb/tests/olap/load/lib/clickbench.py

@@ -2,7 +2,7 @@
 import pytest
 from .conftest import LoadSuiteBase
 from os import getenv
-from ydb.tests.olap.lib.ydb_cli import WorkloadType, YdbCliHelper
+from ydb.tests.olap.lib.ydb_cli import WorkloadType, YdbCliHelper, CheckCanonicalPolicy
 from ydb.tests.olap.lib.ydb_cluster import YdbCluster
 from ydb.tests.olap.lib.utils import get_external_param
 
@@ -29,7 +29,7 @@ def do_setup_class(cls):
                         iterations=1,
                         workload_type=cls.workload_type,
                         timeout=cls._get_timeout(query_num),
-                        check_canonical=True
+                        check_canonical=CheckCanonicalPolicy.ERROR
                     )
                     cls.process_query_result(result=result, query_num=query_num, iterations=1, upload=False)
             except BaseException:

ydb/tests/olap/load/lib/conftest.py

@@ -2,7 +2,7 @@
 import pytest
 import allure
 import json
-from ydb.tests.olap.lib.ydb_cli import YdbCliHelper, WorkloadType
+from ydb.tests.olap.lib.ydb_cli import YdbCliHelper, WorkloadType, CheckCanonicalPolicy
 from ydb.tests.olap.lib.ydb_cluster import YdbCluster
 from ydb.tests.olap.lib.allure_utils import allure_test_description
 from ydb.tests.olap.lib.results_processor import ResultsProcessor
@@ -23,7 +23,7 @@ def __init__(self, iterations: Optional[int] = None, timeout: Optional[float] =
     workload_type: WorkloadType = None
     timeout: float = 1800.
     refference: str = ''
-    check_canonical: bool = False
+    check_canonical: CheckCanonicalPolicy = CheckCanonicalPolicy.NO
     query_syntax: str = ''
     query_settings: dict[int, LoadSuiteBase.QuerySettings] = {}
     scale: Optional[int] = None
@@ -165,10 +165,12 @@ def _attach_plans(plan: YdbCliHelper.QueryPlan) -> None:
                 statistics=stats,
             )
         if not success:
-            exc = pytest.fail.Exception(error_message)
+            exc = pytest.fail.Exception('\n'.join([error_message, result.warning_message]))
             if result.traceback is not None:
                 exc = exc.with_traceback(result.traceback)
             raise exc
+        if result.warning_message:
+            raise Exception(result.warning_message)
 
     @classmethod
     def setup_class(cls) -> None:

ydb/tests/olap/load/lib/tpcds.py

@@ -2,7 +2,7 @@
 import pytest
 from .conftest import LoadSuiteBase
 from os import getenv
-from ydb.tests.olap.lib.ydb_cli import WorkloadType
+from ydb.tests.olap.lib.ydb_cli import WorkloadType, CheckCanonicalPolicy
 from ydb.tests.olap.lib.utils import get_external_param
 from ydb.tests.olap.lib.ydb_cluster import YdbCluster
 
@@ -46,7 +46,7 @@ def test_tpcds(self, query_num: int):
 
 class TestTpcds1(TpcdsSuiteBase):
     scale: int = 1
-    check_canonical: bool = True
+    check_canonical: bool = CheckCanonicalPolicy.ERROR
     tables_size: dict[str, int] = {
         'call_center': 6,
         'catalog_page': 11718,
@@ -71,7 +71,7 @@ class TestTpcds1(TpcdsSuiteBase):
 
 class TestTpcds10(TpcdsSuiteBase):
     scale: int = 10
-    check_canonical: bool = True
+    check_canonical: bool = CheckCanonicalPolicy.WARNING
     timeout = max(TpcdsSuiteBase.timeout, 300.)
     tables_size: dict[str, int] = {
         'call_center': 24,
@@ -97,7 +97,7 @@ class TestTpcds10(TpcdsSuiteBase):
 
 class TestTpcds100(TpcdsSuiteBase):
     scale: int = 100
-    check_canonical: bool = True
+    check_canonical: bool = CheckCanonicalPolicy.WARNING
     iterations: int = 2
     timeout = max(TpcdsSuiteBase.timeout, 3600.)
     query_settings = {

ydb/tests/olap/load/lib/tpch.py

@@ -2,7 +2,7 @@
 import pytest
 from .conftest import LoadSuiteBase
 from os import getenv
-from ydb.tests.olap.lib.ydb_cli import WorkloadType
+from ydb.tests.olap.lib.ydb_cli import WorkloadType, CheckCanonicalPolicy
 from ydb.tests.olap.lib.utils import get_external_param
 from ydb.tests.olap.lib.ydb_cluster import YdbCluster
 
@@ -50,23 +50,23 @@ class TestTpch1(TpchSuiteBase):
         'lineitem': 6001215,
     }
     scale: int = 1
-    check_canonical: bool = True
+    check_canonical: bool = CheckCanonicalPolicy.ERROR
 
 
 class TestTpch10(TpchSuiteBase):
     tables_size: dict[str, int] = {
         'lineitem': 59986052,
     }
     scale: int = 10
-    check_canonical: bool = True
+    check_canonical: bool = CheckCanonicalPolicy.ERROR
 
 
 class TestTpch100(TpchSuiteBase):
     tables_size: dict[str, int] = {
         'lineitem': 600037902,
     }
     scale: int = 100
-    check_canonical: bool = True
+    check_canonical: bool = CheckCanonicalPolicy.ERROR
     timeout = max(TpchSuiteBase.timeout, 300.)
 
 
@@ -75,7 +75,7 @@ class TestTpch1000(TpchSuiteBase):
         'lineitem': 5999989709,
     }
     scale: int = 1000
-    check_canonical: bool = True
+    check_canonical: bool = CheckCanonicalPolicy.WARNING
     timeout = max(TpchSuiteBase.timeout, 3600.)
 
 
@@ -85,5 +85,5 @@ class TestTpch10000(TpchSuiteBase):
     }
     scale: int = 10000
     iterations: int = 2
-    check_canonical: bool = True
+    check_canonical: bool = CheckCanonicalPolicy.WARNING
     timeout = max(TpchSuiteBase.timeout, 3600.)