YDB FQ: integration tests for MS SQL Server (#7871)

Author: vitalyisaev2

ydb/library/yql/providers/generic/connector/tests/common_test_cases/base.py

@@ -24,6 +24,8 @@ def name(self) -> str:
                 # ClickHouse has two kinds of network protocols: NATIVE and HTTP,
                 # so we append protocol name to the test case name
                 return f'{self.name_}_{EProtocol.Name(self.protocol)}'
+            case EDataSourceKind.MS_SQL_SERVER:
+                return self.name_
             case EDataSourceKind.MYSQL:
                 return self.name_
             case EDataSourceKind.ORACLE:
@@ -39,11 +41,14 @@ def name(self) -> str:
     def database(self) -> Database:
         '''
         For PG/CH we create a distinct database on every test case.
-        For YDB/MySQL we use single predefined database.
+        For YDB/MySQL/Microsoft SQL Server we use single predefined database.
         '''
+        # FIXME: do not hardcode databases here
         match self.data_source_kind:
             case EDataSourceKind.CLICKHOUSE:
                 return Database(self.name, self.data_source_kind)
+            case EDataSourceKind.MS_SQL_SERVER:
+                return Database("master", self.data_source_kind)
             case EDataSourceKind.MYSQL:
                 return Database("db", self.data_source_kind)
             case EDataSourceKind.ORACLE:
@@ -62,6 +67,8 @@ def table_name(self) -> str:
         match self.data_source_kind:
             case EDataSourceKind.CLICKHOUSE:
                 return 't' + make_random_string(8)
+            case EDataSourceKind.MS_SQL_SERVER:
+                return self.name
             case EDataSourceKind.MYSQL:
                 return self.name
             case EDataSourceKind.ORACLE:
@@ -90,6 +97,11 @@ def generic_settings(self) -> GenericSettings:
                         GenericSettings.ClickHouseCluster(database=self.database.name, protocol=EProtocol.NATIVE)
                     ],
                 )
+            case EDataSourceKind.MS_SQL_SERVER:
+                return GenericSettings(
+                    date_time_format=EDateTimeFormat.YQL_FORMAT,
+                    ms_sql_server_clusters=[GenericSettings.MsSQLServerCluster(database=self.database.name)],
+                )
             case EDataSourceKind.MYSQL:
                 return GenericSettings(
                     date_time_format=EDateTimeFormat.YQL_FORMAT,

ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_missing_database.py

@@ -23,6 +23,10 @@ def generic_settings(self) -> GenericSettings:
             if self.service_name is not None:
                 cluster.service_name = self.service_name
 
+        if self.data_source_kind == EDataSourceKind.MS_SQL_SERVER:
+            for cluster in gs.ms_sql_server_clusters:
+                cluster.database = "missing_database"
+
         return gs
 
 

ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_positive_common.py

@@ -98,6 +98,7 @@ def _column_selection(self) -> Sequence[TestCase]:
                     EDataSourceKind.POSTGRESQL,
                     EDataSourceKind.YDB,
                     EDataSourceKind.MYSQL,
+                    EDataSourceKind.MS_SQL_SERVER,
                 ),
             ),
             # SELECT COL1 FROM table
@@ -112,6 +113,7 @@ def _column_selection(self) -> Sequence[TestCase]:
                     # NOTE: YQ-2264: doesn't work for PostgreSQL because of implicit cast to lowercase (COL1 -> col1)
                     EDataSourceKind.YDB,
                     EDataSourceKind.MYSQL,
+                    EDataSourceKind.MS_SQL_SERVER,
                 ),
             ),
             # SELECT col1 FROM table
@@ -135,6 +137,7 @@ def _column_selection(self) -> Sequence[TestCase]:
                     EDataSourceKind.POSTGRESQL,
                     EDataSourceKind.YDB,
                     EDataSourceKind.MYSQL,
+                    EDataSourceKind.MS_SQL_SERVER,
                 ),
             ),
             # SELECT col2, COL1 FROM table
@@ -149,6 +152,7 @@ def _column_selection(self) -> Sequence[TestCase]:
                     # NOTE: YQ-2264: doesn't work for PostgreSQL because of implicit cast to lowercase (COL1 -> col1)
                     EDataSourceKind.YDB,
                     EDataSourceKind.MYSQL,
+                    EDataSourceKind.MS_SQL_SERVER,
                 ),
             ),
             # SELECT col2, col1 FROM table
@@ -173,6 +177,7 @@ def _column_selection(self) -> Sequence[TestCase]:
                     # NOTE: YQ-2264: doesn't work for PostgreSQL because of implicit cast to lowercase (COL1 -> col1)
                     EDataSourceKind.YDB,
                     EDataSourceKind.MYSQL,
+                    EDataSourceKind.MS_SQL_SERVER,
                 ),
             ),
             # Select the same column multiple times with different aliases
@@ -194,6 +199,7 @@ def _column_selection(self) -> Sequence[TestCase]:
                     EDataSourceKind.POSTGRESQL,
                     EDataSourceKind.YDB,
                     EDataSourceKind.MYSQL,
+                    EDataSourceKind.MS_SQL_SERVER,
                 ),
             ),
         )
@@ -291,11 +297,16 @@ def make_test_cases(self, data_source_kind: EDataSourceKind) -> Sequence[TestCas
             EDataSourceKind.POSTGRESQL: [EProtocol.NATIVE],
             EDataSourceKind.YDB: [EProtocol.NATIVE],
             EDataSourceKind.MYSQL: [EProtocol.NATIVE],
+            EDataSourceKind.MS_SQL_SERVER: [EProtocol.NATIVE],
         }
 
         base_test_cases = None
 
-        if data_source_kind in [EDataSourceKind.YDB, EDataSourceKind.MYSQL]:
+        if data_source_kind in [
+            EDataSourceKind.YDB,
+            EDataSourceKind.MYSQL,
+            EDataSourceKind.MS_SQL_SERVER,
+        ]:
             base_test_cases = self._column_selection()
         elif data_source_kind in [EDataSourceKind.CLICKHOUSE, EDataSourceKind.POSTGRESQL]:
             base_test_cases = list(

ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml

@@ -12,7 +12,7 @@ services:
     - 8123
   fq-connector-go:
     container_name: fq-tests-ch-fq-connector-go
-    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.2@sha256:c007dc10b89aa0342bea93d843cca7097e66b2e45310010d9321259aeee453cc
+    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.3@sha256:7b8ccac5aaa14814333ddbb0f91e60039e158018972759506b1f4ce7899b83a9
     ports:
     - 2130
     volumes:

ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/collection.py

@@ -0,0 +1,39 @@
+from typing import Sequence, Mapping
+
+from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_database as select_missing_database
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_table as select_missing_table
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_positive_common as select_positive_common
+import select_datetime
+import select_positive
+
+# import select_positive_with_schema
+
+from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
+
+
+class Collection(object):
+    _test_cases: Mapping[str, Sequence]
+
+    def __init__(self, ss: Settings):
+        self._test_cases = {
+            'select_missing_database': select_missing_database.Factory(ss).make_test_cases(
+                EDataSourceKind.MS_SQL_SERVER
+            ),
+            'select_missing_table': select_missing_table.Factory(ss).make_test_cases(EDataSourceKind.MS_SQL_SERVER),
+            'select_positive': select_positive.Factory().make_test_cases()
+            + select_positive_common.Factory(ss).make_test_cases(EDataSourceKind.MS_SQL_SERVER),
+            'select_datetime': select_datetime.Factory().make_test_cases(),
+        }
+
+    def get(self, key: str) -> Sequence:
+        if key not in self._test_cases:
+            raise ValueError(f'no such test: {key}')
+
+        return self._test_cases[key]
+
+    def ids(self, key: str) -> Sequence[str]:
+        if key not in self._test_cases:
+            raise ValueError(f'no such test: {key}')
+
+        return [tc.name for tc in self._test_cases[key]]

ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/conftest.py

@@ -0,0 +1,16 @@
+from typing import Final
+import pathlib
+
+import pytest
+
+
+from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind
+from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
+
+
+docker_compose_dir: Final = pathlib.Path("ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server")
+
+
+@pytest.fixture
+def settings() -> Settings:
+    return Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EDataSourceKind.MS_SQL_SERVER])

ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/custom.cnf

@@ -0,0 +1,4 @@
+[mysqld]
+character-set-server = utf8
+collation-server = utf8_unicode_ci
+skip-character-set-client-handshake

ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml

@@ -0,0 +1,21 @@
+services:
+  fq-connector-go:
+    container_name: fq-tests-mssql-fq-connector-go
+    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.3@sha256:7b8ccac5aaa14814333ddbb0f91e60039e158018972759506b1f4ce7899b83a9
+    ports:
+    - 2130
+    volumes:
+    - ../../fq-connector-go/:/opt/ydb/cfg/
+  ms_sql_server:
+    build: 
+      context: ./init
+      dockerfile: Dockerfile
+    container_name: fq-tests-mssql-ms_sql_server
+    environment:
+      ACCEPT_EULA: "Y"
+      DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: "false"
+      LANG: ru_RU.UTF-8
+      SA_PASSWORD: Qwerty12345!
+    ports:
+    - 1433:1433
+version: "3.4"

ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/init/Dockerfile

@@ -0,0 +1,17 @@
+FROM mcr.microsoft.com/mssql/server:2022-latest
+
+# Create a config directory
+USER root
+RUN mkdir -p /usr/config
+WORKDIR /usr/config
+
+# Bundle config source
+COPY . /usr/config
+
+# Grant permissions for to our scripts to be executable
+RUN chmod +x /usr/config/entrypoint.sh
+RUN chmod +x /usr/config/configure-db.sh
+
+USER mssql
+
+ENTRYPOINT ["./entrypoint.sh"]

ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/init/configure-db.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -x
+
+# Wait 60 seconds for SQL Server to start up by ensuring that 
+# calling SQLCMD does not return an error code, which will ensure that sqlcmd is accessible
+# and that system and user databases return "0" which means all databases are in an "online" state
+# https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-databases-transact-sql?view=sql-server-2017 
+
+ERRCODE=1
+i=0
+
+while [[ $i -lt 60 ]] && [[ $ERRCODE -ne 0 ]]; do
+	i=$i+1
+	DBSTATUS=$(/opt/mssql-tools18/bin/sqlcmd -C -U sa -P $SA_PASSWORD -Q "SET NOCOUNT ON; Select SUM(state) from sys.databases")
+	ERRCODE=$?
+	echo "$i $DBSTATUS $ERRCODE"
+	sleep 1
+done
+
+if [[ "$ERRCODE" -ne 0 ]]; then 
+	echo "SQL Server took more than 60 seconds to start up or one or more databases are not in an ONLINE state"
+	exit 1
+fi
+
+# Run the setup script to create the DB and the schema in the DB
+/opt/mssql-tools18/bin/sqlcmd -C -S localhost -U sa -P $SA_PASSWORD -d master -i setup.sql