From 92c1551744378fa4dceb4333fb85729050132a6f Mon Sep 17 00:00:00 2001 From: Grigoriy Pisarenko Date: Wed, 18 Sep 2024 14:34:45 +0000 Subject: [PATCH 1/4] Added kqprun recipe --- ydb/tests/tools/kqprun/kqprun.cpp | 10 +-- ydb/tests/tools/kqprun/recipe/__main__.py | 74 +++++++++++++++++++ ydb/tests/tools/kqprun/recipe/ya.make | 12 +++ ydb/tests/tools/kqprun/tests/cfg/config.conf | 3 + .../tools/kqprun/tests/cfg/create_tables.sql | 4 + .../tools/kqprun/tests/cfg/fill_tables.sql | 2 + .../tools/kqprun/tests/test_kqprun_recipe.py | 18 +++++ ydb/tests/tools/kqprun/tests/ya.make | 30 ++++++++ ydb/tests/tools/kqprun/ya.make | 10 ++- 9 files changed, 154 insertions(+), 9 deletions(-) create mode 100644 ydb/tests/tools/kqprun/recipe/__main__.py create mode 100644 ydb/tests/tools/kqprun/recipe/ya.make create mode 100644 ydb/tests/tools/kqprun/tests/cfg/config.conf create mode 100644 ydb/tests/tools/kqprun/tests/cfg/create_tables.sql create mode 100644 ydb/tests/tools/kqprun/tests/cfg/fill_tables.sql create mode 100644 ydb/tests/tools/kqprun/tests/test_kqprun_recipe.py create mode 100644 ydb/tests/tools/kqprun/tests/ya.make diff --git a/ydb/tests/tools/kqprun/kqprun.cpp b/ydb/tests/tools/kqprun/kqprun.cpp index 88802d2aead9..e10acd510f0b 100644 --- a/ydb/tests/tools/kqprun/kqprun.cpp +++ b/ydb/tests/tools/kqprun/kqprun.cpp @@ -317,15 +317,9 @@ void RunArgumentQueries(const TExecutionOptions& executionOptions, NKqpRun::TKqp void RunAsDaemon() { NColorizer::TColors colors = NColorizer::AutoColors(Cout); - Cout << colors.Yellow() << TInstant::Now().ToIsoStringLocal() << " Started reading commands" << colors.Default() << Endl; + Cout << colors.Yellow() << TInstant::Now().ToIsoStringLocal() << " Initialization finished" << colors.Default() << Endl; while (true) { - TString command; - Cin >> command; - - if (command == "exit") { - break; - } - Cerr << colors.Red() << TInstant::Now().ToIsoStringLocal() << " Invalid command '" << command << "'" << colors.Default() << Endl; + pause(); } } diff --git a/ydb/tests/tools/kqprun/recipe/__main__.py b/ydb/tests/tools/kqprun/recipe/__main__.py new file mode 100644 index 000000000000..744f0efe7a58 --- /dev/null +++ b/ydb/tests/tools/kqprun/recipe/__main__.py @@ -0,0 +1,74 @@ +import argparse +import logging +import os + +from library.python.testing.recipe import declare_recipe, set_env +from library.recipes import common as recipes_common +from yatest.common.network import PortManager +from ydb.tests.library.common import yatest_common + + +PID_FILENAME = "kqprun_daemon.pid" +KQPRUN_PATH = os.getenv("KQPRUN_EXECUTABLE") or "ydb/tests/tools/kqprun/kqprun" + + +def is_kqprun_daemon_ready() -> bool: + info = open(yatest_common.output_path("kqprun_daemon.out.log"), "r").read() + return "Initialization finished" in info + + +def build_start_comand(argv: list[str], grpc_port: int) -> tuple[int, list[str]]: + parser = argparse.ArgumentParser() + parser.add_argument("--query", action="append", type=str, default=[]) + parser.add_argument("--config", action='store', type=str, default="ydb/tests/tools/kqprun/kqprun/configuration/app_config.conf") + parser.add_argument("--timeout-ms", action='store', type=int, default=30000) + parsed, _ = parser.parse_known_args(argv) + + cmd = [ + yatest_common.binary_path(KQPRUN_PATH), + "--log-file", yatest_common.output_path("kqprun_daemon.ydb.log"), + "--app-config", yatest_common.source_path(parsed.config), + "--grpc", str(grpc_port), + "--timeout", str(parsed.timeout_ms) + ] + + if parsed.query: + cmd.append("--execution-case") + cmd.append("query") + + for query in parsed.query: + cmd.append("--script-query") + cmd.append(yatest_common.source_path(query)) + + return (parsed.timeout_ms // 500, cmd) + + +def start(argv: list[str]): + logging.debug("Starting kqprun daemon") + + portManager = PortManager() + grpc_port = portManager.get_port() + timeout, cmd = build_start_comand(argv, grpc_port) + + recipes_common.start_daemon( + command=cmd, + environment=None, + is_alive_check=is_kqprun_daemon_ready, + pid_file_name=PID_FILENAME, + timeout=timeout, + daemon_name="kqprun_daemon" + ) + + set_env("KQPRUN_ENDPOINT", f"grpc://localhost:{grpc_port}") + logging.debug(f"kqprun daemon has been started on port: {grpc_port}") + + +def stop(argv: list[str]): + logging.debug("Stop kqprun daemon") + with open(PID_FILENAME, "r") as pidFile: + pid = int(pidFile.read()) + recipes_common.stop_daemon(pid) + + +if __name__ == "__main__": + declare_recipe(start, stop) diff --git a/ydb/tests/tools/kqprun/recipe/ya.make b/ydb/tests/tools/kqprun/recipe/ya.make new file mode 100644 index 000000000000..787459970a34 --- /dev/null +++ b/ydb/tests/tools/kqprun/recipe/ya.make @@ -0,0 +1,12 @@ +PY3_PROGRAM(kqprun_recipe) + +PY_SRCS(__main__.py) + +PEERDIR( + library/python/testing/recipe + library/python/testing/yatest_common + library/recipes/common + ydb/tests/library +) + +END() diff --git a/ydb/tests/tools/kqprun/tests/cfg/config.conf b/ydb/tests/tools/kqprun/tests/cfg/config.conf new file mode 100644 index 000000000000..834d010c72d1 --- /dev/null +++ b/ydb/tests/tools/kqprun/tests/cfg/config.conf @@ -0,0 +1,3 @@ +LogConfig { + DefaultLevel: 5 +} diff --git a/ydb/tests/tools/kqprun/tests/cfg/create_tables.sql b/ydb/tests/tools/kqprun/tests/cfg/create_tables.sql new file mode 100644 index 000000000000..aaa2ed285f70 --- /dev/null +++ b/ydb/tests/tools/kqprun/tests/cfg/create_tables.sql @@ -0,0 +1,4 @@ +CREATE TABLE test_table ( + Key Int32, + PRIMARY KEY (Key) +); diff --git a/ydb/tests/tools/kqprun/tests/cfg/fill_tables.sql b/ydb/tests/tools/kqprun/tests/cfg/fill_tables.sql new file mode 100644 index 000000000000..33afd0e6e28b --- /dev/null +++ b/ydb/tests/tools/kqprun/tests/cfg/fill_tables.sql @@ -0,0 +1,2 @@ +INSERT INTO test_table +SELECT 42 AS Key; diff --git a/ydb/tests/tools/kqprun/tests/test_kqprun_recipe.py b/ydb/tests/tools/kqprun/tests/test_kqprun_recipe.py new file mode 100644 index 000000000000..9f668c145714 --- /dev/null +++ b/ydb/tests/tools/kqprun/tests/test_kqprun_recipe.py @@ -0,0 +1,18 @@ +import os + +from ydb.tests.oss.ydb_sdk_import import ydb + + +class TestKqprunRecipe(object): + def test_query_execution(self): + with ydb.Driver( + endpoint=os.getenv("KQPRUN_ENDPOINT"), + database="/Root" + ) as driver: + driver.wait(timeout=5, fail_fast=True) + + with ydb.QuerySessionPool(driver) as pool: + result_sets = pool.execute_with_retries("SELECT * FROM test_table") + rows = result_sets[0].rows + assert len(rows) == 1 + assert rows[0].Key == 42 diff --git a/ydb/tests/tools/kqprun/tests/ya.make b/ydb/tests/tools/kqprun/tests/ya.make new file mode 100644 index 000000000000..f4e46767c3f3 --- /dev/null +++ b/ydb/tests/tools/kqprun/tests/ya.make @@ -0,0 +1,30 @@ +PY3TEST() + +DATA( + arcadia/ydb/tests/tools/kqprun/tests/cfg +) + +TEST_SRCS( + test_kqprun_recipe.py +) + +PEERDIR( + library/python/testing/recipe + library/python/testing/yatest_common + library/recipes/common + ydb/tests/oss/ydb_sdk_import +) + +DEPENDS( + ydb/tests/tools/kqprun + ydb/tests/tools/kqprun/recipe +) + +USE_RECIPE( + ydb/tests/tools/kqprun/recipe/kqprun_recipe --query ydb/tests/tools/kqprun/tests/cfg/create_tables.sql --query ydb/tests/tools/kqprun/tests/cfg/fill_tables.sql --config ydb/tests/tools/kqprun/tests/cfg/config.conf +) + +TIMEOUT(10) +SIZE(MEDIUM) + +END() diff --git a/ydb/tests/tools/kqprun/ya.make b/ydb/tests/tools/kqprun/ya.make index f4eae31d9147..0f56bceef8bf 100644 --- a/ydb/tests/tools/kqprun/ya.make +++ b/ydb/tests/tools/kqprun/ya.make @@ -1,4 +1,4 @@ -PROGRAM() +PROGRAM(kqprun) SRCS( kqprun.cpp @@ -24,3 +24,11 @@ PEERDIR( YQL_LAST_ABI_VERSION() END() + +RECURSE( + recipe +) + +RECURSE_FOR_TESTS( + tests +) From 3b06c66a2bf48268cb7aed29c678a9513abb79f7 Mon Sep 17 00:00:00 2001 From: Grigoriy Pisarenko Date: Wed, 18 Sep 2024 14:38:15 +0000 Subject: [PATCH 2/4] Fixed test ya.make --- ydb/tests/tools/kqprun/tests/ya.make | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ydb/tests/tools/kqprun/tests/ya.make b/ydb/tests/tools/kqprun/tests/ya.make index f4e46767c3f3..03edde225ae6 100644 --- a/ydb/tests/tools/kqprun/tests/ya.make +++ b/ydb/tests/tools/kqprun/tests/ya.make @@ -21,7 +21,10 @@ DEPENDS( ) USE_RECIPE( - ydb/tests/tools/kqprun/recipe/kqprun_recipe --query ydb/tests/tools/kqprun/tests/cfg/create_tables.sql --query ydb/tests/tools/kqprun/tests/cfg/fill_tables.sql --config ydb/tests/tools/kqprun/tests/cfg/config.conf + ydb/tests/tools/kqprun/recipe/kqprun_recipe + --config ydb/tests/tools/kqprun/tests/cfg/config.conf + --query ydb/tests/tools/kqprun/tests/cfg/create_tables.sql + --query ydb/tests/tools/kqprun/tests/cfg/fill_tables.sql ) TIMEOUT(10) From f1503ed031932ea3a32c63c80c26de6753217975 Mon Sep 17 00:00:00 2001 From: Grigoriy Pisarenko Date: Wed, 18 Sep 2024 14:41:44 +0000 Subject: [PATCH 3/4] Fixed file opening in recipe --- ydb/tests/tools/kqprun/recipe/__main__.py | 4 ++-- ydb/tests/tools/kqprun/tests/ya.make | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/ydb/tests/tools/kqprun/recipe/__main__.py b/ydb/tests/tools/kqprun/recipe/__main__.py index 744f0efe7a58..92ac95ef4964 100644 --- a/ydb/tests/tools/kqprun/recipe/__main__.py +++ b/ydb/tests/tools/kqprun/recipe/__main__.py @@ -13,8 +13,8 @@ def is_kqprun_daemon_ready() -> bool: - info = open(yatest_common.output_path("kqprun_daemon.out.log"), "r").read() - return "Initialization finished" in info + with open(yatest_common.output_path("kqprun_daemon.out.log"), "r") as outFile: + return "Initialization finished" in outFile.read() def build_start_comand(argv: list[str], grpc_port: int) -> tuple[int, list[str]]: diff --git a/ydb/tests/tools/kqprun/tests/ya.make b/ydb/tests/tools/kqprun/tests/ya.make index 03edde225ae6..2ce3ec5b0521 100644 --- a/ydb/tests/tools/kqprun/tests/ya.make +++ b/ydb/tests/tools/kqprun/tests/ya.make @@ -9,9 +9,6 @@ TEST_SRCS( ) PEERDIR( - library/python/testing/recipe - library/python/testing/yatest_common - library/recipes/common ydb/tests/oss/ydb_sdk_import ) @@ -27,7 +24,6 @@ USE_RECIPE( --query ydb/tests/tools/kqprun/tests/cfg/fill_tables.sql ) -TIMEOUT(10) SIZE(MEDIUM) END() From 4db92cb9e9c5961af5173f9469c825a8fa0a5495 Mon Sep 17 00:00:00 2001 From: Grigoriy Pisarenko Date: Thu, 19 Sep 2024 07:39:14 +0000 Subject: [PATCH 4/4] Fixed timeout calculation --- ydb/tests/tools/kqprun/recipe/__main__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ydb/tests/tools/kqprun/recipe/__main__.py b/ydb/tests/tools/kqprun/recipe/__main__.py index 92ac95ef4964..154bf127a33c 100644 --- a/ydb/tests/tools/kqprun/recipe/__main__.py +++ b/ydb/tests/tools/kqprun/recipe/__main__.py @@ -10,6 +10,7 @@ PID_FILENAME = "kqprun_daemon.pid" KQPRUN_PATH = os.getenv("KQPRUN_EXECUTABLE") or "ydb/tests/tools/kqprun/kqprun" +INITIALIZATION_IMEOUT_RATIO = 2 def is_kqprun_daemon_ready() -> bool: @@ -40,7 +41,7 @@ def build_start_comand(argv: list[str], grpc_port: int) -> tuple[int, list[str]] cmd.append("--script-query") cmd.append(yatest_common.source_path(query)) - return (parsed.timeout_ms // 500, cmd) + return (parsed.timeout_ms, cmd) def start(argv: list[str]): @@ -48,14 +49,14 @@ def start(argv: list[str]): portManager = PortManager() grpc_port = portManager.get_port() - timeout, cmd = build_start_comand(argv, grpc_port) + timeout_ms, cmd = build_start_comand(argv, grpc_port) recipes_common.start_daemon( command=cmd, environment=None, is_alive_check=is_kqprun_daemon_ready, pid_file_name=PID_FILENAME, - timeout=timeout, + timeout=INITIALIZATION_IMEOUT_RATIO * (timeout_ms // 1000), daemon_name="kqprun_daemon" )