From ae53eb120fc5560a481ef481993cd4727aa57a85 Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Mon, 2 Dec 2024 12:04:44 +0300 Subject: [PATCH 1/3] Enable kafka api by env variable (#11892) --- ydb/public/tools/lib/cmds/__init__.py | 13 +++++++++++++ ydb/tests/library/harness/kikimr_config.py | 8 ++++++++ 2 files changed, 21 insertions(+) diff --git a/ydb/public/tools/lib/cmds/__init__.py b/ydb/public/tools/lib/cmds/__init__.py index 6832ed6d0cbd..58168424219c 100644 --- a/ydb/public/tools/lib/cmds/__init__.py +++ b/ydb/public/tools/lib/cmds/__init__.py @@ -349,6 +349,10 @@ def deploy(arguments): if 'YDB_EXPERIMENTAL_PG' in os.environ: optionals['pg_compatible_expirement'] = True + if _is_env_option_enabled('YDB_KAFKA_API_ENABLED'): + kafka_api_port = int(os.environ.get("YDB_KAFKA_PROXY_PORT", "9092")) + optionals['kafka_api_port'] = kafka_api_port + configuration = KikimrConfigGenerator( parse_erasure(arguments), arguments.ydb_binary_path, @@ -542,3 +546,12 @@ def start_recipe(args): def stop_recipe(args): arguments = produce_arguments(args) cleanup(arguments) + + +def _is_true_string(s): + s = s.upper() + return s in ['1', 'TRUE', 'YES', 'ON'] + + +def _is_env_option_enabled(name): + return _is_true_string(os.environ.get(name, "FALSE")) diff --git a/ydb/tests/library/harness/kikimr_config.py b/ydb/tests/library/harness/kikimr_config.py index ea0dbb67733d..6b6def14934e 100644 --- a/ydb/tests/library/harness/kikimr_config.py +++ b/ydb/tests/library/harness/kikimr_config.py @@ -166,6 +166,7 @@ def __init__( pg_compatible_expirement=False, generic_connector_config=None, # typing.Optional[TGenericConnectorConfig] pgwire_port=None, + kafka_api_port=None, ): if extra_feature_flags is None: extra_feature_flags = [] @@ -430,6 +431,13 @@ def __init__( self.yaml_config["feature_flags"]["enable_external_data_sources"] = True self.yaml_config["feature_flags"]["enable_script_execution_operations"] = True + if kafka_api_port is not None: + kafka_proxy_config = dict() + kafka_proxy_config["enable_kafka_proxy"] = True + kafka_proxy_config["listening_port"] = kafka_api_port + + self.yaml_config["kafka_proxy_config"] = kafka_proxy_config + @property def pdisks_info(self): return self._pdisks_info From ea3fbbc92600b00fdbba84e6ad461392fedb54ab Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Tue, 3 Dec 2024 16:19:51 +0300 Subject: [PATCH 2/3] use YDB_KAFKA_PROXY_PORT for enable kafka (#12242) --- ydb/public/tools/lib/cmds/__init__.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/ydb/public/tools/lib/cmds/__init__.py b/ydb/public/tools/lib/cmds/__init__.py index 58168424219c..93f0c3a89b7a 100644 --- a/ydb/public/tools/lib/cmds/__init__.py +++ b/ydb/public/tools/lib/cmds/__init__.py @@ -349,8 +349,8 @@ def deploy(arguments): if 'YDB_EXPERIMENTAL_PG' in os.environ: optionals['pg_compatible_expirement'] = True - if _is_env_option_enabled('YDB_KAFKA_API_ENABLED'): - kafka_api_port = int(os.environ.get("YDB_KAFKA_PROXY_PORT", "9092")) + kafka_api_port = int(os.environ.get("YDB_KAFKA_PROXY_PORT", "0")) + if kafka_api_port != 0: optionals['kafka_api_port'] = kafka_api_port configuration = KikimrConfigGenerator( @@ -546,12 +546,3 @@ def start_recipe(args): def stop_recipe(args): arguments = produce_arguments(args) cleanup(arguments) - - -def _is_true_string(s): - s = s.upper() - return s in ['1', 'TRUE', 'YES', 'ON'] - - -def _is_env_option_enabled(name): - return _is_true_string(os.environ.get(name, "FALSE")) From b7dc5935640349b13d00a8ede329600b6752cffb Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Tue, 3 Jun 2025 12:17:31 +0300 Subject: [PATCH 3/3] start kafka port by default in local ydb (#19194) --- .github/docker/Dockerfile | 1 + .github/docker/files/initialize_local_ydb | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/docker/Dockerfile b/.github/docker/Dockerfile index d266c29a933a..e3dc58d4003a 100644 --- a/.github/docker/Dockerfile +++ b/.github/docker/Dockerfile @@ -44,6 +44,7 @@ COPY --from=builder \ EXPOSE ${GRPC_TLS_PORT:-2135} EXPOSE ${GRPC_PORT:-2136} EXPOSE ${MON_PORT:-8765} +EXPOSE ${YDB_KAFKA_PROXY_PORT:-9092} HEALTHCHECK --start-period=60s --interval=1s CMD sh ./health_check diff --git a/.github/docker/files/initialize_local_ydb b/.github/docker/files/initialize_local_ydb index 0484aa53316e..339ef370e84b 100644 --- a/.github/docker/files/initialize_local_ydb +++ b/.github/docker/files/initialize_local_ydb @@ -8,6 +8,7 @@ export YDB_GRPC_ENABLE_TLS="true" export GRPC_TLS_PORT=${GRPC_TLS_PORT:-2135} export GRPC_PORT=${GRPC_PORT:-2136} export YDB_GRPC_TLS_DATA_PATH="/ydb_certs" +export YDB_KAFKA_PROXY_PORT=${YDB_KAFKA_PROXY_PORT:-9092} /local_ydb deploy --ydb-working-dir /ydb_data --ydb-binary-path /ydbd --fixed-ports --dont-use-log-files;