From 9d9707c175ccff307ff02a7aac55d72965aeaa56 Mon Sep 17 00:00:00 2001
From: Grigoriy Pisarenko <grigoriypisar@ydb.tech>
Date: Tue, 4 Mar 2025 09:40:05 +0000
Subject: [PATCH 1/3] Fixed scripts

---
 ydb/tests/tools/fqrun/flame_graph.sh          |  2 +-
 ydb/tests/tools/kqprun/README.md              | 12 ++-
 .../configuration/prometheus_config.yaml      | 75 +++++++++++++++++++
 ydb/tests/tools/kqprun/flame_graph.sh         | 31 --------
 ydb/tests/tools/kqprun/scripts/flame_graph.sh | 31 ++++++++
 .../tools/kqprun/scripts/start_connector.sh   | 16 ++++
 .../tools/kqprun/scripts/start_prometheus.sh  | 29 +++++++
 7 files changed, 163 insertions(+), 33 deletions(-)
 create mode 100644 ydb/tests/tools/kqprun/configuration/prometheus_config.yaml
 delete mode 100755 ydb/tests/tools/kqprun/flame_graph.sh
 create mode 100755 ydb/tests/tools/kqprun/scripts/flame_graph.sh
 create mode 100755 ydb/tests/tools/kqprun/scripts/start_connector.sh
 create mode 100755 ydb/tests/tools/kqprun/scripts/start_prometheus.sh

diff --git a/ydb/tests/tools/fqrun/flame_graph.sh b/ydb/tests/tools/fqrun/flame_graph.sh
index 9b002f622749..8f88eb745bc7 100755
--- a/ydb/tests/tools/fqrun/flame_graph.sh
+++ b/ydb/tests/tools/fqrun/flame_graph.sh
@@ -9,4 +9,4 @@ fi
 
 SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
 
-$SCRIPT_DIR/../kqprun/flame_graph.sh ${1:-'30'} ${2:-''} fqrun
+$SCRIPT_DIR/../kqprun/scripts/flame_graph.sh ${1:-'30'} ${2:-''} fqrun
diff --git a/ydb/tests/tools/kqprun/README.md b/ydb/tests/tools/kqprun/README.md
index 225a58264a25..213494f00f7a 100644
--- a/ydb/tests/tools/kqprun/README.md
+++ b/ydb/tests/tools/kqprun/README.md
@@ -8,7 +8,17 @@ For profiling memory allocations build kqprun with ya make flag `-D PROFILE_MEMO
 
 * `flame_graph.sh` - script for collecting flame graphs in svg format, usage:
     ```(bash)
-    ./flame_graph.sh [graph collection time in seconds] [use sudo]
+    ./scripts/flame_graph.sh [graph collection time in seconds] [use sudo]
+    ```
+
+* `start_prometheus.sh` - script for starting prometheus web UI, can be used for counters visualisation (kqprun should be runned with flag `-M <monitoring port>`), usage:
+    ```(bash)
+    ./scripts/start_prometheus.sh <monitoring port> <web UI port> [config path]
+    ```
+
+* `start_connector.sh` - script for starting local FQ connector, usage:
+    ```(bash)
+    ./scripts/start_connector.sh <connector port>
     ```
 
 ## Examples
diff --git a/ydb/tests/tools/kqprun/configuration/prometheus_config.yaml b/ydb/tests/tools/kqprun/configuration/prometheus_config.yaml
new file mode 100644
index 000000000000..8ff36ef3e934
--- /dev/null
+++ b/ydb/tests/tools/kqprun/configuration/prometheus_config.yaml
@@ -0,0 +1,75 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+
+scrape_configs:
+  - job_name: "ydb_dynamic"
+    metrics_path: "/counters/counters=ydb/name_label=name/prometheus"
+    static_configs:
+      - targets: ["localhost:${TARGET_PORT}"]
+        labels:
+          container: ydb-dynamic
+    metric_relabel_configs:
+      - source_labels: ["__name__"]
+        target_label: "__name__"
+        replacement: "ydb_$1"
+  - job_name: "utils_dynamic"
+    metrics_path: "/counters/counters=utils/prometheus"
+    static_configs:
+      - targets: ["localhost:${TARGET_PORT}"]
+        labels:
+          container: ydb-dynamic
+    metric_relabel_configs:
+      - source_labels: ["__name__"]
+        target_label: "__name__"
+        replacement: "utils_$1"
+  - job_name: "kqp_dynamic"
+    metrics_path: "/counters/counters=kqp/prometheus"
+    static_configs:
+      - targets: ["localhost:${TARGET_PORT}"]
+        labels:
+          container: ydb-dynamic
+    metric_relabel_configs:
+      - source_labels: ["__name__"]
+        target_label: "__name__"
+        replacement: "kqp_$1"
+  - job_name: "tablets_dynamic"
+    metrics_path: "/counters/counters=tablets/prometheus"
+    static_configs:
+      - targets: ["localhost:${TARGET_PORT}"]
+        labels:
+          container: ydb-dynamic
+    metric_relabel_configs:
+      - source_labels: ["__name__"]
+        target_label: "__name__"
+        replacement: "tablets_$1"
+  - job_name: "proxy_dynamic"
+    metrics_path: "/counters/counters=proxy/prometheus"
+    static_configs:
+      - targets: ["localhost:${TARGET_PORT}"]
+        labels:
+          container: ydb-dynamic
+    metric_relabel_configs:
+      - source_labels: ["__name__"]
+        target_label: "__name__"
+        replacement: "proxy_$1"
+  - job_name: "dsproxynode_dynamic"
+    metrics_path: "/counters/counters=dsproxynode/prometheus"
+    static_configs:
+      - targets: ["localhost:${TARGET_PORT}"]
+        labels:
+          container: ydb-dynamic
+    metric_relabel_configs:
+      - source_labels: ["__name__"]
+        target_label: "__name__"
+        replacement: "dsproxynode_$1"
+  - job_name: "ic_dynamic"
+    metrics_path: "/counters/counters=interconnect/prometheus"
+    static_configs:
+      - targets: ["localhost:${TARGET_PORT}"]
+        labels:
+          container: ydb-dynamic
+    metric_relabel_configs:
+      - source_labels: ["__name__"]
+        target_label: "__name__"
+        replacement: "interconnect_$1"
diff --git a/ydb/tests/tools/kqprun/flame_graph.sh b/ydb/tests/tools/kqprun/flame_graph.sh
deleted file mode 100755
index 25467038bfe6..000000000000
--- a/ydb/tests/tools/kqprun/flame_graph.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/usr/bin/env bash
-
-set -eux
-
-if [ $# -gt 3 ]; then
-    echo "Too many arguments"
-    exit -1
-fi
-
-SUDO=""
-
-if [ ${2:-''} ]; then
-    SUDO="sudo"
-fi
-
-function cleanup {
-    $SUDO rm ./profdata
-    rm ./profdata.txt
-}
-trap cleanup EXIT
-
-TARGER_PID=$(pgrep -u $USER ${3:-'kqprun'})
-
-$SUDO perf record -F 50 --call-graph dwarf -g --proc-map-timeout=10000 --pid $TARGER_PID -v -o profdata -- sleep ${1:-'30'}
-$SUDO perf script -i profdata > profdata.txt
-
-SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
-
-flame_graph_tool="$SCRIPT_DIR/../../../../contrib/tools/flame-graph/"
-
-${flame_graph_tool}/stackcollapse-perf.pl profdata.txt | ${flame_graph_tool}/flamegraph.pl > profdata.svg
diff --git a/ydb/tests/tools/kqprun/scripts/flame_graph.sh b/ydb/tests/tools/kqprun/scripts/flame_graph.sh
new file mode 100755
index 000000000000..667c7a5e1114
--- /dev/null
+++ b/ydb/tests/tools/kqprun/scripts/flame_graph.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -eux
+
+if [ $# -gt 3 ]; then
+    echo "Too many arguments"
+    exit -1
+fi
+
+SUDO=""
+
+if [ ${2:-''} ]; then
+    SUDO="sudo"
+fi
+
+SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
+
+function cleanup {
+    $SUDO rm $SCRIPT_DIR/profdata
+    rm $SCRIPT_DIR/profdata.txt
+}
+trap cleanup EXIT
+
+TARGER_PID=$(pgrep -u $USER ${3:-'kqprun'})
+
+$SUDO perf record -F 50 --call-graph dwarf -g --proc-map-timeout=10000 --pid $TARGER_PID -v -o $SCRIPT_DIR/profdata -- sleep ${1:-'30'}
+$SUDO perf script -i $SCRIPT_DIR/profdata > $SCRIPT_DIR/profdata.txt
+
+flame_graph_tool="$SCRIPT_DIR/../../../../../contrib/tools/flame-graph/"
+
+${flame_graph_tool}/stackcollapse-perf.pl $SCRIPT_DIR/profdata.txt | ${flame_graph_tool}/flamegraph.pl > ./profdata.svg
diff --git a/ydb/tests/tools/kqprun/scripts/start_connector.sh b/ydb/tests/tools/kqprun/scripts/start_connector.sh
new file mode 100755
index 000000000000..376628ab9641
--- /dev/null
+++ b/ydb/tests/tools/kqprun/scripts/start_connector.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -eux
+
+if [ $# -le 0 ]; then
+    echo "Please provide connector port (value of Generic::Connector::Endpoint::Port in config)"
+    exit -1
+fi
+
+if [ $# -gt 2 ]; then
+    echo "Too many arguments"
+    exit -1
+fi
+
+docker pull ghcr.io/ydb-platform/fq-connector-go:latest
+docker run --rm --name=fq-connector-go-$1 --network host ghcr.io/ydb-platform/fq-connector-go:latest --connector-port=$1 --metrics-port=$(($1 + 1)) --pprof-port=$(($1 + 2))
diff --git a/ydb/tests/tools/kqprun/scripts/start_prometheus.sh b/ydb/tests/tools/kqprun/scripts/start_prometheus.sh
new file mode 100755
index 000000000000..0d7090df2a60
--- /dev/null
+++ b/ydb/tests/tools/kqprun/scripts/start_prometheus.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -eux
+
+if [ $# -le 0 ]; then
+    echo "Please provide monitoring port to listen (value of -M argument)"
+    exit -1
+fi
+
+if [ $# -le 1 ]; then
+    echo "Please provide prometheus web ui port"
+    exit -1
+fi
+
+if [ $# -gt 3 ]; then
+    echo "Too many arguments"
+    exit -1
+fi
+
+SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
+
+function cleanup {
+    rm $SCRIPT_DIR/prometheus_config.yaml
+}
+trap cleanup EXIT
+
+sed "s/\${TARGET_PORT}/$1/g" "${3:-$SCRIPT_DIR/../configuration/prometheus_config.yaml}" > $SCRIPT_DIR/prometheus_config.yaml
+
+docker run --rm --name=prometheus-$1-$2 --network host -v $SCRIPT_DIR/prometheus_config.yaml:/etc/prometheus/prometheus.yml prom/prometheus --config.file=/etc/prometheus/prometheus.yml --web.listen-address=:$2

From f5f260dec349c6cc798476a71483265be0fadff8 Mon Sep 17 00:00:00 2001
From: Grigoriy Pisarenko <grigoriypisar@ydb.tech>
Date: Tue, 4 Mar 2025 12:55:25 +0000
Subject: [PATCH 2/3] Supported graphana

---
 ydb/tests/tools/kqprun/README.md              | 23 +++++++--
 .../kqprun/configuration/app_config.conf      |  1 +
 .../tools/kqprun/scripts/cleanup_docker.sh    | 16 ++++++
 .../tools/kqprun/scripts/start_connector.sh   |  7 ++-
 .../tools/kqprun/scripts/start_grafana.sh     | 49 +++++++++++++++++++
 .../tools/kqprun/scripts/start_prometheus.sh  |  6 ++-
 6 files changed, 97 insertions(+), 5 deletions(-)
 create mode 100755 ydb/tests/tools/kqprun/scripts/cleanup_docker.sh
 create mode 100755 ydb/tests/tools/kqprun/scripts/start_grafana.sh

diff --git a/ydb/tests/tools/kqprun/README.md b/ydb/tests/tools/kqprun/README.md
index 213494f00f7a..e014905b5f49 100644
--- a/ydb/tests/tools/kqprun/README.md
+++ b/ydb/tests/tools/kqprun/README.md
@@ -11,16 +11,33 @@ For profiling memory allocations build kqprun with ya make flag `-D PROFILE_MEMO
     ./scripts/flame_graph.sh [graph collection time in seconds] [use sudo]
     ```
 
-* `start_prometheus.sh` - script for starting prometheus web UI, can be used for counters visualisation (kqprun should be runned with flag `-M <monitoring port>`), usage:
+* `start_prometheus.sh` - start prometheus web UI, can be used for counters visualisation (kqprun should be runned with flag `-M <monitoring port>`), usage:
     ```(bash)
-    ./scripts/start_prometheus.sh <monitoring port> <web UI port> [config path]
+    ./scripts/start_prometheus.sh <kqprun monitoring port> <web UI port> [config path]
     ```
+    Prometheus UI available on `http://localhost:<web UI port>/targets`
 
-* `start_connector.sh` - script for starting local FQ connector, usage:
+* `start_grafana.sh` - start grafana other existing prometheus, usage:
+    ```(bash)
+    ./scripts/start_grafana.sh <prometheus port> <web UI port> [additional dashboards dirs]
+    ```
+
+    Command for starting prometheus and grafana:
+    ```(bash)
+    ./scripts/start_prometheus.sh 32000 32001 && ./scripts/start_grafana.sh 32001 32002
+    ```
+    Where 32000 - kqprun monitoring port, graphana UI available on http://localhost:32002 (login: `admin`, password: `admin`)
+
+* `start_connector.sh` - start local FQ connector, usage:
     ```(bash)
     ./scripts/start_connector.sh <connector port>
     ```
 
+* `cleanup_docker.sh` - stop created docker containers, usege:
+    ```(bash)
+    ./scripts/cleanup_docker.sh [name filter]
+    ```
+
 ## Examples
 
 ### Queries
diff --git a/ydb/tests/tools/kqprun/configuration/app_config.conf b/ydb/tests/tools/kqprun/configuration/app_config.conf
index 8dcb892766a8..fab94d893562 100644
--- a/ydb/tests/tools/kqprun/configuration/app_config.conf
+++ b/ydb/tests/tools/kqprun/configuration/app_config.conf
@@ -55,6 +55,7 @@ FeatureFlags {
   EnableExternalSourceSchemaInference: true
   EnableTempTables: true
   EnableReplaceIfExistsForExternalEntities: true
+  EnableResourcePoolsCounters: true
 }
 
 KQPConfig {
diff --git a/ydb/tests/tools/kqprun/scripts/cleanup_docker.sh b/ydb/tests/tools/kqprun/scripts/cleanup_docker.sh
new file mode 100755
index 000000000000..ba7342996e08
--- /dev/null
+++ b/ydb/tests/tools/kqprun/scripts/cleanup_docker.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -eux
+
+if [ $# -gt 1 ]; then
+    echo "Too many arguments"
+    exit -1
+fi
+
+KQPRUN_CONTAINERS=$(docker container ls -q --filter name=${1:-$USER-kqprun-*})
+
+if [ "$KQPRUN_CONTAINERS" ]; then
+    docker container stop $KQPRUN_CONTAINERS
+else
+    echo "Nothing to cleanup"
+fi
diff --git a/ydb/tests/tools/kqprun/scripts/start_connector.sh b/ydb/tests/tools/kqprun/scripts/start_connector.sh
index 376628ab9641..f010b3b7237a 100755
--- a/ydb/tests/tools/kqprun/scripts/start_connector.sh
+++ b/ydb/tests/tools/kqprun/scripts/start_connector.sh
@@ -12,5 +12,10 @@ if [ $# -gt 2 ]; then
     exit -1
 fi
 
+SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
+CONTAINER_NAME=$USER-kqprun-fq-connector-go-$1
+
+$SCRIPT_DIR/cleanup_docker.sh $CONTAINER_NAME
 docker pull ghcr.io/ydb-platform/fq-connector-go:latest
-docker run --rm --name=fq-connector-go-$1 --network host ghcr.io/ydb-platform/fq-connector-go:latest --connector-port=$1 --metrics-port=$(($1 + 1)) --pprof-port=$(($1 + 2))
+docker run -d --rm --name=$CONTAINER_NAME --network host ghcr.io/ydb-platform/fq-connector-go:latest --connector-port=$1 --metrics-port=$(($1 + 1)) --pprof-port=$(($1 + 2))
+docker container ls --filter name=$CONTAINER_NAME
diff --git a/ydb/tests/tools/kqprun/scripts/start_grafana.sh b/ydb/tests/tools/kqprun/scripts/start_grafana.sh
new file mode 100755
index 000000000000..4fc707f73312
--- /dev/null
+++ b/ydb/tests/tools/kqprun/scripts/start_grafana.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+set -eux
+
+if [ $# -le 0 ]; then
+    echo "Please provide prometheus port"
+    exit -1
+fi
+
+if [ $# -le 1 ]; then
+    echo "Please provide grafana web ui port"
+    exit -1
+fi
+
+SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
+DASHBOARDS_DIRS="$SCRIPT_DIR/../../../../deploy/helm/ydb-prometheus/dashboards/"
+if [ $# -gt 2 ]; then
+    DASHBOARDS_DIRS="$DASHBOARDS_DIRS $3"
+fi
+
+if [ $# -gt 3 ]; then
+    echo "Too many arguments"
+    exit -1
+fi
+
+CONTAINER_NAME=$USER-kqprun-grafana-$1-$2
+
+$SCRIPT_DIR/cleanup_docker.sh $CONTAINER_NAME
+docker run -d --rm --name=$CONTAINER_NAME --network host -e "GF_SERVER_HTTP_PORT=$2" grafana/grafana-oss
+docker container ls --filter name=$CONTAINER_NAME
+
+echo "Wait grafana is up"
+
+bash -ceux "while ! nc -v -z -w 1 localhost $2 ; do sleep 2 ; done"
+
+echo "Initialization of datasource and dashboards"
+
+GRAFANA_API="http://admin:admin@localhost:$2/api"
+
+curl -X POST -H "Content-Type: application/json" ${GRAFANA_API}/datasources --data-ascii "{ \"name\": \"prometheus_datasource_$1\", \"type\": \"prometheus\", \"url\": \"http://localhost:$1\", \"access\": \"proxy\" }"
+curl -X POST -H "Content-Type: application/json" ${GRAFANA_API}/folders --data-ascii '{ "uid": "ydb", "title": "YDB" }'
+
+for DASHBOARDS_PATH in $DASHBOARDS_DIRS; do
+    for DASH in $(ls $DASHBOARDS_PATH); do
+        cat $DASHBOARDS_PATH/${DASH} | jq '{ folderUid: "ydb", dashboard: . }' | curl -X POST -H "Content-Type: application/json" ${GRAFANA_API}/dashboards/db -d @-
+    done
+done
+
+echo "Initialization finished successfully"
diff --git a/ydb/tests/tools/kqprun/scripts/start_prometheus.sh b/ydb/tests/tools/kqprun/scripts/start_prometheus.sh
index 0d7090df2a60..db197fd277ef 100755
--- a/ydb/tests/tools/kqprun/scripts/start_prometheus.sh
+++ b/ydb/tests/tools/kqprun/scripts/start_prometheus.sh
@@ -26,4 +26,8 @@ trap cleanup EXIT
 
 sed "s/\${TARGET_PORT}/$1/g" "${3:-$SCRIPT_DIR/../configuration/prometheus_config.yaml}" > $SCRIPT_DIR/prometheus_config.yaml
 
-docker run --rm --name=prometheus-$1-$2 --network host -v $SCRIPT_DIR/prometheus_config.yaml:/etc/prometheus/prometheus.yml prom/prometheus --config.file=/etc/prometheus/prometheus.yml --web.listen-address=:$2
+CONTAINER_NAME=$USER-kqprun-prometheus-$1-$2
+
+$SCRIPT_DIR/cleanup_docker.sh $CONTAINER_NAME
+docker run -d --rm --name=$CONTAINER_NAME --network host -v $SCRIPT_DIR/prometheus_config.yaml:/etc/prometheus/prometheus.yml prom/prometheus --config.file=/etc/prometheus/prometheus.yml --web.listen-address=:$2
+docker container ls --filter name=$CONTAINER_NAME

From 8048809f5acf4dc25b2bfb2932836f8ef92bd968 Mon Sep 17 00:00:00 2001
From: Grigoriy Pisarenko <grigoriypisar@ydb.tech>
Date: Tue, 4 Mar 2025 13:34:54 +0000
Subject: [PATCH 3/3] Fixed issues

---
 ydb/tests/tools/kqprun/scripts/start_grafana.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ydb/tests/tools/kqprun/scripts/start_grafana.sh b/ydb/tests/tools/kqprun/scripts/start_grafana.sh
index 4fc707f73312..ed5f54b0ee01 100755
--- a/ydb/tests/tools/kqprun/scripts/start_grafana.sh
+++ b/ydb/tests/tools/kqprun/scripts/start_grafana.sh
@@ -31,7 +31,7 @@ docker container ls --filter name=$CONTAINER_NAME
 
 echo "Wait grafana is up"
 
-bash -ceux "while ! nc -v -z -w 1 localhost $2 ; do sleep 2 ; done"
+(set -eux; while ! nc -v -z -w 1 localhost $2; do sleep 2; done)
 
 echo "Initialization of datasource and dashboards"
 
@@ -40,9 +40,9 @@ GRAFANA_API="http://admin:admin@localhost:$2/api"
 curl -X POST -H "Content-Type: application/json" ${GRAFANA_API}/datasources --data-ascii "{ \"name\": \"prometheus_datasource_$1\", \"type\": \"prometheus\", \"url\": \"http://localhost:$1\", \"access\": \"proxy\" }"
 curl -X POST -H "Content-Type: application/json" ${GRAFANA_API}/folders --data-ascii '{ "uid": "ydb", "title": "YDB" }'
 
-for DASHBOARDS_PATH in $DASHBOARDS_DIRS; do
-    for DASH in $(ls $DASHBOARDS_PATH); do
-        cat $DASHBOARDS_PATH/${DASH} | jq '{ folderUid: "ydb", dashboard: . }' | curl -X POST -H "Content-Type: application/json" ${GRAFANA_API}/dashboards/db -d @-
+for DASHBOARDS_DIR in $DASHBOARDS_DIRS; do
+    for DASH in "$DASHBOARDS_DIR"/*; do
+        jq '{ folderUid: "ydb", dashboard: . }' < "$DASH" | curl -X POST -H "Content-Type: application/json" ${GRAFANA_API}/dashboards/db -d @-
     done
 done