diff --git a/ydb/tests/tools/fqrun/flame_graph.sh b/ydb/tests/tools/fqrun/flame_graph.sh index 9b002f622749..8f88eb745bc7 100755 --- a/ydb/tests/tools/fqrun/flame_graph.sh +++ b/ydb/tests/tools/fqrun/flame_graph.sh @@ -9,4 +9,4 @@ fi SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) -$SCRIPT_DIR/../kqprun/flame_graph.sh ${1:-'30'} ${2:-''} fqrun +$SCRIPT_DIR/../kqprun/scripts/flame_graph.sh ${1:-'30'} ${2:-''} fqrun diff --git a/ydb/tests/tools/kqprun/README.md b/ydb/tests/tools/kqprun/README.md index 225a58264a25..e014905b5f49 100644 --- a/ydb/tests/tools/kqprun/README.md +++ b/ydb/tests/tools/kqprun/README.md @@ -8,7 +8,34 @@ For profiling memory allocations build kqprun with ya make flag `-D PROFILE_MEMO * `flame_graph.sh` - script for collecting flame graphs in svg format, usage: ```(bash) - ./flame_graph.sh [graph collection time in seconds] [use sudo] + ./scripts/flame_graph.sh [graph collection time in seconds] [use sudo] + ``` + +* `start_prometheus.sh` - start prometheus web UI, can be used for counters visualisation (kqprun should be runned with flag `-M `), usage: + ```(bash) + ./scripts/start_prometheus.sh [config path] + ``` + Prometheus UI available on `http://localhost:/targets` + +* `start_grafana.sh` - start grafana other existing prometheus, usage: + ```(bash) + ./scripts/start_grafana.sh [additional dashboards dirs] + ``` + + Command for starting prometheus and grafana: + ```(bash) + ./scripts/start_prometheus.sh 32000 32001 && ./scripts/start_grafana.sh 32001 32002 + ``` + Where 32000 - kqprun monitoring port, graphana UI available on http://localhost:32002 (login: `admin`, password: `admin`) + +* `start_connector.sh` - start local FQ connector, usage: + ```(bash) + ./scripts/start_connector.sh + ``` + +* `cleanup_docker.sh` - stop created docker containers, usege: + ```(bash) + ./scripts/cleanup_docker.sh [name filter] ``` ## Examples diff --git a/ydb/tests/tools/kqprun/configuration/app_config.conf b/ydb/tests/tools/kqprun/configuration/app_config.conf index 8dcb892766a8..fab94d893562 100644 --- a/ydb/tests/tools/kqprun/configuration/app_config.conf +++ b/ydb/tests/tools/kqprun/configuration/app_config.conf @@ -55,6 +55,7 @@ FeatureFlags { EnableExternalSourceSchemaInference: true EnableTempTables: true EnableReplaceIfExistsForExternalEntities: true + EnableResourcePoolsCounters: true } KQPConfig { diff --git a/ydb/tests/tools/kqprun/configuration/prometheus_config.yaml b/ydb/tests/tools/kqprun/configuration/prometheus_config.yaml new file mode 100644 index 000000000000..8ff36ef3e934 --- /dev/null +++ b/ydb/tests/tools/kqprun/configuration/prometheus_config.yaml @@ -0,0 +1,75 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + +scrape_configs: + - job_name: "ydb_dynamic" + metrics_path: "/counters/counters=ydb/name_label=name/prometheus" + static_configs: + - targets: ["localhost:${TARGET_PORT}"] + labels: + container: ydb-dynamic + metric_relabel_configs: + - source_labels: ["__name__"] + target_label: "__name__" + replacement: "ydb_$1" + - job_name: "utils_dynamic" + metrics_path: "/counters/counters=utils/prometheus" + static_configs: + - targets: ["localhost:${TARGET_PORT}"] + labels: + container: ydb-dynamic + metric_relabel_configs: + - source_labels: ["__name__"] + target_label: "__name__" + replacement: "utils_$1" + - job_name: "kqp_dynamic" + metrics_path: "/counters/counters=kqp/prometheus" + static_configs: + - targets: ["localhost:${TARGET_PORT}"] + labels: + container: ydb-dynamic + metric_relabel_configs: + - source_labels: ["__name__"] + target_label: "__name__" + replacement: "kqp_$1" + - job_name: "tablets_dynamic" + metrics_path: "/counters/counters=tablets/prometheus" + static_configs: + - targets: ["localhost:${TARGET_PORT}"] + labels: + container: ydb-dynamic + metric_relabel_configs: + - source_labels: ["__name__"] + target_label: "__name__" + replacement: "tablets_$1" + - job_name: "proxy_dynamic" + metrics_path: "/counters/counters=proxy/prometheus" + static_configs: + - targets: ["localhost:${TARGET_PORT}"] + labels: + container: ydb-dynamic + metric_relabel_configs: + - source_labels: ["__name__"] + target_label: "__name__" + replacement: "proxy_$1" + - job_name: "dsproxynode_dynamic" + metrics_path: "/counters/counters=dsproxynode/prometheus" + static_configs: + - targets: ["localhost:${TARGET_PORT}"] + labels: + container: ydb-dynamic + metric_relabel_configs: + - source_labels: ["__name__"] + target_label: "__name__" + replacement: "dsproxynode_$1" + - job_name: "ic_dynamic" + metrics_path: "/counters/counters=interconnect/prometheus" + static_configs: + - targets: ["localhost:${TARGET_PORT}"] + labels: + container: ydb-dynamic + metric_relabel_configs: + - source_labels: ["__name__"] + target_label: "__name__" + replacement: "interconnect_$1" diff --git a/ydb/tests/tools/kqprun/flame_graph.sh b/ydb/tests/tools/kqprun/flame_graph.sh deleted file mode 100755 index 25467038bfe6..000000000000 --- a/ydb/tests/tools/kqprun/flame_graph.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -set -eux - -if [ $# -gt 3 ]; then - echo "Too many arguments" - exit -1 -fi - -SUDO="" - -if [ ${2:-''} ]; then - SUDO="sudo" -fi - -function cleanup { - $SUDO rm ./profdata - rm ./profdata.txt -} -trap cleanup EXIT - -TARGER_PID=$(pgrep -u $USER ${3:-'kqprun'}) - -$SUDO perf record -F 50 --call-graph dwarf -g --proc-map-timeout=10000 --pid $TARGER_PID -v -o profdata -- sleep ${1:-'30'} -$SUDO perf script -i profdata > profdata.txt - -SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) - -flame_graph_tool="$SCRIPT_DIR/../../../../contrib/tools/flame-graph/" - -${flame_graph_tool}/stackcollapse-perf.pl profdata.txt | ${flame_graph_tool}/flamegraph.pl > profdata.svg diff --git a/ydb/tests/tools/kqprun/scripts/cleanup_docker.sh b/ydb/tests/tools/kqprun/scripts/cleanup_docker.sh new file mode 100755 index 000000000000..ba7342996e08 --- /dev/null +++ b/ydb/tests/tools/kqprun/scripts/cleanup_docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -eux + +if [ $# -gt 1 ]; then + echo "Too many arguments" + exit -1 +fi + +KQPRUN_CONTAINERS=$(docker container ls -q --filter name=${1:-$USER-kqprun-*}) + +if [ "$KQPRUN_CONTAINERS" ]; then + docker container stop $KQPRUN_CONTAINERS +else + echo "Nothing to cleanup" +fi diff --git a/ydb/tests/tools/kqprun/scripts/flame_graph.sh b/ydb/tests/tools/kqprun/scripts/flame_graph.sh new file mode 100755 index 000000000000..667c7a5e1114 --- /dev/null +++ b/ydb/tests/tools/kqprun/scripts/flame_graph.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -eux + +if [ $# -gt 3 ]; then + echo "Too many arguments" + exit -1 +fi + +SUDO="" + +if [ ${2:-''} ]; then + SUDO="sudo" +fi + +SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) + +function cleanup { + $SUDO rm $SCRIPT_DIR/profdata + rm $SCRIPT_DIR/profdata.txt +} +trap cleanup EXIT + +TARGER_PID=$(pgrep -u $USER ${3:-'kqprun'}) + +$SUDO perf record -F 50 --call-graph dwarf -g --proc-map-timeout=10000 --pid $TARGER_PID -v -o $SCRIPT_DIR/profdata -- sleep ${1:-'30'} +$SUDO perf script -i $SCRIPT_DIR/profdata > $SCRIPT_DIR/profdata.txt + +flame_graph_tool="$SCRIPT_DIR/../../../../../contrib/tools/flame-graph/" + +${flame_graph_tool}/stackcollapse-perf.pl $SCRIPT_DIR/profdata.txt | ${flame_graph_tool}/flamegraph.pl > ./profdata.svg diff --git a/ydb/tests/tools/kqprun/scripts/start_connector.sh b/ydb/tests/tools/kqprun/scripts/start_connector.sh new file mode 100755 index 000000000000..f010b3b7237a --- /dev/null +++ b/ydb/tests/tools/kqprun/scripts/start_connector.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -eux + +if [ $# -le 0 ]; then + echo "Please provide connector port (value of Generic::Connector::Endpoint::Port in config)" + exit -1 +fi + +if [ $# -gt 2 ]; then + echo "Too many arguments" + exit -1 +fi + +SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +CONTAINER_NAME=$USER-kqprun-fq-connector-go-$1 + +$SCRIPT_DIR/cleanup_docker.sh $CONTAINER_NAME +docker pull ghcr.io/ydb-platform/fq-connector-go:latest +docker run -d --rm --name=$CONTAINER_NAME --network host ghcr.io/ydb-platform/fq-connector-go:latest --connector-port=$1 --metrics-port=$(($1 + 1)) --pprof-port=$(($1 + 2)) +docker container ls --filter name=$CONTAINER_NAME diff --git a/ydb/tests/tools/kqprun/scripts/start_grafana.sh b/ydb/tests/tools/kqprun/scripts/start_grafana.sh new file mode 100755 index 000000000000..ed5f54b0ee01 --- /dev/null +++ b/ydb/tests/tools/kqprun/scripts/start_grafana.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -eux + +if [ $# -le 0 ]; then + echo "Please provide prometheus port" + exit -1 +fi + +if [ $# -le 1 ]; then + echo "Please provide grafana web ui port" + exit -1 +fi + +SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +DASHBOARDS_DIRS="$SCRIPT_DIR/../../../../deploy/helm/ydb-prometheus/dashboards/" +if [ $# -gt 2 ]; then + DASHBOARDS_DIRS="$DASHBOARDS_DIRS $3" +fi + +if [ $# -gt 3 ]; then + echo "Too many arguments" + exit -1 +fi + +CONTAINER_NAME=$USER-kqprun-grafana-$1-$2 + +$SCRIPT_DIR/cleanup_docker.sh $CONTAINER_NAME +docker run -d --rm --name=$CONTAINER_NAME --network host -e "GF_SERVER_HTTP_PORT=$2" grafana/grafana-oss +docker container ls --filter name=$CONTAINER_NAME + +echo "Wait grafana is up" + +(set -eux; while ! nc -v -z -w 1 localhost $2; do sleep 2; done) + +echo "Initialization of datasource and dashboards" + +GRAFANA_API="http://admin:admin@localhost:$2/api" + +curl -X POST -H "Content-Type: application/json" ${GRAFANA_API}/datasources --data-ascii "{ \"name\": \"prometheus_datasource_$1\", \"type\": \"prometheus\", \"url\": \"http://localhost:$1\", \"access\": \"proxy\" }" +curl -X POST -H "Content-Type: application/json" ${GRAFANA_API}/folders --data-ascii '{ "uid": "ydb", "title": "YDB" }' + +for DASHBOARDS_DIR in $DASHBOARDS_DIRS; do + for DASH in "$DASHBOARDS_DIR"/*; do + jq '{ folderUid: "ydb", dashboard: . }' < "$DASH" | curl -X POST -H "Content-Type: application/json" ${GRAFANA_API}/dashboards/db -d @- + done +done + +echo "Initialization finished successfully" diff --git a/ydb/tests/tools/kqprun/scripts/start_prometheus.sh b/ydb/tests/tools/kqprun/scripts/start_prometheus.sh new file mode 100755 index 000000000000..db197fd277ef --- /dev/null +++ b/ydb/tests/tools/kqprun/scripts/start_prometheus.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -eux + +if [ $# -le 0 ]; then + echo "Please provide monitoring port to listen (value of -M argument)" + exit -1 +fi + +if [ $# -le 1 ]; then + echo "Please provide prometheus web ui port" + exit -1 +fi + +if [ $# -gt 3 ]; then + echo "Too many arguments" + exit -1 +fi + +SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) + +function cleanup { + rm $SCRIPT_DIR/prometheus_config.yaml +} +trap cleanup EXIT + +sed "s/\${TARGET_PORT}/$1/g" "${3:-$SCRIPT_DIR/../configuration/prometheus_config.yaml}" > $SCRIPT_DIR/prometheus_config.yaml + +CONTAINER_NAME=$USER-kqprun-prometheus-$1-$2 + +$SCRIPT_DIR/cleanup_docker.sh $CONTAINER_NAME +docker run -d --rm --name=$CONTAINER_NAME --network host -v $SCRIPT_DIR/prometheus_config.yaml:/etc/prometheus/prometheus.yml prom/prometheus --config.file=/etc/prometheus/prometheus.yml --web.listen-address=:$2 +docker container ls --filter name=$CONTAINER_NAME