Skip to content

Commit 5f24705

Browse files
GrigoriyPAGrigoriyPA
committed
Added scripts for running prometeus and connector
1 parent d492c81 commit 5f24705

File tree

6 files changed

+188
-0
lines changed

6 files changed

+188
-0
lines changed

‎ydb/tests/tools/kqprun/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,4 @@ udfs
1010
*.txt
1111
*.svg
1212
*.old
13+
*.yaml

‎ydb/tests/tools/kqprun/configuration/connector_config.yaml

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
connector_server:
2+
endpoint:
3+
host: "0.0.0.0"
4+
port: ${TARGET_PORT}
5+
6+
logger:
7+
log_level: INFO
8+
enable_sql_query_logging: false
9+
10+
paging:
11+
bytes_per_page: 4194304
12+
prefetch_queue_capacity: 2
13+
14+
conversion:
15+
use_unsafe_converters: true
16+
17+
data_source_default: &data_source_default_var
18+
open_connection_timeout: 5s
19+
ping_connection_timeout: 5s
20+
exponential_backoff:
21+
initial_interval: 500ms
22+
randomization_factor: 0.5
23+
multiplier: 1.5
24+
max_interval: 20s
25+
max_elapsed_time: 1m
26+
27+
datasources:
28+
clickhouse:
29+
<<: *data_source_default_var
30+
31+
greenplum:
32+
<<: *data_source_default_var
33+
34+
ms_sql_server:
35+
<<: *data_source_default_var
36+
37+
mysql:
38+
<<: *data_source_default_var
39+
result_chan_capacity: 1024
40+
41+
postgresql:
42+
<<: *data_source_default_var
43+
44+
oracle:
45+
<<: *data_source_default_var
46+
47+
mongodb:
48+
<<: *data_source_default_var
49+
count_docs_to_deduce_schema: 5
50+
51+
ydb:
52+
<<: *data_source_default_var
53+
use_underlay_network_for_dedicated_databases: false

‎ydb/tests/tools/kqprun/configuration/prometheus_config.yaml

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
global:
2+
scrape_interval: 15s
3+
evaluation_interval: 15s
4+
5+
scrape_configs:
6+
- job_name: "ydb_dynamic"
7+
metrics_path: "/counters/counters=ydb/name_label=name/prometheus"
8+
static_configs:
9+
- targets: ["localhost:${TARGET_PORT}"]
10+
labels:
11+
container: ydb-dynamic
12+
metric_relabel_configs:
13+
- source_labels: ["__name__"]
14+
target_label: "__name__"
15+
replacement: "ydb_$1"
16+
- job_name: "utils_dynamic"
17+
metrics_path: "/counters/counters=utils/prometheus"
18+
static_configs:
19+
- targets: ["localhost:${TARGET_PORT}"]
20+
labels:
21+
container: ydb-dynamic
22+
metric_relabel_configs:
23+
- source_labels: ["__name__"]
24+
target_label: "__name__"
25+
replacement: "utils_$1"
26+
- job_name: "kqp_dynamic"
27+
metrics_path: "/counters/counters=kqp/prometheus"
28+
static_configs:
29+
- targets: ["localhost:${TARGET_PORT}"]
30+
labels:
31+
container: ydb-dynamic
32+
metric_relabel_configs:
33+
- source_labels: ["__name__"]
34+
target_label: "__name__"
35+
replacement: "kqp_$1"
36+
- job_name: "tablets_dynamic"
37+
metrics_path: "/counters/counters=tablets/prometheus"
38+
static_configs:
39+
- targets: ["localhost:${TARGET_PORT}"]
40+
labels:
41+
container: ydb-dynamic
42+
metric_relabel_configs:
43+
- source_labels: ["__name__"]
44+
target_label: "__name__"
45+
replacement: "tablets_$1"
46+
- job_name: "proxy_dynamic"
47+
metrics_path: "/counters/counters=proxy/prometheus"
48+
static_configs:
49+
- targets: ["localhost:${TARGET_PORT}"]
50+
labels:
51+
container: ydb-dynamic
52+
metric_relabel_configs:
53+
- source_labels: ["__name__"]
54+
target_label: "__name__"
55+
replacement: "proxy_$1"
56+
- job_name: "dsproxynode_dynamic"
57+
metrics_path: "/counters/counters=dsproxynode/prometheus"
58+
static_configs:
59+
- targets: ["localhost:${TARGET_PORT}"]
60+
labels:
61+
container: ydb-dynamic
62+
metric_relabel_configs:
63+
- source_labels: ["__name__"]
64+
target_label: "__name__"
65+
replacement: "dsproxynode_$1"
66+
- job_name: "ic_dynamic"
67+
metrics_path: "/counters/counters=interconnect/prometheus"
68+
static_configs:
69+
- targets: ["localhost:${TARGET_PORT}"]
70+
labels:
71+
container: ydb-dynamic
72+
metric_relabel_configs:
73+
- source_labels: ["__name__"]
74+
target_label: "__name__"
75+
replacement: "interconnect_$1"

‎ydb/tests/tools/kqprun/flame_graph.sh

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,11 @@
22

33
set -eux
44

5+
if [ $# -gt 1 ]; then
6+
echo "Too many arguments"
7+
exit -1
8+
fi
9+
510
kqprun_pid=$(pgrep -u $USER kqprun)
611

712
sudo perf record -F 50 --call-graph dwarf -g --proc-map-timeout=10000 --pid $kqprun_pid -v -o profdata -- sleep ${1:-'30'}

‎ydb/tests/tools/kqprun/start_connector.sh

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/usr/bin/env bash
2+
3+
set -eux
4+
5+
if [ $# -le 0 ]; then
6+
echo "Please provide connector port (value of Generic::Connector::Endpoint::Port in config)"
7+
exit -1
8+
fi
9+
10+
if [ $# -gt 1 ]; then
11+
echo "Too many arguments"
12+
exit -1
13+
fi
14+
15+
if netstat -tuln | grep ":$1"; then
16+
echo "Can not start connector on port $1, port already in use"
17+
exit -1
18+
fi
19+
20+
cp ${3:-'./configuration/connector_config.yaml'} ./connector_config.yaml
21+
sed -i "s/\${TARGET_PORT}/$1/g" ./connector_config.yaml
22+
23+
docker run --rm --name=fq-connector-go-$1 --network host -v ./connector_config.yaml:/opt/ydb/cfg/fq-connector-go.yaml ghcr.io/ydb-platform/fq-connector-go:latest

‎ydb/tests/tools/kqprun/start_prometheus.sh

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/usr/bin/env bash
2+
3+
set -eux
4+
5+
if [ $# -le 0 ]; then
6+
echo "Please provide monitoring port to listen (value of -M argument)"
7+
exit -1
8+
fi
9+
10+
if [ $# -le 1 ]; then
11+
echo "Please provide prometheus web ui port"
12+
exit -1
13+
fi
14+
15+
if [ $# -gt 3 ]; then
16+
echo "Too many arguments"
17+
exit -1
18+
fi
19+
20+
echo "Checking monitoring port $1"
21+
nc -v -z -w 5 localhost $1
22+
23+
if netstat -tuln | grep ":$2"; then
24+
echo "Can not start prometheus on port $2, port already in use"
25+
exit -1
26+
fi
27+
28+
cp ${3:-'./configuration/prometheus_config.yaml'} ./prometheus_config.yaml
29+
sed -i "s/\${TARGET_PORT}/$1/g" ./prometheus_config.yaml
30+
31+
docker run --rm --name=prometeus-$1 --network host -v ./prometheus_config.yaml:/etc/prometheus/prometheus.yml prom/prometheus --config.file=/etc/prometheus/prometheus.yml --web.listen-address=:$2

0 commit comments

Comments
 (0)