应用IO性能问题求助

[xydz]

查看线程

ps -eLf | grep python

root 9 1 25 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 26 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 27 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 28 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 29 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 30 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 31 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 32 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 33 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 34 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 35 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 36 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 37 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 38 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 39 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 40 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 41 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 42 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 43 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 44 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 45 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 46 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 47 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 48 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 49 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 50 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 51 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 52 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 53 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 54 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 55 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 56 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 57 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 58 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 59 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 60 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 61 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 62 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 63 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 64 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 65 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 66 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 67 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 68 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 69 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 70 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 71 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 72 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 73 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 74 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 75 0 104 22:12 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 80 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 81 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 82 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 83 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 84 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 85 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 86 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 87 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 88 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 89 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 90 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 91 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 92 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 93 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 94 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 95 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 96 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 97 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 98 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 99 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 100 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 101 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 102 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 103 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 104 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 105 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 106 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 121 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 127 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 128 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 129 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 130 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 131 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 132 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 133 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 134 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 135 0 104 22:13 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 9 1 136 0 104 22:14 ? 00:00:00 /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
root 144 137 144 0 1 22:17 pts/0 00:00:00 grep python

查看线程堆栈信息

 py-spy dump --pid 9

Process 9: /opt/.uv.venv/bin/python /opt/.uv.venv/bin/dbgpt start webserver --config /app/configs/bff/online.toml
Python v3.11.11 (/usr/local/bin/python3.11)

Thread 9 (idle): "MainThread"
select (selectors.py:468)
_run_once (asyncio/base_events.py:1898)
run_forever (asyncio/base_events.py:608)
run_until_complete (asyncio/base_events.py:641)
run (asyncio/runners.py:118)
run (asyncio/runners.py:190)
run (uvicorn/server.py:67)
run (uvicorn/main.py:580)
run_uvicorn (dbgpt_app/dbgpt_server.py:224)
run_webserver (dbgpt_app/dbgpt_server.py:272)
start_webserver (dbgpt_app/_cli.py:25)
wrapper (dbgpt/model/cli.py:458)
invoke (click/core.py:788)
invoke (click/core.py:1443)
invoke (click/core.py:1697)
invoke (click/core.py:1697)
main (click/core.py:1082)
call (click/core.py:1161)
main (dbgpt/cli/cli_scripts.py:229)
(dbgpt:10)
Thread 75 (idle): "Thread-1 (_flush_to_storages)"
wait (threading.py:331)
get (queue.py:180)
_flush_to_storages (dbgpt/util/tracer/span_storage.py:78)
run (threading.py:982)
_bootstrap_inner (threading.py:1045)
_bootstrap (threading.py:1002)
Thread 80 (idle): "APIMixin-HealthCheck"
_heartbeat_checker (dbgpt/util/api_utils.py:151)
run (threading.py:982)
_bootstrap_inner (threading.py:1045)
_bootstrap (threading.py:1002)
Thread 81 (idle): "HealthCheck-Dedicated_0"
_worker (concurrent/futures/thread.py:81)
run (threading.py:982)
_bootstrap_inner (threading.py:1045)
_bootstrap (threading.py:1002)
Thread 82 (idle): "APIMixin-HealthCheck"
_heartbeat_checker (dbgpt/util/api_utils.py:151)
run (threading.py:982)
_bootstrap_inner (threading.py:1045)
_bootstrap (threading.py:1002)
Thread 83 (idle): "HealthCheck-Dedicated_0"
_worker (concurrent/futures/thread.py:81)
run (threading.py:982)
_bootstrap_inner (threading.py:1045)
_bootstrap (threading.py:1002)
Thread 84 (idle): "DBSummaryInit"
readinto (socket.py:718)
_read_bytes (pymysql/connections.py:782)
_read_packet (pymysql/connections.py:744)
_get_server_information (pymysql/connections.py:1098)
connect (pymysql/connections.py:668)
init (pymysql/connections.py:361)
connect (sqlalchemy/engine/default.py:616)
connect (sqlalchemy/engine/create.py:643)
__connect (sqlalchemy/pool/base.py:896)
init (sqlalchemy/pool/base.py:674)
_create_connection (sqlalchemy/pool/base.py:390)
_do_get (sqlalchemy/pool/impl.py:177)
checkout (sqlalchemy/pool/base.py:712)
_checkout (sqlalchemy/pool/base.py:1263)
connect (sqlalchemy/pool/base.py:449)
raw_connection (sqlalchemy/engine/base.py:3303)
init (sqlalchemy/engine/base.py:145)
connect (sqlalchemy/engine/base.py:3279)
_init_engine (sqlalchemy/engine/reflection.py:247)
_construct (sqlalchemy/engine/reflection.py:236)
_engine_insp (sqlalchemy/engine/reflection.py:303)
inspect (sqlalchemy/inspection.py:140)
init (dbgpt/datasource/rdbms/base.py:152)
init (dbgpt_ext/datasource/rdbms/conn_mysql.py:63)
from_uri (dbgpt/datasource/rdbms/base.py:214)
from_parameters (dbgpt/datasource/rdbms/base.py:180)
create_connector (dbgpt_ext/datasource/rdbms/conn_mysql.py:39)
create_connector (dbgpt_serve/datasource/manages/connector_manager.py:219)
get_connector (dbgpt_serve/datasource/manages/connector_manager.py:190)
init (dbgpt_ext/rag/summary/rdbms_db_summary.py:68)
create_summary_client (dbgpt_serve/datasource/service/db_summary_client.py:157)
db_summary_embedding (dbgpt_serve/datasource/service/db_summary_client.py:51)
init_db_summary (dbgpt_serve/datasource/service/db_summary_client.py:88)
run (threading.py:982)
_bootstrap_inner (threading.py:1045)
_bootstrap (threading.py:1002)
Thread 85 (idle): "AppScheduler"
_scheduler (dbgpt_app/initialization/scheduler.py:51)
run (threading.py:982)
_bootstrap_inner (threading.py:1045)
_bootstrap (threading.py:1002)
Thread 121 (idle): "asyncio_0"
_worker (concurrent/futures/thread.py:81)
run (threading.py:982)
_bootstrap_inner (threading.py:1045)
_bootstrap (threading.py:1002)
Thread 127 (idle): "asyncio_1"
_worker (concurrent/futures/thread.py:81)
run (threading.py:982)
_bootstrap_inner (threading.py:1045)
_bootstrap (threading.py:1002)
Thread 128 (idle): "asyncio_2"
_worker (concurrent/futures/thread.py:81)
run (threading.py:982)
_bootstrap_inner (threading.py:1045)
_bootstrap (threading.py:1002)
Thread 129 (idle): "asyncio_3"
_worker (concurrent/futures/thread.py:81)
run (threading.py:982)
_bootstrap_inner (threading.py:1045)
_bootstrap (threading.py:1002)
Thread 130 (idle): "asyncio_4"
_worker (concurrent/futures/thread.py:81)
run (threading.py:982)
_bootstrap_inner (threading.py:1045)
_bootstrap (threading.py:1002)
Thread 131 (idle): "asyncio_5"
_worker (concurrent/futures/thread.py:81)
run (threading.py:982)
_bootstrap_inner (threading.py:1045)
_bootstrap (threading.py:1002)
Thread 132 (idle): "asyncio_6"
_worker (concurrent/futures/thread.py:81)
run (threading.py:982)
_bootstrap_inner (threading.py:1045)
_bootstrap (threading.py:1002)
Thread 133 (idle): "asyncio_7"
_worker (concurrent/futures/thread.py:81)
run (threading.py:982)
_bootstrap_inner (threading.py:1045)
_bootstrap (threading.py:1002)
Thread 134 (idle): "asyncio_8"
_worker (concurrent/futures/thread.py:81)
run (threading.py:982)
_bootstrap_inner (threading.py:1045)
_bootstrap (threading.py:1002)
Thread 136 (idle): "AnyIO worker thread"
wait (threading.py:327)
get (queue.py:171)
run (anyio/_backends/_asyncio.py:956)
_bootstrap_inner (threading.py:1045)
_bootstrap (threading.py:1002)

部署在docker 4核8G 发现问题如下：

• 一份一份添加文档同步RAG(graph)造成IO打满，健康检测超时，容器杀死。
• 异步框架中使用同步数据库操作，导致大量线程阻塞和资源浪费。

更多版本信息

• DB-GPT version 0.7.3
• Python 3.11.11