2
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
3
4
4
import asyncio
5
- from typing import Any , Optional
5
+ from typing import Optional
6
6
7
7
import uvicorn
8
8
from fastapi import FastAPI
9
9
10
10
from vllm import envs
11
11
from vllm .config import VllmConfig
12
+ from vllm .distributed .kv_transfer .kv_connector .v1 .base import (
13
+ KVConnectorHandshakeMetadata )
12
14
from vllm .logger import init_logger
13
15
14
16
logger = init_logger (__name__ )
@@ -32,8 +34,9 @@ def _setup_routes(self):
32
34
@self .app .get ("/get_kv_connector_metadata/{dp_rank}/{tp_rank}" )
33
35
async def get_kv_connector_metadata (dp_rank : Optional [int ] = None ,
34
36
tp_rank : Optional [int ] = None ):
35
- kv_meta : Optional [dict [str , dict [str , dict [str , Any ]]]] = (
36
- self .vllm_config .cache_config .transfer_handshake_metadata )
37
+ kv_meta : Optional [dict [int , dict [
38
+ int , KVConnectorHandshakeMetadata ]]] = (
39
+ self .vllm_config .cache_config .transfer_handshake_metadata )
37
40
38
41
if kv_meta is None :
39
42
return None
@@ -71,7 +74,8 @@ async def start_async(self):
71
74
self .server = uvicorn .Server (config )
72
75
73
76
# start the server in a background task
74
- asyncio .create_task (self .server .serve ())
77
+ if self .server is not None :
78
+ asyncio .create_task (self .server .serve ())
75
79
logger .info ("NIXL side channel server started successfully" )
76
80
77
81
async def stop_async (self ):
0 commit comments