Invoking the dialogue interface returned an Internal Server Error.

[monkdharma]

When I performed an inference task, the following error message returned. Has anyone encountered this error and how should it be resolved?

VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226] Exception in worker VllmWorkerProcess while processing method start_worker_execution_loop: [Errno 2] No such file or directory: '/root/.triton/cache/dcb308a6649aa9071d14ae03263f5a63/fused_moe_kernel.cubin.tmp.pid_88_942500', Traceback (most recent call last):
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/vllm/executor/multiproc_worker_utils.py", line 223, in _run_worker_process
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     output = executor(*args, **kwargs)
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 115, in decorate_context
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     return func(*args, **kwargs)
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/vllm/worker/worker_base.py", line 64, in start_worker_execution_loop
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     output = self.execute_model(execute_model_req=None)
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/vllm/worker/worker_base.py", line 271, in execute_model
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     output = self.model_runner.execute_model(
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 115, in decorate_context
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     return func(*args, **kwargs)
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/vllm/worker/model_runner.py", line 1243, in execute_model
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     hidden_or_intermediate_states = model_executable(
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     return self._call_impl(*args, **kwargs)
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1541, in _call_impl
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     return forward_call(*args, **kwargs)
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/mixtral.py", line 348, in forward
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     hidden_states = self.model(input_ids, positions, kv_caches,
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     return self._call_impl(*args, **kwargs)
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1541, in _call_impl
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     return forward_call(*args, **kwargs)
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/mixtral.py", line 276, in forward
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     hidden_states, residual = layer(positions, hidden_states,
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     return self._call_impl(*args, **kwargs)
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1541, in _call_impl
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     return forward_call(*args, **kwargs)
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/mixtral.py", line 232, in forward
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     hidden_states = self.block_sparse_moe(hidden_states)
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     return self._call_impl(*args, **kwargs)
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1541, in _call_impl
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     return forward_call(*args, **kwargs)
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/mixtral.py", line 95, in forward
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     final_hidden_states = self.experts(hidden_states, router_logits)
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     return self._call_impl(*args, **kwargs)
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1541, in _call_impl
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     return forward_call(*args, **kwargs)
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/layers/fused_moe/layer.py", line 186, in forward
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     final_hidden_states = self.quant_method.apply(
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/layers/fused_moe/layer.py", line 68, in apply
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     return fused_moe(x,
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/layers/fused_moe/fused_moe.py", line 574, in fused_moe
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     return fused_experts(hidden_states,
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/layers/fused_moe/fused_moe.py", line 506, in fused_experts
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     invoke_fused_moe_kernel(intermediate_cache2,
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/layers/fused_moe/fused_moe.py", line 246, in invoke_fused_moe_kernel
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     fused_moe_kernel[grid](
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/triton/runtime/jit.py", line 167, in <lambda>
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/triton/runtime/jit.py", line 416, in run
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     self.cache[device][key] = compile(
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/triton/compiler/compiler.py", line 202, in compile
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     return CompiledKernel(so_path, metadata_group.get(metadata_filename))
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/triton/compiler/compiler.py", line 230, in __init__
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     self.asm = {
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/local/lib/python3.10/dist-packages/triton/compiler/compiler.py", line 231, in <dictcomp>
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     file.suffix[1:]: file.read_bytes() if file.suffix[1:] == driver.binary_ext else file.read_text()
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/lib/python3.10/pathlib.py", line 1134, in read_text
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     with self.open(mode='r', encoding=encoding, errors=errors) as f:
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]   File "/usr/lib/python3.10/pathlib.py", line 1119, in open
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226]     return self._accessor.open(self, mode, buffering, encoding, errors,
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226] FileNotFoundError: [Errno 2] No such file or directory: '/root/.triton/cache/dcb308a6649aa9071d14ae03263f5a63/fused_moe_kernel.cubin.tmp.pid_88_942500'
(VllmWorkerProcess pid=89) ERROR 07-15 07:59:54 multiproc_worker_utils.py:226] 

INFO:     100.89.61.1:45862 - "POST /v1/completions HTTP/1.1" 500 Internal Server Error
ERROR:    Exception in ASGI application
Traceback (most recent call last):
  File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_llm_engine.py", line 589, in run_engine_loop
    done, _ = await asyncio.wait(
  File "/usr/lib/python3.10/asyncio/tasks.py", line 384, in wait
    return await _wait(fs, timeout, return_when, loop)
  File "/usr/lib/python3.10/asyncio/tasks.py", line 491, in _wait
    await waiter
asyncio.exceptions.CancelledError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_llm_engine.py", line 43, in _log_task_completion
    return_value = task.result()
  File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_llm_engine.py", line 588, in run_engine_loop
    async with asyncio_timeout(ENGINE_ITERATION_TIMEOUT_S):
  File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_timeout.py", line 95, in __aexit__
    self._do_exit(exc_type)
  File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_timeout.py", line 178, in _do_exit
    raise asyncio.TimeoutError
asyncio.exceptions.TimeoutError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/local/lib/python3.10/dist-packages/uvicorn/protocols/http/httptools_impl.py", line 399, in run_asgi
    result = await app(  # type: ignore[func-returns-value]
  File "/usr/local/lib/python3.10/dist-packages/uvicorn/middleware/proxy_headers.py", line 70, in __call__
    return await self.app(scope, receive, send)
  File "/usr/local/lib/python3.10/dist-packages/fastapi/applications.py", line 1054, in __call__
    await super().__call__(scope, receive, send)
  File "/usr/local/lib/python3.10/dist-packages/starlette/applications.py", line 123, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/usr/local/lib/python3.10/dist-packages/starlette/middleware/errors.py", line 186, in __call__
    raise exc
  File "/usr/local/lib/python3.10/dist-packages/starlette/middleware/errors.py", line 164, in __call__
    await self.app(scope, receive, _send)
  File "/usr/local/lib/python3.10/dist-packages/starlette/middleware/cors.py", line 85, in __call__
    await self.app(scope, receive, send)
  File "/usr/local/lib/python3.10/dist-packages/starlette/middleware/exceptions.py", line 65, in __call__
    await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
  File "/usr/local/lib/python3.10/dist-packages/starlette/_exception_handler.py", line 64, in wrapped_app
    raise exc
  File "/usr/local/lib/python3.10/dist-packages/starlette/_exception_handler.py", line 53, in wrapped_app
    await app(scope, receive, sender)
  File "/usr/local/lib/python3.10/dist-packages/starlette/routing.py", line 756, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/usr/local/lib/python3.10/dist-packages/starlette/routing.py", line 776, in app
    await route.handle(scope, receive, send)
  File "/usr/local/lib/python3.10/dist-packages/starlette/routing.py", line 297, in handle
    await self.app(scope, receive, send)
  File "/usr/local/lib/python3.10/dist-packages/starlette/routing.py", line 77, in app
    await wrap_app_handling_exceptions(app, request)(scope, receive, send)
  File "/usr/local/lib/python3.10/dist-packages/starlette/_exception_handler.py", line 64, in wrapped_app
    raise exc
  File "/usr/local/lib/python3.10/dist-packages/starlette/_exception_handler.py", line 53, in wrapped_app
    await app(scope, receive, sender)
  File "/usr/local/lib/python3.10/dist-packages/starlette/routing.py", line 72, in app
    response = await func(request)
  File "/usr/local/lib/python3.10/dist-packages/fastapi/routing.py", line 278, in app
    raw_response = await run_endpoint_function(
  File "/usr/local/lib/python3.10/dist-packages/fastapi/routing.py", line 191, in run_endpoint_function
    return await dependant.call(**values)
  File "/usr/local/lib/python3.10/dist-packages/vllm/entrypoints/openai/api_server.py", line 147, in create_completion
    generator = await openai_serving_completion.create_completion(
  File "/usr/local/lib/python3.10/dist-packages/vllm/entrypoints/openai/serving_completion.py", line 181, in create_completion
    async for i, res in result_generator:
  File "/usr/local/lib/python3.10/dist-packages/vllm/utils.py", line 319, in consumer
    raise e
  File "/usr/local/lib/python3.10/dist-packages/vllm/utils.py", line 310, in consumer
    raise item
  File "/usr/local/lib/python3.10/dist-packages/vllm/utils.py", line 294, in producer
    async for item in iterator:
  File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_llm_engine.py", line 746, in generate
    async for output in self._process_request(
  File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_llm_engine.py", line 845, in _process_request
    stream = await self.add_request(
  File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_llm_engine.py", line 654, in add_request
    self.start_background_loop()
  File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_llm_engine.py", line 476, in start_background_loop
    raise AsyncEngineDeadError(
vllm.engine.async_llm_engine.AsyncEngineDeadError: Background loop has errored already.