Timeout error of `ComputeSession.complete` SDK

[jopemachine]

Summary  

Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "/home/jopemachine/.local/backend.ai/repos/main/abcd.py", line 11, in <module>
    test()
    ~~~~^^
  File "/home/jopemachine/.local/backend.ai/repos/main/abcd.py", line 6, in test
    result = session.ComputeSession("6f9e897a-cd75-4616-98c1-c2a3fc46ee9f").complete("")
  File "/home/jopemachine/.local/backend.ai/repos/main/src/ai/backend/client/func/base.py", line 34, in _method
    return _api_session.worker_thread.execute(coro)
           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^
  File "/home/jopemachine/.local/backend.ai/repos/main/src/ai/backend/client/session.py", line 192, in execute
    raise result
  File "/home/jopemachine/.local/backend.ai/repos/main/src/ai/backend/client/session.py", line 172, in run
    result = ctx.run(loop.run_until_complete, coro)  # type: ignore
  File "/home/jopemachine/.pyenv/versions/3.13.3/lib/python3.13/asyncio/base_events.py", line 719, in run_until_complete
    return future.result()
           ~~~~~~~~~~~~~^^
  File "/home/jopemachine/.local/backend.ai/repos/main/src/ai/backend/client/func/session.py", line 732, in complete
    async with rqst.fetch() as resp:
               ~~~~~~~~~~^^
  File "/home/jopemachine/.local/backend.ai/repos/main/src/ai/backend/client/request.py", line 576, in __aenter__
    raise BackendAPIError(raw_resp.status, raw_resp.reason or "", msg)
ai.backend.client.exceptions.BackendAPIError: BackendAPIError(500, 'Internal Server Error', {'type': 'https://api.backend.ai/probs/kernel-execution-failed', 'title': 'Executing user code in the kernel has failed.', 'error_code': 'kernel_execute_internal-error', 'msg': 'Traceback (most recent call last):\n  File "/home/jopemachine/.local/backend.ai/repos/main/src/ai/backend/manager/server.py", line 342, in exception_middleware\n    resp = await handler(request)\n           ^^^^^^^^^^^^^^^^^^^^^^\n  File "/home/jopemachine/.local/backend.ai/repos/main/src/ai/backend/manager/server.py", line 304, in api_middleware\n    resp = await _handler(request)\n           ^^^^^^^^^^^^^^^^^^^^^^^\n  File "/home/jopemachine/.local/backend.ai/repos/main/src/ai/backend/common/metrics/http.py", line 39, in metric_middleware\n    resp = await handler(request)\n           ^^^^^^^^^^^^^^^^^^^^^^\n  File "/home/jopemachine/.local/backend.ai/repos/main/src/ai/backend/manager/api/auth.py", line 593, in auth_middleware\n    return await handler(request)\n           ^^^^^^^^^^^^^^^^^^^^^^\n  File "/home/jopemachine/.local/backend.ai/repos/main/dist/export/python/virtualenvs/python-default/3.13.3/lib/python3.13/site-packages/aiotools/func.py", line 23, in wrapped\n    return await coro(*args, *cargs, **kwargs, **ckwargs)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File "/home/jopemachine/.local/backend.ai/repos/main/src/ai/backend/manager/api/ratelimit.py", line 73, in rlim_middleware\n    response = await handler(request)\n               ^^^^^^^^^^^^^^^^^^^^^^\n  File "/home/jopemachine/.local/backend.ai/repos/main/dist/export/python/virtualenvs/python-default/3.13.3/lib/python3.13/site-packages/aiohttp/web_middlewares.py", line 117, in impl\n    return await handler(request)\n           ^^^^^^^^^^^^^^^^^^^^^^\n  File "/home/jopemachine/.local/backend.ai/repos/main/src/ai/backend/manager/api/manager.py", line 80, in wrapped\n    return await handler(request, *args, **kwargs)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File "/home/jopemachine/.local/backend.ai/repos/main/src/ai/backend/manager/api/auth.py", line 600, in wrapped\n    return await handler(request, *args, **kwargs)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File "/home/jopemachine/.local/backend.ai/repos/main/src/ai/backend/manager/api/session.py", line 1258, in complete\n    action_result = await root_ctx.processors.session.complete.wait_for_complete(\n                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n    ...<6 lines>...\n    )\n    ^\n  File "/home/jopemachine/.local/backend.ai/repos/main/src/ai/backend/manager/actions/processor.py", line 74, in wait_for_complete\n    return await self._run(action)\n           ^^^^^^^^^^^^^^^^^^^^^^^\n  File "/home/jopemachine/.local/backend.ai/repos/main/src/ai/backend/manager/actions/processor.py", line 43, in _run\n    result = await self._func(action)\n             ^^^^^^^^^^^^^^^^^^^^^^^^\n  File "/home/jopemachine/.local/backend.ai/repos/main/src/ai/backend/manager/services/session/service.py", line 291, in complete\n    await self._agent_registry.get_completions(session, code, opts=options),\n    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File "/home/jopemachine/.local/backend.ai/repos/main/src/ai/backend/manager/registry.py", line 2817, in get_completions\n    async with handle_session_exception(self.db, "execute", session.id):\n               ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File "/home/jopemachine/.pyenv/versions/3.13.3/lib/python3.13/contextlib.py", line 235, in __aexit__\n    await self.gen.athrow(value)\n  File "/home/jopemachine/.local/backend.ai/repos/main/src/ai/backend/manager/models/session.py", line 470, in handle_session_exception\n    raise exc_class("TIMEOUT") from None\nai.backend.manager.errors.exceptions.KernelExecutionFailed: None\n'})

Example code

from ai.backend.client.session import Session


def test():
    with Session() as session:
        result = session.ComputeSession("6f9e897a-cd75-4616-98c1-c2a3fc46ee9f").complete("")
        print(result)


if __name__ == "__main__":
    test()