Node setup sometimes hangs, but retrying helps

[blokhin]

# yasetnode 192.168.1.147~16
Setup host...
INFO:RemoteMachine:root@192.168.1.147:CPUs count: 32

here we have veeeeery long waiting...

^C
Traceback (most recent call last):
  File "/usr/lib/python3.11/asyncio/runners.py", line 118, in run
    return self._loop.run_until_complete(task)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.11/asyncio/base_events.py", line 653, in run_until_complete
    return future.result()
           ^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/dist-packages/yascheduler/utils.py", line 436, in _manage_node
    await machine.setup_node(config.engines)
  File "/usr/local/lib/python3.11/dist-packages/yascheduler/remote_machine/remote_machine.py", line 378, in setup_node
    await retry(self.adapter.setup_node)(
  File "/usr/local/lib/python3.11/dist-packages/backoff/_async.py", line 151, in retry
    ret = await target(*args, **kwargs)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/dist-packages/yascheduler/remote_machine/linux_methods.py", line 202, in linux_setup_deb_node
    await run(f"{apt_cmd} upgrade", check=True)
  File "/usr/local/lib/python3.11/dist-packages/backoff/_async.py", line 151, in retry
    ret = await target(*args, **kwargs)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/dist-packages/yascheduler/remote_machine/remote_machine.py", line 334, in run
    return await self.adapter.run(
           ^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/dist-packages/yascheduler/remote_machine/common.py", line 34, in run
    return await conn.run(command, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/dist-packages/asyncssh/connection.py", line 4325, in run
    return await process.wait(check, timeout)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/dist-packages/asyncssh/process.py", line 1478, in wait
    await asyncio.wait_for(self.communicate(), timeout)
  File "/usr/lib/python3.11/asyncio/tasks.py", line 442, in wait_for
    return await fut
           ^^^^^^^^^
  File "/usr/local/lib/python3.11/dist-packages/asyncssh/process.py", line 1367, in communicate
    await self.wait_closed()
  File "/usr/local/lib/python3.11/dist-packages/asyncssh/process.py", line 1134, in wait_closed
    await self._chan.wait_closed()
  File "/usr/local/lib/python3.11/dist-packages/asyncssh/channel.py", line 786, in wait_closed
    await self._close_event.wait()
  File "/usr/lib/python3.11/asyncio/locks.py", line 213, in wait
    await fut
asyncio.exceptions.CancelledError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/bin/yasetnode", line 8, in <module>
    sys.exit(manage_node())
             ^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/dist-packages/yascheduler/utils.py", line 446, in manage_node
    asyncio.run(_manage_node())
  File "/usr/lib/python3.11/asyncio/runners.py", line 190, in run
    return runner.run(main)
           ^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.11/asyncio/runners.py", line 123, in run
    raise KeyboardInterrupt()
KeyboardInterrupt