Add support for OpenSSH "limits" extension

This commit adds client and server support for the OpenSSH "limits"
extension, which allows the client to query server limits such as the
maximum supported read and write size, improving performance between
clients and servers that can support larger sizes.

When AsyncSSH is acting as a server, it advertises support for up to
4 MiB reads and write, and a large enough packet size to hold such
write requests. As a client, it will query these values from servers
supporting the extension and automatically default to the largest
supported size.

When a server does not support this extension, AsyncSSH will fall back
to a "safe" maxmium size of 16 KiB for both reads and writes.

SCP has also been adjusted from a default size of 16 KiB to 256 KiB,
which seemed to be the sweet spot after some local performance testing.

As before, callers can always choose to override this default with
the block_size parameter on calls to open() or to the higher-level
get/put/copy functions, but generally speaking this should not be
necessary.

Author: ronf

asyncssh/scp.py

@@ -42,7 +42,7 @@
 from .sftp import SFTPAttrs, SFTPGlob, SFTPName, SFTPServer, SFTPServerFS
 from .sftp import SFTPFileProtocol, SFTPError, SFTPFailure, SFTPBadMessage
 from .sftp import SFTPConnectionLost, SFTPErrorHandler, SFTPProgressHandler
-from .sftp import SFTP_BLOCK_SIZE, local_fs
+from .sftp import local_fs
 
 
 if TYPE_CHECKING:
@@ -57,6 +57,9 @@
 _SCPConnPath = Union[Tuple[_SCPConn, _SCPPath], _SCPConn, _SCPPath]
 
 
+_SCP_BLOCK_SIZE = 256*1024    # 256 KiB
+
+
 class _SCPFSProtocol(Protocol):
     """Protocol for accessing a filesystem during an SCP copy"""
 
@@ -409,7 +412,7 @@ class _SCPSource(_SCPHandler):
 
     def __init__(self, fs: _SCPFSProtocol, reader: 'SSHReader[bytes]',
                  writer: 'SSHWriter[bytes]', preserve: bool, recurse: bool,
-                 block_size: int = SFTP_BLOCK_SIZE,
+                 block_size: int = _SCP_BLOCK_SIZE,
                  progress_handler: SFTPProgressHandler = None,
                  error_handler: SFTPErrorHandler = None, server: bool = False):
         super().__init__(reader, writer, error_handler, server)
@@ -568,7 +571,7 @@ class _SCPSink(_SCPHandler):
 
     def __init__(self, fs: _SCPFSProtocol, reader: 'SSHReader[bytes]',
                  writer: 'SSHWriter[bytes]', must_be_dir: bool, preserve: bool,
-                 recurse: bool, block_size: int = SFTP_BLOCK_SIZE,
+                 recurse: bool, block_size: int = _SCP_BLOCK_SIZE,
                  progress_handler: SFTPProgressHandler = None,
                  error_handler: SFTPErrorHandler = None, server: bool = False):
         super().__init__(reader, writer, error_handler, server)
@@ -736,7 +739,7 @@ def __init__(self, src_reader: 'SSHReader[bytes]',
                  src_writer: 'SSHWriter[bytes]',
                  dst_reader: 'SSHReader[bytes]',
                  dst_writer: 'SSHWriter[bytes]',
-                 block_size: int = SFTP_BLOCK_SIZE,
+                 block_size: int = _SCP_BLOCK_SIZE,
                  progress_handler: SFTPProgressHandler = None,
                  error_handler: SFTPErrorHandler = None):
         self._source = _SCPHandler(src_reader, src_writer)
@@ -898,7 +901,7 @@ async def run(self) -> None:
 
 async def scp(srcpaths: Union[_SCPConnPath, Sequence[_SCPConnPath]],
               dstpath: _SCPConnPath = None, *, preserve: bool = False,
-              recurse: bool = False, block_size: int = SFTP_BLOCK_SIZE,
+              recurse: bool = False, block_size: int = _SCP_BLOCK_SIZE,
               progress_handler: SFTPProgressHandler = None,
               error_handler: SFTPErrorHandler = None, **kwargs) -> None:
     """Copy files using SCP
@@ -955,7 +958,7 @@ async def scp(srcpaths: Union[_SCPConnPath, Sequence[_SCPConnPath]],
        SFTP instead.
 
        The block_size value controls the size of read and write operations
-       issued to copy the files. It defaults to 16 KB.
+       issued to copy the files. It defaults to 256 KB.
 
        If progress_handler is specified, it will be called after each
        block of a file is successfully copied. The arguments passed to