Merge PRs #661, #693, #695 and #696 into devel

Author: webknjaz

docs/changelog-fragments/661.bugfix.rst

@@ -0,0 +1 @@
+Uploading large files over SCP no longer fails -- by :user:`Jakuje`.

docs/requirements.txt

@@ -304,9 +304,9 @@ requests==2.32.3 \
     --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \
     --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6
     # via sphinx
-setuptools-scm==8.2.1 \
-    --hash=sha256:0d234a45ca9e4cb7379886b827558d952d1d8f730c3957e7f217b2ec77fe4402 \
-    --hash=sha256:51cfdd1deefc9b8c08d1a61e940a59c4dec39eb6c285d33fa2f1b4be26c7874d
+setuptools-scm==8.2.0 \
+    --hash=sha256:136e2b1d393d709d2bcf26f275b8dec06c48b811154167b0fd6bb002aad17d6d \
+    --hash=sha256:a18396a1bc0219c974d1a74612b11f9dce0d5bd8b1dc55c65f6ac7fd609e8c28
     # via -r docs/requirements.in
 snowballstemmer==2.2.0 \
     --hash=sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1 \
@@ -419,9 +419,9 @@ zipp==3.21.0 \
     #   importlib-resources
 
 # The following packages are considered to be unsafe in a requirements file:
-setuptools==75.8.0 \
-    --hash=sha256:c5afc8f407c626b8313a86e10311dd3f661c6cd9c09d4bf8c15c0e11f9f2b0e6 \
-    --hash=sha256:e3982f444617239225d675215d51f6ba05f845d4eec313da4418fdbb56fb27e3
+setuptools==77.0.3 \
+    --hash=sha256:583b361c8da8de57403743e756609670de6fb2345920e36dc5c2d914c319c945 \
+    --hash=sha256:67122e78221da5cf550ddd04cf8742c8fe12094483749a792d56cd669d6cf58c
     # via
     #   incremental
     #   setuptools-scm

src/pylibsshext/scp.pyx

@@ -24,7 +24,7 @@ from pylibsshext.errors cimport LibsshSCPException
 from pylibsshext.session cimport get_libssh_session
 
 
-SCP_MAX_CHUNK = 65536
+SCP_MAX_CHUNK = 65_536  # 64kB
 
 
 cdef class SCP:
@@ -74,15 +74,25 @@ cdef class SCP:
                 )
 
             try:
+                # Read buffer
+                read_buffer_size = min(file_size, SCP_MAX_CHUNK)
+
                 # Begin to send to the file
                 rc = libssh.ssh_scp_push_file(scp, filename_b, file_size, file_mode)
                 if rc != libssh.SSH_OK:
                     raise LibsshSCPException("Can't open remote file: %s" % self._get_ssh_error_str())
 
-                # Write to the open file
-                rc = libssh.ssh_scp_write(scp, PyBytes_AS_STRING(f.read()), file_size)
-                if rc != libssh.SSH_OK:
-                    raise LibsshSCPException("Can't write to remote file: %s" % self._get_ssh_error_str())
+                remaining_bytes_to_read = file_size
+                while remaining_bytes_to_read > 0:
+                    # Read the chunk from local file
+                    read_bytes = min(remaining_bytes_to_read, read_buffer_size)
+                    read_buffer = f.read(read_bytes)
+
+                    # Write to the open file
+                    rc = libssh.ssh_scp_write(scp, PyBytes_AS_STRING(read_buffer), read_bytes)
+                    if rc != libssh.SSH_OK:
+                        raise LibsshSCPException("Can't write to remote file: %s" % self._get_ssh_error_str())
+                    remaining_bytes_to_read -= read_bytes
             finally:
                 libssh.ssh_scp_close(scp)
                 libssh.ssh_scp_free(scp)

tests/unit/channel_test.py

@@ -109,7 +109,7 @@ def test_send_signal(ssh_channel):
         ssh_channel.poll(timeout=POLL_TIMEOUT)
         output += ssh_channel.recv().decode('utf-8')
 
-    # Send SIGINT
+    # Send SIGUSR1
     ssh_channel.send_signal(signal.SIGUSR1)
 
     rc = -1

tests/unit/scp_test.py

@@ -5,11 +5,11 @@
 import os
 import random
 import string
-import uuid
 
 import pytest
 
 from pylibsshext.errors import LibsshSCPException
+from pylibsshext.scp import SCP_MAX_CHUNK
 
 
 @pytest.fixture
@@ -22,11 +22,22 @@ def ssh_scp(ssh_client_session):
         del scp  # noqa: WPS420
 
 
-@pytest.fixture
-def transmit_payload():
-    """Generate a binary test payload."""
-    uuid_name = uuid.uuid4()
-    return 'Hello, {name!s}'.format(name=uuid_name).encode()
+@pytest.fixture(
+    params=(32, SCP_MAX_CHUNK + 1),
+    ids=('small-payload', 'large-payload'),
+)
+def transmit_payload(request: pytest.FixtureRequest):
+    """Generate a binary test payload.
+
+    The choice 32 is arbitrary small value.
+
+    The choice SCP_CHUNK_SIZE + 1 (64kB + 1B) is meant to be 1B larger than the chunk
+    size used in :file:`scp.pyx` to make sure we excercise at least two rounds of
+    reading/writing.
+    """
+    payload_len = request.param
+    random_bytes = [ord(random.choice(string.printable)) for _ in range(payload_len)]
+    return bytes(random_bytes)
 
 
 @pytest.fixture
@@ -91,32 +102,3 @@ def test_get_existing_local(pre_existing_file_path, src_path, ssh_scp, transmit_
     """Check that SCP file download works and overwrites local file if it exists."""
     ssh_scp.get(str(src_path), str(pre_existing_file_path))
     assert pre_existing_file_path.read_bytes() == transmit_payload
-
-
-@pytest.fixture
-def large_payload():
-    """Generate a large 65537 byte (64kB+1B) test payload."""
-    random_char_kilobyte = [ord(random.choice(string.printable)) for _ in range(1024)]
-    full_bytes_number = 64
-    a_64kB_chunk = bytes(random_char_kilobyte * full_bytes_number)
-    the_last_byte = random.choice(random_char_kilobyte).to_bytes(length=1, byteorder='big')
-    return a_64kB_chunk + the_last_byte
-
-
-@pytest.fixture
-def src_path_large(tmp_path, large_payload):
-    """Return a remote path that to a 65537 byte-sized file.
-
-    Typical single-read chunk size is 64kB in ``libssh`` so
-    the test needs a file that would overflow that to trigger
-    the read loop.
-    """
-    path = tmp_path / 'large.txt'
-    path.write_bytes(large_payload)
-    return path
-
-
-def test_get_large(dst_path, src_path_large, ssh_scp, large_payload):
-    """Check that SCP file download gets over 64kB of data."""
-    ssh_scp.get(str(src_path_large), str(dst_path))
-    assert dst_path.read_bytes() == large_payload