Support sftp in wrap file

mesonbuild/wrap/wrap.py

@@ -720,6 +720,29 @@ def get_data(self, urlstring: str) -> T.Tuple[str, str]:
             resp = open_wrapdburl(urlstring, allow_insecure=self.allow_insecure, have_opt=self.wrap_frontend)
         elif WHITELIST_SUBDOMAIN in urlstring:
             raise WrapException(f'{urlstring} may be a WrapDB-impersonating URL')
+        elif url.scheme == 'sftp':
+            sftp = shutil.which('sftp')
+            if sftp is None:
+                raise WrapException('Scheme sftp is not available. Install sftp to enable it.')
+            with tempfile.TemporaryDirectory() as workdir, \
+                    tempfile.NamedTemporaryFile(mode='wb', dir=self.cachedir, delete=False) as tmpfile:
+                args = []
+                if 'source_hostkey' in self.wrap.values:
+                    with open(os.path.join(workdir, 'known_hosts'), 'w', encoding='utf-8') as f:
+                        f.write(f'[{url.hostname}]:{url.port} {self.wrap.get("source_hostkey")}')
+                    args += ['-o', 'UserKnownHostsFile=known_hosts']
+                if 'source_identityfile' in self.wrap.values:
+                    args += ['-o', f'IdentityFile={self.wrap.get("source_identityfile")}']
+                # Older versions of the sftp client cannot handle URLs, hence the splitting of url below
+                if url.port:
+                    args += ['-P', f'{url.port}']
+                user = f'{url.username}@' if url.username else ''
+                command = [sftp, '-o', 'KbdInteractiveAuthentication=no', *args, f'{user}{url.hostname}:{url.path[1:]}']
+                subprocess.run(command, cwd=workdir, check=True)
+                downloaded = os.path.join(workdir, os.path.basename(url.path))
+                tmpfile.close()
+                shutil.move(downloaded, tmpfile.name)
+                return self.hash_file(tmpfile.name), tmpfile.name
         else:
             headers = {
                 'User-Agent': f'mesonbuild/{coredata.version}',
@@ -744,7 +767,7 @@ def get_data(self, urlstring: str) -> T.Tuple[str, str]:
                 resp = urllib.request.urlopen(req, timeout=REQ_TIMEOUT)
             except OSError as e:
                 mlog.log(str(e))
-                raise WrapException(f'could not get {urlstring} is the internet available?')
+                raise WrapException(f'could not get {urlstring}; is the internet available?')
         with contextlib.closing(resp) as resp, tmpfile as tmpfile:
             try:
                 dlsize = int(resp.info()['Content-Length'])
@@ -775,14 +798,17 @@ def get_data(self, urlstring: str) -> T.Tuple[str, str]:
             hashvalue = h.hexdigest()
         return hashvalue, tmpfile.name
 
+    def hash_file(self, path: str) -> str:
+        h = hashlib.sha256()
+        with open(path, 'rb') as f:
+            h.update(f.read())
+        return h.hexdigest()
+
     def check_hash(self, what: str, path: str, hash_required: bool = True) -> None:
         if what + '_hash' not in self.wrap.values and not hash_required:
             return
         expected = self.wrap.get(what + '_hash').lower()
-        h = hashlib.sha256()
-        with open(path, 'rb') as f:
-            h.update(f.read())
-        dhash = h.hexdigest()
+        dhash = self.hash_file(path)
         if dhash != expected:
             raise WrapException(f'Incorrect hash for {what}:\n {expected} expected\n {dhash} actual.')
 

test cases/unit/130 wrap file sftp/foo/foo.c

@@ -0,0 +1,5 @@
+#include "foo.h"
+
+int foo_double(int x) {
+    return 2 * x;
+}

test cases/unit/130 wrap file sftp/foo/foo.h

@@ -0,0 +1,6 @@
+#ifndef FOO_H
+#define FOO_H
+
+int foo_double(int x);
+
+#endif

test cases/unit/130 wrap file sftp/foo/meson.build

@@ -0,0 +1,5 @@
+project('foo', 'c')
+
+libfoo = library('foo', ['foo.c', 'foo.h'])
+
+libfoo_dep = declare_dependency(include_directories: '.', link_with: libfoo)

test cases/unit/130 wrap file sftp/recreate_foo_archive.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+# This script can be used to recreate the test archive.
+
+import tarfile
+import time
+
+def set_mtime(tarinfo):
+    tarinfo.mtime = 0
+    return tarinfo
+
+# To avoid updated timestamps alone causing a diff in the archive, make sure
+# the gzip header has mtime 0, and also set mtime 0 for the members.
+time.time = lambda: 0
+with tarfile.open('foo.tar.gz', 'w:gz') as archive:
+    archive.add('foo/foo.c', filter=set_mtime)
+    archive.add('foo/foo.h', filter=set_mtime)
+    archive.add('foo/meson.build', filter=set_mtime)

test cases/unit/130 wrap file sftp/top/doubler.c

@@ -0,0 +1,6 @@
+#include "foo.h"
+
+int main(int argc, char **argv) {
+    (void)foo_double(argc);
+    return 0;
+}

test cases/unit/130 wrap file sftp/top/meson.build

@@ -0,0 +1,6 @@
+project('proj', 'c')
+
+libfoo_proj = subproject('foo')
+libfoo_dep = libfoo_proj.get_variable('libfoo_dep')
+
+executable('doubler', 'doubler.c', dependencies: [libfoo_dep])

unittests/allplatformstests.py

@@ -13,10 +13,13 @@
 import pickle
 import zipfile, tarfile
 import sys
+import socket
+import stat
+import hashlib
 from unittest import mock, SkipTest, skipIf, skipUnless, expectedFailure
 from contextlib import contextmanager
 from glob import glob
-from pathlib import (PurePath, Path)
+from pathlib import (PurePath, Path, PureWindowsPath)
 import typing as T
 
 import mesonbuild.mlog
@@ -5373,3 +5376,128 @@ def test_fortran_new_module_in_dep(self) -> None:
         output = entry['output']
 
         self.build(output, extra_args=['-j1'])
+
+    @skipIfNoExecutable('sshd')
+    @skipIfNoExecutable('sftp')
+    # Not tested on Windows, since there is not yet an OpenSSH server available
+    def test_wrap_file_sftp(self) -> None:
+        testdir = Path(self.unit_test_dir) / '130 wrap file sftp'
+
+        def write_file(path, contents):
+            with open(path, 'w', encoding='utf-8') as f:
+                f.write(contents)
+
+        def read_file(path):
+            with open(path, 'r', encoding='utf-8') as f:
+                return f.read()
+
+        def generate_key(path):
+            subprocess.run(['ssh-keygen', '-f', path, '-N', ''], check=True)
+            os.chmod(path, stat.S_IREAD | stat.S_IWRITE)
+            with open(path.with_suffix('.pub'), 'r', encoding='utf-8') as f:
+                return f.read()
+
+        def find_free_port():
+            with socket.socket() as sock:
+                sock.bind(('', 0))
+                return sock.getsockname()[1]
+
+        def generate_wrap_file(tmpdir, ssh_server_port, user_key_path, host_public_key, source_hash):
+            if mesonbuild.environment.detect_msys2_arch():
+                user_key_path = to_msys_path(user_key_path)
+            (tmpdir / 'subprojects').mkdir()
+            write_file(tmpdir / 'subprojects' / 'foo.wrap',
+                textwrap.dedent(f'''\
+                    [wrap-file]
+                    directory = foo
+
+                    source_url = sftp://localhost:{ssh_server_port}/foo.tar.gz
+                    source_filename = foo.tar.gz
+                    source_hash = {source_hash}
+                    source_hostkey = {host_public_key}
+                    source_identityfile = {user_key_path}
+                '''))
+
+        def generate_sshd_config(sshdir, user_public_key, ssh_server_port, sftpdir):
+            authorized_keys_path = sshdir / 'authorized_keys'
+            write_file(authorized_keys_path, user_public_key)
+            if mesonbuild.environment.detect_msys2_arch():
+                authorized_keys_path = to_msys_path(authorized_keys_path)
+                sftpdir = to_msys_path(sftpdir)
+            sshd_config_path = sshdir / 'sshd_config'
+            write_file(sshd_config_path,
+                textwrap.dedent(f'''\
+                    ListenAddress localhost:{ssh_server_port}
+                    PidFile "{sshdir / 'sshd_pid'}"
+                    AuthorizedKeysFile "{authorized_keys_path}"
+                    Subsystem sftp internal-sftp -d {sftpdir}
+                    PasswordAuthentication no
+                    StrictModes no
+                '''))
+            return sshd_config_path
+
+        def to_msys_path(path):
+            # Convert A:\b\c to /a/b/c
+            path = PureWindowsPath(str(path))
+            return f'/{path.drive[:1].lower()}/{path.relative_to(path.drive + "/").as_posix()}'
+
+        def start_sshd(config_path, host_key_path):
+            if mesonbuild.environment.detect_msys2_arch():
+                config_path = to_msys_path(config_path)
+                host_key_path = to_msys_path(host_key_path)
+            sshd_path = shutil.which('sshd')
+            sshd = subprocess.Popen([sshd_path, '-f', config_path, '-h', host_key_path, '-D'])
+            try:
+                sshd.wait(1)
+                return None
+            except subprocess.TimeoutExpired:
+                # It seems sshd started successfully
+                return sshd
+
+        def hash_file(path):
+            h = hashlib.sha256()
+            with open(path, 'rb') as f:
+                h.update(f.read())
+            return h.hexdigest()
+
+        # In Ubuntu Bionic, the privilege separation directory /run/sshd is
+        # created when the service is started. Since this has never happened in
+        # the docker images in the CI environment, create it manually.
+        if is_linux() and 'CI' in os.environ:
+            os.makedirs('/run/sshd', mode=0o755, exist_ok=True)
+        builddir = Path(self.builddir)
+        workdir = builddir / 'work'
+        sftpdir = builddir / 'sftp'
+        sshdir = builddir / 'ssh'
+        sftpdir.mkdir()
+        sshdir.mkdir()
+        shutil.copytree(testdir / 'top', workdir)
+        shutil.copy(testdir / 'foo.tar.gz', sftpdir)
+        source_hash = hash_file(testdir / 'foo.tar.gz')
+        host_key_path = sshdir / 'host_key'
+        user_key_path = sshdir / 'user_key'
+        host_public_key = generate_key(host_key_path)
+        user_public_key = generate_key(user_key_path)
+
+        # As there is no reliable way to avoid the port being taken between
+        # us finding a free port and starting the server, support a number
+        # of retries.
+        attempts = 0
+        while attempts < 3:
+            port = find_free_port()
+            sshd_config_path = generate_sshd_config(sshdir, user_public_key, port, sftpdir)
+            sshd = start_sshd(sshd_config_path, host_key_path)
+            if sshd is None:
+                print(f'Failed to start sshd, probably due to port being taken. Trying again.')
+                attempts += 1
+                continue
+            generate_wrap_file(workdir, port, user_key_path, host_public_key, source_hash)
+            try:
+                self.new_builddir()  # Ensure builddir is not parent or workdir
+                self.init(str(workdir))
+                self.build()
+                return
+            finally:
+                sshd.terminate()
+                sshd.wait()
+        raise self.fail(f'Failed to start sshd after {attempts} attempts.')