Merge remote-tracking branch 'origin/master' into edge

Author: jonasbardino

Makefile

@@ -64,6 +64,8 @@ else
 ./envhelp/py3.depends: $(REQS_PATH)
 endif
 	@rm -f ./envhelp/py3.depends
+	@echo "upgrading venv pip as required for some dependencies"
+	@./envhelp/venv/bin/pip3 install --upgrade pip
 	@echo "installing dependencies from $(REQS_PATH)"
 	@./envhelp/venv/bin/pip3 install -r $(REQS_PATH)
 ifeq ($(MIG_ENV),'local')

envhelp/docker/Dockerfile.python2

@@ -2,7 +2,7 @@ FROM python:2
 
 WORKDIR /usr/src/app
 
-COPY requirements.txt ./
-RUN pip install --no-cache-dir -r requirements.txt
+COPY requirements.txt local-requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt -r local-requirements.txt
 
 CMD [ "python", "--version" ]

local-requirements.txt

@@ -1,3 +1,9 @@
 # migrid dependencies on a format suitable for pip install as described on
 # https://pip.pypa.io/en/stable/reference/requirement-specifiers/
-autopep8
+# This list is mainly used to specify addons needed for the unit tests.
+# We only need autopep8 on py 3 as it's used in 'make fmt' (with py3)
+autopep8;python_version >= "3"
+# We need paramiko for the ssh unit tests
+# NOTE: paramiko-3.0.0 dropped python2 and python3.6 support
+paramiko;python_version >= "3.7"
+paramiko<3;python_version < "3.7"

mig/shared/compat.py

@@ -34,7 +34,13 @@
 from past.builtins import basestring
 
 import codecs
+import io
 import sys
+# NOTE: StringIO is only available in python2
+try:
+    import StringIO
+except ImportError:
+    StringIO = None
 
 PY2 = sys.version_info[0] < 3
 _TYPE_UNICODE = type(u"")
@@ -72,3 +78,15 @@ def ensure_native_string(string_or_bytes):
     else:
         textual_output = string_or_bytes
     return textual_output
+
+
+def NativeStringIO(initial_value=''):
+    """Mock StringIO pseudo-class to create a StringIO matching the native
+    string coding form. That is a BytesIO with utf8 on python 2 and unicode
+    StringIO otherwise. Optional string helpers are automatically converted
+    accordingly.
+    """
+    if PY2 and StringIO is not None:
+        return StringIO.StringIO(initial_value)
+    else:
+        return io.StringIO(initial_value)

mig/shared/ssh.py

@@ -4,7 +4,7 @@
 # --- BEGIN_HEADER ---
 #
 # ssh - remote command wrappers using ssh/scp
-# Copyright (C) 2003-2023  The MiG Project lead by Brian Vinter
+# Copyright (C) 2003-2024  The MiG Project lead by Brian Vinter
 #
 # This file is part of MiG.
 #
@@ -35,14 +35,6 @@
 import sys
 import tempfile
 
-# NOTE: we would prefer the modern io.BytesIO but it fails in python2 during
-#       rsa_key.write_private_key(string_io_obj) with a Paramiko error:
-#       TypeError: 'unicode' does not have the buffer interface
-try:
-    from cStringIO import StringIO as LegacyStringIO
-except ImportError:
-    from io import BytesIO as LegacyStringIO
-
 try:
     import paramiko
 except ImportError:
@@ -51,6 +43,7 @@
 
 from mig.shared.base import client_id_dir, force_utf8
 from mig.shared.conf import get_resource_exe, get_configuration_object
+from mig.shared.compat import NativeStringIO
 from mig.shared.defaults import ssh_conf_dir
 from mig.shared.safeeval import subprocess_popen, subprocess_pipe
 
@@ -95,8 +88,8 @@ def parse_pub_key(public_key):
         msg = 'Invalid ssh public key: (%s)' % public_key
         raise ValueError(msg)
 
-    head, tail = public_key.split(' ')[ssh_type_idx:2+ssh_type_idx]
-    bits = base64.decodestring(tail)
+    head, tail = public_key.split(' ')[ssh_type_idx:2 + ssh_type_idx]
+    bits = base64.b64decode(tail)
     msg = paramiko.Message(bits)
     parse_key = type_map[head]
     return parse_key(msg)
@@ -511,7 +504,12 @@ def generate_ssh_rsa_key_pair(size=2048, public_key_prefix='',
         raise Exception("You need paramiko to provide the ssh/sftp service")
     rsa_key = paramiko.RSAKey.generate(size)
 
-    string_io_obj = LegacyStringIO()
+    # NOTE: we would prefer the modern io.BytesIO on python2 but it fails
+    #       during rsa_key.write_private_key(string_io_obj) with a Paramiko
+    #       TypeError: 'unicode' does not have the buffer interface.
+    #       Use compat NativeStringIO to pick a suitable StringIO for the
+    #       active platform.
+    string_io_obj = NativeStringIO()
     rsa_key.write_private_key(string_io_obj)
 
     private_key = string_io_obj.getvalue()

tests/test_mig_shared_ssh.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+#
+# --- BEGIN_HEADER ---
+#
+# test_mig_shared_ssh - unit test of the corresponding mig shared module
+# Copyright (C) 2003-2024  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+# USA.
+#
+# --- END_HEADER ---
+#
+
+"""Unit tests for the migrid module pointed to in the filename"""
+
+import os
+import sys
+
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__))))
+
+from tests.support import TEST_OUTPUT_DIR, MigTestCase, FakeConfiguration, \
+    cleanpath, testmain
+from mig.shared.ssh import supported_pub_key_parsers, parse_pub_key, \
+    generate_ssh_rsa_key_pair
+
+
+class MigSharedSsh(MigTestCase):
+    """Wrap unit tests for the corresponding module"""
+
+    def test_ssh_key_generate_and_parse(self):
+        parsers = supported_pub_key_parsers()
+        # NOTE: should return a non-empty dict of algos and parsers
+        self.assertTrue(parsers)
+        self.assertTrue('ssh-rsa' in parsers)
+
+        # Generate common sized keys and parse the result
+        for keysize in (2048, 3072, 4096):
+            (priv_key, pub_key) = generate_ssh_rsa_key_pair(size=keysize)
+            self.assertTrue(priv_key)
+            self.assertTrue(pub_key)
+
+            # NOTE: parse_pub_key expects a native string so we use this case
+            try:
+                parsed = parse_pub_key(pub_key)
+            except ValueError as vae:
+                #print("Error in parsing pub key: %r" % vae)
+                parsed = None
+            self.assertTrue(parsed is not None)
+
+            (priv_key, pub_key) = generate_ssh_rsa_key_pair(size=keysize,
+                                                            encode_utf8=True)
+            self.assertTrue(priv_key)
+            self.assertTrue(pub_key)
+
+
+if __name__ == '__main__':
+    testmain()