Manually merge PR108 to handle missing bits of ssh key generation on py3. It is needed for cloud and jupyter integration at least. Added unit tests for it. Rework dependency handling to include the paramiko installlation needed in shared.ssh and therefore now also for the unit test of it. That requires a more recent pip and careful version selection on certain legacy platforms.

git-svn-id: svn+ssh://svn.code.sf.net/p/migrid/code/trunk@6114 b75ad72c-e7d7-11dd-a971-7dbc132099af

Author: jonasbardino

Makefile

@@ -64,6 +64,8 @@ else
 ./envhelp/py3.depends: $(REQS_PATH)
 endif
 	@rm -f ./envhelp/py3.depends
+	@echo "upgrading venv pip as required for some dependencies"
+	@./envhelp/venv/bin/pip3 install --upgrade pip
 	@echo "installing dependencies from $(REQS_PATH)"
 	@./envhelp/venv/bin/pip3 install -r $(REQS_PATH)
 ifeq ($(MIG_ENV),'local')

envhelp/docker/Dockerfile.python2

@@ -2,7 +2,7 @@ FROM python:2
 
 WORKDIR /usr/src/app
 
-COPY requirements.txt ./
-RUN pip install --no-cache-dir -r requirements.txt
+COPY requirements.txt local-requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt -r local-requirements.txt
 
 CMD [ "python", "--version" ]

local-requirements.txt

@@ -1,3 +1,9 @@
 # migrid dependencies on a format suitable for pip install as described on
 # https://pip.pypa.io/en/stable/reference/requirement-specifiers/
-autopep8
+# This list is mainly used to specify addons needed for the unit tests.
+# We only need autopep8 on py 3 as it's used in 'make fmt' (with py3)
+autopep8;python_version >= "3"
+# We need paramiko for the ssh unit tests
+# NOTE: paramiko-3.0.0 dropped python2 and python3.6 support
+paramiko;python_version >= "3.7"
+paramiko<3;python_version < "3.7"

mig/shared/compat.py

@@ -34,7 +34,13 @@
 from past.builtins import basestring
 
 import codecs
+import io
 import sys
+# NOTE: StringIO is only available in python2
+try:
+    import StringIO
+except ImportError:
+    StringIO = None
 
 PY2 = sys.version_info[0] < 3
 _TYPE_UNICODE = type(u"")
@@ -72,3 +78,15 @@ def ensure_native_string(string_or_bytes):
     else:
         textual_output = string_or_bytes
     return textual_output
+
+
+def NativeStringIO(initial_value=''):
+    """Mock StringIO pseudo-class to create a StringIO matching the native
+    string coding form. That is a BytesIO with utf8 on python 2 and unicode
+    StringIO otherwise. Optional string helpers are automatically converted
+    accordingly.
+    """
+    if PY2 and StringIO is not None:
+        return StringIO.StringIO(initial_value)
+    else:
+        return io.StringIO(initial_value)

mig/shared/ssh.py

@@ -4,7 +4,7 @@
 # --- BEGIN_HEADER ---
 #
 # ssh - remote command wrappers using ssh/scp
-# Copyright (C) 2003-2023  The MiG Project lead by Brian Vinter
+# Copyright (C) 2003-2024  The MiG Project lead by Brian Vinter
 #
 # This file is part of MiG.
 #
@@ -35,14 +35,6 @@
 import sys
 import tempfile
 
-# NOTE: we would prefer the modern io.BytesIO but it fails in python2 during
-#       rsa_key.write_private_key(string_io_obj) with a Paramiko error:
-#       TypeError: 'unicode' does not have the buffer interface
-try:
-    from cStringIO import StringIO as LegacyStringIO
-except ImportError:
-    from io import BytesIO as LegacyStringIO
-
 try:
     import paramiko
 except ImportError:
@@ -51,6 +43,7 @@
 
 from mig.shared.base import client_id_dir, force_utf8
 from mig.shared.conf import get_resource_exe, get_configuration_object
+from mig.shared.compat import NativeStringIO
 from mig.shared.defaults import ssh_conf_dir
 from mig.shared.safeeval import subprocess_popen, subprocess_pipe
 
@@ -95,8 +88,8 @@ def parse_pub_key(public_key):
         msg = 'Invalid ssh public key: (%s)' % public_key
         raise ValueError(msg)
 
-    head, tail = public_key.split(' ')[ssh_type_idx:2+ssh_type_idx]
-    bits = base64.decodestring(tail)
+    head, tail = public_key.split(' ')[ssh_type_idx:2 + ssh_type_idx]
+    bits = base64.b64decode(tail)
     msg = paramiko.Message(bits)
     parse_key = type_map[head]
     return parse_key(msg)
@@ -511,7 +504,12 @@ def generate_ssh_rsa_key_pair(size=2048, public_key_prefix='',
         raise Exception("You need paramiko to provide the ssh/sftp service")
     rsa_key = paramiko.RSAKey.generate(size)
 
-    string_io_obj = LegacyStringIO()
+    # NOTE: we would prefer the modern io.BytesIO on python2 but it fails
+    #       during rsa_key.write_private_key(string_io_obj) with a Paramiko
+    #       TypeError: 'unicode' does not have the buffer interface.
+    #       Use compat NativeStringIO to pick a suitable StringIO for the
+    #       active platform.
+    string_io_obj = NativeStringIO()
     rsa_key.write_private_key(string_io_obj)
 
     private_key = string_io_obj.getvalue()

tests/test_mig_shared_ssh.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+#
+# --- BEGIN_HEADER ---
+#
+# test_mig_shared_ssh - unit test of the corresponding mig shared module
+# Copyright (C) 2003-2024  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+# USA.
+#
+# --- END_HEADER ---
+#
+
+"""Unit tests for the migrid module pointed to in the filename"""
+
+import os
+import sys
+
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__))))
+
+from tests.support import TEST_OUTPUT_DIR, MigTestCase, FakeConfiguration, \
+    cleanpath, testmain
+from mig.shared.ssh import supported_pub_key_parsers, parse_pub_key, \
+    generate_ssh_rsa_key_pair
+
+
+class MigSharedSsh(MigTestCase):
+    """Wrap unit tests for the corresponding module"""
+
+    def test_ssh_key_generate_and_parse(self):
+        parsers = supported_pub_key_parsers()
+        # NOTE: should return a non-empty dict of algos and parsers
+        self.assertTrue(parsers)
+        self.assertTrue('ssh-rsa' in parsers)
+
+        # Generate common sized keys and parse the result
+        for keysize in (2048, 3072, 4096):
+            (priv_key, pub_key) = generate_ssh_rsa_key_pair(size=keysize)
+            self.assertTrue(priv_key)
+            self.assertTrue(pub_key)
+
+            # NOTE: parse_pub_key expects a native string so we use this case
+            try:
+                parsed = parse_pub_key(pub_key)
+            except ValueError as vae:
+                #print("Error in parsing pub key: %r" % vae)
+                parsed = None
+            self.assertTrue(parsed is not None)
+
+            (priv_key, pub_key) = generate_ssh_rsa_key_pair(size=keysize,
+                                                            encode_utf8=True)
+            self.assertTrue(priv_key)
+            self.assertTrue(pub_key)
+
+
+if __name__ == '__main__':
+    testmain()