Make responding with binary data work under Py3.

.github/workflows/ci.yml

@@ -35,7 +35,7 @@ jobs:
           make dependencies
       - name: Run tests
         run: |
-          make test
+          make unittest
 
   python3-rocky9ish:
     runs-on: ubuntu-22.04
@@ -51,7 +51,7 @@ jobs:
           make dependencies
       - name: Run tests
         run: |
-          make test
+          make unittest
 
   python3-rocky8ish:
     runs-on: ubuntu-20.04
@@ -67,7 +67,7 @@ jobs:
           make dependencies
       - name: Run tests
         run: |
-          make test
+          make unittest
 
   python2-latest:
     runs-on: ubuntu-latest
@@ -80,8 +80,7 @@ jobs:
         uses: actions/checkout@v4
       - name: Setup environment
         run: |
-          pip install --no-cache-dir -r requirements.txt -r local-requirements.txt
+          make PYTHON_BIN=python PY=2 dependencies
       - name: Run tests
         run: |
-          PYTHON_BIN=python ./envhelp/makeconfig test --python2
-          MIG_ENV='local' python -m unittest discover -s tests/
+          make PYTHON_BIN=python PY=2 unittest

Makefile

@@ -1,11 +1,21 @@
 ifndef MIG_ENV
 	MIG_ENV = 'local'
 endif
-ifeq ($(PY),2)
+
+ifndef PY
+	PY = 3
+endif
+
+LOCAL_PYTHON_BIN = './envhelp/lpython'
+
+ifdef PYTHON_BIN
+	LOCAL_PYTHON_BIN = $(PYTHON_BIN)
+else ifeq ($(PY),2)
 	PYTHON_BIN = './envhelp/python2'
 else
 	PYTHON_BIN = './envhelp/python3'
 endif
+
 ifeq ($(ALLDEPS),1)
 	REQS_PATH = ./recommended.txt
 else
@@ -17,21 +27,23 @@ info:
 	@echo
 	@echo "The following should help you get started:"
 	@echo
-	@echo "'make test'      - run the test suite"
-	@echo "'make PY=2 test' - run the test suite (python 2)"
+	@echo "'make test'      - run the test suite (default python 3)"
+	@echo "'make PY=2 test' - run the test suite (default python 2)"
+	@echo "'make unittest'  - execute tests locally for development"
 
 .PHONY: fmt
 fmt:
 ifneq ($(MIG_ENV),'local')
 	@echo "unavailable outside local development environment"
 	@exit 1
 endif
-	$(PYTHON_BIN) -m autopep8 --ignore E402 -i
+	$(LOCAL_PYTHON_BIN) -m autopep8 --ignore E402 -i
 
 .PHONY: clean
 clean:
 	@rm -f ./envhelp/py2.imageid
-	@rm -f ./envhelp/py3.depends
+	@rm -f ./envhelp/py3.imageid
+	@rm -f ./envhelp/local.depends
 
 .PHONY: distclean
 distclean: clean
@@ -44,37 +56,41 @@ distclean: clean
 test: dependencies testconfig
 	@$(PYTHON_BIN) -m unittest discover -s tests/
 
+.PHONY: unittest
+unittest: dependencies testconfig
+	@$(LOCAL_PYTHON_BIN) -m unittest discover -s tests/
+
 .PHONY: dependencies
-dependencies: ./envhelp/venv/pyvenv.cfg ./envhelp/py3.depends
+ifeq ($(PY),2)
+dependencies: ./envhelp/local.depends
+else
+dependencies: ./envhelp/venv/pyvenv.cfg ./envhelp/local.depends
+endif
 
 .PHONY: testconfig
 testconfig: ./envhelp/output/testconfs
 
 ./envhelp/output/testconfs:
-	@./envhelp/makeconfig test --python2
+	@./envhelp/makeconfig test --docker
 	@./envhelp/makeconfig test
-	@mkdir -p ./envhelp/output/certs
-	@mkdir -p ./envhelp/output/state
-	@mkdir -p ./envhelp/output/state/log
 
 ifeq ($(MIG_ENV),'local')
-./envhelp/py3.depends: $(REQS_PATH) local-requirements.txt
+./envhelp/local.depends: $(REQS_PATH) local-requirements.txt
 else
-./envhelp/py3.depends: $(REQS_PATH)
+./envhelp/local.depends: $(REQS_PATH)
 endif
-	@rm -f ./envhelp/py3.depends
-	@echo "upgrading venv pip as required for some dependencies"
-	@./envhelp/venv/bin/pip3 install --upgrade pip
 	@echo "installing dependencies from $(REQS_PATH)"
-	@./envhelp/venv/bin/pip3 install -r $(REQS_PATH)
+	@$(LOCAL_PYTHON_BIN) -m pip install -r $(REQS_PATH)
 ifeq ($(MIG_ENV),'local')
 	@echo ""
 	@echo "installing development dependencies"
-	@./envhelp/venv/bin/pip3 install -r local-requirements.txt
+	@$(LOCAL_PYTHON_BIN) -m pip install -r local-requirements.txt
 endif
-	@touch ./envhelp/py3.depends
+	@touch ./envhelp/local.depends
 
 ./envhelp/venv/pyvenv.cfg:
 	@echo "provisioning environment"
 	@/usr/bin/env python3 -m venv ./envhelp/venv
-	@rm -f ./envhelp/py3.depends
+	@rm -f ./envhelp/local.depends
+	@echo "upgrading venv pip as required for some dependencies"
+	@./envhelp/venv/bin/pip3 install --upgrade pip

envhelp/docker/Dockerfile.py3

@@ -0,0 +1,8 @@
+FROM python:3.9
+
+WORKDIR /usr/src/app
+
+COPY requirements.txt local-requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt -r local-requirements.txt
+
+CMD [ "python", "--version" ]

envhelp/docker/Dockerfile.pyver

@@ -0,0 +1,9 @@
+ARG pyver
+FROM python:${pyver}
+
+WORKDIR /usr/src/app
+
+COPY requirements.txt local-requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt -r local-requirements.txt
+
+CMD [ "python", "--version" ]

envhelp/dpython

@@ -0,0 +1,88 @@
+#!/bin/sh
+#
+# --- BEGIN_HEADER ---
+#
+# dpython - wrapper to invoke a containerised python
+# Copyright (C) 2003-2024  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+# USA.
+#
+# --- END_HEADER ---
+#
+
+set -e
+
+SCRIPT_PATH=$(realpath "$0")
+SCRIPT_BASE=$(dirname -- "$SCRIPT_PATH")
+MIG_BASE=$(realpath "$SCRIPT_BASE/..")
+
+if [ -n "${PY}" ]; then
+    PYVER="$PY"
+    PYTHON_SUFFIX="py$PY"
+    DOCKER_FILE_SUFFIX="$PYTHON_SUFFIX"
+elif [ -n "${PYVER}" ]; then
+    PY=3
+    PYTHON_SUFFIX="pyver-$PYVER"
+    DOCKER_FILE_SUFFIX="pyver"
+else
+    echo "No python version specified - please supply a PY env var"
+    exit 1
+fi
+
+DOCKER_FILE="$SCRIPT_BASE/docker/Dockerfile.$DOCKER_FILE_SUFFIX"
+DOCKER_IMAGEID_FILE="$SCRIPT_BASE/$PYTHON_SUFFIX.imageid"
+
+# NOTE: portable dynamic lookup with docker as default and fallback to podman
+DOCKER_BIN=$(command -v docker || command -v podman || echo "")
+if [ -z "${DOCKER_BIN}" ]; then
+    echo "No docker binary found - cannot use for python $PY tests"
+    exit 1
+fi
+
+# default PYTHONPATH such that directly executing files in the repo "just works"
+# NOTE: this is hard-coded to the mount point used within the container
+PYTHONPATH='/usr/src/app'
+
+# default any variables for container development
+MIG_ENV=${MIG_ENV:-'docker'}
+
+# determine if the image has changed
+echo -n "validating python $PY container.. "
+
+# load a previously written docker image id if present
+IMAGEID_STORED=$(cat "$DOCKER_IMAGEID_FILE" 2>/dev/null || echo "")
+
+IMAGEID=$(${DOCKER_BIN} build -f "$DOCKER_FILE" . -q --build-arg "pyver=$PYVER")
+if [ "$IMAGEID" != "$IMAGEID_STORED" ]; then
+    echo "rebuilt for changes"
+
+    # reset the image id so the next call finds no changes
+    echo "$IMAGEID" > "$DOCKER_IMAGEID_FILE"
+else
+    echo "no changes needed"
+fi
+
+echo "using image id $IMAGEID"
+
+# execute python2 within the image passing the supplied arguments
+
+${DOCKER_BIN} run -it --rm \
+    --mount "type=bind,source=$MIG_BASE,target=/usr/src/app" \
+    --env "PYTHONPATH=$PYTHONPATH" \
+    --env "MIG_ENV=$MIG_ENV" \
+    "$IMAGEID" python$PY $@

envhelp/lpython

@@ -0,0 +1,46 @@
+#!/bin/sh
+#
+# --- BEGIN_HEADER ---
+#
+# python3 - wrapper to invoke a local python3 virtual environment
+# Copyright (C) 2003-2024  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+# USA.
+#
+# --- END_HEADER ---
+#
+
+set -e
+
+SCRIPT_PATH=$(realpath "$0")
+SCRIPT_BASE=$(dirname -- "$SCRIPT_PATH")
+MIG_BASE=$(realpath "$SCRIPT_BASE/..")
+
+PYTHON_BIN=${PYTHON_BIN:-"$SCRIPT_BASE/venv/bin/python3"}
+if [ ! -f "${PYTHON_BIN}" ]; then
+    echo "No python binary found - perhaps the virtual env was not created"
+    exit 1
+fi
+
+# default PYTHONPATH such that directly executing files in the repo "just works"
+PYTHONPATH=${PYTHONPATH:-"$MIG_BASE"}
+
+# default any variables for local development
+MIG_ENV=${MIG_ENV:-'local'}
+
+PYTHONPATH="$PYTHONPATH" MIG_ENV="$MIG_ENV" "$PYTHON_BIN" "$@"

envhelp/makeconfig.py

@@ -38,8 +38,9 @@
 
 from mig.shared.install import MIG_BASE, generate_confs
 
-_LOCAL_ENVHELP_OUTPUT_DIR = os.path.realpath(
-    os.path.join(os.path.dirname(__file__), "output"))
+_LOCAL_MIG_BASE = os.path.normpath(
+    os.path.join(os.path.dirname(__file__), ".."))
+_LOCAL_ENVHELP_OUTPUT_DIR = os.path.join(_LOCAL_MIG_BASE, "envhelp/output")
 _MAKECONFIG_ALLOWED = ["local", "test"]
 
 
@@ -51,21 +52,27 @@ def _at(sequence, index=-1, default=None):
         return default
 
 
-def write_testconfig(env_name, is_py2=False):
-    confs_name = 'confs' if env_name == 'local' else '%sconfs' % (env_name,)
-    confs_suffix = 'py2' if is_py2 else 'py3'
+def write_testconfig(env_name, is_docker=False):
+    is_predefined = env_name == 'test'
+    confs_name = '%sconfs' % (env_name,)
+    if is_predefined:
+        confs_suffix = 'docker' if is_docker else 'local'
+    else:
+        confs_suffix = 'py3'
 
     overrides = {
         'destination': os.path.join(_LOCAL_ENVHELP_OUTPUT_DIR, confs_name),
         'destination_suffix': "-%s" % (confs_suffix,),
     }
 
-    # determine the paths by which we will access the various configured dirs
-    if is_py2:
+    # determine the paths b which we will access the various configured dirs
+    #  the tests output directory - when invoked within
+
+    if is_predefined and is_docker:
         env_mig_base = '/usr/src/app'
     else:
-        env_mig_base = MIG_BASE
-    conf_dir_path = os.path.join(env_mig_base, "envhelp/output")
+        env_mig_base = _LOCAL_MIG_BASE
+    conf_dir_path = os.path.join(env_mig_base, "tests/output")
 
     overrides.update(**{
         'mig_code': os.path.join(conf_dir_path, 'mig'),
@@ -85,7 +92,7 @@ def write_testconfig(env_name, is_py2=False):
 
 def main_(argv):
     env_name = _at(argv, index=1, default='')
-    arg_is_py2 = '--python2' in argv
+    arg_is_docker = '--docker' in argv
 
     if env_name == '':
         raise RuntimeError(
@@ -94,7 +101,7 @@ def main_(argv):
         raise RuntimeError('environment must be one of %s' %
                            (_MAKECONFIG_ALLOWED,))
 
-    write_testconfig(env_name, is_py2=arg_is_py2)
+    write_testconfig(env_name, is_docker=arg_is_docker)
 
 
 def main(argv=sys.argv):