Rework python version handling associated testconfig path handling.

A previous revision fixed an immediate problem with test path handling
but knowingly left some things on the table - in particular a split
between container based handling of Python 2 and the local execution
of Python 3 (meaning a potentially inconsistent version relative to
what the project considers officially supported).

Address this entirely: rework the default behaviour of `make test` to
container based execution and use a consistent baseline Python 3.

In order to continue to support rapid local iteration, provide a
separate `make unittest` target which will execute the test suite
locally and is also used as a mechanism to run other supporting tools.

The clean use of containers necessitated various changes that make path
arguments within the testconfig non-overlapping and better isolated.
Adjust all paths generated within the test suite to be always from the
base root instead of relative the output directory. Opt to patch the
makeconfig generator rather than fiddle with generateconfs again.

While here also add support for an environment variable overried that
allows execution of the test suite against arbitrary python 3 versions.

Author: albu-diku

.github/workflows/ci.yml

@@ -35,7 +35,7 @@ jobs:
           make dependencies
       - name: Run tests
         run: |
-          make test
+          make unittest
 
   python3-rocky9ish:
     runs-on: ubuntu-22.04
@@ -51,7 +51,7 @@ jobs:
           make dependencies
       - name: Run tests
         run: |
-          make test
+          make unittest
 
   python3-rocky8ish:
     runs-on: ubuntu-20.04
@@ -67,7 +67,7 @@ jobs:
           make dependencies
       - name: Run tests
         run: |
-          make test
+          make unittest
 
   python2-latest:
     runs-on: ubuntu-latest
@@ -80,8 +80,7 @@ jobs:
         uses: actions/checkout@v4
       - name: Setup environment
         run: |
-          pip install --no-cache-dir -r requirements.txt -r local-requirements.txt
+          make PYTHON_BIN=python PY=2 dependencies
       - name: Run tests
         run: |
-          PYTHON_BIN=python ./envhelp/makeconfig test --python2
-          MIG_ENV='local' python -m unittest discover -s tests/
+          make PYTHON_BIN=python PY=2 unittest

Makefile

@@ -1,11 +1,21 @@
 ifndef MIG_ENV
 	MIG_ENV = 'local'
 endif
-ifeq ($(PY),2)
+
+ifndef PY
+	PY = 3
+endif
+
+LOCAL_PYTHON_BIN = './envhelp/lpython'
+
+ifdef PYTHON_BIN
+	LOCAL_PYTHON_BIN = $(PYTHON_BIN)
+else ifeq ($(PY),2)
 	PYTHON_BIN = './envhelp/python2'
 else
 	PYTHON_BIN = './envhelp/python3'
 endif
+
 ifeq ($(ALLDEPS),1)
 	REQS_PATH = ./recommended.txt
 else
@@ -17,21 +27,23 @@ info:
 	@echo
 	@echo "The following should help you get started:"
 	@echo
-	@echo "'make test'      - run the test suite"
-	@echo "'make PY=2 test' - run the test suite (python 2)"
+	@echo "'make test'      - run the test suite (default python 3)"
+	@echo "'make PY=2 test' - run the test suite (default python 2)"
+	@echo "'make unittest'  - execute tests locally for development"
 
 .PHONY: fmt
 fmt:
 ifneq ($(MIG_ENV),'local')
 	@echo "unavailable outside local development environment"
 	@exit 1
 endif
-	$(PYTHON_BIN) -m autopep8 --ignore E402 -i
+	$(LOCAL_PYTHON_BIN) -m autopep8 --ignore E402 -i
 
 .PHONY: clean
 clean:
 	@rm -f ./envhelp/py2.imageid
-	@rm -f ./envhelp/py3.depends
+	@rm -f ./envhelp/py3.imageid
+	@rm -f ./envhelp/local.depends
 
 .PHONY: distclean
 distclean: clean
@@ -44,37 +56,41 @@ distclean: clean
 test: dependencies testconfig
 	@$(PYTHON_BIN) -m unittest discover -s tests/
 
+.PHONY: unittest
+unittest: dependencies testconfig
+	@$(LOCAL_PYTHON_BIN) -m unittest discover -s tests/
+
 .PHONY: dependencies
-dependencies: ./envhelp/venv/pyvenv.cfg ./envhelp/py3.depends
+ifeq ($(PY),2)
+dependencies: ./envhelp/local.depends
+else
+dependencies: ./envhelp/venv/pyvenv.cfg ./envhelp/local.depends
+endif
 
 .PHONY: testconfig
 testconfig: ./envhelp/output/testconfs
 
 ./envhelp/output/testconfs:
-	@./envhelp/makeconfig test --python2
+	@./envhelp/makeconfig test --docker
 	@./envhelp/makeconfig test
-	@mkdir -p ./envhelp/output/certs
-	@mkdir -p ./envhelp/output/state
-	@mkdir -p ./envhelp/output/state/log
 
 ifeq ($(MIG_ENV),'local')
-./envhelp/py3.depends: $(REQS_PATH) local-requirements.txt
+./envhelp/local.depends: $(REQS_PATH) local-requirements.txt
 else
-./envhelp/py3.depends: $(REQS_PATH)
+./envhelp/local.depends: $(REQS_PATH)
 endif
-	@rm -f ./envhelp/py3.depends
-	@echo "upgrading venv pip as required for some dependencies"
-	@./envhelp/venv/bin/pip3 install --upgrade pip
 	@echo "installing dependencies from $(REQS_PATH)"
-	@./envhelp/venv/bin/pip3 install -r $(REQS_PATH)
+	@$(LOCAL_PYTHON_BIN) -m pip install -r $(REQS_PATH)
 ifeq ($(MIG_ENV),'local')
 	@echo ""
 	@echo "installing development dependencies"
-	@./envhelp/venv/bin/pip3 install -r local-requirements.txt
+	@$(LOCAL_PYTHON_BIN) -m pip install -r local-requirements.txt
 endif
-	@touch ./envhelp/py3.depends
+	@touch ./envhelp/local.depends
 
 ./envhelp/venv/pyvenv.cfg:
 	@echo "provisioning environment"
 	@/usr/bin/env python3 -m venv ./envhelp/venv
-	@rm -f ./envhelp/py3.depends
+	@rm -f ./envhelp/local.depends
+	@echo "upgrading venv pip as required for some dependencies"
+	@./envhelp/venv/bin/pip3 install --upgrade pip

envhelp/docker/Dockerfile.python2 renamed to envhelp/docker/Dockerfile.py2

@@ -0,0 +1,8 @@
+FROM python:3.9
+
+WORKDIR /usr/src/app
+
+COPY requirements.txt local-requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt -r local-requirements.txt
+
+CMD [ "python", "--version" ]

envhelp/docker/Dockerfile.py3

@@ -0,0 +1,9 @@
+ARG pyver
+FROM python:${pyver}
+
+WORKDIR /usr/src/app
+
+COPY requirements.txt local-requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt -r local-requirements.txt
+
+CMD [ "python", "--version" ]

envhelp/docker/Dockerfile.pyver

@@ -0,0 +1,88 @@
+#!/bin/sh
+#
+# --- BEGIN_HEADER ---
+#
+# dpython - wrapper to invoke a containerised python
+# Copyright (C) 2003-2024  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+# USA.
+#
+# --- END_HEADER ---
+#
+
+set -e
+
+SCRIPT_PATH=$(realpath "$0")
+SCRIPT_BASE=$(dirname -- "$SCRIPT_PATH")
+MIG_BASE=$(realpath "$SCRIPT_BASE/..")
+
+if [ -n "${PY}" ]; then
+    PYVER="$PY"
+    PYTHON_SUFFIX="py$PY"
+    DOCKER_FILE_SUFFIX="$PYTHON_SUFFIX"
+elif [ -n "${PYVER}" ]; then
+    PY=3
+    PYTHON_SUFFIX="pyver-$PYVER"
+    DOCKER_FILE_SUFFIX="pyver"
+else
+    echo "No python version specified - please supply a PY env var"
+    exit 1
+fi
+
+DOCKER_FILE="$SCRIPT_BASE/docker/Dockerfile.$DOCKER_FILE_SUFFIX"
+DOCKER_IMAGEID_FILE="$SCRIPT_BASE/$PYTHON_SUFFIX.imageid"
+
+# NOTE: portable dynamic lookup with docker as default and fallback to podman
+DOCKER_BIN=$(command -v docker || command -v podman || echo "")
+if [ -z "${DOCKER_BIN}" ]; then
+    echo "No docker binary found - cannot use for python $PY tests"
+    exit 1
+fi
+
+# default PYTHONPATH such that directly executing files in the repo "just works"
+# NOTE: this is hard-coded to the mount point used within the container
+PYTHONPATH='/usr/src/app'
+
+# default any variables for container development
+MIG_ENV=${MIG_ENV:-'docker'}
+
+# determine if the image has changed
+echo -n "validating python $PY container.. "
+
+# load a previously written docker image id if present
+IMAGEID_STORED=$(cat "$DOCKER_IMAGEID_FILE" 2>/dev/null || echo "")
+
+IMAGEID=$(${DOCKER_BIN} build -f "$DOCKER_FILE" . -q --build-arg "pyver=$PYVER")
+if [ "$IMAGEID" != "$IMAGEID_STORED" ]; then
+    echo "rebuilt for changes"
+
+    # reset the image id so the next call finds no changes
+    echo "$IMAGEID" > "$DOCKER_IMAGEID_FILE"
+else
+    echo "no changes needed"
+fi
+
+echo "using image id $IMAGEID"
+
+# execute python2 within the image passing the supplied arguments
+
+${DOCKER_BIN} run -it --rm \
+    --mount "type=bind,source=$MIG_BASE,target=/usr/src/app" \
+    --env "PYTHONPATH=$PYTHONPATH" \
+    --env "MIG_ENV=$MIG_ENV" \
+    "$IMAGEID" python$PY $@

envhelp/dpython

@@ -0,0 +1,46 @@
+#!/bin/sh
+#
+# --- BEGIN_HEADER ---
+#
+# python3 - wrapper to invoke a local python3 virtual environment
+# Copyright (C) 2003-2024  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+# USA.
+#
+# --- END_HEADER ---
+#
+
+set -e
+
+SCRIPT_PATH=$(realpath "$0")
+SCRIPT_BASE=$(dirname -- "$SCRIPT_PATH")
+MIG_BASE=$(realpath "$SCRIPT_BASE/..")
+
+PYTHON_BIN=${PYTHON_BIN:-"$SCRIPT_BASE/venv/bin/python3"}
+if [ ! -f "${PYTHON_BIN}" ]; then
+    echo "No python binary found - perhaps the virtual env was not created"
+    exit 1
+fi
+
+# default PYTHONPATH such that directly executing files in the repo "just works"
+PYTHONPATH=${PYTHONPATH:-"$MIG_BASE"}
+
+# default any variables for local development
+MIG_ENV=${MIG_ENV:-'local'}
+
+PYTHONPATH="$PYTHONPATH" MIG_ENV="$MIG_ENV" "$PYTHON_BIN" "$@"

envhelp/lpython

@@ -38,8 +38,9 @@
 
 from mig.shared.install import MIG_BASE, generate_confs
 
-_LOCAL_ENVHELP_OUTPUT_DIR = os.path.realpath(
-    os.path.join(os.path.dirname(__file__), "output"))
+_LOCAL_MIG_BASE = os.path.normpath(
+    os.path.join(os.path.dirname(__file__), ".."))
+_LOCAL_ENVHELP_OUTPUT_DIR = os.path.join(_LOCAL_MIG_BASE, "envhelp/output")
 _MAKECONFIG_ALLOWED = ["local", "test"]
 
 
@@ -51,21 +52,27 @@ def _at(sequence, index=-1, default=None):
         return default
 
 
-def write_testconfig(env_name, is_py2=False):
-    confs_name = 'confs' if env_name == 'local' else '%sconfs' % (env_name,)
-    confs_suffix = 'py2' if is_py2 else 'py3'
+def write_testconfig(env_name, is_docker=False):
+    is_predefined = env_name == 'test'
+    confs_name = '%sconfs' % (env_name,)
+    if is_predefined:
+        confs_suffix = 'docker' if is_docker else 'local'
+    else:
+        confs_suffix = 'py3'
 
     overrides = {
         'destination': os.path.join(_LOCAL_ENVHELP_OUTPUT_DIR, confs_name),
         'destination_suffix': "-%s" % (confs_suffix,),
     }
 
-    # determine the paths by which we will access the various configured dirs
-    if is_py2:
+    # determine the paths b which we will access the various configured dirs
+    #  the tests output directory - when invoked within
+
+    if is_predefined and is_docker:
         env_mig_base = '/usr/src/app'
     else:
-        env_mig_base = MIG_BASE
-    conf_dir_path = os.path.join(env_mig_base, "envhelp/output")
+        env_mig_base = _LOCAL_MIG_BASE
+    conf_dir_path = os.path.join(env_mig_base, "tests/output")
 
     overrides.update(**{
         'mig_code': os.path.join(conf_dir_path, 'mig'),
@@ -85,7 +92,7 @@ def write_testconfig(env_name, is_py2=False):
 
 def main_(argv):
     env_name = _at(argv, index=1, default='')
-    arg_is_py2 = '--python2' in argv
+    arg_is_docker = '--docker' in argv
 
     if env_name == '':
         raise RuntimeError(
@@ -94,7 +101,7 @@ def main_(argv):
         raise RuntimeError('environment must be one of %s' %
                            (_MAKECONFIG_ALLOWED,))
 
-    write_testconfig(env_name, is_py2=arg_is_py2)
+    write_testconfig(env_name, is_docker=arg_is_docker)
 
 
 def main(argv=sys.argv):