Skip to content

Commit 4d7344d

Browse files
jonasbardinojonasbardino
committed
Manually merged the majority of PR134 except the github hooks, which are only
part of the specific github branches. Reworks python version handling associated testconfig path handling. Credits for the original unit test framework and this unification work to move all tests into containers goes out to Alex Burke (albu-diku). Thanks, Alex! :) I only did a bit of minor polish as explained in #134 git-svn-id: svn+ssh://svn.code.sf.net/p/migrid/code/trunk@6195 b75ad72c-e7d7-11dd-a971-7dbc132099af
1 parent 910f848 commit 4d7344d

17 files changed

+349
-144
lines changed

‎Makefile

Lines changed: 35 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,21 @@
11
ifndef MIG_ENV
22
MIG_ENV = 'local'
33
endif
4-
ifeq ($(PY),2)
4+
5+
ifndef PY
6+
PY = 3
7+
endif
8+
9+
LOCAL_PYTHON_BIN = './envhelp/lpython'
10+
11+
ifdef PYTHON_BIN
12+
LOCAL_PYTHON_BIN = $(PYTHON_BIN)
13+
else ifeq ($(PY),2)
514
PYTHON_BIN = './envhelp/python2'
615
else
716
PYTHON_BIN = './envhelp/python3'
817
endif
18+
919
ifeq ($(ALLDEPS),1)
1020
REQS_PATH = ./recommended.txt
1121
else
@@ -17,21 +27,23 @@ info:
1727
@echo
1828
@echo "The following should help you get started:"
1929
@echo
20-
@echo "'make test' - run the test suite"
21-
@echo "'make PY=2 test' - run the test suite (python 2)"
30+
@echo "'make test' - run the test suite (default python 3)"
31+
@echo "'make PY=2 test' - run the test suite (default python 2)"
32+
@echo "'make unittest' - execute tests locally for development"
2233

2334
.PHONY: fmt
2435
fmt:
2536
ifneq ($(MIG_ENV),'local')
2637
@echo "unavailable outside local development environment"
2738
@exit 1
2839
endif
29-
$(PYTHON_BIN) -m autopep8 --ignore E402 -i
40+
$(LOCAL_PYTHON_BIN) -m autopep8 --ignore E402 -i
3041

3142
.PHONY: clean
3243
clean:
3344
@rm -f ./envhelp/py2.imageid
34-
@rm -f ./envhelp/py3.depends
45+
@rm -f ./envhelp/py3.imageid
46+
@rm -f ./envhelp/local.depends
3547

3648
.PHONY: distclean
3749
distclean: clean
@@ -44,37 +56,41 @@ distclean: clean
4456
test: dependencies testconfig
4557
@$(PYTHON_BIN) -m unittest discover -s tests/
4658

59+
.PHONY: unittest
60+
unittest: dependencies testconfig
61+
@$(LOCAL_PYTHON_BIN) -m unittest discover -s tests/
62+
4763
.PHONY: dependencies
48-
dependencies: ./envhelp/venv/pyvenv.cfg ./envhelp/py3.depends
64+
ifeq ($(PY),2)
65+
dependencies: ./envhelp/local.depends
66+
else
67+
dependencies: ./envhelp/venv/pyvenv.cfg ./envhelp/local.depends
68+
endif
4969

5070
.PHONY: testconfig
5171
testconfig: ./envhelp/output/testconfs
5272

5373
./envhelp/output/testconfs:
54-
@./envhelp/makeconfig test --python2
74+
@./envhelp/makeconfig test --docker
5575
@./envhelp/makeconfig test
56-
@mkdir -p ./envhelp/output/certs
57-
@mkdir -p ./envhelp/output/state
58-
@mkdir -p ./envhelp/output/state/log
5976

6077
ifeq ($(MIG_ENV),'local')
61-
./envhelp/py3.depends: $(REQS_PATH) local-requirements.txt
78+
./envhelp/local.depends: $(REQS_PATH) local-requirements.txt
6279
else
63-
./envhelp/py3.depends: $(REQS_PATH)
80+
./envhelp/local.depends: $(REQS_PATH)
6481
endif
65-
@rm -f ./envhelp/py3.depends
66-
@echo "upgrading venv pip as required for some dependencies"
67-
@./envhelp/venv/bin/pip3 install --upgrade pip
6882
@echo "installing dependencies from $(REQS_PATH)"
69-
@./envhelp/venv/bin/pip3 install -r $(REQS_PATH)
83+
@$(LOCAL_PYTHON_BIN) -m pip install -r $(REQS_PATH)
7084
ifeq ($(MIG_ENV),'local')
7185
@echo ""
7286
@echo "installing development dependencies"
73-
@./envhelp/venv/bin/pip3 install -r local-requirements.txt
87+
@$(LOCAL_PYTHON_BIN) -m pip install -r local-requirements.txt
7488
endif
75-
@touch ./envhelp/py3.depends
89+
@touch ./envhelp/local.depends
7690

7791
./envhelp/venv/pyvenv.cfg:
7892
@echo "provisioning environment"
7993
@/usr/bin/env python3 -m venv ./envhelp/venv
80-
@rm -f ./envhelp/py3.depends
94+
@rm -f ./envhelp/local.depends
95+
@echo "upgrading venv pip as required for some dependencies"
96+
@./envhelp/venv/bin/pip3 install --upgrade pip

‎envhelp/docker/Dockerfile.py2

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# NOTE: we use upstream python-2.7 image to mimic the version on CentOS/RHEL 7
2+
FROM python:2
3+
4+
WORKDIR /usr/src/app
5+
6+
COPY requirements.txt local-requirements.txt ./
7+
RUN pip install --no-cache-dir -r requirements.txt -r local-requirements.txt
8+
9+
CMD [ "python", "--version" ]

‎envhelp/docker/Dockerfile.py3

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# NOTE: we use upstream python-3.9 image to mimic the version on RHEL/Rocky 9
2+
FROM python:3.9
3+
4+
WORKDIR /usr/src/app
5+
6+
COPY requirements.txt local-requirements.txt ./
7+
RUN pip install --no-cache-dir -r requirements.txt -r local-requirements.txt
8+
9+
CMD [ "python", "--version" ]

‎envhelp/docker/Dockerfile.python2 renamed to ‎envhelp/docker/Dockerfile.pyver

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
FROM python:2
1+
ARG pyver
2+
FROM python:${pyver}
23

34
WORKDIR /usr/src/app
45

‎envhelp/dpython

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
#!/bin/sh
2+
#
3+
# --- BEGIN_HEADER ---
4+
#
5+
# dpython - wrapper to invoke a containerised python
6+
# Copyright (C) 2003-2024 The MiG Project by the Science HPC Center at UCPH
7+
#
8+
# This file is part of MiG.
9+
#
10+
# MiG is free software: you can redistribute it and/or modify
11+
# it under the terms of the GNU General Public License as published by
12+
# the Free Software Foundation; either version 2 of the License, or
13+
# (at your option) any later version.
14+
#
15+
# MiG is distributed in the hope that it will be useful,
16+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
17+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18+
# GNU General Public License for more details.
19+
#
20+
# You should have received a copy of the GNU General Public License
21+
# along with this program; if not, write to the Free Software
22+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
23+
# USA.
24+
#
25+
# --- END_HEADER ---
26+
#
27+
28+
set -e
29+
30+
SCRIPT_PATH=$(realpath "$0")
31+
SCRIPT_BASE=$(dirname -- "$SCRIPT_PATH")
32+
MIG_BASE=$(realpath "$SCRIPT_BASE/..")
33+
34+
if [ -n "${PY}" ]; then
35+
PYVER="$PY"
36+
PYTHON_SUFFIX="py$PY"
37+
DOCKER_FILE_SUFFIX="$PYTHON_SUFFIX"
38+
elif [ -n "${PYVER}" ]; then
39+
PY=3
40+
PYTHON_SUFFIX="pyver-$PYVER"
41+
DOCKER_FILE_SUFFIX="pyver"
42+
else
43+
echo "No python version specified - please supply a PY env var"
44+
exit 1
45+
fi
46+
47+
DOCKER_FILE="$SCRIPT_BASE/docker/Dockerfile.$DOCKER_FILE_SUFFIX"
48+
DOCKER_IMAGEID_FILE="$SCRIPT_BASE/$PYTHON_SUFFIX.imageid"
49+
50+
# NOTE: portable dynamic lookup with docker as default and fallback to podman
51+
DOCKER_BIN=$(command -v docker || command -v podman || echo "")
52+
if [ -z "${DOCKER_BIN}" ]; then
53+
echo "No docker binary found - cannot use for python $PY tests"
54+
exit 1
55+
fi
56+
57+
# default PYTHONPATH such that directly executing files in the repo "just works"
58+
# NOTE: this is hard-coded to the mount point used within the container
59+
PYTHONPATH='/usr/src/app'
60+
61+
# default any variables for container development
62+
MIG_ENV=${MIG_ENV:-'docker'}
63+
64+
# determine if the image has changed
65+
echo -n "validating python $PY container.. "
66+
67+
# load a previously written docker image id if present
68+
IMAGEID_STORED=$(cat "$DOCKER_IMAGEID_FILE" 2>/dev/null || echo "")
69+
70+
IMAGEID=$(${DOCKER_BIN} build -f "$DOCKER_FILE" . -q --build-arg "pyver=$PYVER")
71+
if [ "$IMAGEID" != "$IMAGEID_STORED" ]; then
72+
echo "rebuilt for changes"
73+
74+
# reset the image id so the next call finds no changes
75+
echo "$IMAGEID" > "$DOCKER_IMAGEID_FILE"
76+
else
77+
echo "no changes needed"
78+
fi
79+
80+
echo "using image id $IMAGEID"
81+
82+
# execute python2 within the image passing the supplied arguments
83+
84+
${DOCKER_BIN} run -it --rm \
85+
--mount "type=bind,source=$MIG_BASE,target=/usr/src/app" \
86+
--env "PYTHONPATH=$PYTHONPATH" \
87+
--env "MIG_ENV=$MIG_ENV" \
88+
"$IMAGEID" python$PY $@

‎envhelp/lpython

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/bin/sh
2+
#
3+
# --- BEGIN_HEADER ---
4+
#
5+
# python3 - wrapper to invoke a local python3 virtual environment
6+
# Copyright (C) 2003-2024 The MiG Project by the Science HPC Center at UCPH
7+
#
8+
# This file is part of MiG.
9+
#
10+
# MiG is free software: you can redistribute it and/or modify
11+
# it under the terms of the GNU General Public License as published by
12+
# the Free Software Foundation; either version 2 of the License, or
13+
# (at your option) any later version.
14+
#
15+
# MiG is distributed in the hope that it will be useful,
16+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
17+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18+
# GNU General Public License for more details.
19+
#
20+
# You should have received a copy of the GNU General Public License
21+
# along with this program; if not, write to the Free Software
22+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
23+
# USA.
24+
#
25+
# --- END_HEADER ---
26+
#
27+
28+
set -e
29+
30+
SCRIPT_PATH=$(realpath "$0")
31+
SCRIPT_BASE=$(dirname -- "$SCRIPT_PATH")
32+
MIG_BASE=$(realpath "$SCRIPT_BASE/..")
33+
34+
PYTHON_BIN=${PYTHON_BIN:-"$SCRIPT_BASE/venv/bin/python3"}
35+
if [ ! -f "${PYTHON_BIN}" ]; then
36+
echo "No python binary found - perhaps the virtual env was not created"
37+
exit 1
38+
fi
39+
40+
# default PYTHONPATH such that directly executing files in the repo "just works"
41+
PYTHONPATH=${PYTHONPATH:-"$MIG_BASE"}
42+
43+
# default any variables for local development
44+
MIG_ENV=${MIG_ENV:-'local'}
45+
46+
PYTHONPATH="$PYTHONPATH" MIG_ENV="$MIG_ENV" "$PYTHON_BIN" "$@"

‎envhelp/makeconfig.py

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,9 @@
3838

3939
from mig.shared.install import MIG_BASE, generate_confs
4040

41-
_LOCAL_ENVHELP_OUTPUT_DIR = os.path.realpath(
42-
os.path.join(os.path.dirname(__file__), "output"))
41+
_LOCAL_MIG_BASE = os.path.normpath(
42+
os.path.join(os.path.dirname(__file__), ".."))
43+
_LOCAL_ENVHELP_OUTPUT_DIR = os.path.join(_LOCAL_MIG_BASE, "envhelp/output")
4344
_MAKECONFIG_ALLOWED = ["local", "test"]
4445

4546

@@ -51,21 +52,27 @@ def _at(sequence, index=-1, default=None):
5152
return default
5253

5354

54-
def write_testconfig(env_name, is_py2=False):
55-
confs_name = 'confs' if env_name == 'local' else '%sconfs' % (env_name,)
56-
confs_suffix = 'py2' if is_py2 else 'py3'
55+
def write_testconfig(env_name, is_docker=False):
56+
is_predefined = env_name == 'test'
57+
confs_name = '%sconfs' % (env_name,)
58+
if is_predefined:
59+
confs_suffix = 'docker' if is_docker else 'local'
60+
else:
61+
confs_suffix = 'py3'
5762

5863
overrides = {
5964
'destination': os.path.join(_LOCAL_ENVHELP_OUTPUT_DIR, confs_name),
6065
'destination_suffix': "-%s" % (confs_suffix,),
6166
}
6267

6368
# determine the paths by which we will access the various configured dirs
64-
if is_py2:
69+
# the tests output directory - when invoked within
70+
71+
if is_predefined and is_docker:
6572
env_mig_base = '/usr/src/app'
6673
else:
67-
env_mig_base = MIG_BASE
68-
conf_dir_path = os.path.join(env_mig_base, "envhelp/output")
74+
env_mig_base = _LOCAL_MIG_BASE
75+
conf_dir_path = os.path.join(env_mig_base, "tests/output")
6976

7077
overrides.update(**{
7178
'mig_code': os.path.join(conf_dir_path, 'mig'),
@@ -85,7 +92,7 @@ def write_testconfig(env_name, is_py2=False):
8592

8693
def main_(argv):
8794
env_name = _at(argv, index=1, default='')
88-
arg_is_py2 = '--python2' in argv
95+
arg_is_docker = '--docker' in argv
8996

9097
if env_name == '':
9198
raise RuntimeError(
@@ -94,7 +101,7 @@ def main_(argv):
94101
raise RuntimeError('environment must be one of %s' %
95102
(_MAKECONFIG_ALLOWED,))
96103

97-
write_testconfig(env_name, is_py2=arg_is_py2)
104+
write_testconfig(env_name, is_docker=arg_is_docker)
98105

99106

100107
def main(argv=sys.argv):

‎envhelp/python2

Lines changed: 1 addition & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -29,46 +29,5 @@ set -e
2929

3030
SCRIPT_PATH=$(realpath "$0")
3131
SCRIPT_BASE=$(dirname -- "$SCRIPT_PATH")
32-
DOCKER_BASE="$SCRIPT_BASE/docker"
33-
DOCKER_IMAGEID_FILE="$SCRIPT_BASE/py2.imageid"
34-
# NOTE: portable dynamic lookup with docker as default and fallback to podman
35-
DOCKER_BIN=$(command -v docker || command -v podman || echo "")
3632

37-
if [ -z "${DOCKER_BIN}" ]; then
38-
echo "No docker binary found - cannot use for python2 tests"
39-
exit 1
40-
fi
41-
42-
# default PYTHONPATH such that directly executing files in the repo "just works"
43-
# NOTE: this is hard-coded to the mount point used within the container
44-
PYTHONPATH='/usr/app/src'
45-
46-
# default any variables for local development
47-
MIG_ENV=${MIG_ENV:-'local'}
48-
49-
# determine if the image has changed
50-
echo -n "validating container.. "
51-
52-
# load a previously written docker image id if present
53-
IMAGEID_STORED=$(cat "$DOCKER_IMAGEID_FILE" 2>/dev/null || echo "")
54-
55-
IMAGEID=$(${DOCKER_BIN} build -f "$DOCKER_BASE/Dockerfile.python2" . -q)
56-
if [ "$IMAGEID" != "$IMAGEID_STORED" ]; then
57-
echo "rebuilt for changes"
58-
59-
# reset the image id so the next call finds no changes
60-
echo "$IMAGEID" > "$DOCKER_IMAGEID_FILE"
61-
else
62-
echo "no changes needed"
63-
fi
64-
65-
echo "running with MIG_ENV='$MIG_ENV' under python 2"
66-
echo
67-
68-
# execute python2 within the image passing the supplied arguments
69-
70-
${DOCKER_BIN} run -it --rm \
71-
--mount type=bind,source=.,target=/usr/src/app \
72-
--env "PYTHONPATH=$PYTHON_PATH" \
73-
--env "MIG_ENV=$MIG_ENV" \
74-
"$IMAGEID" python2 "$@"
33+
PY=2 $SCRIPT_BASE/dpython "$@"

0 commit comments

Comments
 (0)