Skip to content

Commit d386b88

Browse files
jjhurseyjjhursey
committed
Runtime Testing harness
Signed-off-by: Joshua Hursey <jhursey@us.ibm.com>
1 parent 08ab736 commit d386b88

File tree

7 files changed

+256
-0
lines changed

7 files changed

+256
-0
lines changed

‎runtime/.ci-configure

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#
2+
# Open MPI is built with the following options:
3+
# ./configure --prefix=/opt/ci/support/exports/ompi --disable-cuda --disable-nvml --with-cuda=no --without-hcoll
4+
#
5+
# Additional options can be added by listing them.
6+
# Options can be listed as either:
7+
# - One option per line
8+
# - Multiple options per line
9+
# Note: Line continutions are not supported
10+
#
11+
# Enable Debug
12+
--enable-debug
13+
# With Python Bindings
14+
# Need to install Cython on the CI machine
15+
#--enable-python-bindings

‎runtime/.ci-tests

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Start with the basics
2+
hello_world

‎runtime/README.md

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# Test suite for Open MPI runtime
2+
3+
This test suite is meant to be able to be run stand-alone or under CI.
4+
5+
All of the tests that are intended for CI must be listed in the `.ci-tests` file.
6+
7+
If the Open MPI build needs additional `configure` options those can be added to the `.ci-configure` file.
8+
9+
## Running tests stand alone
10+
11+
1. Make sure that Open MPI and other required libraries are in your `PATH`/`LD_LIBRARY_PATH`
12+
2. Drop into a directory:
13+
- Use the `build.sh` script to build any test articles
14+
- Use the `run.sh` script to run the test program
15+
16+
17+
## CI Environment Variables
18+
19+
The CI infrastructure defines the following environment variables to be used in the test programs. These are defined during the `run.sh` phase and not the `build.sh` phase.
20+
21+
* `CI_HOSTFILE` : Absolute path to the hostfile for this run.
22+
* `CI_NUM_NODES` : Number of nodes in this cluster.
23+
* `CI_OMPI_SRC` : top level directory of the Open MPI repository checkout.
24+
* `CI_OMPI_TESTS_PUBLIC_DIR` : Top level directory of the [Open MPI Public Test](https://github.com/open-mpi/ompi-tests-public) repository checkout
25+
* `OMPI_ROOT` : Open MPI install directory.
26+
27+
28+
### Adding a new test for CI
29+
30+
1. Create a directory with your test.
31+
- **Note**: Please make your test scripts such that they can be easily run with or without the CI environment variables.
32+
2. Create a build script named `build.sh`
33+
- CI will call this exactly one time (with a timeout in case it hangs).
34+
- If the script returns `0` then it is considered successful. Otherwise it is considered failed.
35+
3. Create a run script named `run.sh`
36+
- The script is responsible for running your test including any runtime setup/shutdown and test result inspection.
37+
- CI will call this exactly one time (with a timeout in case it hangs).
38+
- If the script returns `0` then it is considered successful. Otherwise it is considered failed.
39+
4. Add your directory name to the `.ci-tests` file in this directory in the order that they should be executed.
40+
- Note that adding the directory is not sufficient to have CI run the test, it must be in the file.
41+
- Comments (starting with `#`) are allowed.

‎runtime/bin/cleanup-scrub-local.sh

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
3+
PROGS="prte prted prun mpirun timeout"
4+
5+
clean_files()
6+
{
7+
FILES=("pmix-*" "core*" "openmpi-sessions-*" "pmix_dstor_*" "ompi.*" "prrte.*" )
8+
9+
for fn in ${FILES[@]}; do
10+
find /tmp/ -maxdepth 1 \
11+
-user $USER -a \
12+
-name $fn \
13+
-exec rm -rf {} \;
14+
15+
if [ -n "$TMPDIR" ] ; then
16+
find $TMPDIR -maxdepth 1 \
17+
-user $USER -a \
18+
-name $fn \
19+
-exec rm -rf {} \;
20+
fi
21+
done
22+
}
23+
24+
killall -q ${PROGS} > /dev/null
25+
clean_files
26+
killall -q -9 ${PROGS} > /dev/null
27+
28+
exit 0
29+
30+

‎runtime/bin/cleanup.sh

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/bin/bash
2+
3+
clean_server()
4+
{
5+
SERVER=$1
6+
ITER=$2
7+
MAX=$3
8+
QUIET=$4
9+
10+
SCRIPTDIR=$PWD/`dirname $0`/
11+
12+
if [[ $QUIET == 0 ]] ; then
13+
echo "Cleaning server ($ITER / $MAX): $SERVER"
14+
fi
15+
ssh -oBatchMode=yes ${SERVER} ${SCRIPTDIR}/cleanup-scrub-local.sh
16+
}
17+
18+
if [[ "x" != "x$CI_HOSTFILE" && -f "$CI_HOSTFILE" ]] ; then
19+
ALLHOSTS=(`cat $CI_HOSTFILE | sort | uniq`)
20+
else
21+
ALLHOSTS=(`hostname`)
22+
fi
23+
LEN=${#ALLHOSTS[@]}
24+
25+
# Use a background mode if running at scale
26+
USE_BG=0
27+
if [ $LEN -gt 10 ] ; then
28+
USE_BG=1
29+
fi
30+
31+
for (( i=0; i<${LEN}; i++ ));
32+
do
33+
if [ $USE_BG == 1 ] ; then
34+
if [ $(($i % 100)) == 0 ] ; then
35+
echo "| $i"
36+
else
37+
if [ $(($i % 10)) == 0 ] ; then
38+
echo -n "|"
39+
else
40+
echo -n "."
41+
fi
42+
fi
43+
fi
44+
45+
if [ $USE_BG == 1 ] ; then
46+
clean_server ${ALLHOSTS[$i]} $i $LEN $USE_BG &
47+
sleep 0.25
48+
else
49+
clean_server ${ALLHOSTS[$i]} $i $LEN $USE_BG
50+
echo "-------------------------"
51+
fi
52+
done
53+
54+
if [ $USE_BG == 1 ] ; then
55+
echo ""
56+
echo "------------------------- Waiting"
57+
wait
58+
fi
59+
60+
echo "------------------------- Done"
61+
62+
exit 0

‎runtime/hello_world/build.sh

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash -e
2+
3+
# Wrapper compiler
4+
_MPICC=mpicc
5+
echo "=========================="
6+
echo "Wrapper compiler: $_MPICC"
7+
echo "=========================="
8+
${_MPICC} --showme
9+
10+
echo "=========================="
11+
echo "Building MPI Hello World"
12+
echo "=========================="
13+
cp ${CI_OMPI_SRC}/examples/hello_c.c .
14+
${_MPICC} hello_c.c -o hello
15+
16+
echo "=========================="
17+
echo "Success"
18+
echo "=========================="
19+
exit 0

‎runtime/hello_world/run.sh

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
#!/bin/bash -xe
2+
3+
# Final return value
4+
FINAL_RTN=0
5+
6+
# Number of nodes - for accounting/verification purposes
7+
# Default: 1
8+
NUM_NODES=${CI_NUM_NODES:-1}
9+
10+
if [ "x" != "x${CI_HOSTFILE}" ] ; then
11+
ARG_HOSTFILE="--hostfile ${CI_HOSTFILE}"
12+
else
13+
ARG_HOSTFILE=""
14+
fi
15+
16+
_shutdown()
17+
{
18+
# ---------------------------------------
19+
# Cleanup
20+
# ---------------------------------------
21+
22+
exit $FINAL_RTN
23+
}
24+
25+
# ---------------------------------------
26+
# Run the test - Hostname
27+
# ---------------------------------------
28+
echo "=========================="
29+
echo "Test: hostname"
30+
echo "=========================="
31+
mpirun ${ARG_HOSTFILE} --map-by ppr:5:node hostname 2>&1 | tee output-hn.txt
32+
33+
# ---------------------------------------
34+
# Verify the results
35+
# ---------------------------------------
36+
ERRORS=`grep ERROR output-hn.txt | wc -l`
37+
if [[ $ERRORS -ne 0 ]] ; then
38+
echo "ERROR: Error string detected in the output"
39+
FINAL_RTN=1
40+
_shutdown
41+
fi
42+
43+
LINES=`wc -l output-hn.txt | awk '{print $1}'`
44+
if [[ $LINES -ne $(( 5 * $NUM_NODES )) ]] ; then
45+
echo "ERROR: Incorrect number of lines of output"
46+
FINAL_RTN=2
47+
_shutdown
48+
fi
49+
50+
if [ $FINAL_RTN == 0 ] ; then
51+
echo "Success - hostname"
52+
fi
53+
54+
55+
# ---------------------------------------
56+
# Run the test - Hello World
57+
# ---------------------------------------
58+
echo "=========================="
59+
echo "Test: Hello World"
60+
echo "=========================="
61+
mpirun ${ARG_HOSTFILE} --map-by ppr:5:node ./hello 2>&1 | tee output.txt
62+
63+
# ---------------------------------------
64+
# Verify the results
65+
# ---------------------------------------
66+
ERRORS=`grep ERROR output.txt | wc -l`
67+
if [[ $ERRORS -ne 0 ]] ; then
68+
echo "ERROR: Error string detected in the output"
69+
FINAL_RTN=1
70+
_shutdown
71+
fi
72+
73+
LINES=`wc -l output.txt | awk '{print $1}'`
74+
if [[ $LINES -ne $(( 5 * $NUM_NODES )) ]] ; then
75+
echo "ERROR: Incorrect number of lines of output"
76+
FINAL_RTN=2
77+
_shutdown
78+
fi
79+
80+
if [ $FINAL_RTN == 0 ] ; then
81+
echo "Success - hello world"
82+
fi
83+
84+
echo "=========================="
85+
echo "Success"
86+
echo "=========================="
87+
_shutdown

0 commit comments

Comments
 (0)