Skip to content

Commit 7592eb3

Browse files
authored
Merge pull request #8069 from bwbarrett/feature/3rdparty-packaging
Rework how Open MPI integrates with 3rd party packages
2 parents 1f5ed0b + a30eb69 commit 7592eb3

File tree

284 files changed

+1780
-78140
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

284 files changed

+1780
-78140
lines changed

.gitignore

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,15 @@
4444

4545
Makefile
4646
Makefile.in
47+
48+
# hwloc and pmix have been de-modularized, but still uses parts of the
49+
# MCA system due to the amount of glue code that ended up in the
50+
# framework base. Until that is cleaned up, do not ignore the
51+
# hard-coded static-components.h file.
4752
static-components.h
53+
!opal/mca/hwloc/base/static-components.h
54+
!opal/mca/pmix/base/static-components.h
55+
4856
*\\#
4957
config.cache
5058
aclocal.m4
@@ -90,6 +98,14 @@ ltsugar.m4
9098
ltversion.m4
9199
ltoptions.m4
92100

101+
# Libevent and hwloc are included as a tarball. Ignore any expanded
102+
# tarballs, since they are not included in git. Do not ignore the
103+
# tarballs themselves, and those are artifacts we will store in git.
104+
3rd-party/libevent-*
105+
!3rd-party/libevent-*.tar.*
106+
3rd-party/hwloc-*
107+
!3rd-party/hwloc-*.tar.*
108+
93109
config/project_list.m4
94110
config/autogen_found_items.m4
95111
config/opal_get_version.sh
@@ -367,11 +383,6 @@ opal/mca/event/libevent*/libevent/libevent_openssl.pc
367383
opal/mca/event/libevent*/libevent/libevent_pthreads.pc
368384
opal/mca/event/libevent*/libevent/include/event2/event-config.h
369385

370-
opal/mca/hwloc/hwloc*/hwloc/include/hwloc/autogen/config.h
371-
opal/mca/hwloc/hwloc*/hwloc/include/private/autogen/config.h
372-
opal/mca/hwloc/base/static-components.h.new.extern
373-
opal/mca/hwloc/base/static-components.h.new.struct
374-
375386
opal/mca/installdirs/config/install_dirs.h
376387

377388
!opal/mca/pmix/pmix*/openpmix/AUTHORS

.gitmodules

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,8 @@
1-
[submodule "hwloc-20x"]
2-
path = opal/mca/hwloc/hwloc2/hwloc
3-
url = https://github.com/open-mpi/hwloc.git
41
[submodule "prrte"]
5-
path = prrte
2+
path = 3rd-party/prrte
63
url = https://github.com/openpmix/prrte
74
branch = master
85
[submodule "openpmix"]
9-
path = opal/mca/pmix/pmix4x/openpmix
6+
path = 3rd-party/openpmix
107
url = https://github.com/openpmix/openpmix.git
118
branch = master

3rd-party/Makefile.am

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#
2+
# Copyright (c) 2020 Amazon.com, Inc. or its affiliates.
3+
# All Rights reserved.
4+
# $COPYRIGHT$
5+
#
6+
# Additional copyrights may follow
7+
#
8+
# $HEADER$
9+
#
10+
# This Makefile exists (as opposed to being part of the top level
11+
# Makefile) solely to have a blank check-recursive rule. Otherwise, the
12+
# stability of Open MPI's ability to run "make check" or "make
13+
# distcheck" is dependent on the ability of our 3rd-party packages to
14+
# do the same. Libevent's tests are not stable enough for that kind
15+
# of choice.
16+
17+
SUBDIRS = $(OPAL_3RDPARTY_SUBDIRS)
18+
DIST_SUBDIRS = $(OPAL_3RDPARTY_DIST_SUBDIRS)
19+
EXTRA_DIST = $(OPAL_3RDPARTY_EXTRA_DIST)
20+
21+
distclean-local:
22+
rm -rf $(OPAL_3RDPARTY_DISTCLEAN_DIRS)
23+
24+
check:
25+
@echo "auto-recursing into 3rd-party packages for check disabled"
26+
27+
check-recursive:
28+
@echo "auto-recursing into 3rd-party packages for check disabled"

3rd-party/hwloc-2.2.0.tar.gz

6.43 MB
Binary file not shown.
1.05 MB
Binary file not shown.

3rd-party/openpmix

Submodule openpmix added at 952a986

3rd-party/prrte

Submodule prrte added at 545863e

LICENSE

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ Copyright (c) 2014-2015 Hewlett-Packard Development Company, LP. All
5353
rights reserved.
5454
Copyright (c) 2013-2017 Research Organization for Information Science (RIST).
5555
All rights reserved.
56-
Copyright (c) 2017-2018 Amazon.com, Inc. or its affiliates. All Rights
56+
Copyright (c) 2017-2020 Amazon.com, Inc. or its affiliates. All Rights
5757
reserved.
5858
Copyright (c) 2018 DataDirect Networks. All rights reserved.
5959
Copyright (c) 2018-2019 Triad National Security, LLC. All rights reserved.

Makefile.am

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
# Copyright (c) 2006-2016 Cisco Systems, Inc. All rights reserved.
1313
# Copyright (c) 2012-2015 Los Alamos National Security, Inc. All rights reserved.
1414
# Copyright (c) 2014-2019 Intel, Inc. All rights reserved.
15-
# Copyright (c) 2017-2018 Amazon.com, Inc. or its affiliates.
15+
# Copyright (c) 2017-2020 Amazon.com, Inc. or its affiliates.
1616
# All Rights reserved.
1717
# Copyright (c) 2020 IBM Corporation. All rights reserved.
1818
# $COPYRIGHT$
@@ -22,15 +22,8 @@
2222
# $HEADER$
2323
#
2424

25-
SUBDIRS = config contrib $(MCA_PROJECT_SUBDIRS) test
26-
if OMPI_WANT_PRRTE
27-
SUBDIRS += prrte
28-
endif
29-
30-
DIST_SUBDIRS = config contrib $(MCA_PROJECT_DIST_SUBDIRS) test
31-
if OMPI_WANT_PRRTE
32-
DIST_SUBDIRS += prrte
33-
endif
25+
SUBDIRS = config contrib 3rd-party $(MCA_PROJECT_SUBDIRS) test
26+
DIST_SUBDIRS = config contrib 3rd-party $(MCA_PROJECT_DIST_SUBDIRS) test
3427
EXTRA_DIST = README INSTALL VERSION Doxyfile LICENSE autogen.pl README.JAVA.txt AUTHORS
3528

3629
include examples/Makefile.include
@@ -50,7 +43,8 @@ install-exec-hook:
5043
--brief \
5144
--top_builddir=$(top_builddir) \
5245
--top_srcdir=$(top_srcdir) \
53-
--objext=$(OBJEXT); \
46+
--objext=$(OBJEXT) \
47+
--skipdir=3rd-party ; \
5448
fi
5549

5650
ACLOCAL_AMFLAGS = -I config

autogen.pl

Lines changed: 88 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
# Copyright (c) 2015-2019 Research Organization for Information Science
99
# and Technology (RIST). All rights reserved.
1010
# Copyright (c) 2015-2020 IBM Corporation. All rights reserved.
11+
# Copyright (c) 2020 Amazon.com, Inc. or its affiliates.
12+
# All Rights reserved.
1113
#
1214
# $COPYRIGHT$
1315
#
@@ -49,6 +51,7 @@
4951
my $no_orte_arg = 0;
5052
my $no_prrte_arg = 0;
5153
my $no_oshmem_arg = 0;
54+
my $no_3rdparty_arg = "";
5255
my $quiet_arg = 0;
5356
my $debug_arg = 0;
5457
my $help_arg = 0;
@@ -72,6 +75,10 @@
7275
my $ompi_automake_search = "automake";
7376
my $ompi_libtoolize_search = "libtoolize;glibtoolize";
7477

78+
# version of packages we ship as tarballs
79+
my $libevent_version="2.1.12-stable";
80+
my $hwloc_version="2.2.0";
81+
7582
# One-time setup
7683
my $username;
7784
my $hostname;
@@ -1125,6 +1132,7 @@ sub in_tarball {
11251132
"no-orte" => \$no_orte_arg,
11261133
"no-prrte" => \$no_prrte_arg,
11271134
"no-oshmem" => \$no_oshmem_arg,
1135+
"no-3rdparty=s" => \$no_3rdparty_arg,
11281136
"quiet|q" => \$quiet_arg,
11291137
"debug|d" => \$debug_arg,
11301138
"help|h" => \$help_arg,
@@ -1143,6 +1151,7 @@ sub in_tarball {
11431151
--no-orte | -no-orte Do not build Open MPI's runtime support (alias for --no-prrte)
11441152
--no-prrte | -no-prrte Do not build Open MPI's runtime support
11451153
--no-oshmem | -no-oshmem Do not build the OSHMEM layer
1154+
--no-3rdparty <package> Do not build the listed 3rd-party package (comma separtated list)
11461155
--quiet | -q Do not display normal verbose output
11471156
--debug | -d Output lots of debug information
11481157
--help | -h This help list
@@ -1170,10 +1179,6 @@ sub in_tarball {
11701179
$no_ompi_arg = 1;
11711180
debug "No ompi subdirectory found - will not build MPI layer\n";
11721181
}
1173-
if (! -e "prrte") {
1174-
$no_prrte_arg = 1;
1175-
debug "No prrte subdirectory found - will not build PRRTE\n";
1176-
}
11771182
if (! -e "oshmem") {
11781183
$no_oshmem_arg = 1;
11791184
debug "No oshmem subdirectory found - will not build OSHMEM\n";
@@ -1406,9 +1411,6 @@ sub in_tarball {
14061411
foreach my $p (@$projects) {
14071412
$m4 .= "m4_define([project_$p->{name}], [1])\n";
14081413
}
1409-
if (!$no_prrte_arg) {
1410-
$m4 .= "m4_define([project_prrte], [1])\n";
1411-
}
14121414

14131415
$m4 .= "\ndnl Project names
14141416
m4_define([project_name_long], [$project_name_long])
@@ -1419,6 +1421,85 @@ sub in_tarball {
14191421

14201422
#---------------------------------------------------------------------------
14211423

1424+
# Handle 3rd-party packages
1425+
++$step;
1426+
verbose "\n$step. Setup for 3rd-party packages\n";
1427+
1428+
my @enabled_3rdparty_packages = ();
1429+
my @disabled_3rdparty_packages = split(/,/, $no_3rdparty_arg);
1430+
if ($no_prrte_arg) {
1431+
push(@disabled_3rdparty_packages, "prrte");
1432+
}
1433+
1434+
$m4 .= "\n$dnl_line
1435+
$dnl_line
1436+
$dnl_line
1437+
1438+
dnl 3rd-party package information\n";
1439+
1440+
# these are fairly one-off, so we did not try to do anything
1441+
# generic. Sorry :).
1442+
1443+
verbose "=== Libevent\n";
1444+
if ("libevent" ~~ @disabled_3rdparty_packages) {
1445+
verbose "--- Libevent disabled\n";
1446+
} else {
1447+
my $libevent_directory = "libevent-" . $libevent_version;
1448+
my $libevent_tarball = $libevent_directory . ".tar.gz";
1449+
if (! -f "3rd-party/" . $libevent_tarball) {
1450+
my_die("Could not find libevent tarball\n");
1451+
}
1452+
$m4 .= "m4_define([package_libevent], [1])\n";
1453+
$m4 .= "m4_define([libevent_tarball], [" . $libevent_tarball . "])\n";
1454+
$m4 .= "m4_define([libevent_directory], [" . $libevent_directory . "])\n";
1455+
verbose "--- Libevent enabled (" . $libevent_version . ")\n";
1456+
}
1457+
1458+
verbose "=== hwloc\n";
1459+
if ("hwloc" ~~ @disabled_3rdparty_packages) {
1460+
verbose "--- hwloc disabled\n";
1461+
} else {
1462+
my $hwloc_directory = "hwloc-" . $hwloc_version;
1463+
my $hwloc_tarball = $hwloc_directory . ".tar.gz";
1464+
if (! -f "3rd-party/" . $hwloc_tarball) {
1465+
my_die("Could not find hwloc tarball\n");
1466+
}
1467+
$m4 .= "m4_define([package_hwloc], [1])\n";
1468+
$m4 .= "m4_define([hwloc_tarball], [" . $hwloc_tarball . "])\n";
1469+
$m4 .= "m4_define([hwloc_directory], [" . $hwloc_directory . "])\n";
1470+
verbose "--- hwloc enabled\n";
1471+
}
1472+
1473+
verbose "=== PMIx\n";
1474+
if ("pmix" ~~ @disabled_3rdparty_packages) {
1475+
verbose "--- PMIx disabled\n";
1476+
} else {
1477+
# sanity check pmix files exist
1478+
if (! -f "3rd-party/openpmix/configure.ac") {
1479+
my_die("Could not find pmix files\n");
1480+
}
1481+
push(@subdirs, "3rd-party/openpmix/");
1482+
$m4 .= "m4_define([package_pmix], [1])\n";
1483+
verbose "--- PMIx enabled\n";
1484+
}
1485+
1486+
verbose "=== PRRTE\n";
1487+
if ("prrte" ~~ @disabled_3rdparty_packages) {
1488+
verbose "--- PRRTE disabled\n";
1489+
} else {
1490+
# sanity check prrte files exist
1491+
if (! -f "3rd-party/prrte/configure.ac") {
1492+
my_die("Could not find pmix files\n");
1493+
}
1494+
push(@subdirs, "3rd-party/prrte/");
1495+
$m4 .= "m4_define([package_prrte], [1])\n";
1496+
verbose "--- PRRTE enabled\n";
1497+
}
1498+
1499+
$m4 .= "\n";
1500+
1501+
#---------------------------------------------------------------------------
1502+
14221503
# Find MPI extensions and contribs
14231504
if (!$no_ompi_arg) {
14241505
++$step;
@@ -1436,10 +1517,6 @@ sub in_tarball {
14361517
++$step;
14371518
verbose "\n$step. Processing autogen.subdirs directories\n";
14381519

1439-
if (!$no_prrte_arg) {
1440-
process_autogen_subdirs(".");
1441-
}
1442-
14431520
if ($#subdirs >= 0) {
14441521
foreach my $d (@subdirs) {
14451522
process_subdir($d);

0 commit comments

Comments
 (0)