Skip to content

Commit f6c2b34

Browse files
hjelmnawlauria
hjelmn
authored and
awlauria
committed
smsc/cma: add support for Linux CMA single-copy
This commit adds a component to the shared-memory single-copy framework to support Linux Cross Memory Attach (CMA). This component supports copy_to and copy_from without registration. Signed-off-by: Nathan Hjelm <hjelmn@google.com> (cherry picked from commit d74e95c)
1 parent 7ee20e3 commit f6c2b34

File tree

7 files changed

+518
-0
lines changed

7 files changed

+518
-0
lines changed

‎opal/mca/smsc/cma/Makefile.am

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#
2+
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3+
# University Research and Technology
4+
# Corporation. All rights reserved.
5+
# Copyright (c) 2004-2009 The University of Tennessee and The University
6+
# of Tennessee Research Foundation. All rights
7+
# reserved.
8+
# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
9+
# University of Stuttgart. All rights reserved.
10+
# Copyright (c) 2004-2005 The Regents of the University of California.
11+
# All rights reserved.
12+
# Copyright (c) 2009-2014 Cisco Systems, Inc. All rights reserved.
13+
# Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
14+
# reserved.
15+
# Copyright (c) 2017 IBM Corporation. All rights reserved.
16+
# Copyright (c) 2020-2021 Google, LLC. All rights reserved.
17+
# $COPYRIGHT$
18+
#
19+
# Additional copyrights may follow
20+
#
21+
# $HEADER$
22+
#
23+
24+
EXTRA_DIST = post_configure.sh
25+
26+
AM_CPPFLAGS = $(smsc_cma_CPPFLAGS)
27+
28+
libmca_smsc_cma_la_sources = \
29+
smsc_cma_component.c \
30+
smsc_cma_module.c \
31+
smsc_cma_internal.h \
32+
smsc_cma.h
33+
34+
# Make the output library in this directory, and name it either
35+
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
36+
# (for static builds).
37+
38+
if MCA_BUILD_opal_smsc_cma_DSO
39+
component_noinst =
40+
component_install = mca_smsc_cma.la
41+
else
42+
component_noinst = libmca_smsc_cma.la
43+
component_install =
44+
endif
45+
46+
mcacomponentdir = $(opallibdir)
47+
mcacomponent_LTLIBRARIES = $(component_install)
48+
mca_smsc_cma_la_SOURCES = $(libmca_smsc_cma_la_sources)
49+
mca_smsc_cma_la_LDFLAGS = -module -avoid-version $(smsc_cma_LDFLAGS)
50+
mca_smsc_cma_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
51+
$(smsc_cma_LIBS)
52+
53+
noinst_LTLIBRARIES = $(component_noinst)
54+
libmca_smsc_cma_la_SOURCES = $(libmca_smsc_cma_la_sources)
55+
libmca_smsc_cma_la_LIBADD = $(smsc_cma_LIBS)
56+
libmca_smsc_cma_la_LDFLAGS = -module -avoid-version $(smsc_cma_LDFLAGS)

‎opal/mca/smsc/cma/configure.m4

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# -*- shell-script -*-
2+
#
3+
# Copyright (c) 2009 The University of Tennessee and The University
4+
# of Tennessee Research Foundation. All rights
5+
# reserved.
6+
# Copyright (c) 2009-2016 Cisco Systems, Inc. All rights reserved.
7+
# Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
8+
# reserved.
9+
# Copyright (c) 2015 Research Organization for Information Science
10+
# and Technology (RIST). All rights reserved.
11+
# Copyright (c) 2021 Google, LLC. All rights reserved.
12+
# $COPYRIGHT$
13+
#
14+
# Additional copyrights may follow
15+
#
16+
# $HEADER$
17+
#
18+
19+
# MCA_smsc_cma_CONFIG([action-if-can-compile],
20+
# [action-if-cant-compile])
21+
# ------------------------------------------------
22+
AC_DEFUN([MCA_opal_smsc_cma_CONFIG],[
23+
AC_CONFIG_FILES([opal/mca/smsc/cma/Makefile])
24+
25+
OPAL_CHECK_CMA([smsc_cma], [AC_CHECK_HEADER([sys/prctl.h]) $1], [$2])
26+
27+
AC_SUBST([smsc_cma_CFLAGS])
28+
AC_SUBST([smsc_cma_CPPFLAGS])
29+
AC_SUBST([smsc_cma_LDFLAGS])
30+
AC_SUBST([smsc_cma_LIBS])
31+
])dnl

‎opal/mca/smsc/cma/post_configure.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
DIRECT_CALL_HEADER="opal/mca/smsc/cma/smsc_cma.h"

‎opal/mca/smsc/cma/smsc_cma.h

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2+
/*
3+
* Copyright (c) 2021 Google, Inc. All rights reserved.
4+
* $COPYRIGHT$
5+
*
6+
* Additional copyrights may follow
7+
*
8+
* $HEADER$
9+
*/
10+
11+
#ifndef OPAL_MCA_SMSC_CMA_SMSC_CMA_H
12+
#define OPAL_MCA_SMSC_CMA_SMSC_CMA_H
13+
14+
#include "opal_config.h"
15+
16+
#include "opal/mca/smsc/smsc.h"
17+
18+
mca_smsc_endpoint_t *mca_smsc_cma_get_endpoint(opal_proc_t *peer_proc);
19+
void mca_smsc_cma_return_endpoint(mca_smsc_endpoint_t *endpoint);
20+
21+
int mca_smsc_cma_copy_to(mca_smsc_endpoint_t *endpoint, void *local_address, void *remote_address,
22+
size_t size, void *reg_handle);
23+
int mca_smsc_cma_copy_from(mca_smsc_endpoint_t *endpoint, void *local_address, void *remote_address,
24+
size_t size, void *reg_handle);
25+
26+
/* unsupported interfaces defined to support MCA direct */
27+
void *mca_smsc_cma_map_peer_region(mca_smsc_endpoint_t *endpoint, uint64_t flags,
28+
void *remote_address, size_t size, void **local_mapping);
29+
void mca_smsc_cma_unmap_peer_region(void *ctx);
30+
void *mca_smsc_cma_register_region(void *local_address, size_t size);
31+
void mca_smsc_cma_deregister_region(void *reg_data);
32+
33+
#endif /* OPAL_MCA_SMSC_CMA_SMSC_CMA_H */

‎opal/mca/smsc/cma/smsc_cma_component.c

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2+
/*
3+
* Copyright (c) 2021 Google, Inc. All rights reserved.
4+
* $COPYRIGHT$
5+
*
6+
* Additional copyrights may follow
7+
*
8+
* $HEADER$
9+
*/
10+
#include "opal_config.h"
11+
12+
#include "opal/mca/smsc/base/base.h"
13+
#include "opal/mca/smsc/cma/smsc_cma_internal.h"
14+
15+
#include <fcntl.h>
16+
#include <stdio.h>
17+
#include <sys/prctl.h>
18+
#include <sys/stat.h>
19+
#include <sys/types.h>
20+
#include <unistd.h>
21+
22+
static int mca_smsc_cma_component_register(void);
23+
static int mca_smsc_cma_component_open(void);
24+
static int mca_smsc_cma_component_close(void);
25+
static int mca_smsc_cma_component_query(void);
26+
static mca_smsc_module_t *mca_smsc_cma_component_enable(void);
27+
28+
#define MCA_SMSC_CMA_DEFAULT_PRIORITY 37
29+
static const int mca_smsc_cma_default_priority = MCA_SMSC_CMA_DEFAULT_PRIORITY;
30+
31+
mca_smsc_component_t mca_smsc_cma_component = {
32+
.smsc_version = {
33+
MCA_SMSC_DEFAULT_VERSION("cma"),
34+
.mca_open_component = mca_smsc_cma_component_open,
35+
.mca_close_component = mca_smsc_cma_component_close,
36+
.mca_register_component_params = mca_smsc_cma_component_register,
37+
},
38+
.priority = MCA_SMSC_CMA_DEFAULT_PRIORITY,
39+
.query = mca_smsc_cma_component_query,
40+
.enable = mca_smsc_cma_component_enable,
41+
};
42+
43+
static int mca_smsc_cma_component_register(void)
44+
{
45+
mca_smsc_base_register_default_params(&mca_smsc_cma_component, mca_smsc_cma_default_priority);
46+
return OPAL_SUCCESS;
47+
}
48+
49+
static int mca_smsc_cma_component_open(void)
50+
{
51+
/* nothing to do */
52+
return OPAL_SUCCESS;
53+
}
54+
55+
static int mca_smsc_cma_component_close(void)
56+
{
57+
/* nothing to do */
58+
return OPAL_SUCCESS;
59+
}
60+
61+
/*
62+
* mca_btl_sm_parse_proc_ns_user() tries to get the user namespace ID
63+
* of the current process.
64+
* Returns the ID of the user namespace. In the case of an error '0' is returned.
65+
*/
66+
ino_t mca_smsc_cma_get_user_ns_id(void)
67+
{
68+
struct stat buf;
69+
70+
if (0 > stat("/proc/self/ns/user", &buf)) {
71+
/*
72+
* Something went wrong, probably an old kernel that does not support namespaces
73+
* simply assume all processes are in the same user namespace and return 0
74+
*/
75+
return 0;
76+
}
77+
78+
return buf.st_ino;
79+
}
80+
81+
static int mca_smsc_cma_send_modex(void)
82+
{
83+
mca_smsc_cma_modex_t modex;
84+
85+
modex.pid = getpid();
86+
modex.user_ns_id = mca_smsc_cma_get_user_ns_id();
87+
88+
int rc;
89+
OPAL_MODEX_SEND(rc, PMIX_LOCAL, &mca_smsc_cma_component.smsc_version, &modex, sizeof(modex));
90+
return rc;
91+
}
92+
93+
static int mca_smsc_cma_component_query(void)
94+
{
95+
/* Check if we have the proper permissions for CMA */
96+
char buffer = '0';
97+
bool cma_happy = false;
98+
99+
/* check system setting for current ptrace scope */
100+
int fd = open("/proc/sys/kernel/yama/ptrace_scope", O_RDONLY);
101+
if (0 <= fd) {
102+
int ret = read(fd, &buffer, 1);
103+
if (ret < 0) {
104+
opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT,
105+
opal_smsc_base_framework.framework_output,
106+
"mca_smsc_cma_component_query: could not read ptrace_scope. "
107+
"assuming ptrace scope is 0");
108+
}
109+
close(fd);
110+
}
111+
112+
/* ptrace scope 0 will allow an attach from any of the process owner's
113+
* processes. ptrace scope 1 limits attachers to the process tree
114+
* starting at the parent of this process. */
115+
if ('0' != buffer) {
116+
#if defined PR_SET_PTRACER
117+
/* try setting the ptrace scope to allow attach */
118+
int ret = prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0);
119+
if (0 == ret) {
120+
cma_happy = true;
121+
}
122+
#endif
123+
} else {
124+
cma_happy = true;
125+
}
126+
127+
if (!cma_happy) {
128+
opal_output_verbose(MCA_BASE_VERBOSE_COMPONENT, opal_smsc_base_framework.framework_output,
129+
"mca_smsc_cma_component_query: could not select for use. insufficient "
130+
"ptrace permissions.");
131+
mca_smsc_cma_component.priority = -1;
132+
return OPAL_ERR_NOT_AVAILABLE;
133+
}
134+
135+
mca_smsc_cma_send_modex();
136+
137+
return OPAL_SUCCESS;
138+
}
139+
140+
static mca_smsc_module_t *mca_smsc_cma_component_enable(void)
141+
{
142+
if (0 > mca_smsc_cma_component.priority) {
143+
return NULL;
144+
}
145+
146+
return &mca_smsc_cma_module;
147+
}

‎opal/mca/smsc/cma/smsc_cma_internal.h

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2+
/*
3+
* Copyright (c) 2021 Google, Inc. All rights reserved.
4+
* $COPYRIGHT$
5+
*
6+
* Additional copyrights may follow
7+
*
8+
* $HEADER$
9+
*/
10+
11+
#ifndef OPAL_MCA_SMSC_CMA_SMSC_CMA_INTERNAL_H
12+
#define OPAL_MCA_SMSC_CMA_SMSC_CMA_INTERNAL_H
13+
14+
#include "opal/mca/smsc/cma/smsc_cma.h"
15+
16+
struct mca_smsc_cma_modex_t {
17+
pid_t pid;
18+
ino_t user_ns_id;
19+
};
20+
21+
typedef struct mca_smsc_cma_modex_t mca_smsc_cma_modex_t;
22+
23+
struct mca_smsc_cma_endpoint_t {
24+
mca_smsc_endpoint_t super;
25+
pid_t pid;
26+
};
27+
28+
typedef struct mca_smsc_cma_endpoint_t mca_smsc_cma_endpoint_t;
29+
30+
OBJ_CLASS_DECLARATION(mca_smsc_cma_endpoint_t);
31+
32+
extern mca_smsc_module_t mca_smsc_cma_module;
33+
extern mca_smsc_component_t mca_smsc_cma_component;
34+
35+
ino_t mca_smsc_cma_get_user_ns_id(void);
36+
37+
#endif /* OPAL_MCA_SMSC_CMA_SMSC_CMA_INTERNAL_H */

0 commit comments

Comments
 (0)