Skip to content

Commit 1217a95

Browse files
author
Luke Robison
committed
coll/tuned: Refactor logging, and add coll_tuned_verbose mca parameter
Previously the coll_tuned output stream didn't use opal_output_set_verbosity at all. This made it difficult to distinguish problems from noisy logging. Signed-off-by: Luke Robison <lrbison@amazon.com>
1 parent e973840 commit 1217a95

21 files changed

+226
-135
lines changed

‎ompi/mca/coll/tuned/coll_tuned.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@
3232

3333
BEGIN_C_DECLS
3434

35+
#define COLL_TUNED_TRACING_VERBOSE 50
36+
3537
/* these are the same across all modules and are loaded at component query time */
3638
extern int ompi_coll_tuned_stream;
3739
extern int ompi_coll_tuned_priority;

‎ompi/mca/coll/tuned/coll_tuned_allgather_decision.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ int ompi_coll_tuned_allgather_intra_do_this(const void *sbuf, size_t scount,
135135
mca_coll_base_module_t *module,
136136
int algorithm, int faninout, int segsize)
137137
{
138-
OPAL_OUTPUT((ompi_coll_tuned_stream,
138+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
139139
"coll:tuned:allgather_intra_do_this selected algorithm %d topo faninout %d segsize %d",
140140
algorithm, faninout, segsize));
141141
switch (algorithm) {
@@ -176,7 +176,7 @@ int ompi_coll_tuned_allgather_intra_do_this(const void *sbuf, size_t scount,
176176
rbuf, rcount, rdtype,
177177
comm, module);
178178
} /* switch */
179-
OPAL_OUTPUT((ompi_coll_tuned_stream,
179+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
180180
"coll:tuned:allgather_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
181181
algorithm, ompi_coll_tuned_forced_max_algorithms[ALLGATHER]));
182182
return (MPI_ERR_ARG);

‎ompi/mca/coll/tuned/coll_tuned_allgatherv_decision.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ int ompi_coll_tuned_allgatherv_intra_do_this(const void *sbuf, size_t scount,
135135
int algorithm, int faninout,
136136
int segsize)
137137
{
138-
OPAL_OUTPUT((ompi_coll_tuned_stream,
138+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
139139
"coll:tuned:allgatherv_intra_do_this selected algorithm %d topo faninout %d segsize %d",
140140
algorithm, faninout, segsize));
141141

@@ -169,7 +169,7 @@ int ompi_coll_tuned_allgatherv_intra_do_this(const void *sbuf, size_t scount,
169169
rbuf, rcounts, rdispls, rdtype,
170170
comm, module);
171171
} /* switch */
172-
OPAL_OUTPUT((ompi_coll_tuned_stream,
172+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
173173
"coll:tuned:allgatherv_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
174174
algorithm, ompi_coll_tuned_forced_max_algorithms[ALLGATHERV]));
175175
return (MPI_ERR_ARG);

‎ompi/mca/coll/tuned/coll_tuned_allreduce_decision.c

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -132,8 +132,9 @@ int ompi_coll_tuned_allreduce_intra_do_this(const void *sbuf, void *rbuf, size_t
132132
mca_coll_base_module_t *module,
133133
int algorithm, int faninout, int segsize)
134134
{
135-
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:allreduce_intra_do_this algorithm %d topo fan in/out %d segsize %d",
136-
algorithm, faninout, segsize));
135+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
136+
"coll:tuned:allreduce_intra_do_this algorithm %d topo fan in/out %d segsize %d",
137+
algorithm, faninout, segsize));
137138

138139
switch (algorithm) {
139140
case (0):
@@ -153,7 +154,8 @@ int ompi_coll_tuned_allreduce_intra_do_this(const void *sbuf, void *rbuf, size_t
153154
case (7):
154155
return ompi_coll_base_allreduce_intra_allgather_reduce(sbuf, rbuf, count, dtype, op, comm, module);
155156
} /* switch */
156-
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:allreduce_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
157-
algorithm, ompi_coll_tuned_forced_max_algorithms[ALLREDUCE]));
157+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
158+
"coll:tuned:allreduce_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
159+
algorithm, ompi_coll_tuned_forced_max_algorithms[ALLREDUCE]));
158160
return (MPI_ERR_ARG);
159161
}

‎ompi/mca/coll/tuned/coll_tuned_alltoall_decision.c

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -167,8 +167,9 @@ int ompi_coll_tuned_alltoall_intra_do_this(const void *sbuf, size_t scount,
167167
int algorithm, int faninout, int segsize,
168168
int max_requests)
169169
{
170-
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:alltoall_intra_do_this selected algorithm %d topo faninout %d segsize %d",
171-
algorithm, faninout, segsize));
170+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
171+
"coll:tuned:alltoall_intra_do_this selected algorithm %d topo faninout %d segsize %d",
172+
algorithm, faninout, segsize));
172173

173174
switch (algorithm) {
174175
case (0):
@@ -184,7 +185,8 @@ int ompi_coll_tuned_alltoall_intra_do_this(const void *sbuf, size_t scount,
184185
case (5):
185186
return ompi_coll_base_alltoall_intra_two_procs(sbuf, scount, sdtype, rbuf, rcount, rdtype, comm, module);
186187
} /* switch */
187-
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:alltoall_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
188-
algorithm, ompi_coll_tuned_forced_max_algorithms[ALLTOALL]));
188+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
189+
"coll:tuned:alltoall_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
190+
algorithm, ompi_coll_tuned_forced_max_algorithms[ALLTOALL]));
189191
return (MPI_ERR_ARG);
190192
}

‎ompi/mca/coll/tuned/coll_tuned_alltoallv_decision.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ int ompi_coll_tuned_alltoallv_intra_do_this(const void *sbuf, ompi_count_array_t
9898
mca_coll_base_module_t *module,
9999
int algorithm)
100100
{
101-
OPAL_OUTPUT((ompi_coll_tuned_stream,
101+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
102102
"coll:tuned:alltoallv_intra_do_this selected algorithm %d ",
103103
algorithm));
104104

@@ -116,7 +116,7 @@ int ompi_coll_tuned_alltoallv_intra_do_this(const void *sbuf, ompi_count_array_t
116116
rbuf, rcounts, rdisps, rdtype,
117117
comm, module);
118118
} /* switch */
119-
OPAL_OUTPUT((ompi_coll_tuned_stream,
119+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
120120
"coll:tuned:alltoall_intra_do_this attempt to select "
121121
"algorithm %d when only 0-%d is valid.",
122122
algorithm, ompi_coll_tuned_forced_max_algorithms[ALLTOALLV]));

‎ompi/mca/coll/tuned/coll_tuned_barrier_decision.c

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ int ompi_coll_tuned_barrier_intra_do_this (struct ompi_communicator_t *comm,
9393
mca_coll_base_module_t *module,
9494
int algorithm, int faninout, int segsize)
9595
{
96-
OPAL_OUTPUT((ompi_coll_tuned_stream,
96+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
9797
"coll:tuned:barrier_intra_do_this selected algorithm %d topo fanin/out%d",
9898
algorithm, faninout));
9999

@@ -106,7 +106,8 @@ int ompi_coll_tuned_barrier_intra_do_this (struct ompi_communicator_t *comm,
106106
case (5): return ompi_coll_base_barrier_intra_two_procs(comm, module);
107107
case (6): return ompi_coll_base_barrier_intra_tree(comm, module);
108108
} /* switch */
109-
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:barrier_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
110-
algorithm, ompi_coll_tuned_forced_max_algorithms[BARRIER]));
109+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
110+
"coll:tuned:barrier_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
111+
algorithm, ompi_coll_tuned_forced_max_algorithms[BARRIER]));
111112
return (MPI_ERR_ARG);
112113
}

‎ompi/mca/coll/tuned/coll_tuned_bcast_decision.c

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -142,8 +142,9 @@ int ompi_coll_tuned_bcast_intra_do_this(void *buf, size_t count,
142142
mca_coll_base_module_t *module,
143143
int algorithm, int faninout, int segsize)
144144
{
145-
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:bcast_intra_do_this algorithm %d topo faninout %d segsize %d",
146-
algorithm, faninout, segsize));
145+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
146+
"coll:tuned:bcast_intra_do_this algorithm %d topo faninout %d segsize %d",
147+
algorithm, faninout, segsize));
147148

148149
switch (algorithm) {
149150
case (0):
@@ -168,7 +169,8 @@ int ompi_coll_tuned_bcast_intra_do_this(void *buf, size_t count,
168169
case (9):
169170
return ompi_coll_base_bcast_intra_scatter_allgather_ring(buf, count, dtype, root, comm, module, segsize);
170171
} /* switch */
171-
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:bcast_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
172-
algorithm, ompi_coll_tuned_forced_max_algorithms[BCAST]));
172+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
173+
"coll:tuned:bcast_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
174+
algorithm, ompi_coll_tuned_forced_max_algorithms[BCAST]));
173175
return (MPI_ERR_ARG);
174176
}

‎ompi/mca/coll/tuned/coll_tuned_component.c

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ char* ompi_coll_tuned_dynamic_rules_filename = (char*) NULL;
5858
int ompi_coll_tuned_init_tree_fanout = 4;
5959
int ompi_coll_tuned_init_chain_fanout = 4;
6060
int ompi_coll_tuned_init_max_requests = 128;
61+
int ompi_coll_tuned_verbose = 0;
6162

6263
/* Set it to the same value as intermediate msg by default, so it does not affect
6364
* default algorithm selection. Changing this value will force using linear with
@@ -193,6 +194,15 @@ static int tuned_register(void)
193194
MCA_BASE_VAR_SCOPE_ALL,
194195
&ompi_coll_tuned_dynamic_rules_filename);
195196

197+
ompi_coll_tuned_verbose = 0;
198+
(void) mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
199+
"verbose",
200+
"Verbosity of the tuned coll component",
201+
MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
202+
OPAL_INFO_LVL_9,
203+
MCA_BASE_VAR_SCOPE_ALL,
204+
&ompi_coll_tuned_verbose);
205+
196206
/* register forced params */
197207
ompi_coll_tuned_allreduce_intra_check_forced_init(&ompi_coll_tuned_forced_params[ALLREDUCE]);
198208
ompi_coll_tuned_alltoall_intra_check_forced_init(&ompi_coll_tuned_forced_params[ALLTOALL]);
@@ -216,11 +226,10 @@ static int tuned_open(void)
216226
{
217227
int rc;
218228

219-
#if OPAL_ENABLE_DEBUG
220-
if (ompi_coll_base_framework.framework_verbose) {
229+
if (ompi_coll_tuned_verbose) {
221230
ompi_coll_tuned_stream = opal_output_open(NULL);
231+
opal_output_set_verbosity(ompi_coll_tuned_stream, ompi_coll_tuned_verbose);
222232
}
223-
#endif /* OPAL_ENABLE_DEBUG */
224233

225234
/* now check that the user hasn't overrode any of the decision functions if dynamic rules are enabled */
226235
/* the user can redo this before every comm dup/create if they like */
@@ -233,20 +242,24 @@ static int tuned_open(void)
233242
/* by default DISABLE dynamic rules and instead use fixed [if based] rules */
234243
if (ompi_coll_tuned_use_dynamic_rules) {
235244
if( ompi_coll_tuned_dynamic_rules_filename ) {
236-
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:component_open Reading collective rules file [%s]",
237-
ompi_coll_tuned_dynamic_rules_filename));
245+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
246+
"coll:tuned:component_open Reading collective rules file [%s]",
247+
ompi_coll_tuned_dynamic_rules_filename));
238248
rc = ompi_coll_tuned_read_rules_config_file( ompi_coll_tuned_dynamic_rules_filename,
239249
&(mca_coll_tuned_component.all_base_rules));
240250
if( rc == OPAL_SUCCESS ) {
241-
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:module_open Read a valid rules file"));
251+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
252+
"coll:tuned:module_open Read a valid rules file"));
242253
} else {
243-
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:module_open Reading collective rules file failed\n"));
254+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
255+
"coll:tuned:module_open Reading collective rules file failed\n"));
244256
mca_coll_tuned_component.all_base_rules = NULL;
245257
}
246258
}
247259
}
248260

249-
OPAL_OUTPUT((ompi_coll_tuned_stream, "coll:tuned:component_open: done!"));
261+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
262+
"coll:tuned:component_open: done!"));
250263

251264
return OMPI_SUCCESS;
252265
}
@@ -255,12 +268,14 @@ static int tuned_open(void)
255268
/* i.e. alg table and dynamic changeable rules if allocated etc */
256269
static int tuned_close(void)
257270
{
258-
OPAL_OUTPUT((ompi_coll_tuned_stream, "coll:tuned:component_close: called"));
271+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
272+
"coll:tuned:component_close: called"));
259273

260274
/* dealloc alg table if allocated */
261275
/* dealloc dynamic changeable rules if allocated */
262276

263-
OPAL_OUTPUT((ompi_coll_tuned_stream, "coll:tuned:component_close: done!"));
277+
OPAL_OUTPUT_VERBOSE((COLL_TUNED_TRACING_VERBOSE, ompi_coll_tuned_stream,
278+
"coll:tuned:component_close: done!"));
264279

265280
if( NULL != mca_coll_tuned_component.all_base_rules ) {
266281
ompi_coll_tuned_free_all_rules(mca_coll_tuned_component.all_base_rules);

0 commit comments

Comments
 (0)