Skip to content

Commit a0117ab

Browse files
authored
[SYCL] Add internal debugging environment variables to append to compile/link options (#11981)
Add two new environment variables to append to compile or link options. This is only intended for SYCL developers, and end users should use the '-X' family of dpcpp options. We are adding this because we have had some people hit unexpected issues because if `SYCL_PROGRAM_COMPILE_OPTIONS` is set, any options that would normally be added internally by the runtime and invisibly to the user (`ze-opt-disable`, `ze-opt-level`, etc) are not added, so someone doing A/B testing would not be comparing apples to apples with the envvar set vs not set. --------- Signed-off-by: Sarnie, Nick <nick.sarnie@intel.com>
1 parent 00a7e36 commit a0117ab

File tree

5 files changed

+46
-9
lines changed

5 files changed

+46
-9
lines changed

sycl/doc/EnvironmentVariables.md

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -227,8 +227,10 @@ variables in production code.</span>
227227
| `SYCL_DISABLE_EXECUTION_GRAPH_CLEANUP` | Any(\*) | Disable regular cleanup of enqueued (or finished, in case of host tasks) non-leaf command nodes. If disabled, command nodes will be cleaned up only during the destruction of the last remaining memory object used by them. |
228228
| `SYCL_DISABLE_POST_ENQUEUE_CLEANUP` (deprecated) | Any(\*) | Use `SYCL_DISABLE_EXECUTION_GRAPH_CLEANUP` instead. |
229229
| `SYCL_DEVICELIB_INHIBIT_NATIVE` | String of device library extensions (separated by a whitespace) | Do not rely on device native support for devicelib extensions listed in this option. |
230-
| `SYCL_PROGRAM_COMPILE_OPTIONS` | String of valid OpenCL compile options | Override compile options for all programs. |
231-
| `SYCL_PROGRAM_LINK_OPTIONS` | String of valid OpenCL link options | Override link options for all programs. |
230+
| `SYCL_PROGRAM_COMPILE_OPTIONS` | String of valid compile options | Override compile options for all programs. |
231+
| `SYCL_PROGRAM_LINK_OPTIONS` | String of valid link options | Override link options for all programs. |
232+
| `SYCL_PROGRAM_APPEND_COMPILE_OPTIONS` | String of valid compile options | Append to the end of compile options for all programs. |
233+
| `SYCL_PROGRAM_APPEND_LINK_OPTIONS` | String of valid link options | Append to the end of link options for all programs. |
232234
| `SYCL_USE_KERNEL_SPV` | Path to the SPIR-V binary | Load device image from the specified file. If runtime is unable to read the file, `sycl::runtime_error` exception is thrown. The image is assumed to have been created using the `-fno-sycl-dead-args-optimization` option. |
233235
| `SYCL_DUMP_IMAGES` | Any(\*) | Dump device image binaries to file. Control has no effect if `SYCL_USE_KERNEL_SPV` is set. |
234236
| `SYCL_HOST_UNIFIED_MEMORY` | Integer | Enforce host unified memory support or lack of it for the execution graph builder. If set to 0, it is enforced as not supported by all devices. If set to 1, it is enforced as supported by all devices. |

sycl/source/detail/config.def

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ CONFIG(SYCL_DEVICELIB_NO_FALLBACK, 1, __SYCL_DEVICELIB_NO_FALLBACK)
2222
CONFIG(SYCL_DEVICE_FILTER, 1024, __SYCL_DEVICE_FILTER)
2323
CONFIG(SYCL_PROGRAM_LINK_OPTIONS, 64, __SYCL_PROGRAM_LINK_OPTIONS)
2424
CONFIG(SYCL_PROGRAM_COMPILE_OPTIONS, 64, __SYCL_PROGRAM_COMPILE_OPTIONS)
25+
CONFIG(SYCL_PROGRAM_APPEND_LINK_OPTIONS, 64, __SYCL_PROGRAM_APPEND_LINK_OPTIONS)
26+
CONFIG(SYCL_PROGRAM_APPEND_COMPILE_OPTIONS, 64, __SYCL_PROGRAM_APPEND_COMPILE_OPTIONS)
2527
CONFIG(SYCL_HOST_UNIFIED_MEMORY, 1, __SYCL_HOST_UNIFIED_MEMORY)
2628
// 260 (Windows limit) - 12 (filename) - 84 (cache directory structure)
2729
CONFIG(SYCL_CACHE_DIR, 164, __SYCL_CACHE_DIR)

sycl/source/detail/program_manager/program_manager.cpp

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -530,6 +530,26 @@ static void appendCompileOptionsFromImage(std::string &CompileOpts,
530530
}
531531
}
532532

533+
static void
534+
appendCompileEnvironmentVariablesThatAppend(std::string &CompileOpts) {
535+
static const char *AppendCompileOptsEnv =
536+
SYCLConfig<SYCL_PROGRAM_APPEND_COMPILE_OPTIONS>::get();
537+
if (AppendCompileOptsEnv) {
538+
if (!CompileOpts.empty())
539+
CompileOpts += " ";
540+
CompileOpts += AppendCompileOptsEnv;
541+
}
542+
}
543+
static void appendLinkEnvironmentVariablesThatAppend(std::string &LinkOpts) {
544+
static const char *AppendLinkOptsEnv =
545+
SYCLConfig<SYCL_PROGRAM_APPEND_LINK_OPTIONS>::get();
546+
if (AppendLinkOptsEnv) {
547+
if (!LinkOpts.empty())
548+
LinkOpts += " ";
549+
LinkOpts += AppendLinkOptsEnv;
550+
}
551+
}
552+
533553
static void applyOptionsFromImage(std::string &CompileOpts,
534554
std::string &LinkOpts,
535555
const RTDeviceBinaryImage &Img,
@@ -646,7 +666,9 @@ sycl::detail::pi::PiProgram ProgramManager::getBuiltPIProgram(
646666
&LinkOpts, SpecConsts] {
647667
const PluginPtr &Plugin = ContextImpl->getPlugin();
648668
applyOptionsFromImage(CompileOpts, LinkOpts, Img, {Device}, Plugin);
649-
669+
// Should always come last!
670+
appendCompileEnvironmentVariablesThatAppend(CompileOpts);
671+
appendLinkEnvironmentVariablesThatAppend(LinkOpts);
650672
auto [NativePrg, DeviceCodeWasInCache] = getOrCreatePIProgram(
651673
Img, Context, Device, CompileOpts + LinkOpts, SpecConsts);
652674

@@ -734,6 +756,9 @@ ProgramManager::getOrCreateKernel(const ContextImplPtr &ContextImpl,
734756
std::string CompileOpts, LinkOpts;
735757
SerializedObj SpecConsts;
736758
applyOptionsFromEnvironment(CompileOpts, LinkOpts);
759+
// Should always come last!
760+
appendCompileEnvironmentVariablesThatAppend(CompileOpts);
761+
appendLinkEnvironmentVariablesThatAppend(LinkOpts);
737762
const sycl::detail::pi::PiDevice PiDevice = DeviceImpl->getHandleRef();
738763

739764
auto key = std::make_tuple(std::move(SpecConsts), PiDevice,
@@ -2131,6 +2156,8 @@ ProgramManager::compile(const device_image_plain &DeviceImage,
21312156
applyCompileOptionsFromEnvironment(CompileOptions);
21322157
appendCompileOptionsFromImage(
21332158
CompileOptions, *(InputImpl->get_bin_image_ref()), Devs, Plugin);
2159+
// Should always come last!
2160+
appendCompileEnvironmentVariablesThatAppend(CompileOptions);
21342161
sycl::detail::pi::PiResult Error =
21352162
Plugin->call_nocheck<PiApiKind::piProgramCompile>(
21362163
ObjectImpl->get_program_ref(), /*num devices=*/Devs.size(),
@@ -2169,6 +2196,8 @@ ProgramManager::link(const device_image_plain &DeviceImage,
21692196
appendLinkOptionsFromImage(LinkOptionsStr,
21702197
*(InputImpl->get_bin_image_ref()));
21712198
}
2199+
// Should always come last!
2200+
appendLinkEnvironmentVariablesThatAppend(LinkOptionsStr);
21722201
const context &Context = getSyclObjImpl(DeviceImage)->get_context();
21732202
const ContextImplPtr ContextImpl = getSyclObjImpl(Context);
21742203
const PluginPtr &Plugin = ContextImpl->getPlugin();
@@ -2279,7 +2308,9 @@ device_image_plain ProgramManager::build(const device_image_plain &DeviceImage,
22792308
ContextImplPtr ContextImpl = getSyclObjImpl(Context);
22802309
const PluginPtr &Plugin = ContextImpl->getPlugin();
22812310
applyOptionsFromImage(CompileOpts, LinkOpts, Img, Devs, Plugin);
2282-
2311+
// Should always come last!
2312+
appendCompileEnvironmentVariablesThatAppend(CompileOpts);
2313+
appendLinkEnvironmentVariablesThatAppend(LinkOpts);
22832314
// TODO: Add support for creating non-SPIRV programs from multiple devices.
22842315
if (InputImpl->get_bin_image_ref()->getFormat() !=
22852316
PI_DEVICE_BINARY_TYPE_SPIRV &&

sycl/test-e2e/Config/env_vars.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,14 @@
44
// RUN: %{build} -O0 -o %t.out
55
//
66
// RUN: env SYCL_PROGRAM_COMPILE_OPTIONS="-g" %{run} %t.out
7+
// RUN: env SYCL_PROGRAM_APPEND_COMPILE_OPTIONS="-g" %{run} %t.out
78
//
89
// Now test for invalid options to make sure they are really passed to
910
// a device compiler. Intel GPU runtime doesn't give an error for
1011
// invalid options, so we don't test it here.
1112
//
1213
// RUN: %if cpu %{ env SYCL_PROGRAM_COMPILE_OPTIONS="-enable-link-options -cl-denorms-are-zero" SHOULD_CRASH=1 %{run} %t.out %}
14+
// RUN: %if cpu %{ env SYCL_PROGRAM_APPEND_COMPILE_OPTIONS="-enable-link-options -cl-denorms-are-zero" SHOULD_CRASH=1 %{run} %t.out %}
1315

1416
#include <cassert>
1517
#include <memory>
Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
// REQUIRES: gpu
22
// Disable fallback assert here so, that build process isn't affected
33
// RUN: %{build} -DSYCL_DISABLE_FALLBACK_ASSERT=1 -o %t.out %debug_option
4-
// RUN: env SYCL_PI_TRACE=-1 SYCL_PROGRAM_COMPILE_OPTIONS=-DENV_COMPILE_OPTS SYCL_PROGRAM_LINK_OPTIONS=-DENV_LINK_OPTS %{run} %t.out | FileCheck %s
4+
// RUN: env SYCL_PI_TRACE=-1 SYCL_PROGRAM_COMPILE_OPTIONS=-DENV_COMPILE_OPTS SYCL_PROGRAM_LINK_OPTIONS=-DENV_LINK_OPTS SYCL_PROGRAM_APPEND_COMPILE_OPTIONS=-DENV_APPEND_COMPILE_OPTS SYCL_PROGRAM_APPEND_LINK_OPTIONS=-DENV_APPEND_LINK_OPTS %{run} %t.out | FileCheck %s
55
// Check that options are overrided
66
// RUN: %{build} -DSYCL_DISABLE_FALLBACK_ASSERT=1 -Xsycl-target-linker=spir64 -DBAR -Xsycl-target-frontend=spir64 -DBAR_COMPILE -o %t.out
7-
// RUN: env SYCL_PI_TRACE=-1 SYCL_PROGRAM_COMPILE_OPTIONS=-DENV_COMPILE_OPTS SYCL_PROGRAM_LINK_OPTIONS=-DENV_LINK_OPTS %{run} %t.out | FileCheck %s
7+
// RUN: env SYCL_PI_TRACE=-1 SYCL_PROGRAM_COMPILE_OPTIONS=-DENV_COMPILE_OPTS SYCL_PROGRAM_LINK_OPTIONS=-DENV_LINK_OPTS SYCL_PROGRAM_APPEND_COMPILE_OPTIONS=-DENV_APPEND_COMPILE_OPTS SYCL_PROGRAM_APPEND_LINK_OPTIONS=-DENV_APPEND_LINK_OPTS %{run} %t.out | FileCheck %s
88
// UNSUPPORTED: hip
99

1010
#include "kernel-bundle-merge-options.hpp"
@@ -13,16 +13,16 @@
1313
// CHECK-NEXT: <unknown>
1414
// CHECK-NEXT: <unknown>
1515
// CHECK-NEXT: <unknown>
16-
// CHECK: <const char *>:{{[^bar]*}}-DENV_COMPILE_OPTS{{[^bar]*}}-DENV_LINK_OPTS{{[^bar]*}}
16+
// CHECK: <const char *>:{{[^bar]*}}-DENV_COMPILE_OPTS -DENV_APPEND_COMPILE_OPTS{{[^bar]*}}-DENV_LINK_OPTS -DENV_APPEND_LINK_OPTS{{[^bar]*}}
1717

1818
// CHECK: piProgramCompile(
1919
// CHECK-NEXT: <unknown>
2020
// CHECK-NEXT: <unknown>
2121
// CHECK-NEXT: <unknown>
22-
// CHECK: <const char *>:{{[^bar]*}}-DENV_COMPILE_OPTS{{[^bar]*}}
22+
// CHECK: <const char *>:{{[^bar]*}}-DENV_COMPILE_OPTS -DENV_APPEND_COMPILE_OPTS{{[^bar]*}}
2323

2424
// CHECK: piProgramLink(
2525
// CHECK-NEXT: <unknown>
2626
// CHECK-NEXT: <unknown>
2727
// CHECK-NEXT: <unknown>
28-
// CHECK: <const char *>:{{[^bar]*}}-DENV_LINK_OPTS{{[^bar]*}}
28+
// CHECK: <const char *>:{{[^bar]*}}-DENV_LINK_OPTS -DENV_APPEND_LINK_OPTS{{[^bar]*}}

0 commit comments

Comments
 (0)