Skip to content

[SYCL][UR] Unify logging and leak checking for L0 v1 and v2 #19328

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 3 commits into
base: sycl
Choose a base branch
from
+8 −156
Draft

[SYCL][UR] Unify logging and leak checking for L0 v1 and v2 #19328

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c099481
[SYCL][UR] Unify logging and leak checking for L0 v1 and v2
igchor Jul 7, 2025
8b1ad18
enable L0 leak checker
igchor Jul 7, 2025
30bece8
fix leak in command buffer
igchor Jul 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 1 addition & 23 deletions sycl/ur_win_proxy_loader/ur_win_proxy_loader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,27 +107,6 @@ void *&getDllHandle() {
return dllHandle;
}

static bool shouldLoadL0V2adapter() {
auto SyclEnv = std::getenv("SYCL_UR_USE_LEVEL_ZERO_V2");
auto UREnv = std::getenv("UR_LOADER_USE_LEVEL_ZERO_V2");

try {
if (SyclEnv && std::stoi(SyclEnv) == 1) {
return true;
}
} catch (...) {
}

try {
if (UREnv && std::stoi(UREnv) == 1) {
return true;
}
} catch (...) {
}

return false;
}

/// Load the adapter libraries
void preloadLibraries() {
// Suppress system errors.
Expand Down Expand Up @@ -163,8 +142,7 @@ void preloadLibraries() {
getDllHandle() = loadAdapter(UR_LIBRARY_NAME(loader));
loadAdapter(UR_LIBRARY_NAME(adapter_opencl));
loadAdapter(UR_LIBRARY_NAME(adapter_level_zero));
if (shouldLoadL0V2adapter())
loadAdapter(UR_LIBRARY_NAME(adapter_level_zero_v2));
loadAdapter(UR_LIBRARY_NAME(adapter_level_zero_v2));
loadAdapter(UR_LIBRARY_NAME(adapter_cuda));
loadAdapter(UR_LIBRARY_NAME(adapter_hip));
loadAdapter(UR_LIBRARY_NAME(adapter_native_cpu));
Expand Down
112 changes: 5 additions & 107 deletions unified-runtime/source/adapters/level_zero/adapter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,32 +309,23 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()

if (UrL0Debug & UR_L0_DEBUG_BASIC) {
logger.setLegacySink(std::make_unique<ur_legacy_sink>());
#ifdef UR_ADAPTER_LEVEL_ZERO_V2
setEnvVar("ZEL_ENABLE_LOADER_LOGGING", "1");
setEnvVar("ZEL_LOADER_LOGGING_LEVEL", "trace");
setEnvVar("ZEL_LOADER_LOG_CONSOLE", "1");
setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1");
#endif
};

if (UrL0Debug & UR_L0_DEBUG_VALIDATION) {
setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1");
setEnvVar("ZE_ENABLE_PARAMETER_VALIDATION", "1");
}

PlatformCache.Compute = [](Result<PlatformVec> &result) {
static std::once_flag ZeCallCountInitialized;
try {
std::call_once(ZeCallCountInitialized, []() {
if (UrL0LeaksDebug) {
ZeCallCount = new std::map<std::string, int>;
}
});
} catch (...) {
result = exceptionToResult(std::current_exception());
return;
}
if (UrL0LeaksDebug) {
setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1");
setEnvVar("ZEL_ENABLE_BASIC_LEAK_CHECKER", "1");
}

PlatformCache.Compute = [](Result<PlatformVec> &result) {
uint32_t UserForcedSysManInit = 0;
// Check if the user has disabled the default L0 Env initialization.
const int UrSysManEnvInitEnabled = [&UserForcedSysManInit] {
Expand Down Expand Up @@ -426,7 +417,6 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
useInitDrivers = true;
}

#ifdef UR_ADAPTER_LEVEL_ZERO_V2
if ((loader_version.major == 1 && loader_version.minor < 21) ||
(loader_version.major == 1 && loader_version.minor == 21 &&
loader_version.patch < 2)) {
Expand All @@ -435,7 +425,6 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
"WARNING: Level Zero Loader version is older than 1.21.2. "
"Please update to the latest version for API logging support.\n");
}
#endif
}

if (useInitDrivers) {
Expand Down Expand Up @@ -552,97 +541,6 @@ void globalAdapterOnDemandCleanup() {
}

ur_result_t adapterStateTeardown() {
// Print the balance of various create/destroy native calls.
// The idea is to verify if the number of create(+) and destroy(-) calls are
// matched.
if (ZeCallCount && (UrL0LeaksDebug) != 0) {
bool LeakFound = false;
// clang-format off
//
// The format of this table is such that each row accounts for a
// specific type of objects, and all elements in the raw except the last
// one are allocating objects of that type, while the last element is known
// to deallocate objects of that type.
//
std::vector<std::vector<std::string>> CreateDestroySet = {
{"zeContextCreate", "zeContextDestroy"},
{"zeCommandQueueCreate", "zeCommandQueueDestroy"},
{"zeModuleCreate", "zeModuleDestroy"},
{"zeKernelCreate", "zeKernelDestroy"},
{"zeEventPoolCreate", "zeEventPoolDestroy"},
{"zeCommandListCreateImmediate", "zeCommandListCreate", "zeCommandListDestroy"},
{"zeEventCreate", "zeEventDestroy"},
{"zeFenceCreate", "zeFenceDestroy"},
{"zeImageCreate","zeImageViewCreateExt", "zeImageDestroy"},
{"zeSamplerCreate", "zeSamplerDestroy"},
{"zeMemAllocDevice", "zeMemAllocHost", "zeMemAllocShared", "zeMemFree"},
};

// A sample output aimed below is this:
// ------------------------------------------------------------------------
// zeContextCreate = 1 \---> zeContextDestroy = 1
// zeCommandQueueCreate = 1 \---> zeCommandQueueDestroy = 1
// zeModuleCreate = 1 \---> zeModuleDestroy = 1
// zeKernelCreate = 1 \---> zeKernelDestroy = 1
// zeEventPoolCreate = 1 \---> zeEventPoolDestroy = 1
// zeCommandListCreateImmediate = 1 |
// zeCommandListCreate = 1 \---> zeCommandListDestroy = 1 ---> LEAK = 1
// zeEventCreate = 2 \---> zeEventDestroy = 2
// zeFenceCreate = 1 \---> zeFenceDestroy = 1
// zeImageCreate = 0 \---> zeImageDestroy = 0
// zeSamplerCreate = 0 \---> zeSamplerDestroy = 0
// zeMemAllocDevice = 0 |
// zeMemAllocHost = 1 |
// zeMemAllocShared = 0 \---> zeMemFree = 1
//
// clang-format on
// TODO: use logger to print this messages
std::cerr << "Check balance of create/destroy calls\n";
std::cerr << "----------------------------------------------------------\n";
std::stringstream ss;
for (const auto &Row : CreateDestroySet) {
int diff = 0;
for (auto I = Row.begin(); I != Row.end();) {
const char *ZeName = (*I).c_str();
const auto &ZeCount = (*ZeCallCount)[*I];

bool First = (I == Row.begin());
bool Last = (++I == Row.end());

if (Last) {
ss << " \\--->";
diff -= ZeCount;
} else {
diff += ZeCount;
if (!First) {
ss << " | ";
std::cerr << ss.str() << "\n";
ss.str("");
ss.clear();
}
}
ss << std::setw(30) << std::right << ZeName;
ss << " = ";
ss << std::setw(5) << std::left << ZeCount;
}

if (diff) {
LeakFound = true;
ss << " ---> LEAK = " << diff;
}

std::cerr << ss.str() << '\n';
ss.str("");
ss.clear();
}

ZeCallCount->clear();
delete ZeCallCount;
ZeCallCount = nullptr;
if (LeakFound)
return UR_RESULT_ERROR_INVALID_MEM_OBJECT;
}

// Due to multiple DLLMain definitions with SYCL, register to cleanup the
// Global Adapter after refcnt is 0
#if defined(_WIN32)
Expand Down
23 changes: 0 additions & 23 deletions unified-runtime/source/adapters/level_zero/common.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,6 @@ bool setEnvVar(const char *name, const char *value) {

ZeUSMImportExtension ZeUSMImport;

std::map<std::string, int> *ZeCallCount = nullptr;

void zeParseError(ze_result_t ZeError, const char *&ErrorString) {
switch (ZeError) {
#define ZE_ERRCASE(ERR) \
Expand Down Expand Up @@ -137,31 +135,10 @@ void zeParseError(ze_result_t ZeError, const char *&ErrorString) {
} // switch
}

#ifdef UR_ADAPTER_LEVEL_ZERO_V2
ze_result_t ZeCall::doCall(ze_result_t ZeResult, const char *, const char *,
bool) {
return ZeResult;
}
#else
ze_result_t ZeCall::doCall(ze_result_t ZeResult, const char *ZeName,
const char *ZeArgs, bool TraceError) {
UR_LOG(DEBUG, "ZE ---> {}{}", ZeName, ZeArgs);

if (ZeResult == ZE_RESULT_SUCCESS) {
if (UrL0LeaksDebug) {
++(*ZeCallCount)[ZeName];
}
return ZE_RESULT_SUCCESS;
}

if (TraceError) {
const char *ErrorString = "Unknown";
zeParseError(ZeResult, ErrorString);
UR_LOG(ERR, "Error ({}) in {}", ErrorString, ZeName);
}
return ZeResult;
}
#endif

// Specializations for various L0 structures
template <> ze_structure_type_t getZeStructureType<ze_event_pool_desc_t>() {
Expand Down
3 changes: 0 additions & 3 deletions unified-runtime/source/adapters/level_zero/common.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -371,9 +371,6 @@ class ZeUSMImportExtension {
// Helper wrapper for working with USM import extension in Level Zero.
extern ZeUSMImportExtension ZeUSMImport;

// This will count the calls to Level-Zero
extern std::map<std::string, int> *ZeCallCount;

// Some opencl extensions we know are supported by all Level Zero devices.
constexpr char ZE_SUPPORTED_EXTENSIONS[] =
"cl_khr_il_program cl_khr_subgroups cl_intel_subgroups "
Expand Down
2 changes: 2 additions & 0 deletions unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,8 @@ ur_result_t ur_exp_command_buffer_handle_t_::registerExecutionEventUnlocked(
}

ur_exp_command_buffer_handle_t_::~ur_exp_command_buffer_handle_t_() {
UR_CALL_NOCHECK(commandListManager.lock()->releaseSubmittedKernels());

if (currentExecution) {
currentExecution->release();
}
Expand Down
Loading