[SYCL] Adding new device descriptors as SYCL extensions. (#2897)

This commit adds new device descriptors as SYCL extensions.
These new device descriptors are detailed in
llvm/sycl/docs/extensions/IntelGPU/IntelGPUDeviceInfo.md.
They will provide low-level details about Intel GPU devices.
This support is only provided for the Level Zero Backend.
New aspects have been added to indicate if the support is available.

Signed-off-by: Gail Lyons <gail.lyons@intel.com>

Author: glyons-intel

sycl/doc/EnvironmentVariables.md

@@ -31,6 +31,7 @@ subject to change. Do not rely on these variables in production code.
 | SYCL_PI_LEVEL_ZERO_BATCH_SIZE | Integer | Sets a preferred number of commands to batch into a command list before executing the command list. A value of 0 causes the batch size to be adjusted dynamically. A value greater than 0 specifies fixed size batching, with the batch size set to the specified value. The default is 0. |
 | SYCL_PARALLEL_FOR_RANGE_ROUNDING_TRACE | Any(\*) | Enables tracing of parallel_for invocations with rounded-up ranges. |
 | SYCL_DISABLE_PARALLEL_FOR_RANGE_ROUNDING | Any(\*) | Disables automatic rounding-up of parallel_for invocation ranges. |
+| SYCL_ENABLE_PCI | Integer | When set to 1, enables obtaining the GPU PCI address when using the Level Zero backend. The default is 0. |
 
 `(*) Note: Any means this environment variable is effective when set to any non-null value.`
 

sycl/doc/extensions/IntelGPU/IntelGPUDeviceInfo.md

@@ -1,6 +1,5 @@
 # SYCL(TM) Proposal: Intel's Extensions for Device Information
 
-**IMPORTANT**: This specification is a draft.
 
 **NOTE**: Khronos(R) is a registered trademark and SYCL(TM) is a trademark of the Khronos Group, Inc.
 
@@ -15,13 +14,14 @@ The Feature Test Macro will be defined as:
     #define SYCL_EXT_INTEL_DEVICE_INFO 1
 
 
-
 # PCI Address #
 
 A new device descriptor will be added which will provide the PCI address in BDF format.  BDF format contains the address as: `domain:bus:device.function`.
 
 This new device descriptor is only available for devices in the Level Zero platform, and the matching aspect is only true for those devices. The DPC++ default behavior is to expose GPU devices through the Level Zero platform.
 
+**Note:** The environment variable SYCL\_ENABLE\_PCI must be set to 1 to obtain the PCI address.
+
 
 ## Device Information Descriptors ##
 

sycl/include/CL/sycl.hpp

@@ -26,6 +26,7 @@
 #include <CL/sycl/device_selector.hpp>
 #include <CL/sycl/event.hpp>
 #include <CL/sycl/exception.hpp>
+#include <CL/sycl/feature_test.hpp>
 #include <CL/sycl/group.hpp>
 #include <CL/sycl/handler.hpp>
 #include <CL/sycl/id.hpp>

sycl/include/CL/sycl/aspects.hpp

@@ -30,7 +30,14 @@ enum class aspect {
   usm_host_allocations,
   usm_shared_allocations,
   usm_restricted_shared_allocations,
-  usm_system_allocator
+  usm_system_allocator,
+  ext_intel_pci_address,
+  ext_intel_gpu_eu_count,
+  ext_intel_gpu_eu_simd_width,
+  ext_intel_gpu_slices,
+  ext_intel_gpu_subslices_per_slice,
+  ext_intel_gpu_eu_count_per_subslice,
+  ext_intel_max_mem_bandwidth
 };
 
 } // namespace sycl

sycl/include/CL/sycl/detail/pi.h

@@ -42,6 +42,7 @@
 #define _PI_CONCAT(a, b) _PI_STRING_HELPER(a.b)
 #define _PI_H_VERSION_STRING                                                   \
   _PI_CONCAT(_PI_H_VERSION_MAJOR, _PI_H_VERSION_MINOR)
+
 // TODO: we need a mapping of PI to OpenCL somewhere, and this can be done
 // elsewhere, e.g. in the pi_opencl, but constants/enums mapping is now
 // done here, for efficiency and simplicity.
@@ -264,7 +265,15 @@ typedef enum {
   PI_DEVICE_INFO_USM_CROSS_SHARED_SUPPORT =
       CL_DEVICE_CROSS_DEVICE_SHARED_MEM_CAPABILITIES_INTEL,
   PI_DEVICE_INFO_USM_SYSTEM_SHARED_SUPPORT =
-      CL_DEVICE_SHARED_SYSTEM_MEM_CAPABILITIES_INTEL
+      CL_DEVICE_SHARED_SYSTEM_MEM_CAPABILITIES_INTEL,
+  // These are Intel-specific extensions.
+  PI_DEVICE_INFO_PCI_ADDRESS = 0x10020,
+  PI_DEVICE_INFO_GPU_EU_COUNT = 0x10021,
+  PI_DEVICE_INFO_GPU_EU_SIMD_WIDTH = 0x10022,
+  PI_DEVICE_INFO_GPU_SLICES = 0x10023,
+  PI_DEVICE_INFO_GPU_SUBSLICES_PER_SLICE = 0x10024,
+  PI_DEVICE_INFO_GPU_EU_COUNT_PER_SUBSLICE = 0x10025,
+  PI_DEVICE_INFO_MAX_MEM_BANDWIDTH = 0x10026
 } _pi_device_info;
 
 typedef enum {

sycl/include/CL/sycl/feature_test.hpp

@@ -0,0 +1,18 @@
+//==---- feature_test.hpp - SYCL Feature Test Definitions -----*- C++ -*---==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// ===--------------------------------------------------------------------=== //
+#pragma once
+
+__SYCL_INLINE_NAMESPACE(cl) {
+namespace sycl {
+
+// Feature test macro definitions
+
+#define SYCL_EXT_INTEL_DEVICE_INFO 1
+
+} // namespace sycl
+} // __SYCL_INLINE_NAMESPACE(cl)

sycl/include/CL/sycl/info/device_traits.def

@@ -85,3 +85,10 @@ __SYCL_PARAM_TRAITS_SPEC(device, usm_host_allocations, bool)
 __SYCL_PARAM_TRAITS_SPEC(device, usm_shared_allocations, bool)
 __SYCL_PARAM_TRAITS_SPEC(device, usm_restricted_shared_allocations, bool)
 __SYCL_PARAM_TRAITS_SPEC(device, usm_system_allocator, bool)
+__SYCL_PARAM_TRAITS_SPEC(device, ext_intel_pci_address, string_class)
+__SYCL_PARAM_TRAITS_SPEC(device, ext_intel_gpu_eu_count, pi_uint32)
+__SYCL_PARAM_TRAITS_SPEC(device, ext_intel_gpu_eu_simd_width, pi_uint32)
+__SYCL_PARAM_TRAITS_SPEC(device, ext_intel_gpu_slices, pi_uint32)
+__SYCL_PARAM_TRAITS_SPEC(device, ext_intel_gpu_subslices_per_slice, pi_uint32)
+__SYCL_PARAM_TRAITS_SPEC(device, ext_intel_gpu_eu_count_per_subslice, pi_uint32)
+__SYCL_PARAM_TRAITS_SPEC(device, ext_intel_max_mem_bandwidth, pi_uint64)

sycl/include/CL/sycl/info/info_desc.hpp

@@ -130,7 +130,16 @@ enum class device : cl_device_info {
   usm_host_allocations = PI_USM_HOST_SUPPORT,
   usm_shared_allocations = PI_USM_SINGLE_SHARED_SUPPORT,
   usm_restricted_shared_allocations = PI_USM_CROSS_SHARED_SUPPORT,
-  usm_system_allocator = PI_USM_SYSTEM_SHARED_SUPPORT
+  usm_system_allocator = PI_USM_SYSTEM_SHARED_SUPPORT,
+  // intel extensions
+  ext_intel_pci_address = PI_DEVICE_INFO_PCI_ADDRESS,
+  ext_intel_gpu_eu_count = PI_DEVICE_INFO_GPU_EU_COUNT,
+  ext_intel_gpu_eu_simd_width = PI_DEVICE_INFO_GPU_EU_SIMD_WIDTH,
+  ext_intel_gpu_slices = PI_DEVICE_INFO_GPU_SLICES,
+  ext_intel_gpu_subslices_per_slice = PI_DEVICE_INFO_GPU_SUBSLICES_PER_SLICE,
+  ext_intel_gpu_eu_count_per_subslice =
+      PI_DEVICE_INFO_GPU_EU_COUNT_PER_SUBSLICE,
+  ext_intel_max_mem_bandwidth = PI_DEVICE_INFO_MAX_MEM_BANDWIDTH
 };
 
 enum class device_type : pi_uint64 {

sycl/plugins/cuda/pi_cuda.cpp

@@ -1452,6 +1452,16 @@ pi_result cuda_piDeviceGetInfo(pi_device device, pi_device_info param_name,
     return getInfo(param_value_size, param_value, param_value_size_ret, value);
   }
 
+    // TODO: Investigate if this information is available on CUDA.
+  case PI_DEVICE_INFO_PCI_ADDRESS:
+  case PI_DEVICE_INFO_GPU_EU_COUNT:
+  case PI_DEVICE_INFO_GPU_EU_SIMD_WIDTH:
+  case PI_DEVICE_INFO_GPU_SLICES:
+  case PI_DEVICE_INFO_GPU_SUBSLICES_PER_SLICE:
+  case PI_DEVICE_INFO_GPU_EU_COUNT_PER_SUBSLICE:
+  case PI_DEVICE_INFO_MAX_MEM_BANDWIDTH:
+    return PI_INVALID_VALUE;
+
   default:
     __SYCL_PI_HANDLE_UNKNOWN_PARAM_NAME(param_name);
   }

sycl/plugins/level_zero/pi_level_zero.cpp

@@ -17,10 +17,12 @@
 #include <cstdio>
 #include <cstring>
 #include <memory>
+#include <sstream>
 #include <string>
 #include <thread>
 #include <utility>
 
+#include <level_zero/zes_api.h>
 #include <level_zero/zet_api.h>
 
 #include "usm_allocator.hpp"
@@ -786,6 +788,12 @@ pi_result piPlatformsGet(pi_uint32 NumEntries, pi_platform *Platforms,
     setEnvVar("ZE_ENABLE_PARAMETER_VALIDATION", "1");
   }
 
+  // Enable SYSMAN support for obtaining the PCI address
+  // and maximum memory bandwidth.
+  if (getenv("SYCL_ENABLE_PCI") != nullptr) {
+    setEnvVar("ZES_ENABLE_SYSMAN", "1");
+  }
+
   // TODO: We can still safely recover if something goes wrong during the init.
   // Implement handling segfault using sigaction.
 
@@ -1578,6 +1586,42 @@ pi_result piDeviceGetInfo(pi_device Device, pi_device_info ParamName,
     }
     return ReturnValue(Supported);
   }
+
+    // intel extensions for GPU information
+  case PI_DEVICE_INFO_PCI_ADDRESS: {
+    if (getenv("ZES_ENABLE_SYSMAN") == nullptr) {
+      zePrint("Set SYCL_ENABLE_PCI=1 to obtain PCI data.\n");
+      return PI_INVALID_VALUE;
+    }
+    zes_pci_properties_t ZeDevicePciProperties = {};
+    ZE_CALL(zesDevicePciGetProperties(ZeDevice, &ZeDevicePciProperties));
+    std::stringstream ss;
+    ss << ZeDevicePciProperties.address.domain << ":"
+       << ZeDevicePciProperties.address.bus << ":"
+       << ZeDevicePciProperties.address.device << "."
+       << ZeDevicePciProperties.address.function;
+    return ReturnValue(ss.str().c_str());
+  }
+  case PI_DEVICE_INFO_GPU_EU_COUNT: {
+    pi_uint32 count = Device->ZeDeviceProperties.numEUsPerSubslice *
+                      Device->ZeDeviceProperties.numSubslicesPerSlice *
+                      Device->ZeDeviceProperties.numSlices;
+    return ReturnValue(pi_uint32{count});
+  }
+  case PI_DEVICE_INFO_GPU_EU_SIMD_WIDTH:
+    return ReturnValue(
+        pi_uint32{Device->ZeDeviceProperties.physicalEUSimdWidth});
+  case PI_DEVICE_INFO_GPU_SLICES:
+    return ReturnValue(pi_uint32{Device->ZeDeviceProperties.numSlices});
+  case PI_DEVICE_INFO_GPU_SUBSLICES_PER_SLICE:
+    return ReturnValue(
+        pi_uint32{Device->ZeDeviceProperties.numSubslicesPerSlice});
+  case PI_DEVICE_INFO_GPU_EU_COUNT_PER_SUBSLICE:
+    return ReturnValue(pi_uint32{Device->ZeDeviceProperties.numEUsPerSubslice});
+  case PI_DEVICE_INFO_MAX_MEM_BANDWIDTH:
+    // currently not supported in level zero runtime
+    return PI_INVALID_VALUE;
+
   default:
     zePrint("Unsupported ParamName in piGetDeviceInfo\n");
     zePrint("ParamName=%d(0x%x)\n", ParamName, ParamName);

sycl/plugins/opencl/pi_opencl.cpp

@@ -162,6 +162,29 @@ static pi_result USMSetIndirectAccess(pi_kernel kernel) {
 
 extern "C" {
 
+pi_result piDeviceGetInfo(pi_device device, pi_device_info paramName,
+                          size_t paramValueSize, void *paramValue,
+                          size_t *paramValueSizeRet) {
+  switch (paramName) {
+    // Intel GPU EU device-specific information extensions.
+    // TODO: Check regularly to see if support in enabled in OpenCL.
+  case PI_DEVICE_INFO_PCI_ADDRESS:
+  case PI_DEVICE_INFO_GPU_EU_COUNT:
+  case PI_DEVICE_INFO_GPU_EU_SIMD_WIDTH:
+  case PI_DEVICE_INFO_GPU_SLICES:
+  case PI_DEVICE_INFO_GPU_SUBSLICES_PER_SLICE:
+  case PI_DEVICE_INFO_GPU_EU_COUNT_PER_SUBSLICE:
+  case PI_DEVICE_INFO_MAX_MEM_BANDWIDTH:
+    return PI_INVALID_VALUE;
+
+  default:
+    cl_int result = clGetDeviceInfo(
+        cast<cl_device_id>(device), cast<cl_device_info>(paramName),
+        paramValueSize, paramValue, paramValueSizeRet);
+    return static_cast<pi_result>(result);
+  }
+}
+
 pi_result piPlatformsGet(pi_uint32 num_entries, pi_platform *platforms,
                          pi_uint32 *num_platforms) {
   cl_int result = clGetPlatformIDs(cast<cl_uint>(num_entries),
@@ -1164,7 +1187,7 @@ pi_result piPluginInit(pi_plugin *PluginInit) {
          piextPlatformCreateWithNativeHandle)
   // Device
   _PI_CL(piDevicesGet, piDevicesGet)
-  _PI_CL(piDeviceGetInfo, clGetDeviceInfo)
+  _PI_CL(piDeviceGetInfo, piDeviceGetInfo)
   _PI_CL(piDevicePartition, clCreateSubDevices)
   _PI_CL(piDeviceRetain, clRetainDevice)
   _PI_CL(piDeviceRelease, clReleaseDevice)

sycl/source/detail/device_impl.cpp

@@ -216,6 +216,9 @@ pi_native_handle device_impl::getNative() const {
 }
 
 bool device_impl::has(aspect Aspect) const {
+  size_t return_size = 0;
+  pi_device_type device_type;
+
   switch (Aspect) {
   case aspect::host:
     return is_host();
@@ -253,6 +256,37 @@ bool device_impl::has(aspect Aspect) const {
     return get_info<info::device::usm_restricted_shared_allocations>();
   case aspect::usm_system_allocator:
     return get_info<info::device::usm_system_allocator>();
+  case aspect::ext_intel_pci_address:
+    return getPlugin().call_nocheck<detail::PiApiKind::piDeviceGetInfo>(
+               MDevice, PI_DEVICE_INFO_PCI_ADDRESS, sizeof(pi_device_type),
+               &device_type, &return_size) == PI_SUCCESS;
+  case aspect::ext_intel_gpu_eu_count:
+    return getPlugin().call_nocheck<detail::PiApiKind::piDeviceGetInfo>(
+               MDevice, PI_DEVICE_INFO_GPU_EU_COUNT, sizeof(pi_device_type),
+               &device_type, &return_size) == PI_SUCCESS;
+  case aspect::ext_intel_gpu_eu_simd_width:
+    return getPlugin().call_nocheck<detail::PiApiKind::piDeviceGetInfo>(
+               MDevice, PI_DEVICE_INFO_GPU_EU_SIMD_WIDTH,
+               sizeof(pi_device_type), &device_type,
+               &return_size) == PI_SUCCESS;
+  case aspect::ext_intel_gpu_slices:
+    return getPlugin().call_nocheck<detail::PiApiKind::piDeviceGetInfo>(
+               MDevice, PI_DEVICE_INFO_GPU_SLICES, sizeof(pi_device_type),
+               &device_type, &return_size) == PI_SUCCESS;
+  case aspect::ext_intel_gpu_subslices_per_slice:
+    return getPlugin().call_nocheck<detail::PiApiKind::piDeviceGetInfo>(
+               MDevice, PI_DEVICE_INFO_GPU_SUBSLICES_PER_SLICE,
+               sizeof(pi_device_type), &device_type,
+               &return_size) == PI_SUCCESS;
+  case aspect::ext_intel_gpu_eu_count_per_subslice:
+    return getPlugin().call_nocheck<detail::PiApiKind::piDeviceGetInfo>(
+               MDevice, PI_DEVICE_INFO_GPU_EU_COUNT_PER_SUBSLICE,
+               sizeof(pi_device_type), &device_type,
+               &return_size) == PI_SUCCESS;
+  case aspect::ext_intel_max_mem_bandwidth:
+    // currently not supported
+    return false;
+
   default:
     throw runtime_error("This device aspect has not been implemented yet.",
                         PI_INVALID_DEVICE);

sycl/source/detail/device_info.hpp

@@ -1003,6 +1003,55 @@ template <> struct get_device_info<bool, info::device::usm_system_allocator> {
   }
 };
 
+// Specializations for intel extensions for Level Zero low-level
+// detail device descriptors (not support on host).
+template <>
+inline string_class
+get_device_info_host<info::device::ext_intel_pci_address>() {
+  throw runtime_error(
+      "Obtaining the PCI address is not supported on HOST device",
+      PI_INVALID_DEVICE);
+}
+template <>
+inline cl_uint get_device_info_host<info::device::ext_intel_gpu_eu_count>() {
+  throw runtime_error("Obtaining the EU count is not supported on HOST device",
+                      PI_INVALID_DEVICE);
+}
+template <>
+inline cl_uint
+get_device_info_host<info::device::ext_intel_gpu_eu_simd_width>() {
+  throw runtime_error(
+      "Obtaining the EU SIMD width is not supported on HOST device",
+      PI_INVALID_DEVICE);
+}
+template <>
+inline cl_uint get_device_info_host<info::device::ext_intel_gpu_slices>() {
+  throw runtime_error(
+      "Obtaining the number of slices is not supported on HOST device",
+      PI_INVALID_DEVICE);
+}
+template <>
+inline cl_uint
+get_device_info_host<info::device::ext_intel_gpu_subslices_per_slice>() {
+  throw runtime_error("Obtaining the number of subslices per slice is not "
+                      "supported on HOST device",
+                      PI_INVALID_DEVICE);
+}
+template <>
+inline cl_uint
+get_device_info_host<info::device::ext_intel_gpu_eu_count_per_subslice>() {
+  throw runtime_error(
+      "Obtaining the EU count per subslice is not supported on HOST device",
+      PI_INVALID_DEVICE);
+}
+template <>
+inline cl_ulong
+get_device_info_host<info::device::ext_intel_max_mem_bandwidth>() {
+  throw runtime_error(
+      "Obtaining the maximum memory bandwidth is not supported on HOST device",
+      PI_INVALID_DEVICE);
+}
+
 } // namespace detail
 } // namespace sycl
 } // __SYCL_INLINE_NAMESPACE(cl)

sycl/test/abi/pi_opencl_symbol_check.dump

@@ -7,6 +7,7 @@
 # REQUIRES: linux
 
 piContextCreate
+piDeviceGetInfo
 piDevicesGet
 piEnqueueMemBufferMap
 piEventCreate

sycl/test/abi/sycl_symbols_linux.dump

@@ -4036,6 +4036,13 @@ _ZNK2cl4sycl6device8get_infoILNS0_4info6deviceE4168EEENS3_12param_traitsIS4_XT_E
 _ZNK2cl4sycl6device8get_infoILNS0_4info6deviceE4169EEENS3_12param_traitsIS4_XT_EE11return_typeEv
 _ZNK2cl4sycl6device8get_infoILNS0_4info6deviceE4188EEENS3_12param_traitsIS4_XT_EE11return_typeEv
 _ZNK2cl4sycl6device8get_infoILNS0_4info6deviceE4189EEENS3_12param_traitsIS4_XT_EE11return_typeEv
+_ZNK2cl4sycl6device8get_infoILNS0_4info6deviceE65568EEENS3_12param_traitsIS4_XT_EE11return_typeEv
+_ZNK2cl4sycl6device8get_infoILNS0_4info6deviceE65569EEENS3_12param_traitsIS4_XT_EE11return_typeEv
+_ZNK2cl4sycl6device8get_infoILNS0_4info6deviceE65570EEENS3_12param_traitsIS4_XT_EE11return_typeEv
+_ZNK2cl4sycl6device8get_infoILNS0_4info6deviceE65571EEENS3_12param_traitsIS4_XT_EE11return_typeEv
+_ZNK2cl4sycl6device8get_infoILNS0_4info6deviceE65572EEENS3_12param_traitsIS4_XT_EE11return_typeEv
+_ZNK2cl4sycl6device8get_infoILNS0_4info6deviceE65573EEENS3_12param_traitsIS4_XT_EE11return_typeEv
+_ZNK2cl4sycl6device8get_infoILNS0_4info6deviceE65574EEENS3_12param_traitsIS4_XT_EE11return_typeEv
 _ZNK2cl4sycl6device9getNativeEv
 _ZNK2cl4sycl6kernel11get_contextEv
 _ZNK2cl4sycl6kernel11get_programEv