Skip to content

Commit

Permalink
[SYCL] convert sycl::device to sycl::device* for better handling (#504)
Browse files Browse the repository at this point in the history 
* [SYCL] convert sycl::device to sycl::device* for better handling and setting

* [SYCL] fix ONEAPI warning and build issues

* [SYCL] fix the setDevice method and address comments
  • Loading branch information
@abagusetty
abagusetty committed Nov 5, 2021
1 parent ec8c5de commit 3254e31
Show file tree
Hide file tree
Showing 4 changed files with 54 additions and 29 deletions.
2 changes: 1 addition & 1 deletion src/utilities/_hypre_utilities.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,7 @@ struct hypre_DeviceData
hypre_device_allocator device_allocator;
#endif
#if defined(HYPRE_USING_SYCL)
sycl::device device;
sycl::device* device;
HYPRE_Int device_max_work_group_size;
#else
HYPRE_Int device;
Expand Down
15 changes: 9 additions & 6 deletions src/utilities/device_utils.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -964,14 +964,14 @@ hypre_DeviceDataStream(hypre_DeviceData *data, HYPRE_Int i)
catch (sycl::exception const& ex)
{
std::cout << "Caught asynchronous SYCL exception:" << std::endl
<< ex.what() << ", OpenCL code: " << ex.get_cl_code() << std::endl;
<< ex.what() << ", SYCL code: " << ex.code() << std::endl;
}
}
};

sycl::device syclDev = data->device;
sycl::context syclctxt = sycl::context(syclDev, sycl_asynchandler);
stream = new sycl::queue(syclctxt, syclDev, sycl::property_list{sycl::property::queue::in_order{}});
sycl::device* syclDev = data->device;
sycl::context syclctxt = sycl::context(*syclDev, sycl_asynchandler);
stream = new sycl::queue(syclctxt, *syclDev, sycl::property_list{sycl::property::queue::in_order{}});
data->streams[i] = stream;
}
#endif
Expand Down Expand Up @@ -1019,7 +1019,7 @@ sycl::queue*
hypre_DeviceDataComputeStream(hypre_DeviceData *data)
{
return hypre_DeviceDataStream(data,
hypre_DeviceDataComputeStreamNum(data));
hypre_DeviceDataComputeStreamNum(data));
}

#if defined(HYPRE_USING_CURAND)
Expand Down Expand Up @@ -1228,7 +1228,9 @@ hypre_DeviceDataCreate()
hypre_DeviceData *data = hypre_CTAlloc(hypre_DeviceData, 1, HYPRE_MEMORY_HOST);

#if defined(HYPRE_USING_SYCL)
hypre_DeviceDataDevice(data) = sycl::device(sycl::default_selector{});
/* WM: does the default selector get a GPU if available? Having trouble with getting the device on frank, so temporarily just passing the default selector */
hypre_DeviceDataDevice(data) = nullptr;

hypre_DeviceDataDeviceMaxWorkGroupSize(data) = hypre_DeviceDataDevice(data).get_info<sycl::info::device::max_work_group_size>();
#else
hypre_DeviceDataDevice(data) = 0;
Expand Down Expand Up @@ -1486,6 +1488,7 @@ hypre_bind_device( HYPRE_Int myid,

/* get number of devices on this node */
hypre_GetDeviceCount(&nDevices);
/* TODO: ABB might need to look into this since nDevices are overwritten by 1 */
nDevices = 1;

/* set device */
Expand Down
2 changes: 1 addition & 1 deletion src/utilities/device_utils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ struct hypre_DeviceData
hypre_device_allocator device_allocator;
#endif
#if defined(HYPRE_USING_SYCL)
sycl::device device;
sycl::device* device;
HYPRE_Int device_max_work_group_size;
#else
HYPRE_Int device;
Expand Down
64 changes: 43 additions & 21 deletions src/utilities/general.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,35 @@ hypre_SetDevice(hypre_int device_id, hypre_Handle *hypre_handle_)
#endif

#if defined(HYPRE_USING_SYCL)
/* sycl device set at construction of hypre_DeviceData object */
#elif defined(HYPRE_USING_GPU)
HYPRE_Int nDevices=0;
hypre_GetDeviceCount(&nDevices);
if (device_id > nDevices) {
hypre_printf("ERROR: SYCL device-ID exceed the number of devices on-node... \n");
}

HYPRE_Int local_nDevices=0;
for (int i = 0; i < gpu_devices.size(); i++) {
// multi-tile GPUs
if (gpu_devices[i].get_info<sycl::info::device::partition_max_sub_devices>() > 0) {
auto subDevicesDomainNuma = gpu_devices[i].create_sub_devices<sycl::info::partition_property::partition_by_affinity_domain>(sycl::info::partition_affinity_domain::numa);
for (auto &tile : subDevicesDomainNuma) {
if (local_nDevices == device_id) {
hypre_HandleDevice(hypre_handle_) = &tile;
}
local_nDevices++;
}
}
// single-tile GPUs
else {
if (local_nDevices == device_id) {
hypre_HandleDevice(hypre_handle_) = &(gpu_devices[i]);
}
local_nDevices++;
}
}
#endif

#if defined(HYPRE_USING_GPU) && !defined(HYPRE_USING_SYCL)
if (hypre_handle_)
{
hypre_HandleDevice(hypre_handle_) = device_id;
Expand Down Expand Up @@ -152,25 +179,20 @@ hypre_GetDeviceCount(hypre_int *device_count)
#endif

#if defined(HYPRE_USING_SYCL)
/* WM: todo - doesn't work on frank... commenting out */
/* sycl::platform platform(sycl::gpu_selector{}); */
/* auto const& gpu_devices = platform.get_devices(); */
/* for (int i = 0; i < gpu_devices.size(); i++) */
/* { */
/* if (gpu_devices[i].is_gpu()) */
/* { */
/* if(gpu_devices[i].get_info<sycl::info::device::partition_max_sub_devices>() > 0) */
/* { */
/* auto subDevicesDomainNuma = gpu_devices[i].create_sub_devices<sycl::info::partition_property::partition_by_affinity_domain>( */
/* sycl::info::partition_affinity_domain::numa); */
/* (*device_count) += subDevicesDomainNuma.size(); */
/* } */
/* else */
/* { */
/* (*device_count)++; */
/* } */
/* } */
/* } */
sycl::platform platform(sycl::gpu_selector{});
auto const& gpu_devices = platform.get_devices(sycl::info::device_type::gpu);
for (int i = 0; i < gpu_devices.size(); i++)
{
if(gpu_devices[i].get_info<sycl::info::device::partition_max_sub_devices>() > 0)
{
auto subDevicesDomainNuma = gpu_devices[i].create_sub_devices<sycl::info::partition_property::partition_by_affinity_domain>(sycl::info::partition_affinity_domain::numa);
(*device_count) += subDevicesDomainNuma.size();
}
else
{
(*device_count)++;
}
}
#endif

return hypre_error_flag;
Expand Down

0 comments on commit 3254e31

Please sign in to comment.