-
Notifications
You must be signed in to change notification settings - Fork 188
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[SYCL] convert sycl::device to sycl::device* for better handling #504
Changes from all commits
fe472d9
3097a99
43c3cbd
b81fda9
122a3b1
6754121
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -100,8 +100,35 @@ hypre_SetDevice(hypre_int device_id, hypre_Handle *hypre_handle_) | |
#endif | ||
|
||
#if defined(HYPRE_USING_SYCL) | ||
/* sycl device set at construction of hypre_DeviceData object */ | ||
#elif defined(HYPRE_USING_GPU) | ||
HYPRE_Int nDevices=0; | ||
hypre_GetDeviceCount(&nDevices); | ||
if (device_id > nDevices) { | ||
hypre_printf("ERROR: SYCL device-ID exceed the number of devices on-node... \n"); | ||
} | ||
|
||
HYPRE_Int local_nDevices=0; | ||
for (int i = 0; i < gpu_devices.size(); i++) { | ||
// multi-tile GPUs | ||
if (gpu_devices[i].get_info<sycl::info::device::partition_max_sub_devices>() > 0) { | ||
auto subDevicesDomainNuma = gpu_devices[i].create_sub_devices<sycl::info::partition_property::partition_by_affinity_domain>(sycl::info::partition_affinity_domain::numa); | ||
for (auto &tile : subDevicesDomainNuma) { | ||
if (local_nDevices == device_id) { | ||
hypre_HandleDevice(hypre_handle_) = &tile; | ||
} | ||
local_nDevices++; | ||
} | ||
} | ||
// single-tile GPUs | ||
else { | ||
if (local_nDevices == device_id) { | ||
hypre_HandleDevice(hypre_handle_) = &(gpu_devices[i]); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If you set There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I agree, |
||
} | ||
local_nDevices++; | ||
} | ||
} | ||
#endif | ||
|
||
#if defined(HYPRE_USING_GPU) && !defined(HYPRE_USING_SYCL) | ||
if (hypre_handle_) | ||
{ | ||
hypre_HandleDevice(hypre_handle_) = device_id; | ||
|
@@ -152,25 +179,20 @@ hypre_GetDeviceCount(hypre_int *device_count) | |
#endif | ||
|
||
#if defined(HYPRE_USING_SYCL) | ||
/* WM: todo - doesn't work on frank... commenting out */ | ||
/* sycl::platform platform(sycl::gpu_selector{}); */ | ||
/* auto const& gpu_devices = platform.get_devices(); */ | ||
/* for (int i = 0; i < gpu_devices.size(); i++) */ | ||
/* { */ | ||
/* if (gpu_devices[i].is_gpu()) */ | ||
/* { */ | ||
/* if(gpu_devices[i].get_info<sycl::info::device::partition_max_sub_devices>() > 0) */ | ||
/* { */ | ||
/* auto subDevicesDomainNuma = gpu_devices[i].create_sub_devices<sycl::info::partition_property::partition_by_affinity_domain>( */ | ||
/* sycl::info::partition_affinity_domain::numa); */ | ||
/* (*device_count) += subDevicesDomainNuma.size(); */ | ||
/* } */ | ||
/* else */ | ||
/* { */ | ||
/* (*device_count)++; */ | ||
/* } */ | ||
/* } */ | ||
/* } */ | ||
sycl::platform platform(sycl::gpu_selector{}); | ||
auto const& gpu_devices = platform.get_devices(sycl::info::device_type::gpu); | ||
for (int i = 0; i < gpu_devices.size(); i++) | ||
{ | ||
if(gpu_devices[i].get_info<sycl::info::device::partition_max_sub_devices>() > 0) | ||
{ | ||
auto subDevicesDomainNuma = gpu_devices[i].create_sub_devices<sycl::info::partition_property::partition_by_affinity_domain>(sycl::info::partition_affinity_domain::numa); | ||
(*device_count) += subDevicesDomainNuma.size(); | ||
} | ||
else | ||
{ | ||
(*device_count)++; | ||
} | ||
} | ||
#endif | ||
|
||
return hypre_error_flag; | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
So does this consider each tile to be a separate device? What is the effect of this vs. setting the environment variable that Brian had mentioned:
export ZE_AFFINITY_MASK=0.0
? Does this do the same thing?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It does the same but just generalizes.
ZE_AFFINITY_MASK=0.0
is designed to be set for device ID 0 and tile 0 and not to be a scalable solution.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
That sounds good. Is there ever a case where you would want to treat all tiles as a single device? Either way, I think I'm happy using this for now at least.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Treating a device as 2 separate tiles (explicit scaling) and device as a whole (implicit scaling) are two options. Since majority of the Apps interacting with Hypre are exploring explicit mode, hence the separate tiles. But I want to add implicit scaling too. I will add macros like the following under
HYPRE_USING_SYCL
for a user to choose either explicit or implicit.For e.g.,
HYPRE_SYCL_EXPLICIT
andHYPRE_SYCL_IMPLICIT