-
Notifications
You must be signed in to change notification settings - Fork 188
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[SYCL] convert sycl::device to sycl::device* for better handling #504
Changes from 2 commits
fe472d9
3097a99
43c3cbd
b81fda9
122a3b1
6754121
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -100,7 +100,43 @@ hypre_SetDevice(hypre_int device_id, hypre_Handle *hypre_handle_) | |
#endif | ||
|
||
#if defined(HYPRE_USING_SYCL) | ||
/* sycl device set at construction of hypre_DeviceData object */ | ||
HYPRE_Int nDevices=0; | ||
sycl::platform platform(sycl::gpu_selector{}); | ||
auto gpu_devices = platform.get_devices(sycl::info::device_type::gpu); | ||
for (int i = 0; i < gpu_devices.size(); i++) { | ||
if(gpu_devices[i].get_info<sycl::info::device::partition_max_sub_devices>() > 0) { | ||
auto subDevicesDomainNuma = gpu_devices[i].create_sub_devices<sycl::info::partition_property::partition_by_affinity_domain>(sycl::info::partition_affinity_domain::numa); | ||
nDevices += subDevicesDomainNuma.size(); | ||
} | ||
else { | ||
nDevices++; | ||
} | ||
} | ||
|
||
if (device_id > nDevices) { | ||
hypre_printf("ERROR: SYCL device-ID exceed the number of devices on-node... \n"); | ||
} | ||
|
||
HYPRE_Int local_nDevices=0; | ||
for (int i = 0; i < gpu_devices.size(); i++) { | ||
// multi-tile GPUs | ||
if (gpu_devices[i].get_info<sycl::info::device::partition_max_sub_devices>() > 0) { | ||
auto subDevicesDomainNuma = gpu_devices[i].create_sub_devices<sycl::info::partition_property::partition_by_affinity_domain>(sycl::info::partition_affinity_domain::numa); | ||
for (auto &tile : subDevicesDomainNuma) { | ||
if (local_nDevices == device_id) { | ||
hypre_HandleDevice(hypre_handle_) = &tile; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. So does this consider each tile to be a separate device? What is the effect of this vs. setting the environment variable that Brian had mentioned: There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It does the same but just generalizes. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. That sounds good. Is there ever a case where you would want to treat all tiles as a single device? Either way, I think I'm happy using this for now at least. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Treating a device as 2 separate tiles (explicit scaling) and device as a whole (implicit scaling) are two options. Since majority of the Apps interacting with Hypre are exploring explicit mode, hence the separate tiles. But I want to add implicit scaling too. I will add macros like the following under For e.g., |
||
} | ||
local_nDevices++; | ||
} | ||
} | ||
// single-tile GPUs | ||
else { | ||
if (local_nDevices == device_id) { | ||
hypre_HandleDevice(hypre_handle_) = &(gpu_devices[i]); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If you set There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I agree, |
||
} | ||
local_nDevices++; | ||
} | ||
} | ||
#elif defined(HYPRE_USING_GPU) | ||
if (hypre_handle_) | ||
{ | ||
|
@@ -152,25 +188,20 @@ hypre_GetDeviceCount(hypre_int *device_count) | |
#endif | ||
|
||
#if defined(HYPRE_USING_SYCL) | ||
/* WM: todo - doesn't work on frank... commenting out */ | ||
/* sycl::platform platform(sycl::gpu_selector{}); */ | ||
/* auto const& gpu_devices = platform.get_devices(); */ | ||
/* for (int i = 0; i < gpu_devices.size(); i++) */ | ||
/* { */ | ||
/* if (gpu_devices[i].is_gpu()) */ | ||
/* { */ | ||
/* if(gpu_devices[i].get_info<sycl::info::device::partition_max_sub_devices>() > 0) */ | ||
/* { */ | ||
/* auto subDevicesDomainNuma = gpu_devices[i].create_sub_devices<sycl::info::partition_property::partition_by_affinity_domain>( */ | ||
/* sycl::info::partition_affinity_domain::numa); */ | ||
/* (*device_count) += subDevicesDomainNuma.size(); */ | ||
/* } */ | ||
/* else */ | ||
/* { */ | ||
/* (*device_count)++; */ | ||
/* } */ | ||
/* } */ | ||
/* } */ | ||
sycl::platform platform(sycl::gpu_selector{}); | ||
auto const& gpu_devices = platform.get_devices(sycl::info::device_type::gpu); | ||
for (int i = 0; i < gpu_devices.size(); i++) | ||
{ | ||
if(gpu_devices[i].get_info<sycl::info::device::partition_max_sub_devices>() > 0) | ||
{ | ||
auto subDevicesDomainNuma = gpu_devices[i].create_sub_devices<sycl::info::partition_property::partition_by_affinity_domain>(sycl::info::partition_affinity_domain::numa); | ||
(*device_count) += subDevicesDomainNuma.size(); | ||
} | ||
else | ||
{ | ||
(*device_count)++; | ||
} | ||
} | ||
#endif | ||
|
||
return hypre_error_flag; | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The code above looks like it replicates the new implementation in
hypre_GetDeviceCount()
. Can we just call that here?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Done