Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixing openmp issues with FMS2020 cpu affinity #1148

Merged
merged 3 commits into from
Jul 7, 2020
Merged

Fixing openmp issues with FMS2020 cpu affinity #1148

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7bf4a30
Fixing openmp issues with FMS2020 cpu affinity
nikizadehgfdl Jun 28, 2020
bd57b78
Merge branch 'dev/gfdl' into fix_openmp_affinity_issues_with_FMS2020
adcroft Jun 29, 2020
4647870
Merge branch 'dev/gfdl' into fix_openmp_affinity_issues_with_FMS2020
adcroft Jul 6, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 8 additions & 17 deletions config_src/solo_driver/MOM_driver.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ program MOM_main
use ensemble_manager_mod, only : ensemble_pelist_setup
use mpp_mod, only : set_current_pelist => mpp_set_current_pelist
use time_interp_external_mod, only : time_interp_external_init
use fms_affinity_mod, only : fms_affinity_init, fms_affinity_set,fms_affinity_get

use MOM_ice_shelf, only : initialize_ice_shelf, ice_shelf_end, ice_shelf_CS
use MOM_ice_shelf, only : shelf_calc_flux, add_shelf_forces, ice_shelf_save_restart
Expand Down Expand Up @@ -207,11 +208,10 @@ program MOM_main
character(len=40) :: mod_name = "MOM_main (MOM_driver)" ! This module's name.

integer :: ocean_nthreads = 1
integer :: ncores_per_node = 36
logical :: use_hyper_thread = .false.
integer :: omp_get_num_threads,omp_get_thread_num,get_cpu_affinity,adder,base_cpu
integer :: omp_get_num_threads,omp_get_thread_num
namelist /ocean_solo_nml/ date_init, calendar, months, days, hours, minutes, seconds,&
ocean_nthreads, ncores_per_node, use_hyper_thread
ocean_nthreads, use_hyper_thread

!=====================================================================

Expand Down Expand Up @@ -252,22 +252,13 @@ program MOM_main
endif
endif

!$ call fms_affinity_init
!$ call fms_affinity_set('OCEAN', use_hyper_thread, ocean_nthreads)
!$ call omp_set_num_threads(ocean_nthreads)
!$ base_cpu = get_cpu_affinity()
!$OMP PARALLEL private(adder)
!$ if (use_hyper_thread) then
!$ if (mod(omp_get_thread_num(),2) == 0) then
!$ adder = omp_get_thread_num()/2
!$ else
!$ adder = ncores_per_node + omp_get_thread_num()/2
!$ endif
!$ else
!$ adder = omp_get_thread_num()
!$ endif
!$ call set_cpu_affinity (base_cpu + adder)
!$ write(6,*) " ocean ", base_cpu, get_cpu_affinity(), adder, omp_get_thread_num(), omp_get_num_threads()
!$OMP PARALLEL
!$ write(6,*) "ocean_solo OMPthreading ", fms_affinity_get(), omp_get_thread_num(), omp_get_num_threads()
!$ call flush(6)
!$OMP END PARALLEL
!$OMP END PARALLEL

! Read ocean_solo restart, which can override settings from the namelist.
if (file_exists(trim(dirs%restart_input_dir)//'ocean_solo.res')) then
Expand Down
26 changes: 5 additions & 21 deletions src/framework/MOM_domains.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ module MOM_domains
use mpp_parameter_mod, only : To_East => WUPDATE, To_West => EUPDATE, Omit_Corners => EDGEUPDATE
use mpp_parameter_mod, only : To_North => SUPDATE, To_South => NUPDATE, CENTER
use fms_io_mod, only : file_exist, parse_mask_table
use fms_affinity_mod, only : fms_affinity_init, fms_affinity_set,fms_affinity_get

implicit none ; private

Expand Down Expand Up @@ -1192,7 +1193,6 @@ subroutine MOM_domains_init(MOM_dom, param_file, symmetric, static_memory, &
integer, dimension(4) :: global_indices
!$ integer :: ocean_nthreads ! Number of Openmp threads
!$ integer :: get_cpu_affinity, omp_get_thread_num, omp_get_num_threads
!$ integer :: omp_cores_per_node, adder, base_cpu
!$ logical :: ocean_omp_hyper_thread
integer :: nihalo_dflt, njhalo_dflt
integer :: pe, proc_used
Expand Down Expand Up @@ -1274,6 +1274,7 @@ subroutine MOM_domains_init(MOM_dom, param_file, symmetric, static_memory, &
default=.false.)

#ifndef NOT_SET_AFFINITY
!$ call fms_affinity_init
!$OMP PARALLEL
!$OMP master
!$ ocean_nthreads = omp_get_num_threads()
Expand All @@ -1285,27 +1286,10 @@ subroutine MOM_domains_init(MOM_dom, param_file, symmetric, static_memory, &
!$ default = 1, layoutParam=.true.)
!$ call get_param(param_file, mdl, "OCEAN_OMP_HYPER_THREAD", ocean_omp_hyper_thread, &
!$ "If True, use hyper-threading.", default = .false., layoutParam=.true.)
!$ if (ocean_omp_hyper_thread) then
!$ call get_param(param_file, mdl, "OMP_CORES_PER_NODE", omp_cores_per_node, &
!$ "Number of cores per node needed for hyper-threading.", &
!$ fail_if_missing=.true., layoutParam=.true.)
!$ endif
!$ call fms_affinity_set('OCEAN', ocean_omp_hyper_thread, ocean_nthreads)
!$ call omp_set_num_threads(ocean_nthreads)
!$ base_cpu = get_cpu_affinity()
!$OMP PARALLEL private(adder)
!$ if (ocean_omp_hyper_thread) then
!$ if (mod(omp_get_thread_num(),2) == 0) then
!$ adder = omp_get_thread_num()/2
!$ else
!$ adder = omp_cores_per_node + omp_get_thread_num()/2
!$ endif
!$ else
!$ adder = omp_get_thread_num()
!$ endif
!$ call set_cpu_affinity(base_cpu + adder)
!!$ write(6,*) " ocean ", base_cpu, get_cpu_affinity(), adder, omp_get_thread_num(), omp_get_num_threads()
!!$ call flush(6)
!$OMP END PARALLEL
!$ write(6,*) "MOM_domains_mod OMPthreading ", fms_affinity_get(), omp_get_thread_num(), omp_get_num_threads()
!$ call flush(6)
!$ endif
#endif
call log_param(param_file, mdl, "!SYMMETRIC_MEMORY_", MOM_dom%symmetric, &
Expand Down