Skip to content

Commit

Permalink
Merge branch 'develop' into ACCORD-NWP/to_CY50_rebased
Browse files Browse the repository at this point in the history
  • Loading branch information
samhatfield committed Dec 5, 2024
2 parents 048527f + 5b369ae commit 038023b
Show file tree
Hide file tree
Showing 44 changed files with 1,296 additions and 1,011 deletions.
110 changes: 110 additions & 0 deletions .github/workflows/build-hpc.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
name: build-hpc

# Controls when the action will run
on:

# Trigger the workflow on all pushes to main and develop, except on tag creation
push:
branches:
- main
- develop
tags-ignore:
- '**'

# Trigger the workflow on all pull requests
pull_request: ~

# Allow workflow to be dispatched on demand
workflow_dispatch: ~

env:
ECTRANS_TOOLS: ${{ github.workspace }}/.github/tools
CTEST_PARALLEL_LEVEL: 1
CACHE_SUFFIX: v1 # Increase to force new cache to be created

jobs:
ci-hpc:
name: ci-hpc

strategy:
fail-fast: false # false: try to complete all jobs

matrix:
name:
- ac-gpu nvhpc

include:
- name: ac-gpu nvhpc
site: ac-batch
sbatch_options: |
#SBATCH --time=00:20:00
#SBATCH --nodes=1
#SBATCH --ntasks=2
#SBATCH --cpus-per-task=32
#SBATCH --gpus-per-task=1
#SBATCH --mem=200G
#SBATCH --qos=dg
runs-on: [self-hosted, linux, hpc]
env:
GH_TOKEN: ${{ github.token }}
steps:
- uses: ecmwf-actions/reusable-workflows/ci-hpc-generic@v2
with:
site: ${{ matrix.site }}
troika_user: ${{ secrets.HPC_CI_SSH_USER }}
sbatch_options: ${{ matrix.sbatch_options }}
template_data: |
modules:
- cmake
- ninja
- ecbuild
- prgenv/nvidia
- hpcx-openmpi/2.14.0-cuda
- fftw
cmake_options:
- -DENABLE_MPI=ON
- -DENABLE_ACC=ON
- -DENABLE_GPU=ON
dependencies:
ecmwf-ifs/fiat:
version: develop
cmake_options:
- -DENABLE_MPI=ON
template: |
{% for module in modules %}
module load {{module}}
{% endfor %}
{% for name, options in dependencies.items() %}
mkdir -p {{name}}
pushd {{name}}
git init
git remote add origin ${{ github.server_url }}/{{name}}
git fetch origin {{options['version']}}
git reset --hard FETCH_HEAD
cmake -G Ninja -S . -B build {{ options['cmake_options']|join(' ') }}
cmake --build build
cmake --install build --prefix install
popd
{% endfor %}
mkdir -p ${{ github.repository }}
pushd ${{ github.repository }}
git init
git remote add origin ${{ github.server_url }}/${{ github.repository }}
git fetch origin ${{ github.sha }}
git reset --hard FETCH_HEAD
popd
cmake -G Ninja -S ${{ github.repository }} -B build \
{% for name in dependencies %}
{% set org, proj = name.split('/') %}
-D{{proj}}_ROOT=$PWD/{{name}}/install \
{% endfor %}
{{ cmake_options|join(' ') }}
cmake --build build
ctest --test-dir build
20 changes: 14 additions & 6 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,11 @@ ecbuild_add_option( FEATURE GPU
DESCRIPTION "Compile GPU version of ectrans (Requires OpenACC or sufficient OpenMP offloading support)"
CONDITION (HAVE_HIP OR HAVE_CUDA) AND (HAVE_ACC OR HAVE_OMP) )

# Check CPU or GPU is enabled, and if not, abort
if( (NOT HAVE_CPU) AND (NOT HAVE_GPU) )
ecbuild_critical("Please enable one or both of the CPU and GPU features")
endif()

if( HAVE_GPU )
if( HAVE_ACC )
set( GPU_OFFLOAD "ACC" )
Expand Down Expand Up @@ -142,10 +147,10 @@ ecbuild_add_option( FEATURE GPU_GRAPHS_GEMM
CONDITION HAVE_GPU
DESCRIPTION "Enable graph-based optimisation of Legendre transform GEMM kernel" )

ecbuild_add_option( FEATURE GPU_GRAPHS_FFT
ecbuild_add_option( FEATURE GPU_GRAPHS_FFT
DEFAULT ON
CONDITION HAVE_GPU
DESCRIPTION "Enable graph-based optimisation of FFT kernels" )
DESCRIPTION "Enable graph-based optimisation of FFT kernels" )

if( BUILD_SHARED_LIBS )
set( GPU_STATIC_DEFAULT OFF )
Expand All @@ -171,13 +176,16 @@ ectrans_find_lapack()
ecbuild_add_option( FEATURE TESTS
DEFAULT ON
DESCRIPTION "Enable unit testing"
REQUIRED_PACKAGES "MPI COMPONENTS Fortran"
CONDITION HAVE_CPU )
REQUIRED_PACKAGES "MPI COMPONENTS Fortran" )

### Add sources and tests
### Add sources
include( ectrans_compile_options )
add_subdirectory( src )
add_subdirectory( tests )

### Add tests
if( HAVE_TESTS )
add_subdirectory( tests )
endif()

### Export
if( BUILD_SHARED_LIBS )
Expand Down
166 changes: 83 additions & 83 deletions src/programs/ectrans-benchmark-ifs.F90
Original file line number Diff line number Diff line change
Expand Up @@ -1094,6 +1094,17 @@ subroutine parse_grid(cgrid,ndgl,nloen)

!===================================================================================================

subroutine str2int(str, int, stat)

character(len=*), intent(in) :: str
integer, intent(out) :: int
integer, intent(out) :: stat
read(str, *, iostat=stat) int

end subroutine str2int

!===================================================================================================

function get_int_value(cname, iarg) result(value)

integer :: value
Expand Down Expand Up @@ -1130,6 +1141,78 @@ function get_str_value(cname, iarg) result(value)

!===================================================================================================

subroutine print_help(unit)

integer, optional :: unit
integer :: nout = 6
if (present(unit)) then
nout = unit
endif

write(nout, "(a)") ""

if (jprb == jprd) then
write(nout, "(a)") "NAME ectrans-benchmark-dp"
else
write(nout, "(a)") "NAME ectrans-benchmark-sp"
end if
write(nout, "(a)") ""

write(nout, "(a)") "DESCRIPTION"
write(nout, "(a)") " This program tests ecTrans by transforming fields back and forth&
& between spectral "
if (jprb == jprd) then
write(nout, "(a)") " space and grid-point space (double-precision version)"
else
write(nout, "(a)") " space and grid-point space (single-precision version)"
end if
write(nout, "(a)") ""

write(nout, "(a)") "USAGE"
if (jprb == jprd) then
write(nout, "(a)") " ectrans-benchmark-dp [options]"
else
write(nout, "(a)") " ectrans-benchmark-sp [options]"
end if
write(nout, "(a)") ""

write(nout, "(a)") "OPTIONS"
write(nout, "(a)") " -h, --help Print this message"
write(nout, "(a)") " -v Run with verbose output"
write(nout, "(a)") " -t, --truncation T Run with this triangular spectral truncation&
& (default = 79)"
write(nout, "(a)") " -g, --grid GRID Run with this grid. Possible values: O<N>, F<N>"
write(nout, "(a)") " If not specified, O<N> is used with N=truncation+1&
& (cubic relation)"
write(nout, "(a)") " -n, --niter NITER Run for this many inverse/direct transform&
& iterations (default = 10)"
write(nout, "(a)") " -f, --nfld NFLD Number of scalar fields (default = 1)"
write(nout, "(a)") " -l, --nlev NLEV Number of vertical levels (default = 1)"
write(nout, "(a)") " --vordiv Also transform vorticity-divergence to wind"
write(nout, "(a)") " --scders Compute scalar derivatives (default off)"
write(nout, "(a)") " --uvders Compute uv East-West derivatives (default off). Only&
& when also --vordiv is given"
write(nout, "(a)") " --flt Run with fast Legendre transforms (default off)"
write(nout, "(a)") " --nproma NPROMA Run with NPROMA (default no blocking: NPROMA=ngptot)"
write(nout, "(a)") " --norms Calculate and print spectral norms of transformed&
& fields"
write(nout, "(a)") " The computation of spectral norms will skew overall&
& timings"
write(nout, "(a)") " --meminfo Show diagnostic information from FIAT's ec_meminfo&
& subroutine on memory usage, thread-binding etc."
write(nout, "(a)") " --nprtrv Size of V set in spectral decomposition"
write(nout, "(a)") " --nprtrw Size of W set in spectral decomposition"
write(nout, "(a)") " -c, --check VALUE The multiplier of the machine epsilon used as a&
& tolerance for correctness checking"
write(nout, "(a)") ""
write(nout, "(a)") "DEBUGGING"
write(nout, "(a)") " --dump-values Output gridpoint fields in unformatted binary file"
write(nout, "(a)") ""

end subroutine print_help

!===================================================================================================

subroutine parsing_failed(message)

character(len=*), intent(in) :: message
Expand Down Expand Up @@ -1240,17 +1323,6 @@ function cubic_octahedral_gaussian_grid(nsmax) result(cgrid)

!===================================================================================================

subroutine str2int(str, int, stat)

character(len=*), intent(in) :: str
integer, intent(out) :: int
integer, intent(out) :: stat
read(str, *, iostat=stat) int

end subroutine str2int

!===================================================================================================

subroutine sort(a, n)

real(kind=jprd), intent(inout) :: a(n)
Expand All @@ -1275,78 +1347,6 @@ end subroutine sort

!===================================================================================================

subroutine print_help(unit)

integer, optional :: unit
integer :: nout = 6
if (present(unit)) then
nout = unit
endif

write(nout, "(a)") ""

if (jprb == jprd) then
write(nout, "(a)") "NAME ectrans-benchmark-dp"
else
write(nout, "(a)") "NAME ectrans-benchmark-sp"
end if
write(nout, "(a)") ""

write(nout, "(a)") "DESCRIPTION"
write(nout, "(a)") " This program tests ecTrans by transforming fields back and forth&
& between spectral "
if (jprb == jprd) then
write(nout, "(a)") " space and grid-point space (double-precision version)"
else
write(nout, "(a)") " space and grid-point space (single-precision version)"
end if
write(nout, "(a)") ""

write(nout, "(a)") "USAGE"
if (jprb == jprd) then
write(nout, "(a)") " ectrans-benchmark-dp [options]"
else
write(nout, "(a)") " ectrans-benchmark-sp [options]"
end if
write(nout, "(a)") ""

write(nout, "(a)") "OPTIONS"
write(nout, "(a)") " -h, --help Print this message"
write(nout, "(a)") " -v Run with verbose output"
write(nout, "(a)") " -t, --truncation T Run with this triangular spectral truncation&
& (default = 79)"
write(nout, "(a)") " -g, --grid GRID Run with this grid. Possible values: O<N>, F<N>"
write(nout, "(a)") " If not specified, O<N> is used with N=truncation+1&
& (cubic relation)"
write(nout, "(a)") " -n, --niter NITER Run for this many inverse/direct transform&
& iterations (default = 10)"
write(nout, "(a)") " -f, --nfld NFLD Number of scalar fields (default = 1)"
write(nout, "(a)") " -l, --nlev NLEV Number of vertical levels (default = 1)"
write(nout, "(a)") " --vordiv Also transform vorticity-divergence to wind"
write(nout, "(a)") " --scders Compute scalar derivatives (default off)"
write(nout, "(a)") " --uvders Compute uv East-West derivatives (default off). Only&
& when also --vordiv is given"
write(nout, "(a)") " --flt Run with fast Legendre transforms (default off)"
write(nout, "(a)") " --nproma NPROMA Run with NPROMA (default no blocking: NPROMA=ngptot)"
write(nout, "(a)") " --norms Calculate and print spectral norms of transformed&
& fields"
write(nout, "(a)") " The computation of spectral norms will skew overall&
& timings"
write(nout, "(a)") " --meminfo Show diagnostic information from FIAT's ec_meminfo&
& subroutine on memory usage, thread-binding etc."
write(nout, "(a)") " --nprtrv Size of V set in spectral decomposition"
write(nout, "(a)") " --nprtrw Size of W set in spectral decomposition"
write(nout, "(a)") " -c, --check VALUE The multiplier of the machine epsilon used as a&
& tolerance for correctness checking"
write(nout, "(a)") ""
write(nout, "(a)") "DEBUGGING"
write(nout, "(a)") " --dump-values Output gridpoint fields in unformatted binary file"
write(nout, "(a)") ""

end subroutine print_help

!===================================================================================================

subroutine initialize_spectral_arrays(nsmax, zsp, sp3d)

integer, intent(in) :: nsmax ! Spectral truncation
Expand Down
6 changes: 0 additions & 6 deletions src/trans/common/internal/tpm_dim.F90
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,4 @@ MODULE TPM_DIM
TYPE(DIM_TYPE),ALLOCATABLE,TARGET :: DIM_RESOL(:)
TYPE(DIM_TYPE),POINTER :: R

! flat copies of above
INTEGER(KIND=JPIM) :: R_NSMAX ! Truncation order
INTEGER(KIND=JPIM) :: R_NTMAX ! Truncation order for tendencies
INTEGER(KIND=JPIM) :: R_NDGNH ! Number of rows in northern hemisphere
INTEGER(KIND=JPIM) :: R_NDGL ! Number of rows of latitudes

END MODULE TPM_DIM
24 changes: 0 additions & 24 deletions src/trans/common/internal/tpm_distr.F90
Original file line number Diff line number Diff line change
Expand Up @@ -178,29 +178,5 @@ MODULE TPM_DISTR
TYPE(DISTR_TYPE),ALLOCATABLE,TARGET :: DISTR_RESOL(:)
TYPE(DISTR_TYPE),POINTER :: D

!flat versions of the above
INTEGER(KIND=JPIM) :: D_NUMP ! No. of spectral waves handled by this processor
INTEGER(KIND=JPIM) ,ALLOCATABLE :: D_MYMS(:) ! Wave numbers handled by this PE
INTEGER(KIND=JPIM) ,ALLOCATABLE :: D_NSTAGT0B(:) ! Start adresses for segments within buffer
! (according to processors to whom data
! is going to be sent)
INTEGER(KIND=JPIM) ,ALLOCATABLE :: D_NSTAGT1B(:)
INTEGER(KIND=JPIM) ,ALLOCATABLE :: D_NPROCL(:) ! Process responsible for each lat. (F.S)
INTEGER(KIND=JPIM) ,ALLOCATABLE :: D_NPNTGTB1(:,:)
INTEGER(KIND=JPIM) ,ALLOCATABLE :: D_NASM0(:) ! Address in a spectral array of (m, n=m)
INTEGER(KIND=JPIM) ,ALLOCATABLE :: D_NSTAGTF(:) ! Offset for specific latitude in
INTEGER(KIND=JPIM) :: D_NDGL_FS ! Number of rows of latitudes for which this process is
! performing Fourier Space calculations
INTEGER(KIND=JPIM) ,ALLOCATABLE :: D_MSTABF(:)
INTEGER(KIND=JPIM) ,ALLOCATABLE :: D_NPNTGTB0(:,:)
INTEGER(KIND=JPIM) ,ALLOCATABLE :: D_NPROCM(:) ! Process that does the calc. for certain
INTEGER(KIND=JPIM) ,ALLOCATABLE :: D_NPTRLS(:) ! Pointer to first lat. (F.S)


! The offsets in the input and output arrays to the gemms.
! (1) are the offsets in the "inputs" of dirtrans ("outputs" invtrans)
! (2) are the offsets in the "outputs" of invtrans ("inputs" dirtrans)
INTEGER(KIND=JPIM), POINTER :: D_OFFSETS_GEMM1(:), D_OFFSETS_GEMM2(:)

END MODULE TPM_DISTR

Loading

0 comments on commit 038023b

Please sign in to comment.