BaroWave Scripts

BatchScripts/levante/BaroWave_128Elem.sh

@@ -1,7 +1,10 @@
 #!/bin/bash
 #SBATCH --job-name=my_gpu_job      # Specify job name
 #SBATCH --partition=gpu            # Specify partition name
-#SBATCH --gpus=8
+#SBATCH --gpus-per-node=4
+#SBATCH --nodes=2   # -> 8GPUs
+#SBATCH --tasks-per-node=4
+#SBATCH --cpus-per-task=64
 #SBATCH --exclusive
 #SBATCH --mem=0                    # Request all memory available on all nodes
 #SBATCH --time=00:30:00            # Set a limit on the total run time
@@ -20,5 +23,6 @@ ulimit -s 204800
 
 export JuliaDevice="GPU"
 export JuliaGPU="CUDA"
+export machine="levante"
 export UCX_ERROR_SIGNALS=""
 srun -n 8 gpu_wrapper.sh -n 8 -e "./Jobs/NHSphere/BaroWaveDrySphere_128Elem"

BatchScripts/levante/BaroWave_256Elem.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+#SBATCH --job-name=my_gpu_job      # Specify job name
+#SBATCH --partition=gpu            # Specify partition name
+#SBATCH --gpus-per-node=4
+#SBATCH --nodes=2   # -> 8GPUs
+#SBATCH --tasks-per-node=4
+#SBATCH --cpus-per-task=64
+#SBATCH --exclusive
+#SBATCH --mem=0                    # Request all memory available on all nodes
+#SBATCH --time=00:30:00            # Set a limit on the total run time
+#SBATCH --mail-type=FAIL           # Notify user by email in case of job failure
+#SBATCH --account=bb1143           # Charge resources on this project account
+#SBATCH --output=OutRace           # File name for standard output
+
+set -e
+ulimit -s 204800
+
+# Check GPUs available for the job
+# nvidia-smi
+
+# Check GPUs visible for each task
+# srun -l nvidia-smi
+
+export JuliaDevice="GPU"
+export JuliaGPU="CUDA"
+export machine="levante"
+export UCX_ERROR_SIGNALS=""
+srun -n 8 gpu_wrapper.sh -n 8 -e "./Jobs/NHSphere/BaroWaveDrySphere_256Elem"

Examples/testNHSphere.jl

@@ -116,14 +116,18 @@ ParallelCom.ProcNumber  = ProcNumber
 
 JuliaDevice = get(ENV, "JuliaDevice", "CPU")
 JuliaGPU = get(ENV, "JuliaGPU", "CUDA")
+machine = get(ENV, "machine", "")
 
 if JuliaDevice == "CPU"
   backend = CPU()
 elseif JuliaDevice == "GPU"
   if JuliaGPU == "CUDA"
     backend = CUDABackend()
     CUDA.allowscalar(false)
-    CUDA.device!(MPI.Comm_rank(MPI.COMM_WORLD))
+    if machine == "levante"
+    else
+       CUDA.device!(MPI.Comm_rank(MPI.COMM_WORLD))
+    end
   elseif JuliaGPU == "AMD"
     backend = ROCBackend()
     AMDGPU.allowscalar(false)

Jobs/NHSphere/BaroWaveDrySphere_128Elem

@@ -36,7 +36,7 @@ julia --project Examples/testNHSphere.jl \
   --StretchType="Exp" \
   --GridType="CubedSphere" \
   --nz=64 \
-  --nPanel=32 \
+  --nPanel=128 \
   --H=30000.0 \
   --OrdPoly=3 \
   --HyperVisc=true \

Jobs/NHSphere/BaroWaveDrySphere_256Elem

@@ -0,0 +1,45 @@
+#!/bin/bash
+julia --project Examples/testNHSphere.jl \
+  --Problem="BaroWaveDrySphere" \
+  --FloatTypeBackend="Float32" \
+  --NumberThreadGPU=512 \
+  --NumV=5 \
+  --NumTr=0 \
+  --ProfpBGrd="" \
+  --ProfRhoBGrd="" \
+  --Source=false \
+  --Forcing=false \
+  --Equation="CompressibleShallow" \
+  --State="Dry" \
+  --Buoyancy=true \
+  --Curl=false \
+  --ModelType="VectorInvariant" \
+  --Coriolis=true \
+  --VerticalDiffusion=false \
+  --Upwind=true \
+  --HorLimit=false \
+  --Decomp="EqualArea" \
+  --SimDays=1 \
+  --SimHours=0 \
+  --SimSeconds=0 \
+  --PrintSeconds=0 \
+  --PrintMinutes=0 \
+  --PrintHours=0 \
+  --PrintDays=0 \
+  --StartAverageDays=100 \
+  --Flat=true \
+  --dtau=18.75 \
+  --IntMethod="Rosenbrock" \
+  --Table="SSP-Knoth" \
+  --TopoS="" \
+  --Stretch=true \
+  --StretchType="Exp" \
+  --GridType="CubedSphere" \
+  --nz=64 \
+  --nPanel=256 \
+  --H=30000.0 \
+  --OrdPoly=3 \
+  --HyperVisc=true \
+  --HyperDCurl=7.8125e12 \
+  --HyperDGrad=7.8125e12 \
+  --HyperDDiv=7.8125e12 \