From 89002e384647277ddabbafe89967992d4a72a447 Mon Sep 17 00:00:00 2001 From: Daniel Price Date: Tue, 7 Feb 2023 21:07:51 +1100 Subject: [PATCH 1/4] (inject) bug fix with particle injection+restart --- src/main/initial.F90 | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/main/initial.F90 b/src/main/initial.F90 index 570d700aa..7cef6fea6 100644 --- a/src/main/initial.F90 +++ b/src/main/initial.F90 @@ -189,6 +189,8 @@ subroutine startrun(infile,logfile,evfile,dumpfile,noread) use dust_formation, only:init_nucleation #ifdef INJECT_PARTICLES use inject, only:init_inject,inject_particles + use partinject, only:update_injected_particles + use timestep_ind, only:nbinmax #endif #ifdef KROME use krome_interface, only:initialise_krome @@ -231,6 +233,7 @@ subroutine startrun(infile,logfile,evfile,dumpfile,noread) character(len=7) :: dust_label(maxdusttypes) #ifdef INJECT_PARTICLES character(len=len(dumpfile)) :: file1D + integer :: npart_old #endif @@ -548,16 +551,10 @@ subroutine startrun(infile,logfile,evfile,dumpfile,noread) endif call rename('wind_profile1D.dat',trim(file1D)) endif + npart_old = npart call inject_particles(time,0.,xyzh,vxyzu,xyzmh_ptmass,vxyz_ptmass,& npart,npartoftype,dtinject) -#ifdef GR -! call update_injected_particles(npart_old,npart,istepfrac,nbinmax,time,dtmax,dt,dtinject) - call init_metric(npart,xyzh,metrics,metricderivs) - call prim2consall(npart,xyzh,metrics,vxyzu,dens,pxyzu,use_dens=.false.) - if (iexternalforce > 0 .and. imetric /= imet_minkowski) then - call get_grforce_all(npart,xyzh,metrics,metricderivs,vxyzu,dens,fext,dtextforce) ! Not 100% sure if this is needed here - endif -#endif + call update_injected_particles(npart_old,npart,istepfrac,nbinmax,time,dtmax,dt,dtinject) #endif ! !--set initial chemical abundance values From 3811a19ec89e6d834325826a2784f364c9cd8495 Mon Sep 17 00:00:00 2001 From: Daniel Price Date: Tue, 7 Feb 2023 21:08:24 +1100 Subject: [PATCH 2/4] (energies) floating point exception fixed if accradius1 = 0 --- src/main/energies.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/energies.F90 b/src/main/energies.F90 index f7bcfae93..3e0cbce48 100644 --- a/src/main/energies.F90 +++ b/src/main/energies.F90 @@ -711,7 +711,7 @@ subroutine compute_energies(t) if (track_mass) then accretedmass = ev_data(iev_sum,iev_macc) - ev_data(iev_sum,iev_eacc) = accretedmass/accradius1 ! total accretion energy + if (accradius1 > 0.) ev_data(iev_sum,iev_eacc) = accretedmass/accradius1 ! total accretion energy endif if (track_lum) totlum = ev_data(iev_sum,iev_totlum) From 8f54177476a337d8546ec4cc174bd56347419a13 Mon Sep 17 00:00:00 2001 From: Daniel Price Date: Thu, 9 Feb 2023 10:08:10 +1100 Subject: [PATCH 3/4] (nci) updated docs and queue script to use /g/data instead of /scratch on nci --- build/Makefile_qscripts | 3 +++ build/Makefile_systems | 3 ++- docs/nci.rst | 23 +++++++++++++---------- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/build/Makefile_qscripts b/build/Makefile_qscripts index 7756eae00..fdb3efcdf 100644 --- a/build/Makefile_qscripts +++ b/build/Makefile_qscripts @@ -148,6 +148,9 @@ qscript: ifdef QEXTRA @echo '#PBS '$(QEXTRA) endif + ifdef QSTORAGE + @echo '#PBS -l storage='$(QSTORAGE) + endif @echo '## phantom jobs can be restarted:' @echo '#PBS -r y' ifeq ($(PBSRESUBMIT),yes) diff --git a/build/Makefile_systems b/build/Makefile_systems index 63ca84411..509a0b28b 100644 --- a/build/Makefile_systems +++ b/build/Makefile_systems @@ -151,12 +151,13 @@ ifeq ($(SYSTEM), nci) ifeq ($(MPI),yes) NPAR=32 endif - QPROJECT='fu7' + QPROJECT=$(PROJECT) QNAME='normal' WALLTIME='48:00:00' MPIEXEC='mpiexec -npernode 1' QNODES='ncpus='$(NPAR) QEXTRA='-l other=hyperthread' + QSTORAGE="gdata/$(QPROJECT)" endif ifeq ($(SYSTEM), gfortran) diff --git a/docs/nci.rst b/docs/nci.rst index 3eb571aee..d15cf4485 100644 --- a/docs/nci.rst +++ b/docs/nci.rst @@ -22,10 +22,12 @@ First edit your .bashrc file in your favourite text editor:: Mine has:: export SYSTEM=nci + export PROJECT=wk74 export OMP_STACKSIZE=512M export OMP_NUM_THREADS=32 - export PATH=$PATH:/scratch/fu7/splash/bin - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/scratch/fu7/splash/giza/lib + export SPLASH_DIR=/g/data/$PROJECT/splash + export PATH=$PATH:$SPLASH_DIR/bin + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SPLASH_DIR/giza/lib export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/apps/hdf5/1.10.5/lib export MAXP=2000000 ulimit -s unlimited @@ -33,13 +35,13 @@ Mine has:: If you are using phantom+mcfost, you will need the following lines:: - export MCFOST_DIR=/scratch/fu7/mcfost-src/ + export MCFOST_DIR=/g/data/$PROJECT/mcfost-src/ export MCFOST_AUTO_UPDATE=0 - export MCFOST_INSTALL=/scratch/fu7/mcfost/ + export MCFOST_INSTALL=/g/data/$PROJECT/mcfost/ export MCFOST_GIT=1 export MCFOST_NO_XGBOOST=1 - export MCFOST_LIBS=/scratch/fu7/mcfost - export MCFOST_UTILS=/scratch/fu7/mcfost/utils + export MCFOST_LIBS=/g/data/$PROJECT/mcfost + export MCFOST_UTILS=/g/data/$PROJECT/mcfost/utils export HDF5ROOT=/apps/hdf5/1.10.5/lib Then relevant modules in your .modules file:: @@ -55,12 +57,12 @@ Mine contains:: where the last line is needed for git's large file storage (LFS) to work. -Finally, make a shortcut to the /scratch filesystem:: +Finally, make a shortcut to the /g/data filesystem:: - cd /scratch/fu7 + cd /g/data/$PROJECT mkdir $USER cd - ln -s /scratch/fu7/$USER runs + ln -s /scratch/$PROJECT/$USER runs cd runs pwd -P @@ -104,13 +106,14 @@ should produce something like:: #PBS -l ncpus=48 #PBS -N myrun #PBS -q normal - #PBS -P fu7 + #PBS -P wk74 #PBS -o tde.in.pbsout #PBS -j oe #PBS -m e #PBS -M daniel.price@monash.edu #PBS -l walltime=48:00:00 #PBS -l mem=16G + #PBS -l storage=gdata/wk74 #PBS -l other=hyperthread ## phantom jobs can be restarted: #PBS -r y From 61e1273cf220c37868dec8586084753d0a1cf154 Mon Sep 17 00:00:00 2001 From: Daniel Price Date: Thu, 9 Feb 2023 10:16:22 +1100 Subject: [PATCH 4/4] (damping bcs) build failure with non-ind-timesteps+injection fixed --- src/main/initial.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/initial.F90 b/src/main/initial.F90 index 7cef6fea6..543cf9fad 100644 --- a/src/main/initial.F90 +++ b/src/main/initial.F90 @@ -152,10 +152,10 @@ subroutine startrun(infile,logfile,evfile,dumpfile,noread) h_acc,r_crit,r_crit2,rho_crit,rho_crit_cgs,icreate_sinks, & r_merge_uncond,r_merge_cond,r_merge_uncond2,r_merge_cond2,r_merge2 use timestep, only:time,dt,dtextforce,C_force,dtmax + use timestep_ind, only:istepfrac use timing, only:get_timings #ifdef IND_TIMESTEPS - use timestep, only:dtmax - use timestep_ind, only:istepfrac,ibinnow,maxbins,init_ibin + use timestep_ind, only:ibinnow,maxbins,init_ibin use part, only:ibin,ibin_old,ibin_wake,alphaind use readwrite_dumps, only:dt_read_in #else