Skip to content

Commit

Permalink
Merge pull request #34 from chowland/re-module
Browse files Browse the repository at this point in the history 
Restructuring of multi-resolution additions into clean, concise modules
  • Loading branch information
@chowland
chowland authored Sep 11, 2023
2 parents 6a669a5 + 792f246 commit cf92876
Show file tree
Hide file tree
Showing 81 changed files with 3,685 additions and 1,382 deletions.
136 changes: 72 additions & 64 deletions Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,17 @@
# Choose the machine being used
# Options: PC_GNU, PC_INTEL, (i)SNELLIUS, IRENE(_SKL/_ROME), MARENOSTRUM, SUPERMUC
MACHINE=PC_GNU
# Options: PC, SNELLIUS, IRENE, MARENOSTRUM, SUPERMUC, DISCOVERER
MACHINE=PC
FLAVOUR=GNU
# Modules required for each HPC system as follows:
# SNELLIUS: 2022 foss/2022a HDF5/1.12.2-gompi-2022a
# iSNELLIUS: 2022 intel/2022a FFTW/3.3.10-GCC-11.3.0 HDF5/1.12.2-iimpi-2021a
# IRENE: flavor/hdf5/parallel hdf5 fftw3/gnu
# MARENOSTRUM: fabric intel mkl impi hdf5 fftw szip
# SUPERMUC: fftw hdf5
# SNELLIUS:
# GNU: 2022 foss/2022a HDF5/1.12.2-gompi-2022a
# Intel: 2022 intel/2022a FFTW/3.3.10-GCC-11.3.0 HDF5/1.12.2-iimpi-2021a
# IRENE (Intel): flavor/hdf5/parallel hdf5 fftw3/gnu
# MARENOSTRUM (Intel): fabric intel mkl impi hdf5 fftw szip
# SUPERMUC (Intel): fftw hdf5
# DISCOVERER:
# GNU: hdf5/1/1.14/latest-gcc-openmpi fftw/3/latest-gcc-openmpi lapack
# Intel: hdf5/1/1.14/latest-intel-openmpi fftw/3/latest-gcc-openmpi mkl

#=======================================================================
# Compiler options
Expand All @@ -15,54 +20,60 @@ MACHINE=PC_GNU
# Object and module directory:
OBJDIR=obj

ifeq ($(MACHINE),PC_GNU)
FC = h5pfc -cpp -fdefault-real-8 -fdefault-double-8 -O3
# FC += -pg -fbacktrace -fbounds-check
LDFLAGS = -lfftw3 -llapack -lblas -ldl
ifeq ($(FLAVOUR),GNU)
FC = h5pfc -cpp -fdefault-real-8 -fdefault-double-8 -fallow-argument-mismatch
else
FC = h5pfc -fpp -r8
endif
ifeq ($(MACHINE),PC_INTEL)
FC = h5pfc -fpp -r8 -O3
## Traceback / Debug
# FC += -r8 -O0 -g -traceback -check bounds
# FC += -DSHM -DSHM_DEBUG
LDFLAGS = -lfftw3 -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -lpthread -lhdf5_fortran -lhdf5 -lsz -lz -ldl -lm

ifeq ($(MACHINE),PC)
# GNU Debug Flags
# FC += -O0 -g -fbacktrace -Wall -Wextra
# FC += -Wpedantic
# FC += -Warray-temporaries
# FC += -fcheck=all -finit-real=snan -ffpe-trap=invalid #-std=f2018
# FC += -O0 -pg -fbacktrace -fbounds-check
# Intel Debug Flags
# FC += -O0 -g -traceback -check bounds
ifeq ($(FLAVOUR),GNU)
LDFLAGS = -L$(HOME)/fftw-install/lib -lfftw3 -llapack -ldl
else
LDFLAGS = -lfftw3 -qmkl=sequential
endif
endif
ifeq ($(MACHINE),iSNELLIUS)
FC = h5pfc -fpp -r8 -O3 -align array64byte -fma -ftz -fomit-frame-pointer
LDFLAGS = -lfftw3 -qmkl=sequential
ifeq ($(MACHINE),DISCOVERER)
ifeq ($(FLAVOUR),GNU)
LDFLAGS += -lfftw3 -llapack -ldl
else
LDFLAGS += -lfftw3 -qmkl=sequential
endif
endif
ifeq ($(MACHINE),SNELLIUS)
FC = h5pfc -cpp -fdefault-real-8 -fdefault-double-8 -w -fallow-argument-mismatch
FC += -O2 -march=znver1 -mtune=znver1 -mfma -mavx2 -m3dnow -fomit-frame-pointer
BLAS_LIBS = -lscalapack -lopenblas -ldl
LDFLAGS = -lfftw3 $(BLAS_LIBS)
ifeq ($(FLAVOUR),GNU)
FC += -O2 -march=znver1 -mtune=znver1 -mfma -mavx2 -m3dnow -fomit-frame-pointer
LDFLAGS = -lfftw3 -lopenblas -ldl
else
FC += -align array64byte -fma -ftz -fomit-frame-pointer
LDFLAGS = -lfftw3 -qmkl=sequential
endif
endif
ifeq ($(MACHINE),IRENE_SKL)
FC = h5pfc -fpp -r8 -O3 -mtune=skylake -xCORE-AVX512 -m64 -fPIC $(FFTW3_FFLAGS)
ifeq ($(MACHINE),IRENE)
FC += -mtune=skylake -xCORE-AVX512 -m64 -fPIC $(FFTW3_FFLAGS)
LDFLAGS = $(FFTW3_LDFLAGS) $(MKL_LDFLAGS) -ldl
endif
ifeq ($(MACHINE),IRENE_ROME)
FC = h5pfc -fpp -r8 -O3 -mavx2 $(FFTW3_FFLAGS)
HDF5_LIBS = -lhdf5_fortran -lhdf5 -lz -ldl -lm
LDFLAGS = $(FFTW3_LDFLAGS) $(MKL_LDFLAGS) $(HDF5_LIBS)
endif
ifeq ($(MACHINE),MARENOSTRUM)
FC = h5pfc -fpp -r8 -O3 -mtune=skylake -xCORE-AVX512 -m64 -fPIC $(FFTW_FFLAGS)
FC += -mtune=skylake -xCORE-AVX512 -m64 -fPIC $(FFTW_FFLAGS)
LDFLAGS = $(FFTW_LIBS) -mkl=sequential
endif
ifeq ($(MACHINE),SUPERMUC)
FC = mpif90 -fpp -r8 -O3 $(HDF5_INC)
LDFLAGS = $(FFTW_LIB) $(HDF5_F90_SHLIB) $(HDF5_SHLIB) -qmkl=sequential
endif

ifeq ($(MACHINE),SNELLIUS)
ifeq ($(FLAVOUR),GNU)
FC += -J $(OBJDIR)
else
ifeq ($(MACHINE),PC_GNU)
FC += -J $(OBJDIR)
else
FC += -module $(OBJDIR)
endif
FC += -module $(OBJDIR)
endif

#=======================================================================
Expand All @@ -73,42 +84,31 @@ EXTRA_DIST = transpose_z_to_x.F90 transpose_x_to_z.F90 transpose_x_to_y.F90\
factor.F90 halo.F90 fft_common.F90 alloc.F90 halo_common.F90

# Object files associated with standard flow solver
OBJS = obj/main.o obj/CalcLocalDivergence.o obj/CalcMaxCFL.o \
obj/CalcMeanProfiles.o obj/CheckDivergence.o obj/CorrectPressure.o \
OBJS = obj/main.o obj/CalcMaxCFL.o \
obj/CalcMeanProfiles.o obj/CheckDivergence.o \
obj/CorrectVelocity.o obj/CreateGrid.o obj/CreateInitialConditions.o \
obj/DeallocateVariables.o obj/DebugRoutines.o obj/ExplicitTermsTemp.o \
obj/ExplicitTermsVX.o obj/ExplicitTermsVY.o obj/ExplicitTermsVZ.o \
obj/factorize.o obj/HdfReadContinua.o obj/HdfRoutines.o \
obj/ImplicitAndUpdateTemp.o obj/ImplicitAndUpdateVX.o obj/ImplicitAndUpdateVY.o \
obj/ImplicitAndUpdateVZ.o obj/InitPressureSolver.o obj/InitTimeMarchScheme.o \
obj/ImplicitAndUpdateVZ.o obj/InitTimeMarchScheme.o \
obj/InitVariables.o obj/LocateLargeDivergence.o obj/MakeMovieXCut.o \
obj/MakeMovieYCut.o obj/MakeMovieZCut.o obj/MpiAuxRoutines.o \
obj/QuitRoutine.o obj/ReadInputFile.o obj/ResetLogs.o \
obj/SetTempBCs.o obj/SolveImpEqnUpdate_Temp.o obj/SolveImpEqnUpdate_X.o \
obj/SolveImpEqnUpdate_YZ.o obj/SolvePressureCorrection.o obj/SpecRoutines.o \
obj/SolveImpEqnUpdate_YZ.o obj/SpecRoutines.o \
obj/TimeMarcher.o obj/WriteFlowField.o obj/WriteGridInfo.o \
obj/CalcWriteQ.o obj/GlobalQuantities.o obj/ReadFlowInterp.o

# Object files associated with multiple resolution grids
OBJS += obj/CreateMgrdGrid.o obj/InitMgrdVariables.o \
obj/DeallocateMgrdVariables.o obj/CreateMgrdStencil.o# obj/CreateMgrdStencil.o
obj/DeallocateMgrdVariables.o obj/CreateMgrdStencil.o

# Object files associated with initial condition interpolation
OBJS += obj/CreateNewInputStencil.o obj/CreateOldGrid.o obj/CreateNewSalStencil.o \
obj/InterpInputSal.o obj/InterpInputVel.o obj/InterpSalMgrd.o \
obj/InterpInputSal.o obj/InterpInputVel.o \
obj/InterpVelMgrd.o obj/InitInputVars.o obj/DeallocateInputVars.o \
obj/InterpInputPhi.o# obj/CreateInputStencil.o obj/CreateSalStencil.o

# Object files associated with the salinity field
OBJS += obj/ExplicitTermsSal.o obj/ImplicitAndUpdateSal.o obj/SolveImpEqnUpdate_Sal.o \
obj/UpdateScalarBCs.o obj/CreateICSal.o obj/InitSalVariables.o \
obj/DeallocateSalVariables.o obj/SetSalBCs.o

# Object files associated with the phase-field method
OBJS += obj/AddLatentHeat.o obj/DeallocatePFVariables.o obj/ExplicitTermsPhi.o \
obj/ImplicitAndUpdatePhi.o obj/InitPFVariables.o obj/InterpPhiMgrd.o \
obj/InterpTempMgrd.o obj/SolveImpEqnUpdate_Phi.o obj/CreateICPF.o \
obj/ImmersedBoundary.o
obj/InterpInputPhi.o

# # Object files associated with the immersed boundary method
OBJS += obj/SolveImpEqnUpdate_Temp_ibm.o obj/SolveImpEqnUpdate_X_ibm.o \
Expand All @@ -120,13 +120,15 @@ OBJS += obj/mean_zplane.o

# Module object files
MOBJS = obj/param.o obj/decomp_2d.o obj/AuxiliaryRoutines.o obj/decomp_2d_fft.o \
obj/HermiteInterpolations.o obj/GridModule.o obj/h5_tools.o obj/means.o obj/ibm_param.o
obj/pressure.o obj/HermiteInterpolations.o obj/grid.o obj/h5_tools.o obj/means.o \
obj/ibm_param.o obj/IBMTools.o obj/moisture.o obj/salinity.o obj/phasefield.o

#=======================================================================
# Files that create modules:
#=======================================================================
MFILES = param.F90 decomp_2d.F90 AuxiliaryRoutines.F90 decomp_2d_fft.F90 \
HermiteInterpolations.F90 GridModule.F90 ibm_param.F90
pressure.F90 HermiteInterpolations.F90 grid.F90 ibm_param.F90 IBMTools.F90 \
moisture.F90 salinity.F90 phasefield.F90

#============================================================================
# make PROGRAM
Expand All @@ -151,14 +153,24 @@ $(OBJDIR)/decomp_2d_fft.o: src/flow_solver/2decomp/decomp_2d_fft.F90
$(FC) -c -o $@ $< $(LDFLAGS)
$(OBJDIR)/ibm_param.o: src/ibm/ibm_param.F90
$(FC) -c -o $@ $< $(LDFLAGS)
$(OBJDIR)/GridModule.o: src/flow_solver/GridModule.F90
$(OBJDIR)/grid.o: src/grid.F90
$(FC) -c -o $@ $< $(LDFLAGS)
$(OBJDIR)/HermiteInterpolations.o: src/multires/HermiteInterpolations.F90
$(OBJDIR)/pressure.o: src/pressure.F90
$(FC) -c -o $@ $< $(LDFLAGS)
$(OBJDIR)/HermiteInterpolations.o: src/multires/HermiteInterpolations.F90 obj/ibm_param.o
$(FC) -c -o $@ $< $(LDFLAGS)
$(OBJDIR)/h5_tools.o: src/h5tools/h5_tools.F90
$(FC) -c -o $@ $< $(LDFLAGS)
$(OBJDIR)/means.o: src/h5tools/means.F90 obj/ibm_param.o
$(FC) -c -o $@ $< $(LDFLAGS)
$(OBJDIR)/IBMTools.o: src/ibm/IBMTools.F90
$(FC) -c -o $@ $< $(LDFLAGS)
$(OBJDIR)/salinity.o: src/salinity.F90
$(FC) -c -o $@ $< $(LDFLAGS)
$(OBJDIR)/phasefield.o: src/phasefield.F90 obj/salinity.o
$(FC) -c -o $@ $< $(LDFLAGS)
$(OBJDIR)/moisture.o: src/moisture.F90
$(FC) -c -o $@ $< $(LDFLAGS)
$(OBJDIR)/%.o: src/%.F90 $(MOBJS)
$(FC) -c -o $@ $< $(LDFLAGS)
$(OBJDIR)/%.o: src/flow_solver/%.F90 $(MOBJS)
Expand All @@ -169,10 +181,6 @@ $(OBJDIR)/%.o: src/multires/%.F90 $(MOBJS)
$(FC) -c -o $@ $< $(LDFLAGS)
$(OBJDIR)/%.o: src/multires/IC_interpolation/%.F90 $(MOBJS)
$(FC) -c -o $@ $< $(LDFLAGS)
$(OBJDIR)/%.o: src/multires/phase-field/%.F90 $(MOBJS)
$(FC) -c -o $@ $< $(LDFLAGS)
$(OBJDIR)/%.o: src/multires/salinity/%.F90 $(MOBJS)
$(FC) -c -o $@ $< $(LDFLAGS)
$(OBJDIR)/%.o: src/ibm/%.F90 $(MOBJS)
$(FC) -c -o $@ $< $(LDFLAGS)

Expand Down
55 changes: 55 additions & 0 deletions examples/RainyBenard/bou.in
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
Base grid resolution / number of substeps
NXM NYM NZM NSST(>1=RK,else AB)
96 1024 1 3

Refined grid: on/off and grid size
MULTIRES NXMR NYMR NZMR
0 128 128 1

Salinity and Phase-field (on/off)
FLAGSAL FLAGPF
0 0

Read flow / Reset logs
NREAD IRESET
0 0

Time steps / time limit (wall/flow)
NTST WALLTIMEMAX TMAX
1000000 120 2500.0

Time interval for saving stats/movie frames
TOUT TFRAME SAVE_3D
1.0 5.0 100.0

Domain size (keep ALX3 = 1.0)
ALX3 YLEN ZLEN
1.0 20.0 0.01

Grid stretching parameters
ISTR3 STR3 ISTR3R
6 12.0 3

Physical flow parameters / Free-fall timescale
RAYT PRAT RAYS PRAS FFscaleS
2e5 1.0 1e7 1.0 0

Time stepping parameters
IDTV DT RESID CFLMAX DTMIN DTMAX
1 1e-1 1.e-3 1.0 1e-8 1e-1

Dirichlet/Neumann BC flag on upper(N) and lower(S) walls
inslwS inslwN TfixS TfixN SfixS SfixN
1 1 1 1 1 1

Active/passive scalar flags / Gravitational axis (x=1,y=2,z=3)
active_T active_S gAxis
1 0 1

Wall velocity / Pressure gradient / Melt boundary condition
xplusU xminusU dPdy dPdz MELT
0.0 0.0 0.0 0.0 0

Phase-field parameters
pf_D(*Pe) pf_A pf_S pf_Tm IBM pf_IC
0.1 10.0 10.0 0.0 0 1
24 changes: 24 additions & 0 deletions examples/RainyBenard/humid.in
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
An input file for parameters used in moist convection
---

alpha (Dimensionless saturation parameter)
3.0

beta (Dimensionless lapse rate)
1.2

gamma (Dimensioness latent heat parameter)
0.19

tau (Condensation time scale)
0.375

Diffusive scale for tau? (.true. / .false.)
.false.

Fixed humidity boundary conditions (1 / 0)
top bottom
1 1

Sm (Humid-to-thermal diffusivity ratio)
1.0
12 changes: 8 additions & 4 deletions src/ReadFlowInterp.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ subroutine ReadFlowInterp(prow,pcol)
use local_arrays
use param
use input_grids
use mgrd_arrays
use afid_salinity
use afid_phasefield
use afid_moisture, only: humid
use AuxiliaryRoutines

implicit none
Expand All @@ -13,8 +15,6 @@ subroutine ReadFlowInterp(prow,pcol)

character*70 :: filnam,dsetname

integer :: i, j, k, ic, jc, kc

real :: yleno, zleno

logical :: fexist
Expand Down Expand Up @@ -160,7 +160,11 @@ subroutine ReadFlowInterp(prow,pcol)
inquire(file=filnam, exist=fexist)
if (fexist) then
call HdfReadContinua(nz, ny, nx, xstart(2), xend(2), &
xstart(3), xend(3), 1, pr)
xstart(3), xend(3), 7, pr)
end if
if (moist) then
call HdfReadContinua(nz, ny, nx, xstart(2), xend(2), &
xstart(3), xend(3), 8, humid)
end if
end if

Expand Down
Loading

0 comments on commit cf92876

Please sign in to comment.