From d9e4443c05f520f37dd5208cec4ff2303371a6da Mon Sep 17 00:00:00 2001 From: Javier Principe Date: Wed, 22 May 2024 08:41:04 +0200 Subject: [PATCH 1/5] Changed mumps default options --- src/Solvers/petsc.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Solvers/petsc.jl b/src/Solvers/petsc.jl index 1b01563..4bd12af 100644 --- a/src/Solvers/petsc.jl +++ b/src/Solvers/petsc.jl @@ -15,12 +15,12 @@ function petsc_mumps_setup(ksp) @check_error_code GridapPETSc.PETSC.PCFactorSetMatSolverType(pc[],GridapPETSc.PETSC.MATSOLVERMUMPS) @check_error_code GridapPETSc.PETSC.PCFactorSetUpMatSolverType(pc[]) @check_error_code GridapPETSc.PETSC.PCFactorGetMatrix(pc[],mumpsmat) - @check_error_code GridapPETSc.PETSC.MatMumpsSetIcntl(mumpsmat[], 4, 1) + # @check_error_code GridapPETSc.PETSC.MatMumpsSetIcntl(mumpsmat[], 4, 1) @check_error_code GridapPETSc.PETSC.MatMumpsSetIcntl(mumpsmat[], 7, 0) - @check_error_code GridapPETSc.PETSC.MatMumpsSetIcntl(mumpsmat[], 28, 2) - @check_error_code GridapPETSc.PETSC.MatMumpsSetIcntl(mumpsmat[], 29, 2) - @check_error_code GridapPETSc.PETSC.MatMumpsSetCntl(mumpsmat[], 3, 1.0e-6) - @check_error_code GridapPETSc.PETSC.KSPView(ksp[],C_NULL) + # @check_error_code GridapPETSc.PETSC.MatMumpsSetIcntl(mumpsmat[], 28, 2) + # @check_error_code GridapPETSc.PETSC.MatMumpsSetIcntl(mumpsmat[], 29, 2) + # @check_error_code GridapPETSc.PETSC.MatMumpsSetCntl(mumpsmat[], 3, 1.0e-6) + # @check_error_code GridapPETSc.PETSC.KSPView(ksp[],C_NULL) end # Standalone AMG solver, 5 iterations From c52b9d69ec710f4f9a9bbfbe1b24dc5918e3037d Mon Sep 17 00:00:00 2001 From: Javier Principe Date: Wed, 22 May 2024 08:41:35 +0200 Subject: [PATCH 2/5] New sequential test showing an error --- test/seq/hunt_tests.jl | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/seq/hunt_tests.jl b/test/seq/hunt_tests.jl index 10a1212..d8840a4 100644 --- a/test/seq/hunt_tests.jl +++ b/test/seq/hunt_tests.jl @@ -47,4 +47,17 @@ hunt( solver=:julia, ) +hunt( + nc=(4,4), + np=(1,1), + backend=:sequential, + L=1.0, + B=(0.,20.,0.), + nsums = 100, + debug=false, + vtk=true, + title="hunt", + solver=:li2019, +)' + end # module From 6ca458a36577f5580d92fd42956d643af489a9f3 Mon Sep 17 00:00:00 2001 From: Javier Principe Date: Wed, 22 May 2024 12:14:29 +0200 Subject: [PATCH 3/5] Added mpi tests (only hunt w np=1) --- test/mpi/hunt_li2019_tests.jl | 156 +++++++++++++++++++--------------- test/mpi/hunt_tests.jl | 26 +++--- test/mpi/runtests.jl | 22 +++-- test/mpi/runtests_body.jl | 18 ++++ test/runtests.jl | 2 + test/seq/hunt_tests.jl | 13 --- 6 files changed, 134 insertions(+), 103 deletions(-) create mode 100644 test/mpi/runtests_body.jl diff --git a/test/mpi/hunt_li2019_tests.jl b/test/mpi/hunt_li2019_tests.jl index 49b27dd..e237471 100644 --- a/test/mpi/hunt_li2019_tests.jl +++ b/test/mpi/hunt_li2019_tests.jl @@ -5,76 +5,94 @@ using SparseMatricesCSR using GridapMHD: hunt -# Li2019, MUMPS for Dj -solver = Dict( - :solver => :li2019, - :matrix_type => SparseMatrixCSR{0,PetscScalar,PetscInt}, - :vector_type => Vector{PetscScalar}, - :block_solvers => [:petsc_mumps,:petsc_gmres_schwarz,:petsc_cg_jacobi,:petsc_cg_jacobi], - :petsc_options => "-ksp_error_if_not_converged true -ksp_converged_reason" -) -hunt( - nc=(6,6), - np=(2,2), - backend=:mpi, - L=1.0, - B=(0.,50.,0.), - debug=false, - vtk=true, - title="hunt", - solver=solver, -) +function main(parts) -# Li2019, GMRES + Overlapping Additive Schwarz preconditioner for Dj -solver = Dict( - :solver => :li2019, - :matrix_type => SparseMatrixCSR{0,PetscScalar,PetscInt}, - :vector_type => Vector{PetscScalar}, - :block_solvers => [:petsc_gmres_schwarz,:petsc_gmres_schwarz,:petsc_cg_jacobi,:petsc_cg_jacobi], - :petsc_options => "-ksp_error_if_not_converged true -ksp_converged_reason" -) -hunt( - nc=(4,4), - np=(2,2), - backend=:sequential, - L=1.0, - B=(0.,50.,0.), - debug=false, - vtk=true, - title="hunt", - solver=solver, -) + # Default li2019 + hunt( + nc=(4,4), + np=parts, + backend=:mpi, + L=1.0, + B=(0.0, 20.0, 0.0), + nsums=100, + debug=false, + vtk=true, + title="hunt", + solver=:li2019, + )' -# Li2019, GMRES + Overlapping Additive Schwarz preconditioner for Dj (tunned by options) -petsc_options = """ - -ksp_type gmres - -ksp_rtol 1.0e-5 - -ksp_atol 1.0e-14 - -ksp_converged_reason - -pc_type asm - -pc_asm_overlap 10 - -pc_asm_type restrict - -pc_asm_blocks 32 - -sub_ksp_type preonly - -sub_pc_type lu -""" -solver = Dict( - :solver => :li2019, - :matrix_type => SparseMatrixCSR{0,PetscScalar,PetscInt}, - :vector_type => Vector{PetscScalar}, - :block_solvers => [:petsc_from_options,:petsc_gmres_schwarz,:petsc_cg_jacobi,:petsc_cg_jacobi], - :petsc_options => petsc_options -) -hunt( - nc=(4,4), - np=(2,2), - backend=:sequential, - L=1.0, - B=(0.,50.,0.), - debug=false, - vtk=true, - title="hunt", - solver=solver, -) + # # Li2019, MUMPS for Dj + # solver = Dict( + # :solver => :li2019, + # :matrix_type => SparseMatrixCSR{0,PetscScalar,PetscInt}, + # :vector_type => Vector{PetscScalar}, + # :block_solvers => [:petsc_mumps, :petsc_gmres_schwarz, :petsc_cg_jacobi, :petsc_cg_jacobi], + # :petsc_options => "-ksp_error_if_not_converged true -ksp_converged_reason" + # ) + # hunt( + # nc=(6, 6), + # np=parts, + # backend=:mpi, + # L=1.0, + # B=(0.0, 50.0, 0.0), + # debug=false, + # vtk=true, + # title="hunt", + # solver=solver, + # ) + + # # Li2019, GMRES + Overlapping Additive Schwarz preconditioner for Dj + # solver = Dict( + # :solver => :li2019, + # :matrix_type => SparseMatrixCSR{0,PetscScalar,PetscInt}, + # :vector_type => Vector{PetscScalar}, + # :block_solvers => [:petsc_gmres_schwarz, :petsc_gmres_schwarz, :petsc_cg_jacobi, :petsc_cg_jacobi], + # :petsc_options => "-ksp_error_if_not_converged true -ksp_converged_reason" + # ) + # hunt( + # nc=(4, 4), + # np=parts, + # backend=:sequential, + # L=1.0, + # B=(0.0, 50.0, 0.0), + # debug=false, + # vtk=true, + # title="hunt", + # solver=solver, + # ) + + # # Li2019, GMRES + Overlapping Additive Schwarz preconditioner for Dj (tunned by options) + # petsc_options = """ + # -ksp_type gmres + # -ksp_rtol 1.0e-5 + # -ksp_atol 1.0e-14 + # -ksp_converged_reason + # -pc_type asm + # -pc_asm_overlap 10 + # -pc_asm_type restrict + # -pc_asm_blocks 32 + # -sub_ksp_type preonly + # -sub_pc_type lu + # """ + # solver = Dict( + # :solver => :li2019, + # :matrix_type => SparseMatrixCSR{0,PetscScalar,PetscInt}, + # :vector_type => Vector{PetscScalar}, + # :block_solvers => [:petsc_from_options, :petsc_gmres_schwarz, :petsc_cg_jacobi, :petsc_cg_jacobi], + # :petsc_options => petsc_options + # ) + # hunt( + # nc=(4, 4), + # np=parts, + # backend=:sequential, + # L=1.0, + # B=(0.0, 50.0, 0.0), + # debug=false, + # vtk=true, + # title="hunt", + # solver=solver, + # ) + +end # main end # module \ No newline at end of file diff --git a/test/mpi/hunt_tests.jl b/test/mpi/hunt_tests.jl index ebc0589..0006663 100644 --- a/test/mpi/hunt_tests.jl +++ b/test/mpi/hunt_tests.jl @@ -5,17 +5,19 @@ using SparseMatricesCSR using GridapMHD: hunt -hunt( - nc=(4,4), - np=(2,2), - backend=:mpi, - L=1.0, - B=(0.,50.,0.), - debug=false, - vtk=true, - title="hunt", - solver=:petsc, -) - +function main(parts) + # Default monolithic solver w petsc + hunt( + nc=(4,4), + np=parts, + backend=:mpi, + L=1.0, + B=(0.,50.,0.), + debug=false, + vtk=true, + title="hunt", + solver=:petsc, + ) +end end \ No newline at end of file diff --git a/test/mpi/runtests.jl b/test/mpi/runtests.jl index be35fdf..2805999 100644 --- a/test/mpi/runtests.jl +++ b/test/mpi/runtests.jl @@ -3,18 +3,22 @@ module GridapMHDMPITests using Test using MPI + testdir = @__DIR__ -istest(f) = endswith(f, ".jl") && !(f=="runtests.jl") -testfiles = sort(filter(istest, readdir(testdir))) +# istest(f) = endswith(f, ".jl") && !(f == "runtests.jl") +# testfiles = sort(filter(istest, readdir(testdir))) -MPI.mpiexec() do cmd - for file in testfiles - path = joinpath(testdir,file) - _cmd = `$(cmd) -np 4 --allow-run-as-root --oversubscribe $(Base.julia_cmd()) --project=. $path` - @show _cmd - run(_cmd) - @test true +function run_test(n, file) + MPI.mpiexec() do cmd + path = joinpath(testdir, file) + _cmd = `$(cmd) -np $n --allow-run-as-root --oversubscribe $(Base.julia_cmd()) --project=. $path` + @show _cmd + run(_cmd) + @test true end end +run_test(1,"runtests_body.jl") +# run_test(4,"runtests_body.jl") + end \ No newline at end of file diff --git a/test/mpi/runtests_body.jl b/test/mpi/runtests_body.jl new file mode 100644 index 0000000..69ef5e4 --- /dev/null +++ b/test/mpi/runtests_body.jl @@ -0,0 +1,18 @@ +using MPI + +if ! MPI.Initialized() + MPI.Init() +end +if MPI.Comm_size(MPI.COMM_WORLD) == 4 + parts=(2,2) +elseif MPI.Comm_size(MPI.COMM_WORLD) == 1 + parts=(1,1) +else + MPI.Abort(MPI.COMM_WORLD,0) +end + +# include("hunt_tests.jl") +# HuntTestsMPI.main(parts) + +include("hunt_li2019_tests.jl") +HuntLi2019TestsMPI.main(parts) diff --git a/test/runtests.jl b/test/runtests.jl index 7c34b38..0e79518 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,3 +2,5 @@ using Test include("seq/runtests.jl") + +include("mpi/runtests.jl") diff --git a/test/seq/hunt_tests.jl b/test/seq/hunt_tests.jl index d8840a4..10a1212 100644 --- a/test/seq/hunt_tests.jl +++ b/test/seq/hunt_tests.jl @@ -47,17 +47,4 @@ hunt( solver=:julia, ) -hunt( - nc=(4,4), - np=(1,1), - backend=:sequential, - L=1.0, - B=(0.,20.,0.), - nsums = 100, - debug=false, - vtk=true, - title="hunt", - solver=:li2019, -)' - end # module From e8688ebfb3c388a791e2a48f738975579a414b16 Mon Sep 17 00:00:00 2001 From: Javier Principe Date: Fri, 24 May 2024 09:07:33 +0200 Subject: [PATCH 4/5] Fixed mpi test and added new ones for Li2019 solver --- test/mpi/hunt_li2019_tests.jl | 140 +++++++++++++++++----------------- 1 file changed, 70 insertions(+), 70 deletions(-) diff --git a/test/mpi/hunt_li2019_tests.jl b/test/mpi/hunt_li2019_tests.jl index e237471..9628e72 100644 --- a/test/mpi/hunt_li2019_tests.jl +++ b/test/mpi/hunt_li2019_tests.jl @@ -19,79 +19,79 @@ function main(parts) vtk=true, title="hunt", solver=:li2019, - )' + ) - # # Li2019, MUMPS for Dj - # solver = Dict( - # :solver => :li2019, - # :matrix_type => SparseMatrixCSR{0,PetscScalar,PetscInt}, - # :vector_type => Vector{PetscScalar}, - # :block_solvers => [:petsc_mumps, :petsc_gmres_schwarz, :petsc_cg_jacobi, :petsc_cg_jacobi], - # :petsc_options => "-ksp_error_if_not_converged true -ksp_converged_reason" - # ) - # hunt( - # nc=(6, 6), - # np=parts, - # backend=:mpi, - # L=1.0, - # B=(0.0, 50.0, 0.0), - # debug=false, - # vtk=true, - # title="hunt", - # solver=solver, - # ) + # Li2019, MUMPS for Dj + solver = Dict( + :solver => :li2019, + :matrix_type => SparseMatrixCSR{0,PetscScalar,PetscInt}, + :vector_type => Vector{PetscScalar}, + :block_solvers => [:petsc_mumps, :petsc_gmres_schwarz, :petsc_cg_jacobi, :petsc_cg_jacobi], + :petsc_options => "-ksp_error_if_not_converged true -ksp_converged_reason" + ) + hunt( + nc=(6, 6), + np=parts, + backend=:mpi, + L=1.0, + B=(0.0, 50.0, 0.0), + debug=false, + vtk=true, + title="hunt", + solver=solver, + ) - # # Li2019, GMRES + Overlapping Additive Schwarz preconditioner for Dj - # solver = Dict( - # :solver => :li2019, - # :matrix_type => SparseMatrixCSR{0,PetscScalar,PetscInt}, - # :vector_type => Vector{PetscScalar}, - # :block_solvers => [:petsc_gmres_schwarz, :petsc_gmres_schwarz, :petsc_cg_jacobi, :petsc_cg_jacobi], - # :petsc_options => "-ksp_error_if_not_converged true -ksp_converged_reason" - # ) - # hunt( - # nc=(4, 4), - # np=parts, - # backend=:sequential, - # L=1.0, - # B=(0.0, 50.0, 0.0), - # debug=false, - # vtk=true, - # title="hunt", - # solver=solver, - # ) + # Li2019, GMRES + Overlapping Additive Schwarz preconditioner for Dj + solver = Dict( + :solver => :li2019, + :matrix_type => SparseMatrixCSR{0,PetscScalar,PetscInt}, + :vector_type => Vector{PetscScalar}, + :block_solvers => [:petsc_gmres_schwarz, :petsc_gmres_schwarz, :petsc_cg_jacobi, :petsc_cg_jacobi], + :petsc_options => "-ksp_error_if_not_converged true -ksp_converged_reason" + ) + hunt( + nc=(4, 4), + np=parts, + backend=:sequential, + L=1.0, + B=(0.0, 50.0, 0.0), + debug=false, + vtk=true, + title="hunt", + solver=solver, + ) - # # Li2019, GMRES + Overlapping Additive Schwarz preconditioner for Dj (tunned by options) - # petsc_options = """ - # -ksp_type gmres - # -ksp_rtol 1.0e-5 - # -ksp_atol 1.0e-14 - # -ksp_converged_reason - # -pc_type asm - # -pc_asm_overlap 10 - # -pc_asm_type restrict - # -pc_asm_blocks 32 - # -sub_ksp_type preonly - # -sub_pc_type lu - # """ - # solver = Dict( - # :solver => :li2019, - # :matrix_type => SparseMatrixCSR{0,PetscScalar,PetscInt}, - # :vector_type => Vector{PetscScalar}, - # :block_solvers => [:petsc_from_options, :petsc_gmres_schwarz, :petsc_cg_jacobi, :petsc_cg_jacobi], - # :petsc_options => petsc_options - # ) - # hunt( - # nc=(4, 4), - # np=parts, - # backend=:sequential, - # L=1.0, - # B=(0.0, 50.0, 0.0), - # debug=false, - # vtk=true, - # title="hunt", - # solver=solver, - # ) + # Li2019, GMRES + Overlapping Additive Schwarz preconditioner for Dj (tunned by options) + petsc_options = """ + -ksp_type gmres + -ksp_rtol 1.0e-5 + -ksp_atol 1.0e-14 + -ksp_converged_reason + -pc_type asm + -pc_asm_overlap 10 + -pc_asm_type restrict + -pc_asm_blocks 32 + -sub_ksp_type preonly + -sub_pc_type lu + """ + solver = Dict( + :solver => :li2019, + :matrix_type => SparseMatrixCSR{0,PetscScalar,PetscInt}, + :vector_type => Vector{PetscScalar}, + :block_solvers => [:petsc_from_options, :petsc_gmres_schwarz, :petsc_cg_jacobi, :petsc_cg_jacobi], + :petsc_options => petsc_options + ) + hunt( + nc=(4, 4), + np=parts, + backend=:sequential, + L=1.0, + B=(0.0, 50.0, 0.0), + debug=false, + vtk=true, + title="hunt", + solver=solver, + ) end # main From 0768fc6a0bb0589e8ec10734b33b4bf0846ac37a Mon Sep 17 00:00:00 2001 From: Fernando Roca Date: Mon, 27 May 2024 12:54:21 +0200 Subject: [PATCH 5/5] Revert "Added access to pets_options in hunt.jl" This reverts commit ca33bb8224e69a1d6f2576085fbc25359acb1062. --- src/Applications/hunt.jl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Applications/hunt.jl b/src/Applications/hunt.jl index ad8017b..e30013f 100644 --- a/src/Applications/hunt.jl +++ b/src/Applications/hunt.jl @@ -60,7 +60,6 @@ function _hunt(; jac_assemble = false, solve = true, solver = :julia, - petsc_options = nothing, #Added to have access to non-default petsc options verbose = true, BL_adapted = true, kmap_x = 1, @@ -76,10 +75,6 @@ function _hunt(; :jac_assemble=>jac_assemble, ) - if isa(petsc_options,Nothing) - params[:solver][:petsc_options] = petsc_options - end - # Communicator if isa(distribute,Nothing) @assert isa(rank_partition,Nothing)